From b78f45a98ab52d1c80213618bc1ba2cbbc68eb9d Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 13 Mar 2026 00:02:54 +0000 Subject: [PATCH 01/65] WIP --- .../org/apache/accumulo/core/Constants.java | 1 + .../core/client/admin/InstanceOperations.java | 1 + .../core/clientImpl/ClientContext.java | 3 +- .../apache/accumulo/server/ServerContext.java | 13 +++ .../compaction/CoordinatorLocations.java | 80 ++++++++++++++++ .../server/init/ZooKeeperInitializer.java | 2 + .../apache/accumulo/compactor/Compactor.java | 9 +- .../org/apache/accumulo/manager/Manager.java | 6 ++ .../coordinator/CompactionCoordinator.java | 4 +- .../coordinator/CoordinatorManager.java | 92 +++++++++++++++++++ 10 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocations.java create mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java index eb8ba1059eb..0d1eeffffbc 100644 --- a/core/src/main/java/org/apache/accumulo/core/Constants.java +++ b/core/src/main/java/org/apache/accumulo/core/Constants.java @@ -52,6 +52,7 @@ public class Constants { public static final String ZMANAGER_ASSISTANT_LOCK = ZMANAGERS + "/assistants"; public static final String ZMANAGER_GOAL_STATE = ZMANAGERS + "/goal_state"; public static final String ZMANAGER_TICK = ZMANAGERS + "/tick"; + public static final String ZMANAGER_COORDINATOR = ZMANAGERS + "/coordinators"; public static final String ZGC = "/gc"; public static final String ZGC_LOCK = ZGC + "/lock"; diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java index 72d602704c0..b679edc9efe 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java @@ -247,6 +247,7 @@ Map modifyProperties(Consumer> mapMutator) * @return set of servers of the supplied type * @since 4.0.0 */ + // TODO this needs to change for multiple managers Set getServers(ServerId.Type type); /** diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java index 8b65091c491..4c8ecb15860 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java @@ -1333,7 +1333,8 @@ private static Set createPersistentWatcherPaths() { Constants.ZMANAGER_LOCK, Constants.ZMINI_LOCK, Constants.ZMONITOR_LOCK, Constants.ZNAMESPACES, Constants.ZRECOVERY, Constants.ZSSERVERS, Constants.ZTABLES, Constants.ZTSERVERS, Constants.ZUSERS, RootTable.ZROOT_TABLET, Constants.ZTEST_LOCK, - Constants.ZMANAGER_ASSISTANT_LOCK, Constants.ZRESOURCEGROUPS)) { + Constants.ZMANAGER_ASSISTANT_LOCK, Constants.ZRESOURCEGROUPS, + Constants.ZMANAGER_COORDINATOR)) { pathsToWatch.add(path); } return pathsToWatch; diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java index b12806dc13d..c5b66175cab 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java @@ -66,6 +66,7 @@ import org.apache.accumulo.core.util.AddressUtil; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; +import org.apache.accumulo.server.compaction.CoordinatorLocations; import org.apache.accumulo.server.conf.NamespaceConfiguration; import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.conf.TableConfiguration; @@ -89,6 +90,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; +import com.google.common.net.HostAndPort; /** * Provides a server context for Accumulo server components that operate with the system credentials @@ -119,6 +121,8 @@ public class ServerContext extends ClientContext { private final AtomicBoolean sharedSchedExecutorCreated = new AtomicBoolean(false); private final AtomicBoolean sharedMetadataWriterCreated = new AtomicBoolean(false); private final AtomicBoolean sharedUserWriterCreated = new AtomicBoolean(false); + private final AtomicReference coordinatorLocationsRef = + new AtomicReference<>(); public ServerContext(SiteConfiguration siteConfig) { this(ServerInfo.fromServerConfig(siteConfig), ResourceGroupId.DEFAULT); @@ -528,6 +532,15 @@ public Supplier getSharedUserWriter() { return sharedUserWriter; } + public Map getCoordinatorLocations(boolean useCache) { + var cl = coordinatorLocationsRef.get(); + if (cl == null) { + coordinatorLocationsRef.compareAndSet(null, new CoordinatorLocations(this)); + cl = coordinatorLocationsRef.get(); + } + return cl.getLocations(useCache); + } + @Override public void close() { Preconditions.checkState(!isClosed(), "ServerContext.close was already called."); diff --git a/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocations.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocations.java new file mode 100644 index 00000000000..0338cb74b66 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocations.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.compaction; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.accumulo.core.util.LazySingletons.GSON; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; +import org.apache.accumulo.server.ServerContext; +import org.apache.zookeeper.KeeperException; + +import com.google.common.net.HostAndPort; +import com.google.common.reflect.TypeToken; + +/** + * Reads and writes a map of coordinators to zookeeper. The map contains compactor resource group to + * manager address mappings. + */ +public class CoordinatorLocations { + private final ServerContext context; + + private long lastUpdateCount; + private Map lastLocations; + + public CoordinatorLocations(ServerContext context) { + this.context = context; + } + + public synchronized Map getLocations(boolean useCache) { + var zooCache = context.getZooCache(); + // TODO how frequently does updateCount change? TODO could have an update counter per path + if (lastLocations == null || lastUpdateCount != zooCache.getUpdateCount() || !useCache) { + lastUpdateCount = zooCache.getUpdateCount(); + if (!useCache) { + zooCache.clear(Constants.ZMANAGER_COORDINATOR); + } + byte[] serializedMap = zooCache.get(Constants.ZMANAGER_COORDINATOR); + var type = new TypeToken>() {}.getType(); + Map stringMap = GSON.get().fromJson(new String(serializedMap, UTF_8), type); + Map locations = new HashMap<>(); + stringMap + .forEach((rg, hp) -> locations.put(ResourceGroupId.of(rg), HostAndPort.fromString(hp))); + lastLocations = Map.copyOf(locations); + } + return lastLocations; + } + + public static void setLocations(ZooReaderWriter zk, Map locations) + throws InterruptedException, KeeperException { + Map stringMap = new HashMap<>(locations.size()); + locations.forEach((rg, hp) -> { + stringMap.put(rg.canonical(), hp.toString()); + }); + byte[] serializedMap = GSON.get().toJson(stringMap).getBytes(UTF_8); + zk.putPersistentData(Constants.ZMANAGER_COORDINATOR, serializedMap, + ZooUtil.NodeExistsPolicy.OVERWRITE); + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java index 464a6dae5a6..5389c24eab5 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java @@ -45,6 +45,7 @@ import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.util.tables.TableMapping; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.compaction.CoordinatorLocations; import org.apache.accumulo.server.conf.codec.VersionedPropCodec; import org.apache.accumulo.server.conf.codec.VersionedProperties; import org.apache.accumulo.server.conf.store.ResourceGroupPropKey; @@ -178,6 +179,7 @@ void initialize(final ServerContext context, final String rootTabletDirName, ZooUtil.NodeExistsPolicy.FAIL); zrwChroot.putPersistentData(Constants.ZMANAGER_ASSISTANT_LOCK, EMPTY_BYTE_ARRAY, ZooUtil.NodeExistsPolicy.FAIL); + CoordinatorLocations.setLocations(zrwChroot, Map.of()); } /** diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 66f11b7eb2d..93887ac4835 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -568,13 +568,12 @@ protected TNextCompactionJob getNextJob(Supplier uuid) throws RetriesExcee * @throws TTransportException when unable to get client */ protected CompactionCoordinatorService.Client getCoordinatorClient() throws TTransportException { - var coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(getContext()); - if (coordinatorHost.isEmpty()) { + var coordinatorHost = getContext().getCoordinatorLocations(true).get(getResourceGroup()); + if (coordinatorHost == null) { throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper"); } - LOG.trace("CompactionCoordinator address is: {}", coordinatorHost.orElseThrow()); - return ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost.orElseThrow(), - getContext()); + LOG.trace("CompactionCoordinator address is: {}", coordinatorHost); + return ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost, getContext()); } /** diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 5ff232765c4..8a9b3dbf9d1 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -119,6 +119,7 @@ import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.core.zookeeper.ZcStat; import org.apache.accumulo.manager.compaction.coordinator.CompactionCoordinator; +import org.apache.accumulo.manager.compaction.coordinator.CoordinatorManager; import org.apache.accumulo.manager.fate.FateManager; import org.apache.accumulo.manager.fate.FateWorker; import org.apache.accumulo.manager.merge.FindMergeableRangeTask; @@ -232,6 +233,8 @@ public class Manager extends AbstractServer new AtomicReference<>(); private volatile FateManager fateManager; + private volatile CoordinatorManager coordinatorManager; + static class TServerStatus { // This is the set of tservers that an attempt to gather status from was made final LiveTServersSnapshot snapshot; @@ -1203,6 +1206,9 @@ boolean canSuspendTablets() { // Don't call start the CompactionCoordinator until we have tservers and upgrade is complete. compactionCoordinator.start(); + coordinatorManager = new CoordinatorManager(context); + coordinatorManager.start(); + this.splitter = new Splitter(this); this.splitter.start(); this.fileRangeCache = new FileRangeCache(context); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 19397286222..f895a89466e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -1286,7 +1286,7 @@ private void cleanUpEmptyCompactorPathInZK() { } } - private Set getCompactionServicesConfigurationGroups() + static Set getCompactionServicesConfigurationGroups(ServerContext ctx) throws ReflectiveOperationException, IllegalArgumentException, SecurityException { Set groups = new HashSet<>(); @@ -1357,7 +1357,7 @@ public void cleanUpInternalState() { // Get the set of groups being referenced in the current configuration Set groupsInConfiguration = null; try { - groupsInConfiguration = getCompactionServicesConfigurationGroups(); + groupsInConfiguration = getCompactionServicesConfigurationGroups(ctx); } catch (RuntimeException | ReflectiveOperationException e) { LOG.error( "Error getting groups from the compaction services configuration. Unable to clean up internal state.", diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java new file mode 100644 index 00000000000..c42d37e1449 --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.compaction.coordinator; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.core.util.threads.Threads; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.compaction.CoordinatorLocations; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; +import com.google.common.net.HostAndPort; + +/** + * Handles assigning compactor resource groups to different manager processes. This is run by the + * primary manager to distribute compaction coordination. + */ +public class CoordinatorManager { + + private static final Logger log = LoggerFactory.getLogger(CoordinatorManager.class); + + private final ServerContext context; + + public CoordinatorManager(ServerContext context) { + this.context = context; + } + + private Thread assignmentThread = null; + + private void managerCoordinators() { + while (true) { + try { + var serverIds = context.instanceOperations().getServers(ServerId.Type.MANAGER); + if (!serverIds.isEmpty()) { + var primary = HostAndPort.fromString(serverIds.iterator().next().toHostPortString()); + var compactorGroups = + CompactionCoordinator.getCompactionServicesConfigurationGroups(context); + Map assignments = new HashMap<>(); + compactorGroups.forEach(rg -> assignments.put(rg, primary)); + CoordinatorLocations.setLocations(context.getZooSession().asReaderWriter(), assignments); + log.debug("Set locations {}", assignments); + } + // TODO better exception handling + } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException + | InterruptedException | KeeperException e) { + log.warn("Failed to manage coordinators", e); + } + + // TODO not good for test + UtilWaitThread.sleep(5000); + } + + } + + public synchronized void start() { + Preconditions.checkState(assignmentThread == null); + + assignmentThread = Threads.createCriticalThread("Coordinator Manager", () -> { + try { + managerCoordinators(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + assignmentThread.start(); + } + + // TODO stop() +} From 08796e6faa496b740f708919adcd2f3d694ecf29 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 13 Mar 2026 22:02:56 +0000 Subject: [PATCH 02/65] sends compaction jobs from TGW to coordinator via RPC There is still only one coordinator, but now the TGW and compactors both could talk to multiple coordinators. --- .../thrift/CompactionCoordinatorService.java | 3443 ++++++++++++++++- .../compaction/thrift/CompactorService.java | 36 +- .../thrift/TResolvedCompactionJob.java | 1283 ++++++ .../main/thrift/compaction-coordinator.thrift | 37 + core/src/main/thrift/tabletserver.thrift | 2 +- .../accumulo/manager/TabletGroupWatcher.java | 20 +- .../compaction/CompactionJobClient.java | 140 + .../coordinator/CompactionCoordinator.java | 37 + .../queue/ResolvedCompactionJob.java | 37 + 9 files changed, 4950 insertions(+), 85 deletions(-) create mode 100644 core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TResolvedCompactionJob.java create mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index c9d2708c86b..55e94e1044c 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -47,6 +47,12 @@ public interface Iface { public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException; + public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + + public void addJobs(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.List jobs) throws org.apache.thrift.TException; + + public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + } public interface AsyncIface { @@ -69,6 +75,12 @@ public interface AsyncIface { public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void addJobs(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.List jobs, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -358,6 +370,79 @@ public void send_recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TIn sendBaseOneway("recordCompletion", args); } + @Override + public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + { + send_beginFullJobScan(tinfo, credentials, dataLevel); + recv_beginFullJobScan(); + } + + public void send_beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.thrift.TException + { + beginFullJobScan_args args = new beginFullJobScan_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setDataLevel(dataLevel); + sendBase("beginFullJobScan", args); + } + + public void recv_beginFullJobScan() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + { + beginFullJobScan_result result = new beginFullJobScan_result(); + receiveBase(result, "beginFullJobScan"); + if (result.sec != null) { + throw result.sec; + } + if (result.tnase != null) { + throw result.tnase; + } + return; + } + + @Override + public void addJobs(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.List jobs) throws org.apache.thrift.TException + { + send_addJobs(tinfo, credentials, jobs); + } + + public void send_addJobs(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.List jobs) throws org.apache.thrift.TException + { + addJobs_args args = new addJobs_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setJobs(jobs); + sendBaseOneway("addJobs", args); + } + + @Override + public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + { + send_endFullJobScan(tinfo, credentials, dataLevel); + recv_endFullJobScan(); + } + + public void send_endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.thrift.TException + { + endFullJobScan_args args = new endFullJobScan_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setDataLevel(dataLevel); + sendBase("endFullJobScan", args); + } + + public void recv_endFullJobScan() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + { + endFullJobScan_result result = new endFullJobScan_result(); + receiveBase(result, "endFullJobScan"); + if (result.sec != null) { + throw result.sec; + } + if (result.tnase != null) { + throw result.tnase; + } + return; + } + } public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -768,6 +853,131 @@ public Void getResult() throws org.apache.thrift.TException { } } + @Override + public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + beginFullJobScan_call method_call = new beginFullJobScan_call(tinfo, credentials, dataLevel, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class beginFullJobScan_call extends org.apache.thrift.async.TAsyncMethodCall { + private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; + private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; + private java.lang.String dataLevel; + public beginFullJobScan_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tinfo = tinfo; + this.credentials = credentials; + this.dataLevel = dataLevel; + } + + @Override + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("beginFullJobScan", org.apache.thrift.protocol.TMessageType.CALL, 0)); + beginFullJobScan_args args = new beginFullJobScan_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setDataLevel(dataLevel); + args.write(prot); + prot.writeMessageEnd(); + } + + @Override + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new java.lang.IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_beginFullJobScan(); + return null; + } + } + + @Override + public void addJobs(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.List jobs, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + addJobs_call method_call = new addJobs_call(tinfo, credentials, jobs, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class addJobs_call extends org.apache.thrift.async.TAsyncMethodCall { + private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; + private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; + private java.util.List jobs; + public addJobs_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.List jobs, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, true); + this.tinfo = tinfo; + this.credentials = credentials; + this.jobs = jobs; + } + + @Override + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("addJobs", org.apache.thrift.protocol.TMessageType.ONEWAY, 0)); + addJobs_args args = new addJobs_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setJobs(jobs); + args.write(prot); + prot.writeMessageEnd(); + } + + @Override + public Void getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new java.lang.IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return null; + } + } + + @Override + public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + endFullJobScan_call method_call = new endFullJobScan_call(tinfo, credentials, dataLevel, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class endFullJobScan_call extends org.apache.thrift.async.TAsyncMethodCall { + private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; + private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; + private java.lang.String dataLevel; + public endFullJobScan_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tinfo = tinfo; + this.credentials = credentials; + this.dataLevel = dataLevel; + } + + @Override + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("endFullJobScan", org.apache.thrift.protocol.TMessageType.CALL, 0)); + endFullJobScan_args args = new endFullJobScan_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setDataLevel(dataLevel); + args.write(prot); + prot.writeMessageEnd(); + } + + @Override + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new java.lang.IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_endFullJobScan(); + return null; + } + } + } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -790,6 +1000,9 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { + public beginFullJobScan() { + super("beginFullJobScan"); + } + + @Override + public beginFullJobScan_args getEmptyArgsInstance() { + return new beginFullJobScan_args(); + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + protected boolean rethrowUnhandledExceptions() { + return false; + } + + @Override + public beginFullJobScan_result getResult(I iface, beginFullJobScan_args args) throws org.apache.thrift.TException { + beginFullJobScan_result result = new beginFullJobScan_result(); + try { + iface.beginFullJobScan(args.tinfo, args.credentials, args.dataLevel); + } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + result.sec = sec; + } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { + result.tnase = tnase; + } + return result; + } + } + + public static class addJobs extends org.apache.thrift.ProcessFunction { + public addJobs() { + super("addJobs"); + } + + @Override + public addJobs_args getEmptyArgsInstance() { + return new addJobs_args(); + } + + @Override + protected boolean isOneway() { + return true; + } + + @Override + protected boolean rethrowUnhandledExceptions() { + return false; + } + + @Override + public org.apache.thrift.TBase getResult(I iface, addJobs_args args) throws org.apache.thrift.TException { + iface.addJobs(args.tinfo, args.credentials, args.jobs); + return null; + } + } + + public static class endFullJobScan extends org.apache.thrift.ProcessFunction { + public endFullJobScan() { + super("endFullJobScan"); + } + + @Override + public endFullJobScan_args getEmptyArgsInstance() { + return new endFullJobScan_args(); + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + protected boolean rethrowUnhandledExceptions() { + return false; + } + + @Override + public endFullJobScan_result getResult(I iface, endFullJobScan_args args) throws org.apache.thrift.TException { + endFullJobScan_result result = new endFullJobScan_result(); + try { + iface.endFullJobScan(args.tinfo, args.credentials, args.dataLevel); + } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + result.sec = sec; + } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { + result.tnase = tnase; + } + return result; + } + } + } public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { @@ -1114,6 +1422,9 @@ protected AsyncProcessor(I iface, java.util.Map, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("compactionCompleted_args"); - - private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); - private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); - private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("extent", org.apache.thrift.protocol.TType.STRUCT, (short)4); - private static final org.apache.thrift.protocol.TField STATS_FIELD_DESC = new org.apache.thrift.protocol.TField("stats", org.apache.thrift.protocol.TType.STRUCT, (short)5); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new compactionCompleted_argsStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new compactionCompleted_argsTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required - public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - TINFO((short)1, "tinfo"), - CREDENTIALS((short)2, "credentials"), - EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"), - EXTENT((short)4, "extent"), - STATS((short)5, "stats"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } + public static class beginFullJobScan extends org.apache.thrift.AsyncProcessFunction { + public beginFullJobScan() { + super("beginFullJobScan"); + } + + @Override + public beginFullJobScan_args getEmptyArgsInstance() { + return new beginFullJobScan_args(); + } + + @Override + public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new org.apache.thrift.async.AsyncMethodCallback() { + @Override + public void onComplete(Void o) { + beginFullJobScan_result result = new beginFullJobScan_result(); + try { + fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + } catch (org.apache.thrift.transport.TTransportException e) { + _LOGGER.error("TTransportException writing to internal frame buffer", e); + fb.close(); + } catch (java.lang.Exception e) { + _LOGGER.error("Exception writing to internal frame buffer", e); + onError(e); + } + } + @Override + public void onError(java.lang.Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TSerializable msg; + beginFullJobScan_result result = new beginFullJobScan_result(); + if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { + result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; + result.setSecIsSet(true); + msg = result; + } else if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) { + result.tnase = (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) e; + result.setTnaseIsSet(true); + msg = result; + } else if (e instanceof org.apache.thrift.transport.TTransportException) { + _LOGGER.error("TTransportException inside handler", e); + fb.close(); + return; + } else if (e instanceof org.apache.thrift.TApplicationException) { + _LOGGER.error("TApplicationException inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TApplicationException)e; + } else { + _LOGGER.error("Exception inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + } catch (java.lang.Exception ex) { + _LOGGER.error("Exception writing to internal frame buffer", ex); + fb.close(); + } + } + }; + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + public void start(I iface, beginFullJobScan_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + iface.beginFullJobScan(args.tinfo, args.credentials, args.dataLevel,resultHandler); + } + } + + public static class addJobs extends org.apache.thrift.AsyncProcessFunction { + public addJobs() { + super("addJobs"); + } + + @Override + public addJobs_args getEmptyArgsInstance() { + return new addJobs_args(); + } + + @Override + public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new org.apache.thrift.async.AsyncMethodCallback() { + @Override + public void onComplete(Void o) { + } + @Override + public void onError(java.lang.Exception e) { + if (e instanceof org.apache.thrift.transport.TTransportException) { + _LOGGER.error("TTransportException inside handler", e); + fb.close(); + } else { + _LOGGER.error("Exception inside oneway handler", e); + } + } + }; + } + + @Override + protected boolean isOneway() { + return true; + } + + @Override + public void start(I iface, addJobs_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + iface.addJobs(args.tinfo, args.credentials, args.jobs,resultHandler); + } + } + + public static class endFullJobScan extends org.apache.thrift.AsyncProcessFunction { + public endFullJobScan() { + super("endFullJobScan"); + } + + @Override + public endFullJobScan_args getEmptyArgsInstance() { + return new endFullJobScan_args(); + } + + @Override + public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new org.apache.thrift.async.AsyncMethodCallback() { + @Override + public void onComplete(Void o) { + endFullJobScan_result result = new endFullJobScan_result(); + try { + fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + } catch (org.apache.thrift.transport.TTransportException e) { + _LOGGER.error("TTransportException writing to internal frame buffer", e); + fb.close(); + } catch (java.lang.Exception e) { + _LOGGER.error("Exception writing to internal frame buffer", e); + onError(e); + } + } + @Override + public void onError(java.lang.Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TSerializable msg; + endFullJobScan_result result = new endFullJobScan_result(); + if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { + result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; + result.setSecIsSet(true); + msg = result; + } else if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) { + result.tnase = (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) e; + result.setTnaseIsSet(true); + msg = result; + } else if (e instanceof org.apache.thrift.transport.TTransportException) { + _LOGGER.error("TTransportException inside handler", e); + fb.close(); + return; + } else if (e instanceof org.apache.thrift.TApplicationException) { + _LOGGER.error("TApplicationException inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TApplicationException)e; + } else { + _LOGGER.error("Exception inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + } catch (java.lang.Exception ex) { + _LOGGER.error("Exception writing to internal frame buffer", ex); + fb.close(); + } + } + }; + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + public void start(I iface, endFullJobScan_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + iface.endFullJobScan(args.tinfo, args.credentials, args.dataLevel,resultHandler); + } + } + + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class compactionCompleted_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("compactionCompleted_args"); + + private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("extent", org.apache.thrift.protocol.TType.STRUCT, (short)4); + private static final org.apache.thrift.protocol.TField STATS_FIELD_DESC = new org.apache.thrift.protocol.TField("stats", org.apache.thrift.protocol.TType.STRUCT, (short)5); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new compactionCompleted_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new compactionCompleted_argsTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required + public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TINFO((short)1, "tinfo"), + CREDENTIALS((short)2, "credentials"), + EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"), + EXTENT((short)4, "extent"), + STATS((short)5, "stats"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } } /** @@ -9267,16 +9766,16 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getLongRunningCompa case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { { - org.apache.thrift.protocol.TMap _map28 = iprot.readMapBegin(); - struct.success = new java.util.HashMap(2*_map28.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key29; - @org.apache.thrift.annotation.Nullable TExternalCompactionList _val30; - for (int _i31 = 0; _i31 < _map28.size; ++_i31) + org.apache.thrift.protocol.TMap _map36 = iprot.readMapBegin(); + struct.success = new java.util.HashMap(2*_map36.size); + @org.apache.thrift.annotation.Nullable java.lang.String _key37; + @org.apache.thrift.annotation.Nullable TExternalCompactionList _val38; + for (int _i39 = 0; _i39 < _map36.size; ++_i39) { - _key29 = iprot.readString(); - _val30 = new TExternalCompactionList(); - _val30.read(iprot); - struct.success.put(_key29, _val30); + _key37 = iprot.readString(); + _val38 = new TExternalCompactionList(); + _val38.read(iprot); + struct.success.put(_key37, _val38); } iprot.readMapEnd(); } @@ -9323,10 +9822,10 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getLongRunningComp oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (java.util.Map.Entry _iter32 : struct.success.entrySet()) + for (java.util.Map.Entry _iter40 : struct.success.entrySet()) { - oprot.writeString(_iter32.getKey()); - _iter32.getValue().write(oprot); + oprot.writeString(_iter40.getKey()); + _iter40.getValue().write(oprot); } oprot.writeMapEnd(); } @@ -9374,10 +9873,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getLongRunningCompa if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (java.util.Map.Entry _iter33 : struct.success.entrySet()) + for (java.util.Map.Entry _iter41 : struct.success.entrySet()) { - oprot.writeString(_iter33.getKey()); - _iter33.getValue().write(oprot); + oprot.writeString(_iter41.getKey()); + _iter41.getValue().write(oprot); } } } @@ -9395,16 +9894,16 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getLongRunningCompac java.util.BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { { - org.apache.thrift.protocol.TMap _map34 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT); - struct.success = new java.util.HashMap(2*_map34.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key35; - @org.apache.thrift.annotation.Nullable TExternalCompactionList _val36; - for (int _i37 = 0; _i37 < _map34.size; ++_i37) + org.apache.thrift.protocol.TMap _map42 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT); + struct.success = new java.util.HashMap(2*_map42.size); + @org.apache.thrift.annotation.Nullable java.lang.String _key43; + @org.apache.thrift.annotation.Nullable TExternalCompactionList _val44; + for (int _i45 = 0; _i45 < _map42.size; ++_i45) { - _key35 = iprot.readString(); - _val36 = new TExternalCompactionList(); - _val36.read(iprot); - struct.success.put(_key35, _val36); + _key43 = iprot.readString(); + _val44 = new TExternalCompactionList(); + _val44.read(iprot); + struct.success.put(_key43, _val44); } } struct.setSuccessIsSet(true); @@ -12203,5 +12702,2831 @@ private static S scheme(org.apache. } } + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class beginFullJobScan_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("beginFullJobScan_args"); + + private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField DATA_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("dataLevel", org.apache.thrift.protocol.TType.STRING, (short)3); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new beginFullJobScan_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new beginFullJobScan_argsTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required + public @org.apache.thrift.annotation.Nullable java.lang.String dataLevel; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TINFO((short)1, "tinfo"), + CREDENTIALS((short)2, "credentials"), + DATA_LEVEL((short)3, "dataLevel"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TINFO + return TINFO; + case 2: // CREDENTIALS + return CREDENTIALS; + case 3: // DATA_LEVEL + return DATA_LEVEL; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); + tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); + tmpMap.put(_Fields.DATA_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("dataLevel", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(beginFullJobScan_args.class, metaDataMap); + } + + public beginFullJobScan_args() { + } + + public beginFullJobScan_args( + org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, + org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, + java.lang.String dataLevel) + { + this(); + this.tinfo = tinfo; + this.credentials = credentials; + this.dataLevel = dataLevel; + } + + /** + * Performs a deep copy on other. + */ + public beginFullJobScan_args(beginFullJobScan_args other) { + if (other.isSetTinfo()) { + this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); + } + if (other.isSetCredentials()) { + this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); + } + if (other.isSetDataLevel()) { + this.dataLevel = other.dataLevel; + } + } + + @Override + public beginFullJobScan_args deepCopy() { + return new beginFullJobScan_args(this); + } + + @Override + public void clear() { + this.tinfo = null; + this.credentials = null; + this.dataLevel = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { + return this.tinfo; + } + + public beginFullJobScan_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { + this.tinfo = tinfo; + return this; + } + + public void unsetTinfo() { + this.tinfo = null; + } + + /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ + public boolean isSetTinfo() { + return this.tinfo != null; + } + + public void setTinfoIsSet(boolean value) { + if (!value) { + this.tinfo = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { + return this.credentials; + } + + public beginFullJobScan_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { + this.credentials = credentials; + return this; + } + + public void unsetCredentials() { + this.credentials = null; + } + + /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ + public boolean isSetCredentials() { + return this.credentials != null; + } + + public void setCredentialsIsSet(boolean value) { + if (!value) { + this.credentials = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getDataLevel() { + return this.dataLevel; + } + + public beginFullJobScan_args setDataLevel(@org.apache.thrift.annotation.Nullable java.lang.String dataLevel) { + this.dataLevel = dataLevel; + return this; + } + + public void unsetDataLevel() { + this.dataLevel = null; + } + + /** Returns true if field dataLevel is set (has been assigned a value) and false otherwise */ + public boolean isSetDataLevel() { + return this.dataLevel != null; + } + + public void setDataLevelIsSet(boolean value) { + if (!value) { + this.dataLevel = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case TINFO: + if (value == null) { + unsetTinfo(); + } else { + setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); + } + break; + + case CREDENTIALS: + if (value == null) { + unsetCredentials(); + } else { + setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); + } + break; + + case DATA_LEVEL: + if (value == null) { + unsetDataLevel(); + } else { + setDataLevel((java.lang.String)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case TINFO: + return getTinfo(); + + case CREDENTIALS: + return getCredentials(); + + case DATA_LEVEL: + return getDataLevel(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case TINFO: + return isSetTinfo(); + case CREDENTIALS: + return isSetCredentials(); + case DATA_LEVEL: + return isSetDataLevel(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof beginFullJobScan_args) + return this.equals((beginFullJobScan_args)that); + return false; + } + + public boolean equals(beginFullJobScan_args that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_tinfo = true && this.isSetTinfo(); + boolean that_present_tinfo = true && that.isSetTinfo(); + if (this_present_tinfo || that_present_tinfo) { + if (!(this_present_tinfo && that_present_tinfo)) + return false; + if (!this.tinfo.equals(that.tinfo)) + return false; + } + + boolean this_present_credentials = true && this.isSetCredentials(); + boolean that_present_credentials = true && that.isSetCredentials(); + if (this_present_credentials || that_present_credentials) { + if (!(this_present_credentials && that_present_credentials)) + return false; + if (!this.credentials.equals(that.credentials)) + return false; + } + + boolean this_present_dataLevel = true && this.isSetDataLevel(); + boolean that_present_dataLevel = true && that.isSetDataLevel(); + if (this_present_dataLevel || that_present_dataLevel) { + if (!(this_present_dataLevel && that_present_dataLevel)) + return false; + if (!this.dataLevel.equals(that.dataLevel)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); + if (isSetTinfo()) + hashCode = hashCode * 8191 + tinfo.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); + if (isSetCredentials()) + hashCode = hashCode * 8191 + credentials.hashCode(); + + hashCode = hashCode * 8191 + ((isSetDataLevel()) ? 131071 : 524287); + if (isSetDataLevel()) + hashCode = hashCode * 8191 + dataLevel.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(beginFullJobScan_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTinfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCredentials()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetDataLevel(), other.isSetDataLevel()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDataLevel()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dataLevel, other.dataLevel); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("beginFullJobScan_args("); + boolean first = true; + + sb.append("tinfo:"); + if (this.tinfo == null) { + sb.append("null"); + } else { + sb.append(this.tinfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("credentials:"); + if (this.credentials == null) { + sb.append("null"); + } else { + sb.append(this.credentials); + } + first = false; + if (!first) sb.append(", "); + sb.append("dataLevel:"); + if (this.dataLevel == null) { + sb.append("null"); + } else { + sb.append(this.dataLevel); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (tinfo != null) { + tinfo.validate(); + } + if (credentials != null) { + credentials.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class beginFullJobScan_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public beginFullJobScan_argsStandardScheme getScheme() { + return new beginFullJobScan_argsStandardScheme(); + } + } + + private static class beginFullJobScan_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, beginFullJobScan_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TINFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // CREDENTIALS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // DATA_LEVEL + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.dataLevel = iprot.readString(); + struct.setDataLevelIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, beginFullJobScan_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tinfo != null) { + oprot.writeFieldBegin(TINFO_FIELD_DESC); + struct.tinfo.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.credentials != null) { + oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); + struct.credentials.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.dataLevel != null) { + oprot.writeFieldBegin(DATA_LEVEL_FIELD_DESC); + oprot.writeString(struct.dataLevel); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class beginFullJobScan_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public beginFullJobScan_argsTupleScheme getScheme() { + return new beginFullJobScan_argsTupleScheme(); + } + } + + private static class beginFullJobScan_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, beginFullJobScan_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetTinfo()) { + optionals.set(0); + } + if (struct.isSetCredentials()) { + optionals.set(1); + } + if (struct.isSetDataLevel()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTinfo()) { + struct.tinfo.write(oprot); + } + if (struct.isSetCredentials()) { + struct.credentials.write(oprot); + } + if (struct.isSetDataLevel()) { + oprot.writeString(struct.dataLevel); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, beginFullJobScan_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } + if (incoming.get(1)) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } + if (incoming.get(2)) { + struct.dataLevel = iprot.readString(); + struct.setDataLevelIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class beginFullJobScan_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("beginFullJobScan_result"); + + private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new beginFullJobScan_resultStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new beginFullJobScan_resultTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SEC((short)1, "sec"), + TNASE((short)2, "tnase"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SEC + return SEC; + case 2: // TNASE + return TNASE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); + tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(beginFullJobScan_result.class, metaDataMap); + } + + public beginFullJobScan_result() { + } + + public beginFullJobScan_result( + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, + org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + { + this(); + this.sec = sec; + this.tnase = tnase; + } + + /** + * Performs a deep copy on other. + */ + public beginFullJobScan_result(beginFullJobScan_result other) { + if (other.isSetSec()) { + this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); + } + if (other.isSetTnase()) { + this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); + } + } + + @Override + public beginFullJobScan_result deepCopy() { + return new beginFullJobScan_result(this); + } + + @Override + public void clear() { + this.sec = null; + this.tnase = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { + return this.sec; + } + + public beginFullJobScan_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + this.sec = sec; + return this; + } + + public void unsetSec() { + this.sec = null; + } + + /** Returns true if field sec is set (has been assigned a value) and false otherwise */ + public boolean isSetSec() { + return this.sec != null; + } + + public void setSecIsSet(boolean value) { + if (!value) { + this.sec = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { + return this.tnase; + } + + public beginFullJobScan_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { + this.tnase = tnase; + return this; + } + + public void unsetTnase() { + this.tnase = null; + } + + /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ + public boolean isSetTnase() { + return this.tnase != null; + } + + public void setTnaseIsSet(boolean value) { + if (!value) { + this.tnase = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case SEC: + if (value == null) { + unsetSec(); + } else { + setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); + } + break; + + case TNASE: + if (value == null) { + unsetTnase(); + } else { + setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case SEC: + return getSec(); + + case TNASE: + return getTnase(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case SEC: + return isSetSec(); + case TNASE: + return isSetTnase(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof beginFullJobScan_result) + return this.equals((beginFullJobScan_result)that); + return false; + } + + public boolean equals(beginFullJobScan_result that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_sec = true && this.isSetSec(); + boolean that_present_sec = true && that.isSetSec(); + if (this_present_sec || that_present_sec) { + if (!(this_present_sec && that_present_sec)) + return false; + if (!this.sec.equals(that.sec)) + return false; + } + + boolean this_present_tnase = true && this.isSetTnase(); + boolean that_present_tnase = true && that.isSetTnase(); + if (this_present_tnase || that_present_tnase) { + if (!(this_present_tnase && that_present_tnase)) + return false; + if (!this.tnase.equals(that.tnase)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); + if (isSetSec()) + hashCode = hashCode * 8191 + sec.hashCode(); + + hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); + if (isSetTnase()) + hashCode = hashCode * 8191 + tnase.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(beginFullJobScan_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSec()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTnase()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("beginFullJobScan_result("); + boolean first = true; + + sb.append("sec:"); + if (this.sec == null) { + sb.append("null"); + } else { + sb.append(this.sec); + } + first = false; + if (!first) sb.append(", "); + sb.append("tnase:"); + if (this.tnase == null) { + sb.append("null"); + } else { + sb.append(this.tnase); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class beginFullJobScan_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public beginFullJobScan_resultStandardScheme getScheme() { + return new beginFullJobScan_resultStandardScheme(); + } + } + + private static class beginFullJobScan_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, beginFullJobScan_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SEC + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // TNASE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); + struct.tnase.read(iprot); + struct.setTnaseIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, beginFullJobScan_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.sec != null) { + oprot.writeFieldBegin(SEC_FIELD_DESC); + struct.sec.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.tnase != null) { + oprot.writeFieldBegin(TNASE_FIELD_DESC); + struct.tnase.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class beginFullJobScan_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public beginFullJobScan_resultTupleScheme getScheme() { + return new beginFullJobScan_resultTupleScheme(); + } + } + + private static class beginFullJobScan_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, beginFullJobScan_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetSec()) { + optionals.set(0); + } + if (struct.isSetTnase()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSec()) { + struct.sec.write(oprot); + } + if (struct.isSetTnase()) { + struct.tnase.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, beginFullJobScan_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } + if (incoming.get(1)) { + struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); + struct.tnase.read(iprot); + struct.setTnaseIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class addJobs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("addJobs_args"); + + private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField JOBS_FIELD_DESC = new org.apache.thrift.protocol.TField("jobs", org.apache.thrift.protocol.TType.LIST, (short)3); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new addJobs_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new addJobs_argsTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required + public @org.apache.thrift.annotation.Nullable java.util.List jobs; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TINFO((short)1, "tinfo"), + CREDENTIALS((short)2, "credentials"), + JOBS((short)3, "jobs"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TINFO + return TINFO; + case 2: // CREDENTIALS + return CREDENTIALS; + case 3: // JOBS + return JOBS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); + tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); + tmpMap.put(_Fields.JOBS, new org.apache.thrift.meta_data.FieldMetaData("jobs", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResolvedCompactionJob.class)))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(addJobs_args.class, metaDataMap); + } + + public addJobs_args() { + } + + public addJobs_args( + org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, + org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, + java.util.List jobs) + { + this(); + this.tinfo = tinfo; + this.credentials = credentials; + this.jobs = jobs; + } + + /** + * Performs a deep copy on other. + */ + public addJobs_args(addJobs_args other) { + if (other.isSetTinfo()) { + this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); + } + if (other.isSetCredentials()) { + this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); + } + if (other.isSetJobs()) { + java.util.List __this__jobs = new java.util.ArrayList(other.jobs.size()); + for (TResolvedCompactionJob other_element : other.jobs) { + __this__jobs.add(new TResolvedCompactionJob(other_element)); + } + this.jobs = __this__jobs; + } + } + + @Override + public addJobs_args deepCopy() { + return new addJobs_args(this); + } + + @Override + public void clear() { + this.tinfo = null; + this.credentials = null; + this.jobs = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { + return this.tinfo; + } + + public addJobs_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { + this.tinfo = tinfo; + return this; + } + + public void unsetTinfo() { + this.tinfo = null; + } + + /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ + public boolean isSetTinfo() { + return this.tinfo != null; + } + + public void setTinfoIsSet(boolean value) { + if (!value) { + this.tinfo = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { + return this.credentials; + } + + public addJobs_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { + this.credentials = credentials; + return this; + } + + public void unsetCredentials() { + this.credentials = null; + } + + /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ + public boolean isSetCredentials() { + return this.credentials != null; + } + + public void setCredentialsIsSet(boolean value) { + if (!value) { + this.credentials = null; + } + } + + public int getJobsSize() { + return (this.jobs == null) ? 0 : this.jobs.size(); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Iterator getJobsIterator() { + return (this.jobs == null) ? null : this.jobs.iterator(); + } + + public void addToJobs(TResolvedCompactionJob elem) { + if (this.jobs == null) { + this.jobs = new java.util.ArrayList(); + } + this.jobs.add(elem); + } + + @org.apache.thrift.annotation.Nullable + public java.util.List getJobs() { + return this.jobs; + } + + public addJobs_args setJobs(@org.apache.thrift.annotation.Nullable java.util.List jobs) { + this.jobs = jobs; + return this; + } + + public void unsetJobs() { + this.jobs = null; + } + + /** Returns true if field jobs is set (has been assigned a value) and false otherwise */ + public boolean isSetJobs() { + return this.jobs != null; + } + + public void setJobsIsSet(boolean value) { + if (!value) { + this.jobs = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case TINFO: + if (value == null) { + unsetTinfo(); + } else { + setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); + } + break; + + case CREDENTIALS: + if (value == null) { + unsetCredentials(); + } else { + setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); + } + break; + + case JOBS: + if (value == null) { + unsetJobs(); + } else { + setJobs((java.util.List)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case TINFO: + return getTinfo(); + + case CREDENTIALS: + return getCredentials(); + + case JOBS: + return getJobs(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case TINFO: + return isSetTinfo(); + case CREDENTIALS: + return isSetCredentials(); + case JOBS: + return isSetJobs(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof addJobs_args) + return this.equals((addJobs_args)that); + return false; + } + + public boolean equals(addJobs_args that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_tinfo = true && this.isSetTinfo(); + boolean that_present_tinfo = true && that.isSetTinfo(); + if (this_present_tinfo || that_present_tinfo) { + if (!(this_present_tinfo && that_present_tinfo)) + return false; + if (!this.tinfo.equals(that.tinfo)) + return false; + } + + boolean this_present_credentials = true && this.isSetCredentials(); + boolean that_present_credentials = true && that.isSetCredentials(); + if (this_present_credentials || that_present_credentials) { + if (!(this_present_credentials && that_present_credentials)) + return false; + if (!this.credentials.equals(that.credentials)) + return false; + } + + boolean this_present_jobs = true && this.isSetJobs(); + boolean that_present_jobs = true && that.isSetJobs(); + if (this_present_jobs || that_present_jobs) { + if (!(this_present_jobs && that_present_jobs)) + return false; + if (!this.jobs.equals(that.jobs)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); + if (isSetTinfo()) + hashCode = hashCode * 8191 + tinfo.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); + if (isSetCredentials()) + hashCode = hashCode * 8191 + credentials.hashCode(); + + hashCode = hashCode * 8191 + ((isSetJobs()) ? 131071 : 524287); + if (isSetJobs()) + hashCode = hashCode * 8191 + jobs.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(addJobs_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTinfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCredentials()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetJobs(), other.isSetJobs()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetJobs()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.jobs, other.jobs); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("addJobs_args("); + boolean first = true; + + sb.append("tinfo:"); + if (this.tinfo == null) { + sb.append("null"); + } else { + sb.append(this.tinfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("credentials:"); + if (this.credentials == null) { + sb.append("null"); + } else { + sb.append(this.credentials); + } + first = false; + if (!first) sb.append(", "); + sb.append("jobs:"); + if (this.jobs == null) { + sb.append("null"); + } else { + sb.append(this.jobs); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (tinfo != null) { + tinfo.validate(); + } + if (credentials != null) { + credentials.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class addJobs_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public addJobs_argsStandardScheme getScheme() { + return new addJobs_argsStandardScheme(); + } + } + + private static class addJobs_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, addJobs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TINFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // CREDENTIALS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // JOBS + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list46 = iprot.readListBegin(); + struct.jobs = new java.util.ArrayList(_list46.size); + @org.apache.thrift.annotation.Nullable TResolvedCompactionJob _elem47; + for (int _i48 = 0; _i48 < _list46.size; ++_i48) + { + _elem47 = new TResolvedCompactionJob(); + _elem47.read(iprot); + struct.jobs.add(_elem47); + } + iprot.readListEnd(); + } + struct.setJobsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, addJobs_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tinfo != null) { + oprot.writeFieldBegin(TINFO_FIELD_DESC); + struct.tinfo.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.credentials != null) { + oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); + struct.credentials.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.jobs != null) { + oprot.writeFieldBegin(JOBS_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.jobs.size())); + for (TResolvedCompactionJob _iter49 : struct.jobs) + { + _iter49.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class addJobs_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public addJobs_argsTupleScheme getScheme() { + return new addJobs_argsTupleScheme(); + } + } + + private static class addJobs_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, addJobs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetTinfo()) { + optionals.set(0); + } + if (struct.isSetCredentials()) { + optionals.set(1); + } + if (struct.isSetJobs()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTinfo()) { + struct.tinfo.write(oprot); + } + if (struct.isSetCredentials()) { + struct.credentials.write(oprot); + } + if (struct.isSetJobs()) { + { + oprot.writeI32(struct.jobs.size()); + for (TResolvedCompactionJob _iter50 : struct.jobs) + { + _iter50.write(oprot); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, addJobs_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } + if (incoming.get(1)) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TList _list51 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.jobs = new java.util.ArrayList(_list51.size); + @org.apache.thrift.annotation.Nullable TResolvedCompactionJob _elem52; + for (int _i53 = 0; _i53 < _list51.size; ++_i53) + { + _elem52 = new TResolvedCompactionJob(); + _elem52.read(iprot); + struct.jobs.add(_elem52); + } + } + struct.setJobsIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class endFullJobScan_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("endFullJobScan_args"); + + private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField DATA_LEVEL_FIELD_DESC = new org.apache.thrift.protocol.TField("dataLevel", org.apache.thrift.protocol.TType.STRING, (short)3); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new endFullJobScan_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new endFullJobScan_argsTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required + public @org.apache.thrift.annotation.Nullable java.lang.String dataLevel; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TINFO((short)1, "tinfo"), + CREDENTIALS((short)2, "credentials"), + DATA_LEVEL((short)3, "dataLevel"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TINFO + return TINFO; + case 2: // CREDENTIALS + return CREDENTIALS; + case 3: // DATA_LEVEL + return DATA_LEVEL; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); + tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); + tmpMap.put(_Fields.DATA_LEVEL, new org.apache.thrift.meta_data.FieldMetaData("dataLevel", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(endFullJobScan_args.class, metaDataMap); + } + + public endFullJobScan_args() { + } + + public endFullJobScan_args( + org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, + org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, + java.lang.String dataLevel) + { + this(); + this.tinfo = tinfo; + this.credentials = credentials; + this.dataLevel = dataLevel; + } + + /** + * Performs a deep copy on other. + */ + public endFullJobScan_args(endFullJobScan_args other) { + if (other.isSetTinfo()) { + this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); + } + if (other.isSetCredentials()) { + this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); + } + if (other.isSetDataLevel()) { + this.dataLevel = other.dataLevel; + } + } + + @Override + public endFullJobScan_args deepCopy() { + return new endFullJobScan_args(this); + } + + @Override + public void clear() { + this.tinfo = null; + this.credentials = null; + this.dataLevel = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { + return this.tinfo; + } + + public endFullJobScan_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { + this.tinfo = tinfo; + return this; + } + + public void unsetTinfo() { + this.tinfo = null; + } + + /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ + public boolean isSetTinfo() { + return this.tinfo != null; + } + + public void setTinfoIsSet(boolean value) { + if (!value) { + this.tinfo = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { + return this.credentials; + } + + public endFullJobScan_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { + this.credentials = credentials; + return this; + } + + public void unsetCredentials() { + this.credentials = null; + } + + /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ + public boolean isSetCredentials() { + return this.credentials != null; + } + + public void setCredentialsIsSet(boolean value) { + if (!value) { + this.credentials = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getDataLevel() { + return this.dataLevel; + } + + public endFullJobScan_args setDataLevel(@org.apache.thrift.annotation.Nullable java.lang.String dataLevel) { + this.dataLevel = dataLevel; + return this; + } + + public void unsetDataLevel() { + this.dataLevel = null; + } + + /** Returns true if field dataLevel is set (has been assigned a value) and false otherwise */ + public boolean isSetDataLevel() { + return this.dataLevel != null; + } + + public void setDataLevelIsSet(boolean value) { + if (!value) { + this.dataLevel = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case TINFO: + if (value == null) { + unsetTinfo(); + } else { + setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); + } + break; + + case CREDENTIALS: + if (value == null) { + unsetCredentials(); + } else { + setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); + } + break; + + case DATA_LEVEL: + if (value == null) { + unsetDataLevel(); + } else { + setDataLevel((java.lang.String)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case TINFO: + return getTinfo(); + + case CREDENTIALS: + return getCredentials(); + + case DATA_LEVEL: + return getDataLevel(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case TINFO: + return isSetTinfo(); + case CREDENTIALS: + return isSetCredentials(); + case DATA_LEVEL: + return isSetDataLevel(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof endFullJobScan_args) + return this.equals((endFullJobScan_args)that); + return false; + } + + public boolean equals(endFullJobScan_args that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_tinfo = true && this.isSetTinfo(); + boolean that_present_tinfo = true && that.isSetTinfo(); + if (this_present_tinfo || that_present_tinfo) { + if (!(this_present_tinfo && that_present_tinfo)) + return false; + if (!this.tinfo.equals(that.tinfo)) + return false; + } + + boolean this_present_credentials = true && this.isSetCredentials(); + boolean that_present_credentials = true && that.isSetCredentials(); + if (this_present_credentials || that_present_credentials) { + if (!(this_present_credentials && that_present_credentials)) + return false; + if (!this.credentials.equals(that.credentials)) + return false; + } + + boolean this_present_dataLevel = true && this.isSetDataLevel(); + boolean that_present_dataLevel = true && that.isSetDataLevel(); + if (this_present_dataLevel || that_present_dataLevel) { + if (!(this_present_dataLevel && that_present_dataLevel)) + return false; + if (!this.dataLevel.equals(that.dataLevel)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); + if (isSetTinfo()) + hashCode = hashCode * 8191 + tinfo.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); + if (isSetCredentials()) + hashCode = hashCode * 8191 + credentials.hashCode(); + + hashCode = hashCode * 8191 + ((isSetDataLevel()) ? 131071 : 524287); + if (isSetDataLevel()) + hashCode = hashCode * 8191 + dataLevel.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(endFullJobScan_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTinfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCredentials()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetDataLevel(), other.isSetDataLevel()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetDataLevel()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.dataLevel, other.dataLevel); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("endFullJobScan_args("); + boolean first = true; + + sb.append("tinfo:"); + if (this.tinfo == null) { + sb.append("null"); + } else { + sb.append(this.tinfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("credentials:"); + if (this.credentials == null) { + sb.append("null"); + } else { + sb.append(this.credentials); + } + first = false; + if (!first) sb.append(", "); + sb.append("dataLevel:"); + if (this.dataLevel == null) { + sb.append("null"); + } else { + sb.append(this.dataLevel); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (tinfo != null) { + tinfo.validate(); + } + if (credentials != null) { + credentials.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class endFullJobScan_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public endFullJobScan_argsStandardScheme getScheme() { + return new endFullJobScan_argsStandardScheme(); + } + } + + private static class endFullJobScan_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, endFullJobScan_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TINFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // CREDENTIALS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // DATA_LEVEL + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.dataLevel = iprot.readString(); + struct.setDataLevelIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, endFullJobScan_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tinfo != null) { + oprot.writeFieldBegin(TINFO_FIELD_DESC); + struct.tinfo.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.credentials != null) { + oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); + struct.credentials.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.dataLevel != null) { + oprot.writeFieldBegin(DATA_LEVEL_FIELD_DESC); + oprot.writeString(struct.dataLevel); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class endFullJobScan_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public endFullJobScan_argsTupleScheme getScheme() { + return new endFullJobScan_argsTupleScheme(); + } + } + + private static class endFullJobScan_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, endFullJobScan_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetTinfo()) { + optionals.set(0); + } + if (struct.isSetCredentials()) { + optionals.set(1); + } + if (struct.isSetDataLevel()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTinfo()) { + struct.tinfo.write(oprot); + } + if (struct.isSetCredentials()) { + struct.credentials.write(oprot); + } + if (struct.isSetDataLevel()) { + oprot.writeString(struct.dataLevel); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, endFullJobScan_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } + if (incoming.get(1)) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } + if (incoming.get(2)) { + struct.dataLevel = iprot.readString(); + struct.setDataLevelIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class endFullJobScan_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("endFullJobScan_result"); + + private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new endFullJobScan_resultStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new endFullJobScan_resultTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SEC((short)1, "sec"), + TNASE((short)2, "tnase"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SEC + return SEC; + case 2: // TNASE + return TNASE; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); + tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(endFullJobScan_result.class, metaDataMap); + } + + public endFullJobScan_result() { + } + + public endFullJobScan_result( + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, + org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + { + this(); + this.sec = sec; + this.tnase = tnase; + } + + /** + * Performs a deep copy on other. + */ + public endFullJobScan_result(endFullJobScan_result other) { + if (other.isSetSec()) { + this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); + } + if (other.isSetTnase()) { + this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); + } + } + + @Override + public endFullJobScan_result deepCopy() { + return new endFullJobScan_result(this); + } + + @Override + public void clear() { + this.sec = null; + this.tnase = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { + return this.sec; + } + + public endFullJobScan_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + this.sec = sec; + return this; + } + + public void unsetSec() { + this.sec = null; + } + + /** Returns true if field sec is set (has been assigned a value) and false otherwise */ + public boolean isSetSec() { + return this.sec != null; + } + + public void setSecIsSet(boolean value) { + if (!value) { + this.sec = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { + return this.tnase; + } + + public endFullJobScan_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { + this.tnase = tnase; + return this; + } + + public void unsetTnase() { + this.tnase = null; + } + + /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ + public boolean isSetTnase() { + return this.tnase != null; + } + + public void setTnaseIsSet(boolean value) { + if (!value) { + this.tnase = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case SEC: + if (value == null) { + unsetSec(); + } else { + setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); + } + break; + + case TNASE: + if (value == null) { + unsetTnase(); + } else { + setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case SEC: + return getSec(); + + case TNASE: + return getTnase(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case SEC: + return isSetSec(); + case TNASE: + return isSetTnase(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof endFullJobScan_result) + return this.equals((endFullJobScan_result)that); + return false; + } + + public boolean equals(endFullJobScan_result that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_sec = true && this.isSetSec(); + boolean that_present_sec = true && that.isSetSec(); + if (this_present_sec || that_present_sec) { + if (!(this_present_sec && that_present_sec)) + return false; + if (!this.sec.equals(that.sec)) + return false; + } + + boolean this_present_tnase = true && this.isSetTnase(); + boolean that_present_tnase = true && that.isSetTnase(); + if (this_present_tnase || that_present_tnase) { + if (!(this_present_tnase && that_present_tnase)) + return false; + if (!this.tnase.equals(that.tnase)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); + if (isSetSec()) + hashCode = hashCode * 8191 + sec.hashCode(); + + hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); + if (isSetTnase()) + hashCode = hashCode * 8191 + tnase.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(endFullJobScan_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSec()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTnase()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("endFullJobScan_result("); + boolean first = true; + + sb.append("sec:"); + if (this.sec == null) { + sb.append("null"); + } else { + sb.append(this.sec); + } + first = false; + if (!first) sb.append(", "); + sb.append("tnase:"); + if (this.tnase == null) { + sb.append("null"); + } else { + sb.append(this.tnase); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class endFullJobScan_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public endFullJobScan_resultStandardScheme getScheme() { + return new endFullJobScan_resultStandardScheme(); + } + } + + private static class endFullJobScan_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, endFullJobScan_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SEC + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // TNASE + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); + struct.tnase.read(iprot); + struct.setTnaseIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, endFullJobScan_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.sec != null) { + oprot.writeFieldBegin(SEC_FIELD_DESC); + struct.sec.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.tnase != null) { + oprot.writeFieldBegin(TNASE_FIELD_DESC); + struct.tnase.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class endFullJobScan_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public endFullJobScan_resultTupleScheme getScheme() { + return new endFullJobScan_resultTupleScheme(); + } + } + + private static class endFullJobScan_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, endFullJobScan_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetSec()) { + optionals.set(0); + } + if (struct.isSetTnase()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSec()) { + struct.sec.write(oprot); + } + if (struct.isSetTnase()) { + struct.tnase.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, endFullJobScan_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } + if (incoming.get(1)) { + struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); + struct.tnase.read(iprot); + struct.setTnaseIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + private static void unusedMethod() {} } diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java index 296157617b2..d2f9c7d1aeb 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java @@ -3668,14 +3668,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveCompaction case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list38 = iprot.readListBegin(); - struct.success = new java.util.ArrayList(_list38.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem39; - for (int _i40 = 0; _i40 < _list38.size; ++_i40) + org.apache.thrift.protocol.TList _list54 = iprot.readListBegin(); + struct.success = new java.util.ArrayList(_list54.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem55; + for (int _i56 = 0; _i56 < _list54.size; ++_i56) { - _elem39 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem39.read(iprot); - struct.success.add(_elem39); + _elem55 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem55.read(iprot); + struct.success.add(_elem55); } iprot.readListEnd(); } @@ -3713,9 +3713,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveCompactio oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter41 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter57 : struct.success) { - _iter41.write(oprot); + _iter57.write(oprot); } oprot.writeListEnd(); } @@ -3755,9 +3755,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveCompaction if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter42 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter58 : struct.success) { - _iter42.write(oprot); + _iter58.write(oprot); } } } @@ -3772,14 +3772,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveCompactions java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list43 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.success = new java.util.ArrayList(_list43.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem44; - for (int _i45 = 0; _i45 < _list43.size; ++_i45) + org.apache.thrift.protocol.TList _list59 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.success = new java.util.ArrayList(_list59.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem60; + for (int _i61 = 0; _i61 < _list59.size; ++_i61) { - _elem44 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem44.read(iprot); - struct.success.add(_elem44); + _elem60 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem60.read(iprot); + struct.success.add(_elem60); } } struct.setSuccessIsSet(true); diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TResolvedCompactionJob.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TResolvedCompactionJob.java new file mode 100644 index 00000000000..46eb951ec48 --- /dev/null +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TResolvedCompactionJob.java @@ -0,0 +1,1283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Autogenerated by Thrift Compiler (0.17.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.accumulo.core.compaction.thrift; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) +public class TResolvedCompactionJob implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TResolvedCompactionJob"); + + private static final org.apache.thrift.protocol.TField SELECTED_FATE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("selectedFateId", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField JOB_FILES_FIELD_DESC = new org.apache.thrift.protocol.TField("jobFiles", org.apache.thrift.protocol.TType.LIST, (short)2); + private static final org.apache.thrift.protocol.TField KIND_FIELD_DESC = new org.apache.thrift.protocol.TField("kind", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField COMPACTING_ALL_FIELD_DESC = new org.apache.thrift.protocol.TField("compactingAll", org.apache.thrift.protocol.TType.BOOL, (short)4); + private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("extent", org.apache.thrift.protocol.TType.STRUCT, (short)5); + private static final org.apache.thrift.protocol.TField PRIORITY_FIELD_DESC = new org.apache.thrift.protocol.TField("priority", org.apache.thrift.protocol.TType.I16, (short)6); + private static final org.apache.thrift.protocol.TField GROUP_FIELD_DESC = new org.apache.thrift.protocol.TField("group", org.apache.thrift.protocol.TType.STRING, (short)7); + private static final org.apache.thrift.protocol.TField TABLET_DIR_FIELD_DESC = new org.apache.thrift.protocol.TField("tabletDir", org.apache.thrift.protocol.TType.STRING, (short)8); + private static final org.apache.thrift.protocol.TField OVERLAPS_SELECTED_FILES_FIELD_DESC = new org.apache.thrift.protocol.TField("overlapsSelectedFiles", org.apache.thrift.protocol.TType.BOOL, (short)9); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new TResolvedCompactionJobStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new TResolvedCompactionJobTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable java.lang.String selectedFateId; // required + public @org.apache.thrift.annotation.Nullable java.util.List jobFiles; // required + public @org.apache.thrift.annotation.Nullable java.lang.String kind; // required + public boolean compactingAll; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required + public short priority; // required + public @org.apache.thrift.annotation.Nullable java.lang.String group; // required + public @org.apache.thrift.annotation.Nullable java.lang.String tabletDir; // required + public boolean overlapsSelectedFiles; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SELECTED_FATE_ID((short)1, "selectedFateId"), + JOB_FILES((short)2, "jobFiles"), + KIND((short)3, "kind"), + COMPACTING_ALL((short)4, "compactingAll"), + EXTENT((short)5, "extent"), + PRIORITY((short)6, "priority"), + GROUP((short)7, "group"), + TABLET_DIR((short)8, "tabletDir"), + OVERLAPS_SELECTED_FILES((short)9, "overlapsSelectedFiles"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SELECTED_FATE_ID + return SELECTED_FATE_ID; + case 2: // JOB_FILES + return JOB_FILES; + case 3: // KIND + return KIND; + case 4: // COMPACTING_ALL + return COMPACTING_ALL; + case 5: // EXTENT + return EXTENT; + case 6: // PRIORITY + return PRIORITY; + case 7: // GROUP + return GROUP; + case 8: // TABLET_DIR + return TABLET_DIR; + case 9: // OVERLAPS_SELECTED_FILES + return OVERLAPS_SELECTED_FILES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __COMPACTINGALL_ISSET_ID = 0; + private static final int __PRIORITY_ISSET_ID = 1; + private static final int __OVERLAPSSELECTEDFILES_ISSET_ID = 2; + private byte __isset_bitfield = 0; + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SELECTED_FATE_ID, new org.apache.thrift.meta_data.FieldMetaData("selectedFateId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.JOB_FILES, new org.apache.thrift.meta_data.FieldMetaData("jobFiles", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.tabletserver.thrift.InputFile.class)))); + tmpMap.put(_Fields.KIND, new org.apache.thrift.meta_data.FieldMetaData("kind", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.COMPACTING_ALL, new org.apache.thrift.meta_data.FieldMetaData("compactingAll", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.EXTENT, new org.apache.thrift.meta_data.FieldMetaData("extent", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class))); + tmpMap.put(_Fields.PRIORITY, new org.apache.thrift.meta_data.FieldMetaData("priority", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I16))); + tmpMap.put(_Fields.GROUP, new org.apache.thrift.meta_data.FieldMetaData("group", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.TABLET_DIR, new org.apache.thrift.meta_data.FieldMetaData("tabletDir", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.OVERLAPS_SELECTED_FILES, new org.apache.thrift.meta_data.FieldMetaData("overlapsSelectedFiles", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TResolvedCompactionJob.class, metaDataMap); + } + + public TResolvedCompactionJob() { + } + + public TResolvedCompactionJob( + java.lang.String selectedFateId, + java.util.List jobFiles, + java.lang.String kind, + boolean compactingAll, + org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, + short priority, + java.lang.String group, + java.lang.String tabletDir, + boolean overlapsSelectedFiles) + { + this(); + this.selectedFateId = selectedFateId; + this.jobFiles = jobFiles; + this.kind = kind; + this.compactingAll = compactingAll; + setCompactingAllIsSet(true); + this.extent = extent; + this.priority = priority; + setPriorityIsSet(true); + this.group = group; + this.tabletDir = tabletDir; + this.overlapsSelectedFiles = overlapsSelectedFiles; + setOverlapsSelectedFilesIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public TResolvedCompactionJob(TResolvedCompactionJob other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetSelectedFateId()) { + this.selectedFateId = other.selectedFateId; + } + if (other.isSetJobFiles()) { + java.util.List __this__jobFiles = new java.util.ArrayList(other.jobFiles.size()); + for (org.apache.accumulo.core.tabletserver.thrift.InputFile other_element : other.jobFiles) { + __this__jobFiles.add(new org.apache.accumulo.core.tabletserver.thrift.InputFile(other_element)); + } + this.jobFiles = __this__jobFiles; + } + if (other.isSetKind()) { + this.kind = other.kind; + } + this.compactingAll = other.compactingAll; + if (other.isSetExtent()) { + this.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(other.extent); + } + this.priority = other.priority; + if (other.isSetGroup()) { + this.group = other.group; + } + if (other.isSetTabletDir()) { + this.tabletDir = other.tabletDir; + } + this.overlapsSelectedFiles = other.overlapsSelectedFiles; + } + + @Override + public TResolvedCompactionJob deepCopy() { + return new TResolvedCompactionJob(this); + } + + @Override + public void clear() { + this.selectedFateId = null; + this.jobFiles = null; + this.kind = null; + setCompactingAllIsSet(false); + this.compactingAll = false; + this.extent = null; + setPriorityIsSet(false); + this.priority = 0; + this.group = null; + this.tabletDir = null; + setOverlapsSelectedFilesIsSet(false); + this.overlapsSelectedFiles = false; + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getSelectedFateId() { + return this.selectedFateId; + } + + public TResolvedCompactionJob setSelectedFateId(@org.apache.thrift.annotation.Nullable java.lang.String selectedFateId) { + this.selectedFateId = selectedFateId; + return this; + } + + public void unsetSelectedFateId() { + this.selectedFateId = null; + } + + /** Returns true if field selectedFateId is set (has been assigned a value) and false otherwise */ + public boolean isSetSelectedFateId() { + return this.selectedFateId != null; + } + + public void setSelectedFateIdIsSet(boolean value) { + if (!value) { + this.selectedFateId = null; + } + } + + public int getJobFilesSize() { + return (this.jobFiles == null) ? 0 : this.jobFiles.size(); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Iterator getJobFilesIterator() { + return (this.jobFiles == null) ? null : this.jobFiles.iterator(); + } + + public void addToJobFiles(org.apache.accumulo.core.tabletserver.thrift.InputFile elem) { + if (this.jobFiles == null) { + this.jobFiles = new java.util.ArrayList(); + } + this.jobFiles.add(elem); + } + + @org.apache.thrift.annotation.Nullable + public java.util.List getJobFiles() { + return this.jobFiles; + } + + public TResolvedCompactionJob setJobFiles(@org.apache.thrift.annotation.Nullable java.util.List jobFiles) { + this.jobFiles = jobFiles; + return this; + } + + public void unsetJobFiles() { + this.jobFiles = null; + } + + /** Returns true if field jobFiles is set (has been assigned a value) and false otherwise */ + public boolean isSetJobFiles() { + return this.jobFiles != null; + } + + public void setJobFilesIsSet(boolean value) { + if (!value) { + this.jobFiles = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getKind() { + return this.kind; + } + + public TResolvedCompactionJob setKind(@org.apache.thrift.annotation.Nullable java.lang.String kind) { + this.kind = kind; + return this; + } + + public void unsetKind() { + this.kind = null; + } + + /** Returns true if field kind is set (has been assigned a value) and false otherwise */ + public boolean isSetKind() { + return this.kind != null; + } + + public void setKindIsSet(boolean value) { + if (!value) { + this.kind = null; + } + } + + public boolean isCompactingAll() { + return this.compactingAll; + } + + public TResolvedCompactionJob setCompactingAll(boolean compactingAll) { + this.compactingAll = compactingAll; + setCompactingAllIsSet(true); + return this; + } + + public void unsetCompactingAll() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __COMPACTINGALL_ISSET_ID); + } + + /** Returns true if field compactingAll is set (has been assigned a value) and false otherwise */ + public boolean isSetCompactingAll() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __COMPACTINGALL_ISSET_ID); + } + + public void setCompactingAllIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __COMPACTINGALL_ISSET_ID, value); + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.dataImpl.thrift.TKeyExtent getExtent() { + return this.extent; + } + + public TResolvedCompactionJob setExtent(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent) { + this.extent = extent; + return this; + } + + public void unsetExtent() { + this.extent = null; + } + + /** Returns true if field extent is set (has been assigned a value) and false otherwise */ + public boolean isSetExtent() { + return this.extent != null; + } + + public void setExtentIsSet(boolean value) { + if (!value) { + this.extent = null; + } + } + + public short getPriority() { + return this.priority; + } + + public TResolvedCompactionJob setPriority(short priority) { + this.priority = priority; + setPriorityIsSet(true); + return this; + } + + public void unsetPriority() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __PRIORITY_ISSET_ID); + } + + /** Returns true if field priority is set (has been assigned a value) and false otherwise */ + public boolean isSetPriority() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __PRIORITY_ISSET_ID); + } + + public void setPriorityIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __PRIORITY_ISSET_ID, value); + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getGroup() { + return this.group; + } + + public TResolvedCompactionJob setGroup(@org.apache.thrift.annotation.Nullable java.lang.String group) { + this.group = group; + return this; + } + + public void unsetGroup() { + this.group = null; + } + + /** Returns true if field group is set (has been assigned a value) and false otherwise */ + public boolean isSetGroup() { + return this.group != null; + } + + public void setGroupIsSet(boolean value) { + if (!value) { + this.group = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getTabletDir() { + return this.tabletDir; + } + + public TResolvedCompactionJob setTabletDir(@org.apache.thrift.annotation.Nullable java.lang.String tabletDir) { + this.tabletDir = tabletDir; + return this; + } + + public void unsetTabletDir() { + this.tabletDir = null; + } + + /** Returns true if field tabletDir is set (has been assigned a value) and false otherwise */ + public boolean isSetTabletDir() { + return this.tabletDir != null; + } + + public void setTabletDirIsSet(boolean value) { + if (!value) { + this.tabletDir = null; + } + } + + public boolean isOverlapsSelectedFiles() { + return this.overlapsSelectedFiles; + } + + public TResolvedCompactionJob setOverlapsSelectedFiles(boolean overlapsSelectedFiles) { + this.overlapsSelectedFiles = overlapsSelectedFiles; + setOverlapsSelectedFilesIsSet(true); + return this; + } + + public void unsetOverlapsSelectedFiles() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __OVERLAPSSELECTEDFILES_ISSET_ID); + } + + /** Returns true if field overlapsSelectedFiles is set (has been assigned a value) and false otherwise */ + public boolean isSetOverlapsSelectedFiles() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __OVERLAPSSELECTEDFILES_ISSET_ID); + } + + public void setOverlapsSelectedFilesIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __OVERLAPSSELECTEDFILES_ISSET_ID, value); + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case SELECTED_FATE_ID: + if (value == null) { + unsetSelectedFateId(); + } else { + setSelectedFateId((java.lang.String)value); + } + break; + + case JOB_FILES: + if (value == null) { + unsetJobFiles(); + } else { + setJobFiles((java.util.List)value); + } + break; + + case KIND: + if (value == null) { + unsetKind(); + } else { + setKind((java.lang.String)value); + } + break; + + case COMPACTING_ALL: + if (value == null) { + unsetCompactingAll(); + } else { + setCompactingAll((java.lang.Boolean)value); + } + break; + + case EXTENT: + if (value == null) { + unsetExtent(); + } else { + setExtent((org.apache.accumulo.core.dataImpl.thrift.TKeyExtent)value); + } + break; + + case PRIORITY: + if (value == null) { + unsetPriority(); + } else { + setPriority((java.lang.Short)value); + } + break; + + case GROUP: + if (value == null) { + unsetGroup(); + } else { + setGroup((java.lang.String)value); + } + break; + + case TABLET_DIR: + if (value == null) { + unsetTabletDir(); + } else { + setTabletDir((java.lang.String)value); + } + break; + + case OVERLAPS_SELECTED_FILES: + if (value == null) { + unsetOverlapsSelectedFiles(); + } else { + setOverlapsSelectedFiles((java.lang.Boolean)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case SELECTED_FATE_ID: + return getSelectedFateId(); + + case JOB_FILES: + return getJobFiles(); + + case KIND: + return getKind(); + + case COMPACTING_ALL: + return isCompactingAll(); + + case EXTENT: + return getExtent(); + + case PRIORITY: + return getPriority(); + + case GROUP: + return getGroup(); + + case TABLET_DIR: + return getTabletDir(); + + case OVERLAPS_SELECTED_FILES: + return isOverlapsSelectedFiles(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case SELECTED_FATE_ID: + return isSetSelectedFateId(); + case JOB_FILES: + return isSetJobFiles(); + case KIND: + return isSetKind(); + case COMPACTING_ALL: + return isSetCompactingAll(); + case EXTENT: + return isSetExtent(); + case PRIORITY: + return isSetPriority(); + case GROUP: + return isSetGroup(); + case TABLET_DIR: + return isSetTabletDir(); + case OVERLAPS_SELECTED_FILES: + return isSetOverlapsSelectedFiles(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof TResolvedCompactionJob) + return this.equals((TResolvedCompactionJob)that); + return false; + } + + public boolean equals(TResolvedCompactionJob that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_selectedFateId = true && this.isSetSelectedFateId(); + boolean that_present_selectedFateId = true && that.isSetSelectedFateId(); + if (this_present_selectedFateId || that_present_selectedFateId) { + if (!(this_present_selectedFateId && that_present_selectedFateId)) + return false; + if (!this.selectedFateId.equals(that.selectedFateId)) + return false; + } + + boolean this_present_jobFiles = true && this.isSetJobFiles(); + boolean that_present_jobFiles = true && that.isSetJobFiles(); + if (this_present_jobFiles || that_present_jobFiles) { + if (!(this_present_jobFiles && that_present_jobFiles)) + return false; + if (!this.jobFiles.equals(that.jobFiles)) + return false; + } + + boolean this_present_kind = true && this.isSetKind(); + boolean that_present_kind = true && that.isSetKind(); + if (this_present_kind || that_present_kind) { + if (!(this_present_kind && that_present_kind)) + return false; + if (!this.kind.equals(that.kind)) + return false; + } + + boolean this_present_compactingAll = true; + boolean that_present_compactingAll = true; + if (this_present_compactingAll || that_present_compactingAll) { + if (!(this_present_compactingAll && that_present_compactingAll)) + return false; + if (this.compactingAll != that.compactingAll) + return false; + } + + boolean this_present_extent = true && this.isSetExtent(); + boolean that_present_extent = true && that.isSetExtent(); + if (this_present_extent || that_present_extent) { + if (!(this_present_extent && that_present_extent)) + return false; + if (!this.extent.equals(that.extent)) + return false; + } + + boolean this_present_priority = true; + boolean that_present_priority = true; + if (this_present_priority || that_present_priority) { + if (!(this_present_priority && that_present_priority)) + return false; + if (this.priority != that.priority) + return false; + } + + boolean this_present_group = true && this.isSetGroup(); + boolean that_present_group = true && that.isSetGroup(); + if (this_present_group || that_present_group) { + if (!(this_present_group && that_present_group)) + return false; + if (!this.group.equals(that.group)) + return false; + } + + boolean this_present_tabletDir = true && this.isSetTabletDir(); + boolean that_present_tabletDir = true && that.isSetTabletDir(); + if (this_present_tabletDir || that_present_tabletDir) { + if (!(this_present_tabletDir && that_present_tabletDir)) + return false; + if (!this.tabletDir.equals(that.tabletDir)) + return false; + } + + boolean this_present_overlapsSelectedFiles = true; + boolean that_present_overlapsSelectedFiles = true; + if (this_present_overlapsSelectedFiles || that_present_overlapsSelectedFiles) { + if (!(this_present_overlapsSelectedFiles && that_present_overlapsSelectedFiles)) + return false; + if (this.overlapsSelectedFiles != that.overlapsSelectedFiles) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetSelectedFateId()) ? 131071 : 524287); + if (isSetSelectedFateId()) + hashCode = hashCode * 8191 + selectedFateId.hashCode(); + + hashCode = hashCode * 8191 + ((isSetJobFiles()) ? 131071 : 524287); + if (isSetJobFiles()) + hashCode = hashCode * 8191 + jobFiles.hashCode(); + + hashCode = hashCode * 8191 + ((isSetKind()) ? 131071 : 524287); + if (isSetKind()) + hashCode = hashCode * 8191 + kind.hashCode(); + + hashCode = hashCode * 8191 + ((compactingAll) ? 131071 : 524287); + + hashCode = hashCode * 8191 + ((isSetExtent()) ? 131071 : 524287); + if (isSetExtent()) + hashCode = hashCode * 8191 + extent.hashCode(); + + hashCode = hashCode * 8191 + priority; + + hashCode = hashCode * 8191 + ((isSetGroup()) ? 131071 : 524287); + if (isSetGroup()) + hashCode = hashCode * 8191 + group.hashCode(); + + hashCode = hashCode * 8191 + ((isSetTabletDir()) ? 131071 : 524287); + if (isSetTabletDir()) + hashCode = hashCode * 8191 + tabletDir.hashCode(); + + hashCode = hashCode * 8191 + ((overlapsSelectedFiles) ? 131071 : 524287); + + return hashCode; + } + + @Override + public int compareTo(TResolvedCompactionJob other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetSelectedFateId(), other.isSetSelectedFateId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSelectedFateId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.selectedFateId, other.selectedFateId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetJobFiles(), other.isSetJobFiles()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetJobFiles()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.jobFiles, other.jobFiles); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetKind(), other.isSetKind()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetKind()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.kind, other.kind); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCompactingAll(), other.isSetCompactingAll()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCompactingAll()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compactingAll, other.compactingAll); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetExtent(), other.isSetExtent()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetExtent()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.extent, other.extent); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetPriority(), other.isSetPriority()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPriority()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.priority, other.priority); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetGroup(), other.isSetGroup()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGroup()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.group, other.group); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetTabletDir(), other.isSetTabletDir()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTabletDir()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tabletDir, other.tabletDir); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetOverlapsSelectedFiles(), other.isSetOverlapsSelectedFiles()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetOverlapsSelectedFiles()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.overlapsSelectedFiles, other.overlapsSelectedFiles); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("TResolvedCompactionJob("); + boolean first = true; + + sb.append("selectedFateId:"); + if (this.selectedFateId == null) { + sb.append("null"); + } else { + sb.append(this.selectedFateId); + } + first = false; + if (!first) sb.append(", "); + sb.append("jobFiles:"); + if (this.jobFiles == null) { + sb.append("null"); + } else { + sb.append(this.jobFiles); + } + first = false; + if (!first) sb.append(", "); + sb.append("kind:"); + if (this.kind == null) { + sb.append("null"); + } else { + sb.append(this.kind); + } + first = false; + if (!first) sb.append(", "); + sb.append("compactingAll:"); + sb.append(this.compactingAll); + first = false; + if (!first) sb.append(", "); + sb.append("extent:"); + if (this.extent == null) { + sb.append("null"); + } else { + sb.append(this.extent); + } + first = false; + if (!first) sb.append(", "); + sb.append("priority:"); + sb.append(this.priority); + first = false; + if (!first) sb.append(", "); + sb.append("group:"); + if (this.group == null) { + sb.append("null"); + } else { + sb.append(this.group); + } + first = false; + if (!first) sb.append(", "); + sb.append("tabletDir:"); + if (this.tabletDir == null) { + sb.append("null"); + } else { + sb.append(this.tabletDir); + } + first = false; + if (!first) sb.append(", "); + sb.append("overlapsSelectedFiles:"); + sb.append(this.overlapsSelectedFiles); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (extent != null) { + extent.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TResolvedCompactionJobStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public TResolvedCompactionJobStandardScheme getScheme() { + return new TResolvedCompactionJobStandardScheme(); + } + } + + private static class TResolvedCompactionJobStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, TResolvedCompactionJob struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SELECTED_FATE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.selectedFateId = iprot.readString(); + struct.setSelectedFateIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // JOB_FILES + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list28 = iprot.readListBegin(); + struct.jobFiles = new java.util.ArrayList(_list28.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.InputFile _elem29; + for (int _i30 = 0; _i30 < _list28.size; ++_i30) + { + _elem29 = new org.apache.accumulo.core.tabletserver.thrift.InputFile(); + _elem29.read(iprot); + struct.jobFiles.add(_elem29); + } + iprot.readListEnd(); + } + struct.setJobFilesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // KIND + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.kind = iprot.readString(); + struct.setKindIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // COMPACTING_ALL + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.compactingAll = iprot.readBool(); + struct.setCompactingAllIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 5: // EXTENT + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + struct.extent.read(iprot); + struct.setExtentIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 6: // PRIORITY + if (schemeField.type == org.apache.thrift.protocol.TType.I16) { + struct.priority = iprot.readI16(); + struct.setPriorityIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // GROUP + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.group = iprot.readString(); + struct.setGroupIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 8: // TABLET_DIR + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.tabletDir = iprot.readString(); + struct.setTabletDirIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 9: // OVERLAPS_SELECTED_FILES + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.overlapsSelectedFiles = iprot.readBool(); + struct.setOverlapsSelectedFilesIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, TResolvedCompactionJob struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.selectedFateId != null) { + oprot.writeFieldBegin(SELECTED_FATE_ID_FIELD_DESC); + oprot.writeString(struct.selectedFateId); + oprot.writeFieldEnd(); + } + if (struct.jobFiles != null) { + oprot.writeFieldBegin(JOB_FILES_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.jobFiles.size())); + for (org.apache.accumulo.core.tabletserver.thrift.InputFile _iter31 : struct.jobFiles) + { + _iter31.write(oprot); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.kind != null) { + oprot.writeFieldBegin(KIND_FIELD_DESC); + oprot.writeString(struct.kind); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(COMPACTING_ALL_FIELD_DESC); + oprot.writeBool(struct.compactingAll); + oprot.writeFieldEnd(); + if (struct.extent != null) { + oprot.writeFieldBegin(EXTENT_FIELD_DESC); + struct.extent.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(PRIORITY_FIELD_DESC); + oprot.writeI16(struct.priority); + oprot.writeFieldEnd(); + if (struct.group != null) { + oprot.writeFieldBegin(GROUP_FIELD_DESC); + oprot.writeString(struct.group); + oprot.writeFieldEnd(); + } + if (struct.tabletDir != null) { + oprot.writeFieldBegin(TABLET_DIR_FIELD_DESC); + oprot.writeString(struct.tabletDir); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(OVERLAPS_SELECTED_FILES_FIELD_DESC); + oprot.writeBool(struct.overlapsSelectedFiles); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TResolvedCompactionJobTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public TResolvedCompactionJobTupleScheme getScheme() { + return new TResolvedCompactionJobTupleScheme(); + } + } + + private static class TResolvedCompactionJobTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TResolvedCompactionJob struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetSelectedFateId()) { + optionals.set(0); + } + if (struct.isSetJobFiles()) { + optionals.set(1); + } + if (struct.isSetKind()) { + optionals.set(2); + } + if (struct.isSetCompactingAll()) { + optionals.set(3); + } + if (struct.isSetExtent()) { + optionals.set(4); + } + if (struct.isSetPriority()) { + optionals.set(5); + } + if (struct.isSetGroup()) { + optionals.set(6); + } + if (struct.isSetTabletDir()) { + optionals.set(7); + } + if (struct.isSetOverlapsSelectedFiles()) { + optionals.set(8); + } + oprot.writeBitSet(optionals, 9); + if (struct.isSetSelectedFateId()) { + oprot.writeString(struct.selectedFateId); + } + if (struct.isSetJobFiles()) { + { + oprot.writeI32(struct.jobFiles.size()); + for (org.apache.accumulo.core.tabletserver.thrift.InputFile _iter32 : struct.jobFiles) + { + _iter32.write(oprot); + } + } + } + if (struct.isSetKind()) { + oprot.writeString(struct.kind); + } + if (struct.isSetCompactingAll()) { + oprot.writeBool(struct.compactingAll); + } + if (struct.isSetExtent()) { + struct.extent.write(oprot); + } + if (struct.isSetPriority()) { + oprot.writeI16(struct.priority); + } + if (struct.isSetGroup()) { + oprot.writeString(struct.group); + } + if (struct.isSetTabletDir()) { + oprot.writeString(struct.tabletDir); + } + if (struct.isSetOverlapsSelectedFiles()) { + oprot.writeBool(struct.overlapsSelectedFiles); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TResolvedCompactionJob struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(9); + if (incoming.get(0)) { + struct.selectedFateId = iprot.readString(); + struct.setSelectedFateIdIsSet(true); + } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list33 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.jobFiles = new java.util.ArrayList(_list33.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.InputFile _elem34; + for (int _i35 = 0; _i35 < _list33.size; ++_i35) + { + _elem34 = new org.apache.accumulo.core.tabletserver.thrift.InputFile(); + _elem34.read(iprot); + struct.jobFiles.add(_elem34); + } + } + struct.setJobFilesIsSet(true); + } + if (incoming.get(2)) { + struct.kind = iprot.readString(); + struct.setKindIsSet(true); + } + if (incoming.get(3)) { + struct.compactingAll = iprot.readBool(); + struct.setCompactingAllIsSet(true); + } + if (incoming.get(4)) { + struct.extent = new org.apache.accumulo.core.dataImpl.thrift.TKeyExtent(); + struct.extent.read(iprot); + struct.setExtentIsSet(true); + } + if (incoming.get(5)) { + struct.priority = iprot.readI16(); + struct.setPriorityIsSet(true); + } + if (incoming.get(6)) { + struct.group = iprot.readString(); + struct.setGroupIsSet(true); + } + if (incoming.get(7)) { + struct.tabletDir = iprot.readString(); + struct.setTabletDirIsSet(true); + } + if (incoming.get(8)) { + struct.overlapsSelectedFiles = iprot.readBool(); + struct.setOverlapsSelectedFilesIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + private static void unusedMethod() {} +} + diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index 99d5df10ee9..60e44bd08b3 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -69,6 +69,18 @@ struct TNextCompactionJob { 2:i32 compactorCount } +struct TResolvedCompactionJob { + 1:string selectedFateId + 2:list jobFiles; + 3:string kind + 4:bool compactingAll + 5:data.TKeyExtent extent + 6:i16 priority + 7:string group + 8:string tabletDir + 9:bool overlapsSelectedFiles +} + exception UnknownCompactionIdException {} @@ -179,6 +191,31 @@ service CompactionCoordinatorService { 2:security.TCredentials credentials 3:string externalCompactionId ) + + void beginFullJobScan( + 1:client.TInfo tinfo + 2:security.TCredentials credentials + 3:string dataLevel + )throws( + 1:client.ThriftSecurityException sec + 2:client.ThriftNotActiveServiceException tnase + ) + + oneway void addJobs( + 1:client.TInfo tinfo + 2:security.TCredentials credentials + 3:list jobs + ) + + void endFullJobScan( + 1:client.TInfo tinfo + 2:security.TCredentials credentials + 3:string dataLevel + )throws( + 1:client.ThriftSecurityException sec + 2:client.ThriftNotActiveServiceException tnase + ) + } service CompactorService { diff --git a/core/src/main/thrift/tabletserver.thrift b/core/src/main/thrift/tabletserver.thrift index 1069445ac7b..f66ffefee33 100644 --- a/core/src/main/thrift/tabletserver.thrift +++ b/core/src/main/thrift/tabletserver.thrift @@ -103,7 +103,7 @@ struct InputFile { 3:i64 entries 4:i64 timestamp } - +//TODO this should move outside of the tablet server file, tabletserver no longer has anything to with compactions struct TExternalCompactionJob { 1:string externalCompactionId 2:data.TKeyExtent extent diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index c1192a7421d..904c62f73f1 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -88,6 +88,7 @@ import org.apache.accumulo.core.util.threads.Threads.AccumuloDaemonThread; import org.apache.accumulo.manager.EventCoordinator.Event; import org.apache.accumulo.manager.EventCoordinator.EventScope; +import org.apache.accumulo.manager.compaction.CompactionJobClient; import org.apache.accumulo.manager.recovery.RecoveryManager; import org.apache.accumulo.manager.state.TableCounts; import org.apache.accumulo.manager.state.TableStats; @@ -382,9 +383,10 @@ private boolean processRanges(List ranges) { return false; } - try (var iter = store.iterator(ranges, tabletMgmtParams)) { + try (var iter = store.iterator(ranges, tabletMgmtParams); var compactionJobClient = + new CompactionJobClient(manager.getContext(), store.getLevel(), false)) { long t1 = System.currentTimeMillis(); - manageTablets(iter, tabletMgmtParams, currentTservers, false); + manageTablets(iter, tabletMgmtParams, currentTservers, false, compactionJobClient); long t2 = System.currentTimeMillis(); Manager.log.debug(String.format("[%s]: partial scan time %.2f seconds for %,d ranges", store.name(), (t2 - t1) / 1000., ranges.size())); @@ -491,7 +493,8 @@ private static class TableMgmtStats { private TableMgmtStats manageTablets(Iterator iter, TabletManagementParameters tableMgmtParams, - SortedMap currentTServers, boolean isFullScan) + SortedMap currentTServers, boolean isFullScan, + CompactionJobClient compactionJobClient) throws TException, DistributedStoreException, WalMarkerException, IOException { // When upgrading the Manager needs the TabletGroupWatcher @@ -694,7 +697,7 @@ private TableMgmtStats manageTablets(Iterator iter, var jobs = compactionGenerator.generateJobs(tm, TabletManagementIterator.determineCompactionKinds(actions)); LOG.debug("{} may need compacting adding {} jobs", tm.getExtent(), jobs.size()); - manager.getCompactionCoordinator().addJobs(tm, jobs); + compactionJobClient.addJobs(tm, jobs); } else { LOG.trace("skipping compaction job generation because {} may need splitting.", tm.getExtent()); @@ -827,9 +830,12 @@ public void run() { eventHandler.clearNeedsFullScan(); iter = store.iterator(tableMgmtParams); - manager.getCompactionCoordinator().getJobQueues().beginFullScan(store.getLevel()); - var tabletMgmtStats = manageTablets(iter, tableMgmtParams, currentTServers, true); - manager.getCompactionCoordinator().getJobQueues().endFullScan(store.getLevel()); + TableMgmtStats tabletMgmtStats; + try (var compactionJobClient = + new CompactionJobClient(manager.getContext(), store.getLevel(), true)) { + tabletMgmtStats = + manageTablets(iter, tableMgmtParams, currentTServers, true, compactionJobClient); + } // If currently looking for volume replacements, determine if the next round needs to look. if (lookForTabletsNeedingVolReplacement) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java new file mode 100644 index 00000000000..f55225110c4 --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.compaction; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; +import org.apache.accumulo.core.compaction.thrift.TResolvedCompactionJob; +import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.rpc.ThriftUtil; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.spi.compaction.CompactionJob; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.manager.compaction.queue.ResolvedCompactionJob; +import org.apache.accumulo.server.ServerContext; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.net.HostAndPort; + +/** + * Sends compaction jobs to remote compaction coordinators. + */ +public class CompactionJobClient implements AutoCloseable { + + private static final Logger log = LoggerFactory.getLogger(CompactionJobClient.class); + + private final boolean fullScan; + private final Ample.DataLevel dataLevel; + private final ServerContext context; + + record CoordinatorConnection(CompactionCoordinatorService.Client client, + List jobBuffer) { + } + + private final Map coordinatorLocations; + private final Map coordinatorConnections = new HashMap<>(); + + private static final int BUFFER_SIZE = 1000; + + public CompactionJobClient(ServerContext context, Ample.DataLevel dataLevel, boolean fullScan) { + this.context = context; + this.dataLevel = dataLevel; + this.fullScan = fullScan; + + this.coordinatorLocations = context.getCoordinatorLocations(true); + + var uniqueHosts = new HashSet<>(coordinatorLocations.values()); + for (var hostPort : uniqueHosts) { + try { + var client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, hostPort, context); + if (fullScan) { + client.beginFullJobScan(TraceUtil.traceInfo(), context.rpcCreds(), dataLevel.name()); + } + + coordinatorConnections.put(hostPort, + new CoordinatorConnection(client, new ArrayList<>(BUFFER_SIZE))); + } catch (TException e) { + // TODO only log + throw new RuntimeException(e); + } + } + } + + public void addJobs(TabletMetadata tabletMetadata, Collection jobs) { + for (var job : jobs) { + var resolvedJob = new ResolvedCompactionJob(job, tabletMetadata); + var hostPort = coordinatorLocations.get(resolvedJob.getGroup()); + if (hostPort == null) { + continue; + } + var coordinator = coordinatorConnections.get(hostPort); + if (coordinator == null) { + continue; + } + + coordinator.jobBuffer.add(resolvedJob); + if (coordinator.jobBuffer.size() >= BUFFER_SIZE) { + try { + sendJobs(coordinator); + } catch (TException e) { + log.warn("Failed to send compaction jobs to {}", hostPort, e); + ThriftUtil.returnClient(coordinator.client, context); + // ignore this coordinator for the rest of the session + coordinatorConnections.remove(hostPort); + } + } + } + } + + private void sendJobs(CoordinatorConnection coordinator) throws TException { + List thriftJobs = new ArrayList<>(coordinator.jobBuffer.size()); + for (var job : coordinator.jobBuffer) { + thriftJobs.add(job.toThrift()); + } + coordinator.client.addJobs(TraceUtil.traceInfo(), context.rpcCreds(), thriftJobs); + coordinator.jobBuffer.clear(); + } + + @Override + public void close() { + coordinatorConnections.forEach(((hostAndPort, coordinator) -> { + try { + sendJobs(coordinator); + if (fullScan) { + coordinator.client.endFullJobScan(TraceUtil.traceInfo(), context.rpcCreds(), + dataLevel.name()); + } + } catch (TException e) { + log.warn("Failed to communicate with coordinator {}", hostAndPort, e); + } finally { + ThriftUtil.returnClient(coordinator.client, context); + } + })); + } +} diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index f895a89466e..0df819a56b6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -82,6 +82,7 @@ import org.apache.accumulo.core.compaction.thrift.TExternalCompactionList; import org.apache.accumulo.core.compaction.thrift.TExternalCompactionMap; import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; +import org.apache.accumulo.core.compaction.thrift.TResolvedCompactionJob; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.NamespaceId; @@ -724,6 +725,42 @@ public void registerMetrics(MeterRegistry registry) { queueMetrics.registerMetrics(registry); } + @Override + public void beginFullJobScan(TInfo tinfo, TCredentials credentials, String dataLevel) + throws TException { + if (!security.canPerformSystemActions(credentials)) { + throw new AccumuloSecurityException(credentials.getPrincipal(), + SecurityErrorCode.PERMISSION_DENIED).asThriftException(); + } + jobQueues.beginFullScan(DataLevel.valueOf(dataLevel)); + } + + @Override + public void addJobs(TInfo tinfo, TCredentials credentials, List jobs) + throws TException { + if (!security.canPerformSystemActions(credentials)) { + // this is a oneway method so throwing an exception is not useful + return; + } + + for (var tjob : jobs) { + var job = ResolvedCompactionJob.fromThrift(tjob); + // TODO maybe no longer need to pass a list + jobQueues.add(job.getExtent(), List.of(job)); + } + } + + @Override + public void endFullJobScan(TInfo tinfo, TCredentials credentials, String dataLevel) + throws TException { + if (!security.canPerformSystemActions(credentials)) { + throw new AccumuloSecurityException(credentials.getPrincipal(), + SecurityErrorCode.PERMISSION_DENIED).asThriftException(); + } + jobQueues.endFullScan(DataLevel.valueOf(dataLevel)); + } + + // TODO remove public void addJobs(TabletMetadata tabletMetadata, Collection jobs) { ArrayList resolvedJobs = new ArrayList<>(jobs.size()); for (var job : jobs) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java index 3177b3c0fbf..85b37f34dfb 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java @@ -18,6 +18,8 @@ */ package org.apache.accumulo.manager.compaction.queue; +import static java.util.stream.Collectors.toList; + import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -25,6 +27,7 @@ import java.util.stream.Collectors; import org.apache.accumulo.core.client.admin.compaction.CompactableFile; +import org.apache.accumulo.core.compaction.thrift.TResolvedCompactionJob; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.data.RowRange; @@ -36,6 +39,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.spi.compaction.CompactionJob; import org.apache.accumulo.core.spi.compaction.CompactionKind; +import org.apache.accumulo.core.tabletserver.thrift.InputFile; /** * This class takes a compaction job and the tablet metadata from which is was generated and @@ -61,6 +65,22 @@ public class ResolvedCompactionJob implements CompactionJob { private final String tabletDir; private final boolean overlapsSelectedFiles; + private ResolvedCompactionJob(TResolvedCompactionJob trj) { + selectedFateId = trj.selectedFateId == null ? null : FateId.from(trj.selectedFateId); + jobFiles = new HashMap<>(); + for (var inputFile : trj.jobFiles) { + jobFiles.put(StoredTabletFile.of(inputFile.metadataFileEntry), + new DataFileValue(inputFile.size, inputFile.entries, inputFile.timestamp)); + } + kind = CompactionKind.valueOf(trj.kind); + compactingAll = trj.compactingAll; + extent = KeyExtent.fromThrift(trj.extent); + priority = trj.priority; + group = ResourceGroupId.of(trj.group); + tabletDir = trj.tabletDir; + overlapsSelectedFiles = trj.overlapsSelectedFiles; + } + private static long weigh(RowRange rowRange) { if (rowRange != null) { return weigh(rowRange.asRange()); @@ -211,4 +231,21 @@ public boolean equals(Object o) { public int hashCode() { throw new UnsupportedOperationException(); } + + public TResolvedCompactionJob toThrift() { + // TODO consolidate code to do this conversion + var files = jobFiles.entrySet().stream().map(e -> { + StoredTabletFile file = e.getKey(); + DataFileValue dfv = e.getValue(); + return new InputFile(file.getMetadata(), dfv.getSize(), dfv.getNumEntries(), dfv.getTime()); + }).toList(); + + return new TResolvedCompactionJob(selectedFateId == null ? null : selectedFateId.canonical(), + files, kind.name(), compactingAll, extent.toThrift(), priority, group.canonical(), + tabletDir, overlapsSelectedFiles); + } + + public static ResolvedCompactionJob fromThrift(TResolvedCompactionJob trj) { + return new ResolvedCompactionJob(trj); + } } From e5a5384fdf143a5a038ab3f977508bc07a7b1401 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 17 Mar 2026 01:31:50 +0000 Subject: [PATCH 03/65] First stab at distributing coordinator, its not really working yet --- .../thrift/CompactionCoordinatorService.java | 2521 +++++++++-------- .../compaction/thrift/CompactorService.java | 36 +- .../main/thrift/compaction-coordinator.thrift | 17 +- .../apache/accumulo/compactor/Compactor.java | 3 +- .../org/apache/accumulo/manager/Manager.java | 7 +- .../compaction/CompactionJobClient.java | 10 +- .../coordinator/CompactionCoordinator.java | 28 +- .../coordinator/CoordinatorManager.java | 94 +- .../compaction/queue/CompactionJobQueues.java | 39 +- .../test/ComprehensiveMultiManagerIT.java | 2 + .../test/MultipleManagerCompactionIT.java | 62 + 11 files changed, 1529 insertions(+), 1290 deletions(-) create mode 100644 test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index 55e94e1044c..6dff9868c64 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -29,29 +29,31 @@ public class CompactionCoordinatorService { public interface Iface { - public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public TNextCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public TNextCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public void updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, TCompactionStatusUpdate status, long timestamp) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, TCompactionStatusUpdate status, long timestamp) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public java.util.Map getLongRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public java.util.Map getLongRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException; - public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; public void addJobs(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.List jobs) throws org.apache.thrift.TException; - public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; + + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; } @@ -81,6 +83,8 @@ public interface AsyncIface { public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -106,7 +110,7 @@ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.prot } @Override - public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_compactionCompleted(tinfo, credentials, externalCompactionId, extent, stats); recv_compactionCompleted(); @@ -123,21 +127,18 @@ public void send_compactionCompleted(org.apache.accumulo.core.clientImpl.thrift. sendBase("compactionCompleted", args); } - public void recv_compactionCompleted() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void recv_compactionCompleted() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { compactionCompleted_result result = new compactionCompleted_result(); receiveBase(result, "compactionCompleted"); if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } return; } @Override - public TNextCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public TNextCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_getCompactionJob(tinfo, credentials, groupName, compactor, externalCompactionId); return recv_getCompactionJob(); @@ -154,7 +155,7 @@ public void send_getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TIn sendBase("getCompactionJob", args); } - public TNextCompactionJob recv_getCompactionJob() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public TNextCompactionJob recv_getCompactionJob() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { getCompactionJob_result result = new getCompactionJob_result(); receiveBase(result, "getCompactionJob"); @@ -164,14 +165,11 @@ public TNextCompactionJob recv_getCompactionJob() throws org.apache.accumulo.cor if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getCompactionJob failed: unknown result"); } @Override - public void updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, TCompactionStatusUpdate status, long timestamp) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, TCompactionStatusUpdate status, long timestamp) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_updateCompactionStatus(tinfo, credentials, externalCompactionId, status, timestamp); recv_updateCompactionStatus(); @@ -188,21 +186,18 @@ public void send_updateCompactionStatus(org.apache.accumulo.core.clientImpl.thri sendBase("updateCompactionStatus", args); } - public void recv_updateCompactionStatus() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void recv_updateCompactionStatus() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { updateCompactionStatus_result result = new updateCompactionStatus_result(); receiveBase(result, "updateCompactionStatus"); if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } return; } @Override - public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_compactionFailed(tinfo, credentials, externalCompactionId, extent, exceptionClassName, failureState); recv_compactionFailed(); @@ -220,21 +215,18 @@ public void send_compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TIn sendBase("compactionFailed", args); } - public void recv_compactionFailed() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void recv_compactionFailed() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { compactionFailed_result result = new compactionFailed_result(); receiveBase(result, "compactionFailed"); if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } return; } @Override - public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_getRunningCompactions(tinfo, credentials); return recv_getRunningCompactions(); @@ -248,7 +240,7 @@ public void send_getRunningCompactions(org.apache.accumulo.core.clientImpl.thrif sendBase("getRunningCompactions", args); } - public TExternalCompactionMap recv_getRunningCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public TExternalCompactionMap recv_getRunningCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { getRunningCompactions_result result = new getRunningCompactions_result(); receiveBase(result, "getRunningCompactions"); @@ -258,14 +250,11 @@ public TExternalCompactionMap recv_getRunningCompactions() throws org.apache.acc if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRunningCompactions failed: unknown result"); } @Override - public java.util.Map getLongRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public java.util.Map getLongRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_getLongRunningCompactions(tinfo, credentials); return recv_getLongRunningCompactions(); @@ -279,7 +268,7 @@ public void send_getLongRunningCompactions(org.apache.accumulo.core.clientImpl.t sendBase("getLongRunningCompactions", args); } - public java.util.Map recv_getLongRunningCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public java.util.Map recv_getLongRunningCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { getLongRunningCompactions_result result = new getLongRunningCompactions_result(); receiveBase(result, "getLongRunningCompactions"); @@ -289,14 +278,11 @@ public java.util.Map recv_getLongRunni if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getLongRunningCompactions failed: unknown result"); } @Override - public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_getCompletedCompactions(tinfo, credentials); return recv_getCompletedCompactions(); @@ -310,7 +296,7 @@ public void send_getCompletedCompactions(org.apache.accumulo.core.clientImpl.thr sendBase("getCompletedCompactions", args); } - public TExternalCompactionMap recv_getCompletedCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public TExternalCompactionMap recv_getCompletedCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { getCompletedCompactions_result result = new getCompletedCompactions_result(); receiveBase(result, "getCompletedCompactions"); @@ -320,14 +306,11 @@ public TExternalCompactionMap recv_getCompletedCompactions() throws org.apache.a if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getCompletedCompactions failed: unknown result"); } @Override - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_cancel(tinfo, credentials, externalCompactionId); recv_cancel(); @@ -342,16 +325,13 @@ public void send_cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, sendBase("cancel", args); } - public void recv_cancel() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void recv_cancel() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { cancel_result result = new cancel_result(); receiveBase(result, "cancel"); if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } return; } @@ -371,7 +351,7 @@ public void send_recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TIn } @Override - public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_beginFullJobScan(tinfo, credentials, dataLevel); recv_beginFullJobScan(); @@ -386,16 +366,13 @@ public void send_beginFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TIn sendBase("beginFullJobScan", args); } - public void recv_beginFullJobScan() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void recv_beginFullJobScan() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { beginFullJobScan_result result = new beginFullJobScan_result(); receiveBase(result, "beginFullJobScan"); if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; - } return; } @@ -415,7 +392,7 @@ public void send_addJobs(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, } @Override - public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { send_endFullJobScan(tinfo, credentials, dataLevel); recv_endFullJobScan(); @@ -430,15 +407,38 @@ public void send_endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo sendBase("endFullJobScan", args); } - public void recv_endFullJobScan() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void recv_endFullJobScan() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { endFullJobScan_result result = new endFullJobScan_result(); receiveBase(result, "endFullJobScan"); if (result.sec != null) { throw result.sec; } - if (result.tnase != null) { - throw result.tnase; + return; + } + + @Override + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + { + send_setResourceGroups(tinfo, credentials, groups); + recv_setResourceGroups(); + } + + public void send_setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups) throws org.apache.thrift.TException + { + setResourceGroups_args args = new setResourceGroups_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setGroups(groups); + sendBase("setResourceGroups", args); + } + + public void recv_setResourceGroups() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + { + setResourceGroups_result result = new setResourceGroups_result(); + receiveBase(result, "setResourceGroups"); + if (result.sec != null) { + throw result.sec; } return; } @@ -499,7 +499,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -547,7 +547,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public TNextCompactionJob getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public TNextCompactionJob getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -594,7 +594,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -645,7 +645,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -684,7 +684,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public TExternalCompactionMap getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public TExternalCompactionMap getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -722,7 +722,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public java.util.Map getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public java.util.Map getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -760,7 +760,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public TExternalCompactionMap getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public TExternalCompactionMap getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -801,7 +801,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -884,7 +884,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -967,7 +967,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa } @Override - public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { throw new java.lang.IllegalStateException("Method call not finished!"); } @@ -978,6 +978,48 @@ public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.Thrift } } + @Override + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + setResourceGroups_call method_call = new setResourceGroups_call(tinfo, credentials, groups, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class setResourceGroups_call extends org.apache.thrift.async.TAsyncMethodCall { + private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; + private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; + private java.util.Set groups; + public setResourceGroups_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tinfo = tinfo; + this.credentials = credentials; + this.groups = groups; + } + + @Override + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("setResourceGroups", org.apache.thrift.protocol.TMessageType.CALL, 0)); + setResourceGroups_args args = new setResourceGroups_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setGroups(groups); + args.write(prot); + prot.writeMessageEnd(); + } + + @Override + public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new java.lang.IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + (new Client(prot)).recv_setResourceGroups(); + return null; + } + } + } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -1003,6 +1045,7 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { + public setResourceGroups() { + super("setResourceGroups"); + } + + @Override + public setResourceGroups_args getEmptyArgsInstance() { + return new setResourceGroups_args(); + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + protected boolean rethrowUnhandledExceptions() { + return false; + } + + @Override + public setResourceGroups_result getResult(I iface, setResourceGroups_args args) throws org.apache.thrift.TException { + setResourceGroups_result result = new setResourceGroups_result(); + try { + iface.setResourceGroups(args.tinfo, args.credentials, args.groups); + } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + result.sec = sec; } return result; } @@ -1425,6 +1480,7 @@ protected AsyncProcessor(I iface, java.util.Map extends org.apache.thrift.AsyncProcessFunction { + public setResourceGroups() { + super("setResourceGroups"); + } + + @Override + public setResourceGroups_args getEmptyArgsInstance() { + return new setResourceGroups_args(); + } + + @Override + public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new org.apache.thrift.async.AsyncMethodCallback() { + @Override + public void onComplete(Void o) { + setResourceGroups_result result = new setResourceGroups_result(); + try { + fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + } catch (org.apache.thrift.transport.TTransportException e) { + _LOGGER.error("TTransportException writing to internal frame buffer", e); + fb.close(); + } catch (java.lang.Exception e) { + _LOGGER.error("Exception writing to internal frame buffer", e); + onError(e); + } + } + @Override + public void onError(java.lang.Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TSerializable msg; + setResourceGroups_result result = new setResourceGroups_result(); + if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { + result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; + result.setSecIsSet(true); + msg = result; + } else if (e instanceof org.apache.thrift.transport.TTransportException) { + _LOGGER.error("TTransportException inside handler", e); + fb.close(); + return; + } else if (e instanceof org.apache.thrift.TApplicationException) { + _LOGGER.error("TApplicationException inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TApplicationException)e; + } else { + _LOGGER.error("Exception inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + } catch (java.lang.Exception ex) { + _LOGGER.error("Exception writing to internal frame buffer", ex); + fb.close(); + } + } + }; + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + public void start(I iface, setResourceGroups_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + iface.setResourceGroups(args.tinfo, args.credentials, args.groups,resultHandler); + } + } + } @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) @@ -3078,18 +3164,15 @@ public static class compactionCompleted_result implements org.apache.thrift.TBas private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("compactionCompleted_result"); private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new compactionCompleted_resultStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new compactionCompleted_resultTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); + SEC((short)1, "sec"); private static final java.util.Map byName = new java.util.HashMap(); @@ -3107,8 +3190,6 @@ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -3157,8 +3238,6 @@ public java.lang.String getFieldName() { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compactionCompleted_result.class, metaDataMap); } @@ -3167,12 +3246,10 @@ public compactionCompleted_result() { } public compactionCompleted_result( - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.sec = sec; - this.tnase = tnase; } /** @@ -3182,9 +3259,6 @@ public compactionCompleted_result(compactionCompleted_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -3195,7 +3269,6 @@ public compactionCompleted_result deepCopy() { @Override public void clear() { this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -3223,31 +3296,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public compactionCompleted_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -3259,14 +3307,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -3277,9 +3317,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -3294,8 +3331,6 @@ public boolean isSet(_Fields field) { switch (field) { case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -3322,15 +3357,6 @@ public boolean equals(compactionCompleted_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -3342,10 +3368,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -3367,16 +3389,6 @@ public int compareTo(compactionCompleted_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -3407,14 +3419,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -3469,15 +3473,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, compactionCompleted org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -3499,11 +3494,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, compactionComplete struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -3526,32 +3516,21 @@ public void write(org.apache.thrift.protocol.TProtocol prot, compactionCompleted if (struct.isSetSec()) { optionals.set(0); } - if (struct.isSetTnase()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); + oprot.writeBitSet(optionals, 1); if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, compactionCompleted_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); + java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(1)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -4375,20 +4354,17 @@ public static class getCompactionJob_result implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -4408,8 +4384,6 @@ public static _Fields findByThriftId(int fieldId) { return SUCCESS; case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -4460,8 +4434,6 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TNextCompactionJob.class))); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getCompactionJob_result.class, metaDataMap); } @@ -4471,13 +4443,11 @@ public getCompactionJob_result() { public getCompactionJob_result( TNextCompactionJob success, - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.success = success; this.sec = sec; - this.tnase = tnase; } /** @@ -4490,9 +4460,6 @@ public getCompactionJob_result(getCompactionJob_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -4504,7 +4471,6 @@ public getCompactionJob_result deepCopy() { public void clear() { this.success = null; this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -4557,31 +4523,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public getCompactionJob_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -4601,14 +4542,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -4622,9 +4555,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -4641,8 +4571,6 @@ public boolean isSet(_Fields field) { return isSetSuccess(); case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -4678,15 +4606,6 @@ public boolean equals(getCompactionJob_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -4702,10 +4621,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -4737,16 +4652,6 @@ public int compareTo(getCompactionJob_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -4785,14 +4690,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -4859,15 +4756,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_re org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -4894,11 +4782,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_r struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -4924,25 +4807,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_re if (struct.isSetSec()) { optionals.set(1); } - if (struct.isSetTnase()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); + oprot.writeBitSet(optionals, 2); if (struct.isSetSuccess()) { struct.success.write(oprot); } if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); + java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { struct.success = new TNextCompactionJob(); struct.success.read(iprot); @@ -4953,11 +4830,6 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_res struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(2)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -5780,18 +5652,15 @@ public static class updateCompactionStatus_result implements org.apache.thrift.T private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("updateCompactionStatus_result"); private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new updateCompactionStatus_resultStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new updateCompactionStatus_resultTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); + SEC((short)1, "sec"); private static final java.util.Map byName = new java.util.HashMap(); @@ -5809,8 +5678,6 @@ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -5859,8 +5726,6 @@ public java.lang.String getFieldName() { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(updateCompactionStatus_result.class, metaDataMap); } @@ -5869,12 +5734,10 @@ public updateCompactionStatus_result() { } public updateCompactionStatus_result( - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.sec = sec; - this.tnase = tnase; } /** @@ -5884,9 +5747,6 @@ public updateCompactionStatus_result(updateCompactionStatus_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -5897,7 +5757,6 @@ public updateCompactionStatus_result deepCopy() { @Override public void clear() { this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -5925,31 +5784,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public updateCompactionStatus_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -5961,14 +5795,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -5979,9 +5805,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -5996,8 +5819,6 @@ public boolean isSet(_Fields field) { switch (field) { case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -6024,15 +5845,6 @@ public boolean equals(updateCompactionStatus_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -6044,10 +5856,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -6069,16 +5877,6 @@ public int compareTo(updateCompactionStatus_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -6109,14 +5907,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -6171,15 +5961,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, updateCompactionSta org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -6201,11 +5982,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, updateCompactionSt struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -6228,32 +6004,21 @@ public void write(org.apache.thrift.protocol.TProtocol prot, updateCompactionSta if (struct.isSetSec()) { optionals.set(0); } - if (struct.isSetTnase()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); + oprot.writeBitSet(optionals, 1); if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, updateCompactionStatus_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); + java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(1)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -7202,18 +6967,15 @@ public static class compactionFailed_result implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -7231,8 +6993,6 @@ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -7281,8 +7041,6 @@ public java.lang.String getFieldName() { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compactionFailed_result.class, metaDataMap); } @@ -7291,12 +7049,10 @@ public compactionFailed_result() { } public compactionFailed_result( - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.sec = sec; - this.tnase = tnase; } /** @@ -7306,9 +7062,6 @@ public compactionFailed_result(compactionFailed_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -7319,7 +7072,6 @@ public compactionFailed_result deepCopy() { @Override public void clear() { this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -7347,31 +7099,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public compactionFailed_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -7383,14 +7110,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -7401,9 +7120,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -7418,8 +7134,6 @@ public boolean isSet(_Fields field) { switch (field) { case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -7446,15 +7160,6 @@ public boolean equals(compactionFailed_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -7466,10 +7171,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -7491,16 +7192,6 @@ public int compareTo(compactionFailed_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -7531,14 +7222,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -7593,15 +7276,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, compactionFailed_re org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -7623,11 +7297,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, compactionFailed_r struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -7650,32 +7319,21 @@ public void write(org.apache.thrift.protocol.TProtocol prot, compactionFailed_re if (struct.isSetSec()) { optionals.set(0); } - if (struct.isSetTnase()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); + oprot.writeBitSet(optionals, 1); if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, compactionFailed_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); + java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(1)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -8184,20 +7842,17 @@ public static class getRunningCompactions_result implements org.apache.thrift.TB private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getRunningCompactions_resultStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getRunningCompactions_resultTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable TExternalCompactionMap success; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { SUCCESS((short)0, "success"), - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); + SEC((short)1, "sec"); private static final java.util.Map byName = new java.util.HashMap(); @@ -8217,8 +7872,6 @@ public static _Fields findByThriftId(int fieldId) { return SUCCESS; case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -8269,8 +7922,6 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExternalCompactionMap.class))); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRunningCompactions_result.class, metaDataMap); } @@ -8280,13 +7931,11 @@ public getRunningCompactions_result() { public getRunningCompactions_result( TExternalCompactionMap success, - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.success = success; this.sec = sec; - this.tnase = tnase; } /** @@ -8299,9 +7948,6 @@ public getRunningCompactions_result(getRunningCompactions_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -8313,7 +7959,6 @@ public getRunningCompactions_result deepCopy() { public void clear() { this.success = null; this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -8366,31 +8011,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public getRunningCompactions_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -8410,14 +8030,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -8431,9 +8043,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -8450,8 +8059,6 @@ public boolean isSet(_Fields field) { return isSetSuccess(); case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -8487,15 +8094,6 @@ public boolean equals(getRunningCompactions_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -8511,10 +8109,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -8546,16 +8140,6 @@ public int compareTo(getRunningCompactions_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -8594,14 +8178,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -8668,15 +8244,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getRunningCompactio org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -8703,11 +8270,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getRunningCompacti struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -8733,25 +8295,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getRunningCompactio if (struct.isSetSec()) { optionals.set(1); } - if (struct.isSetTnase()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); + oprot.writeBitSet(optionals, 2); if (struct.isSetSuccess()) { struct.success.write(oprot); } if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, getRunningCompactions_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); + java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { struct.success = new TExternalCompactionMap(); struct.success.read(iprot); @@ -8762,11 +8318,6 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getRunningCompaction struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(2)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -9275,20 +8826,17 @@ public static class getLongRunningCompactions_result implements org.apache.thrif private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.MAP, (short)0); private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getLongRunningCompactions_resultStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getLongRunningCompactions_resultTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable java.util.Map success; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { SUCCESS((short)0, "success"), - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); + SEC((short)1, "sec"); private static final java.util.Map byName = new java.util.HashMap(); @@ -9308,8 +8856,6 @@ public static _Fields findByThriftId(int fieldId) { return SUCCESS; case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -9362,8 +8908,6 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExternalCompactionList.class)))); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getLongRunningCompactions_result.class, metaDataMap); } @@ -9373,13 +8917,11 @@ public getLongRunningCompactions_result() { public getLongRunningCompactions_result( java.util.Map success, - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.success = success; this.sec = sec; - this.tnase = tnase; } /** @@ -9404,9 +8946,6 @@ public getLongRunningCompactions_result(getLongRunningCompactions_result other) if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -9418,7 +8957,6 @@ public getLongRunningCompactions_result deepCopy() { public void clear() { this.success = null; this.sec = null; - this.tnase = null; } public int getSuccessSize() { @@ -9482,31 +9020,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public getLongRunningCompactions_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -9526,14 +9039,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -9547,9 +9052,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -9566,8 +9068,6 @@ public boolean isSet(_Fields field) { return isSetSuccess(); case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -9603,15 +9103,6 @@ public boolean equals(getLongRunningCompactions_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -9627,10 +9118,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -9662,16 +9149,6 @@ public int compareTo(getLongRunningCompactions_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -9710,14 +9187,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -9793,15 +9262,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getLongRunningCompa org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -9836,11 +9296,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getLongRunningComp struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -9866,10 +9321,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getLongRunningCompa if (struct.isSetSec()) { optionals.set(1); } - if (struct.isSetTnase()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); + oprot.writeBitSet(optionals, 2); if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); @@ -9883,15 +9335,12 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getLongRunningCompa if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, getLongRunningCompactions_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); + java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { org.apache.thrift.protocol.TMap _map42 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT); @@ -9913,11 +9362,6 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getLongRunningCompac struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(2)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -10426,20 +9870,17 @@ public static class getCompletedCompactions_result implements org.apache.thrift. private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getCompletedCompactions_resultStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getCompletedCompactions_resultTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable TExternalCompactionMap success; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { SUCCESS((short)0, "success"), - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); + SEC((short)1, "sec"); private static final java.util.Map byName = new java.util.HashMap(); @@ -10459,8 +9900,6 @@ public static _Fields findByThriftId(int fieldId) { return SUCCESS; case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -10511,8 +9950,6 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExternalCompactionMap.class))); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getCompletedCompactions_result.class, metaDataMap); } @@ -10522,13 +9959,11 @@ public getCompletedCompactions_result() { public getCompletedCompactions_result( TExternalCompactionMap success, - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.success = success; this.sec = sec; - this.tnase = tnase; } /** @@ -10541,9 +9976,6 @@ public getCompletedCompactions_result(getCompletedCompactions_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -10555,7 +9987,6 @@ public getCompletedCompactions_result deepCopy() { public void clear() { this.success = null; this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -10608,31 +10039,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public getCompletedCompactions_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -10652,14 +10058,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -10673,9 +10071,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -10692,8 +10087,6 @@ public boolean isSet(_Fields field) { return isSetSuccess(); case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -10729,15 +10122,6 @@ public boolean equals(getCompletedCompactions_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -10753,10 +10137,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -10788,16 +10168,6 @@ public int compareTo(getCompletedCompactions_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -10836,14 +10206,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -10910,15 +10272,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getCompletedCompact org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -10945,11 +10298,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getCompletedCompac struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -10975,25 +10323,19 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getCompletedCompact if (struct.isSetSec()) { optionals.set(1); } - if (struct.isSetTnase()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); + oprot.writeBitSet(optionals, 2); if (struct.isSetSuccess()) { struct.success.write(oprot); } if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, getCompletedCompactions_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); + java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { struct.success = new TExternalCompactionMap(); struct.success.read(iprot); @@ -11004,11 +10346,6 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getCompletedCompacti struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(2)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -11621,18 +10958,15 @@ public static class cancel_result implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -11650,8 +10984,6 @@ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -11700,8 +11032,6 @@ public java.lang.String getFieldName() { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(cancel_result.class, metaDataMap); } @@ -11710,12 +11040,10 @@ public cancel_result() { } public cancel_result( - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.sec = sec; - this.tnase = tnase; } /** @@ -11725,9 +11053,6 @@ public cancel_result(cancel_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -11738,7 +11063,6 @@ public cancel_result deepCopy() { @Override public void clear() { this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -11766,31 +11090,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public cancel_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -11802,14 +11101,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -11820,9 +11111,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -11837,8 +11125,6 @@ public boolean isSet(_Fields field) { switch (field) { case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -11865,15 +11151,6 @@ public boolean equals(cancel_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -11885,10 +11162,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -11910,16 +11183,6 @@ public int compareTo(cancel_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -11950,14 +11213,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -12012,15 +11267,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, cancel_result struc org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -12042,11 +11288,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, cancel_result stru struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -12069,32 +11310,21 @@ public void write(org.apache.thrift.protocol.TProtocol prot, cancel_result struc if (struct.isSetSec()) { optionals.set(0); } - if (struct.isSetTnase()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); + oprot.writeBitSet(optionals, 1); if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, cancel_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); + java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(1)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -13306,18 +12536,15 @@ public static class beginFullJobScan_result implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -13335,8 +12562,6 @@ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -13385,8 +12610,6 @@ public java.lang.String getFieldName() { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(beginFullJobScan_result.class, metaDataMap); } @@ -13395,12 +12618,10 @@ public beginFullJobScan_result() { } public beginFullJobScan_result( - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.sec = sec; - this.tnase = tnase; } /** @@ -13410,9 +12631,6 @@ public beginFullJobScan_result(beginFullJobScan_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -13423,7 +12641,6 @@ public beginFullJobScan_result deepCopy() { @Override public void clear() { this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -13451,31 +12668,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public beginFullJobScan_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -13487,14 +12679,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -13505,9 +12689,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -13522,8 +12703,6 @@ public boolean isSet(_Fields field) { switch (field) { case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -13550,15 +12729,6 @@ public boolean equals(beginFullJobScan_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -13570,10 +12740,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -13595,16 +12761,6 @@ public int compareTo(beginFullJobScan_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -13635,14 +12791,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -13697,15 +12845,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, beginFullJobScan_re org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -13727,11 +12866,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, beginFullJobScan_r struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -13754,32 +12888,21 @@ public void write(org.apache.thrift.protocol.TProtocol prot, beginFullJobScan_re if (struct.isSetSec()) { optionals.set(0); } - if (struct.isSetTnase()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); + oprot.writeBitSet(optionals, 1); if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, beginFullJobScan_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); + java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(1)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } } } @@ -15046,18 +14169,15 @@ public static class endFullJobScan_result implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -15075,8 +14195,6 @@ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { case 1: // SEC return SEC; - case 2: // TNASE - return TNASE; default: return null; } @@ -15125,8 +14243,6 @@ public java.lang.String getFieldName() { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(endFullJobScan_result.class, metaDataMap); } @@ -15135,12 +14251,10 @@ public endFullJobScan_result() { } public endFullJobScan_result( - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this(); this.sec = sec; - this.tnase = tnase; } /** @@ -15150,9 +14264,6 @@ public endFullJobScan_result(endFullJobScan_result other) { if (other.isSetSec()) { this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } } @Override @@ -15163,7 +14274,6 @@ public endFullJobScan_result deepCopy() { @Override public void clear() { this.sec = null; - this.tnase = null; } @org.apache.thrift.annotation.Nullable @@ -15191,31 +14301,6 @@ public void setSecIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public endFullJobScan_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -15227,14 +14312,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - } } @@ -15245,9 +14322,6 @@ public java.lang.Object getFieldValue(_Fields field) { case SEC: return getSec(); - case TNASE: - return getTnase(); - } throw new java.lang.IllegalStateException(); } @@ -15262,8 +14336,6 @@ public boolean isSet(_Fields field) { switch (field) { case SEC: return isSetSec(); - case TNASE: - return isSetTnase(); } throw new java.lang.IllegalStateException(); } @@ -15290,15 +14362,6 @@ public boolean equals(endFullJobScan_result that) { return false; } - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - return true; } @@ -15310,10 +14373,6 @@ public int hashCode() { if (isSetSec()) hashCode = hashCode * 8191 + sec.hashCode(); - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - return hashCode; } @@ -15335,16 +14394,6 @@ public int compareTo(endFullJobScan_result other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -15375,14 +14424,6 @@ public java.lang.String toString() { sb.append(this.sec); } first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; sb.append(")"); return sb.toString(); } @@ -15437,15 +14478,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, endFullJobScan_resu org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -15467,11 +14499,6 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, endFullJobScan_res struct.sec.write(oprot); oprot.writeFieldEnd(); } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -15494,31 +14521,1049 @@ public void write(org.apache.thrift.protocol.TProtocol prot, endFullJobScan_resu if (struct.isSetSec()) { optionals.set(0); } - if (struct.isSetTnase()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); + oprot.writeBitSet(optionals, 1); if (struct.isSetSec()) { struct.sec.write(oprot); } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, endFullJobScan_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); + java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); struct.sec.read(iprot); struct.setSecIsSet(true); } - if (incoming.get(1)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class setResourceGroups_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("setResourceGroups_args"); + + private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField GROUPS_FIELD_DESC = new org.apache.thrift.protocol.TField("groups", org.apache.thrift.protocol.TType.SET, (short)3); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new setResourceGroups_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new setResourceGroups_argsTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required + public @org.apache.thrift.annotation.Nullable java.util.Set groups; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TINFO((short)1, "tinfo"), + CREDENTIALS((short)2, "credentials"), + GROUPS((short)3, "groups"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TINFO + return TINFO; + case 2: // CREDENTIALS + return CREDENTIALS; + case 3: // GROUPS + return GROUPS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); + tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); + tmpMap.put(_Fields.GROUPS, new org.apache.thrift.meta_data.FieldMetaData("groups", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(setResourceGroups_args.class, metaDataMap); + } + + public setResourceGroups_args() { + } + + public setResourceGroups_args( + org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, + org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, + java.util.Set groups) + { + this(); + this.tinfo = tinfo; + this.credentials = credentials; + this.groups = groups; + } + + /** + * Performs a deep copy on other. + */ + public setResourceGroups_args(setResourceGroups_args other) { + if (other.isSetTinfo()) { + this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); + } + if (other.isSetCredentials()) { + this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); + } + if (other.isSetGroups()) { + java.util.Set __this__groups = new java.util.HashSet(other.groups); + this.groups = __this__groups; + } + } + + @Override + public setResourceGroups_args deepCopy() { + return new setResourceGroups_args(this); + } + + @Override + public void clear() { + this.tinfo = null; + this.credentials = null; + this.groups = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { + return this.tinfo; + } + + public setResourceGroups_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { + this.tinfo = tinfo; + return this; + } + + public void unsetTinfo() { + this.tinfo = null; + } + + /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ + public boolean isSetTinfo() { + return this.tinfo != null; + } + + public void setTinfoIsSet(boolean value) { + if (!value) { + this.tinfo = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { + return this.credentials; + } + + public setResourceGroups_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { + this.credentials = credentials; + return this; + } + + public void unsetCredentials() { + this.credentials = null; + } + + /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ + public boolean isSetCredentials() { + return this.credentials != null; + } + + public void setCredentialsIsSet(boolean value) { + if (!value) { + this.credentials = null; + } + } + + public int getGroupsSize() { + return (this.groups == null) ? 0 : this.groups.size(); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Iterator getGroupsIterator() { + return (this.groups == null) ? null : this.groups.iterator(); + } + + public void addToGroups(java.lang.String elem) { + if (this.groups == null) { + this.groups = new java.util.HashSet(); + } + this.groups.add(elem); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Set getGroups() { + return this.groups; + } + + public setResourceGroups_args setGroups(@org.apache.thrift.annotation.Nullable java.util.Set groups) { + this.groups = groups; + return this; + } + + public void unsetGroups() { + this.groups = null; + } + + /** Returns true if field groups is set (has been assigned a value) and false otherwise */ + public boolean isSetGroups() { + return this.groups != null; + } + + public void setGroupsIsSet(boolean value) { + if (!value) { + this.groups = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case TINFO: + if (value == null) { + unsetTinfo(); + } else { + setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); + } + break; + + case CREDENTIALS: + if (value == null) { + unsetCredentials(); + } else { + setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); + } + break; + + case GROUPS: + if (value == null) { + unsetGroups(); + } else { + setGroups((java.util.Set)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case TINFO: + return getTinfo(); + + case CREDENTIALS: + return getCredentials(); + + case GROUPS: + return getGroups(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case TINFO: + return isSetTinfo(); + case CREDENTIALS: + return isSetCredentials(); + case GROUPS: + return isSetGroups(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof setResourceGroups_args) + return this.equals((setResourceGroups_args)that); + return false; + } + + public boolean equals(setResourceGroups_args that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_tinfo = true && this.isSetTinfo(); + boolean that_present_tinfo = true && that.isSetTinfo(); + if (this_present_tinfo || that_present_tinfo) { + if (!(this_present_tinfo && that_present_tinfo)) + return false; + if (!this.tinfo.equals(that.tinfo)) + return false; + } + + boolean this_present_credentials = true && this.isSetCredentials(); + boolean that_present_credentials = true && that.isSetCredentials(); + if (this_present_credentials || that_present_credentials) { + if (!(this_present_credentials && that_present_credentials)) + return false; + if (!this.credentials.equals(that.credentials)) + return false; + } + + boolean this_present_groups = true && this.isSetGroups(); + boolean that_present_groups = true && that.isSetGroups(); + if (this_present_groups || that_present_groups) { + if (!(this_present_groups && that_present_groups)) + return false; + if (!this.groups.equals(that.groups)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); + if (isSetTinfo()) + hashCode = hashCode * 8191 + tinfo.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); + if (isSetCredentials()) + hashCode = hashCode * 8191 + credentials.hashCode(); + + hashCode = hashCode * 8191 + ((isSetGroups()) ? 131071 : 524287); + if (isSetGroups()) + hashCode = hashCode * 8191 + groups.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(setResourceGroups_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTinfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCredentials()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetGroups(), other.isSetGroups()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGroups()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groups, other.groups); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("setResourceGroups_args("); + boolean first = true; + + sb.append("tinfo:"); + if (this.tinfo == null) { + sb.append("null"); + } else { + sb.append(this.tinfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("credentials:"); + if (this.credentials == null) { + sb.append("null"); + } else { + sb.append(this.credentials); + } + first = false; + if (!first) sb.append(", "); + sb.append("groups:"); + if (this.groups == null) { + sb.append("null"); + } else { + sb.append(this.groups); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (tinfo != null) { + tinfo.validate(); + } + if (credentials != null) { + credentials.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class setResourceGroups_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public setResourceGroups_argsStandardScheme getScheme() { + return new setResourceGroups_argsStandardScheme(); + } + } + + private static class setResourceGroups_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, setResourceGroups_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TINFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // CREDENTIALS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // GROUPS + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set54 = iprot.readSetBegin(); + struct.groups = new java.util.HashSet(2*_set54.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem55; + for (int _i56 = 0; _i56 < _set54.size; ++_i56) + { + _elem55 = iprot.readString(); + struct.groups.add(_elem55); + } + iprot.readSetEnd(); + } + struct.setGroupsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, setResourceGroups_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tinfo != null) { + oprot.writeFieldBegin(TINFO_FIELD_DESC); + struct.tinfo.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.credentials != null) { + oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); + struct.credentials.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.groups != null) { + oprot.writeFieldBegin(GROUPS_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRING, struct.groups.size())); + for (java.lang.String _iter57 : struct.groups) + { + oprot.writeString(_iter57); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class setResourceGroups_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public setResourceGroups_argsTupleScheme getScheme() { + return new setResourceGroups_argsTupleScheme(); + } + } + + private static class setResourceGroups_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetTinfo()) { + optionals.set(0); + } + if (struct.isSetCredentials()) { + optionals.set(1); + } + if (struct.isSetGroups()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTinfo()) { + struct.tinfo.write(oprot); + } + if (struct.isSetCredentials()) { + struct.credentials.write(oprot); + } + if (struct.isSetGroups()) { + { + oprot.writeI32(struct.groups.size()); + for (java.lang.String _iter58 : struct.groups) + { + oprot.writeString(_iter58); + } + } + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } + if (incoming.get(1)) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } + if (incoming.get(2)) { + { + org.apache.thrift.protocol.TSet _set59 = iprot.readSetBegin(org.apache.thrift.protocol.TType.STRING); + struct.groups = new java.util.HashSet(2*_set59.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem60; + for (int _i61 = 0; _i61 < _set59.size; ++_i61) + { + _elem60 = iprot.readString(); + struct.groups.add(_elem60); + } + } + struct.setGroupsIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class setResourceGroups_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("setResourceGroups_result"); + + private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new setResourceGroups_resultStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new setResourceGroups_resultTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SEC((short)1, "sec"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SEC + return SEC; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(setResourceGroups_result.class, metaDataMap); + } + + public setResourceGroups_result() { + } + + public setResourceGroups_result( + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) + { + this(); + this.sec = sec; + } + + /** + * Performs a deep copy on other. + */ + public setResourceGroups_result(setResourceGroups_result other) { + if (other.isSetSec()) { + this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); + } + } + + @Override + public setResourceGroups_result deepCopy() { + return new setResourceGroups_result(this); + } + + @Override + public void clear() { + this.sec = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { + return this.sec; + } + + public setResourceGroups_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + this.sec = sec; + return this; + } + + public void unsetSec() { + this.sec = null; + } + + /** Returns true if field sec is set (has been assigned a value) and false otherwise */ + public boolean isSetSec() { + return this.sec != null; + } + + public void setSecIsSet(boolean value) { + if (!value) { + this.sec = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case SEC: + if (value == null) { + unsetSec(); + } else { + setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case SEC: + return getSec(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case SEC: + return isSetSec(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof setResourceGroups_result) + return this.equals((setResourceGroups_result)that); + return false; + } + + public boolean equals(setResourceGroups_result that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_sec = true && this.isSetSec(); + boolean that_present_sec = true && that.isSetSec(); + if (this_present_sec || that_present_sec) { + if (!(this_present_sec && that_present_sec)) + return false; + if (!this.sec.equals(that.sec)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); + if (isSetSec()) + hashCode = hashCode * 8191 + sec.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(setResourceGroups_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSec()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("setResourceGroups_result("); + boolean first = true; + + sb.append("sec:"); + if (this.sec == null) { + sb.append("null"); + } else { + sb.append(this.sec); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class setResourceGroups_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public setResourceGroups_resultStandardScheme getScheme() { + return new setResourceGroups_resultStandardScheme(); + } + } + + private static class setResourceGroups_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, setResourceGroups_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SEC + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, setResourceGroups_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.sec != null) { + oprot.writeFieldBegin(SEC_FIELD_DESC); + struct.sec.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class setResourceGroups_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public setResourceGroups_resultTupleScheme getScheme() { + return new setResourceGroups_resultTupleScheme(); + } + } + + private static class setResourceGroups_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetSec()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSec()) { + struct.sec.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); } } } diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java index c064b95eac2..39bbb673020 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java @@ -3668,14 +3668,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveCompaction case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list54 = iprot.readListBegin(); - struct.success = new java.util.ArrayList(_list54.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem55; - for (int _i56 = 0; _i56 < _list54.size; ++_i56) + org.apache.thrift.protocol.TList _list62 = iprot.readListBegin(); + struct.success = new java.util.ArrayList(_list62.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem63; + for (int _i64 = 0; _i64 < _list62.size; ++_i64) { - _elem55 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem55.read(iprot); - struct.success.add(_elem55); + _elem63 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem63.read(iprot); + struct.success.add(_elem63); } iprot.readListEnd(); } @@ -3713,9 +3713,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveCompactio oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter57 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter65 : struct.success) { - _iter57.write(oprot); + _iter65.write(oprot); } oprot.writeListEnd(); } @@ -3755,9 +3755,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveCompaction if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter58 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter66 : struct.success) { - _iter58.write(oprot); + _iter66.write(oprot); } } } @@ -3772,14 +3772,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveCompactions java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list59 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.success = new java.util.ArrayList(_list59.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem60; - for (int _i61 = 0; _i61 < _list59.size; ++_i61) + org.apache.thrift.protocol.TList _list67 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.success = new java.util.ArrayList(_list67.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem68; + for (int _i69 = 0; _i69 < _list67.size; ++_i69) { - _elem60 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem60.read(iprot); - struct.success.add(_elem60); + _elem68 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem68.read(iprot); + struct.success.add(_elem68); } } struct.setSuccessIsSet(true); diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index 4ebda3807aa..a5011248aeb 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -98,7 +98,6 @@ service CompactionCoordinatorService { 5:tabletserver.TCompactionStats stats )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) /* @@ -112,7 +111,6 @@ service CompactionCoordinatorService { 5:string externalCompactionId )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) /* @@ -126,7 +124,6 @@ service CompactionCoordinatorService { 5:i64 timestamp )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) /* @@ -141,7 +138,6 @@ service CompactionCoordinatorService { 6:TCompactionState failureState )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) /* @@ -152,7 +148,6 @@ service CompactionCoordinatorService { 2:security.TCredentials credentials )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) /* @@ -164,7 +159,6 @@ service CompactionCoordinatorService { 2:security.TCredentials credentials )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) /* @@ -175,7 +169,6 @@ service CompactionCoordinatorService { 2:security.TCredentials credentials )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) void cancel( @@ -184,7 +177,6 @@ service CompactionCoordinatorService { 3:string externalCompactionId )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) oneway void recordCompletion( @@ -199,7 +191,6 @@ service CompactionCoordinatorService { 3:string dataLevel )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) oneway void addJobs( @@ -214,9 +205,15 @@ service CompactionCoordinatorService { 3:string dataLevel )throws( 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase ) + void setResourceGroups( + 1:client.TInfo tinfo + 2:security.TCredentials credentials + 3:set groups + )throws( + 1:client.ThriftSecurityException sec + ) } service CompactorService { diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index cda00e28979..87c03baaee4 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -579,7 +579,8 @@ protected CompactionCoordinatorService.Client getCoordinatorClient() throws TTra if (coordinatorHost == null) { throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper"); } - LOG.trace("CompactionCoordinator address is: {}", coordinatorHost); + // TODO change back to trace + LOG.debug("CompactionCoordinator address is: {}", coordinatorHost); return ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost, getContext()); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 8a9b3dbf9d1..e95b6564d2b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -64,7 +64,6 @@ import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.client.admin.servers.ServerId.Type; -import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.conf.SiteConfiguration; @@ -989,16 +988,12 @@ public void run() { PrimaryManagerThriftServiceWrapper.service(ManagerClientService.Iface.class, ManagerClientService.Processor::new, new ManagerClientServiceHandler(this), this); compactionCoordinator = new CompactionCoordinator(this, this::fateClient); - CompactionCoordinatorService.Iface wrappedCoordinator = - PrimaryManagerThriftServiceWrapper.service(CompactionCoordinatorService.Iface.class, - CompactionCoordinatorService.Processor::new, compactionCoordinator.getThriftService(), - this); // This is not wrapped w/ HighlyAvailableServiceWrapper because it can be run by any manager. FateWorker fateWorker = new FateWorker(context, tserverSet, this::createFateInstance); var processor = ThriftProcessorTypes.getManagerTProcessor(this, fateServiceHandler, - wrappedCoordinator, managerClientHandler, fateWorker, getContext()); + compactionCoordinator.getThriftService(), managerClientHandler, fateWorker, getContext()); try { updateThriftServer(() -> { return TServerUtils.createThriftServer(context, getBindAddress(), diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java index f55225110c4..61a124000ca 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java @@ -54,7 +54,7 @@ public class CompactionJobClient implements AutoCloseable { private final ServerContext context; record CoordinatorConnection(CompactionCoordinatorService.Client client, - List jobBuffer) { + List jobBuffer, HostAndPort address) { } private final Map coordinatorLocations; @@ -78,9 +78,10 @@ public CompactionJobClient(ServerContext context, Ample.DataLevel dataLevel, boo } coordinatorConnections.put(hostPort, - new CoordinatorConnection(client, new ArrayList<>(BUFFER_SIZE))); + new CoordinatorConnection(client, new ArrayList<>(BUFFER_SIZE), hostPort)); } catch (TException e) { // TODO only log + // TODO need to return client or add it throw new RuntimeException(e); } } @@ -91,10 +92,12 @@ public void addJobs(TabletMetadata tabletMetadata, Collection job var resolvedJob = new ResolvedCompactionJob(job, tabletMetadata); var hostPort = coordinatorLocations.get(resolvedJob.getGroup()); if (hostPort == null) { + log.debug("Ignoring job, no coordinator found {}", job.getGroup()); continue; } var coordinator = coordinatorConnections.get(hostPort); if (coordinator == null) { + log.debug("Ignoring job, no connection found {}", job.getGroup()); continue; } @@ -105,7 +108,7 @@ public void addJobs(TabletMetadata tabletMetadata, Collection job } catch (TException e) { log.warn("Failed to send compaction jobs to {}", hostPort, e); ThriftUtil.returnClient(coordinator.client, context); - // ignore this coordinator for the rest of the session + // ignore this coordinator for the rest of the session... coordinatorConnections.remove(hostPort); } } @@ -115,6 +118,7 @@ public void addJobs(TabletMetadata tabletMetadata, Collection job private void sendJobs(CoordinatorConnection coordinator) throws TException { List thriftJobs = new ArrayList<>(coordinator.jobBuffer.size()); for (var job : coordinator.jobBuffer) { + log.debug("Sending job {} {} {}", coordinator.address, job.getGroup(), job.getExtent()); thriftJobs.add(job.toThrift()); } coordinator.client.addJobs(TraceUtil.traceInfo(), context.rpcCreds(), thriftJobs); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index e1ff5f77cec..5a981a674ae 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -20,6 +20,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES; @@ -741,11 +742,14 @@ public void addJobs(TInfo tinfo, TCredentials credentials, List groups) + throws ThriftSecurityException, TException { + if (!security.canPerformSystemActions(credentials)) { + // TODO does not seem like the correct exception, also this code snippet was copied. + throw new AccumuloSecurityException(credentials.getPrincipal(), + SecurityErrorCode.PERMISSION_DENIED).asThriftException(); + } + + // TODO validate that upgrade is complete like FateWorker does + + jobQueues.setResourceGroups(groups.stream().map(ResourceGroupId::of).collect(toSet())); + LOG.debug("Set resource groups to {}", groups); + } + // TODO remove public void addJobs(TabletMetadata tabletMetadata, Collection jobs) { ArrayList resolvedJobs = new ArrayList<>(jobs.size()); @@ -1138,7 +1157,7 @@ protected Set readExternalCompactionIds() { this.ctx.getAmple().readTablets().forLevel(Ample.DataLevel.USER) .filter(new HasExternalCompactionsFilter()).fetch(ECOMP).build()) { return tabletsMetadata.stream().flatMap(tm -> tm.getExternalCompactions().keySet().stream()) - .collect(Collectors.toSet()); + .collect(toSet()); } } @@ -1238,6 +1257,9 @@ public void cancel(TInfo tinfo, TCredentials credentials, String externalCompact TableOperation.COMPACT_CANCEL, TableOperationExceptionType.NOTFOUND, e.getMessage()); } + // TODO could contact all compactors OR could scan the metadata table to find the compactor OR + // could require passing table and end row of compaction + cancelCompactionOnCompactor(runningCompaction.getCompactor(), externalCompactionId); } @@ -1339,6 +1361,8 @@ static Set getCompactionServicesConfigurationGroups(ServerConte public void cleanUpInternalState() { + // TODO needs to be redone + // This method does the following: // // 1. Removes entries from RUNNING_CACHE and LONG_RUNNING_COMPACTIONS_BY_RG that are not really @@ -1363,7 +1387,7 @@ public void cleanUpInternalState() { LONG_RUNNING_COMPACTIONS_BY_RG.values().stream() .flatMap(TimeOrderedRunningCompactionSet::stream) .map(rc -> rc.getJob().getExternalCompactionId()).map(ExternalCompactionId::of) - .collect(Collectors.toSet()))); + .collect(toSet()))); // grab the ids that are listed as running in the metadata table. It important that this is done // after getting the snapshot. diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index c42d37e1449..c3b7e0f517a 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -18,15 +18,25 @@ */ package org.apache.accumulo.manager.compaction.coordinator; +import static java.util.stream.Collectors.toSet; + import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; -import org.apache.accumulo.core.client.admin.servers.ServerId; +import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; +import org.apache.accumulo.core.data.AbstractId; import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.lock.ServiceLockPaths; +import org.apache.accumulo.core.rpc.ThriftUtil; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CoordinatorLocations; +import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,15 +63,69 @@ public CoordinatorManager(ServerContext context) { private void managerCoordinators() { while (true) { try { - var serverIds = context.instanceOperations().getServers(ServerId.Type.MANAGER); - if (!serverIds.isEmpty()) { - var primary = HostAndPort.fromString(serverIds.iterator().next().toHostPortString()); + + var assistants = context.getServerPaths() + .getAssistantManagers(ServiceLockPaths.AddressSelector.all(), true); + if (!assistants.isEmpty()) { var compactorGroups = CompactionCoordinator.getCompactionServicesConfigurationGroups(context); - Map assignments = new HashMap<>(); - compactorGroups.forEach(rg -> assignments.put(rg, primary)); - CoordinatorLocations.setLocations(context.getZooSession().asReaderWriter(), assignments); - log.debug("Set locations {}", assignments); + + int minGroupsPerAssistant = compactorGroups.size() / assistants.size(); + int maxGroupsPerAssistant = + minGroupsPerAssistant + Math.min(compactorGroups.size() % assistants.size(), 1); + + Map currentLocations = + context.getCoordinatorLocations(false); + + // group by coordinator + Map> groupsPerCompactor = new HashMap<>(); + currentLocations.forEach((rg, hp) -> { + groupsPerCompactor.computeIfAbsent(hp, hp2 -> new HashSet<>()).add(rg); + }); + + boolean needsUpdates = !currentLocations.keySet().containsAll(compactorGroups); + // Look for any compactors that are not correct + for (var groups : groupsPerCompactor.values()) { + if (groups.size() < minGroupsPerAssistant || groups.size() > maxGroupsPerAssistant + || !compactorGroups.containsAll(groups)) { + needsUpdates = true; + } + } + + if (needsUpdates) { + // TODO this does not try to keep current assignemnts + // TODO this should ask coordinators what they currently have instead of assuming ZK is + // correct, on failure ZK could be out of sync... otherwise always set RG on + // coordinators to ensure they match ZK + // TODO combine/share code w/ fate for assignments, that is why the todos above are not + // done, this code is temporary hack and it has problems. + Map> updates = new HashMap<>(); + assistants.forEach( + slp -> updates.put(HostAndPort.fromString(slp.getServer()), new HashSet<>())); + + var iter = compactorGroups.iterator(); + updates.values().forEach(groups -> { + while (groups.size() < minGroupsPerAssistant) { + groups.add(iter.next()); + } + }); + + updates.values().forEach(groups -> { + while (iter.hasNext() && groups.size() < maxGroupsPerAssistant) { + groups.add(iter.next()); + } + }); + + updates.forEach(this::setResourceGroups); + Map newLocations = new HashMap<>(); + updates.forEach((hp, groups) -> { + groups.forEach(group -> newLocations.put(group, hp)); + }); + + CoordinatorLocations.setLocations(context.getZooSession().asReaderWriter(), + newLocations); + log.debug("Set locations {}", newLocations); + } } // TODO better exception handling } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException @@ -75,6 +139,20 @@ private void managerCoordinators() { } + private void setResourceGroups(HostAndPort hp, Set groups) { + CompactionCoordinatorService.Client client = null; + try { + client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, hp, context); + client.setResourceGroups(TraceUtil.traceInfo(), context.rpcCreds(), + groups.stream().map(AbstractId::canonical).collect(toSet())); + } catch (TException e) { + // TODO + throw new RuntimeException(e); + } finally { + ThriftUtil.returnClient(client, context); + } + } + public synchronized void start() { Preconditions.checkState(assignmentThread == null); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java index e87c231bed9..d17e57838b1 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java @@ -21,7 +21,9 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap.KeySetView; @@ -50,6 +52,8 @@ public class CompactionJobQueues { private volatile long queueSize; + private final Set allowedGroups = Collections.synchronizedSet(new HashSet<>()); + private final Map currentGenerations; public CompactionJobQueues(long queueSize) { @@ -120,6 +124,16 @@ public long getQueuedJobCount() { return count; } + private CompactionJobPriorityQueue getOrCreateQueue(ResourceGroupId groupId) { + return priorityQueues.computeIfAbsent(groupId, gid -> { + if (allowedGroups.contains(gid)) { + return new CompactionJobPriorityQueue(gid, queueSize, ResolvedCompactionJob.WEIGHER); + } else { + return null; + } + }); + } + /** * Asynchronously get a compaction job from the queue. If the queue currently has jobs then a * completed future will be returned containing the highest priority job in the queue. If the @@ -127,8 +141,11 @@ public long getQueuedJobCount() { * is added to the queue the future will be completed. */ public CompletableFuture getAsync(ResourceGroupId groupId) { - var pq = priorityQueues.computeIfAbsent(groupId, - gid -> new CompactionJobPriorityQueue(gid, queueSize, ResolvedCompactionJob.WEIGHER)); + var pq = getOrCreateQueue(groupId); + if (pq == null) { + // TODO callers will not handle null, is there something besides null to return? + return null; + } return pq.getAsync(); } @@ -149,8 +166,12 @@ private void add(KeyExtent extent, ResourceGroupId groupId, Collection new CompactionJobPriorityQueue(gid, queueSize, ResolvedCompactionJob.WEIGHER)); + var pq = getOrCreateQueue(groupId); + if (pq == null) { + log.trace("Ignored request to add jobs extent:{} group:{} #jobs:{}", extent, groupId, + jobs.size()); + return; + } pq.add(extent, jobs, currentGenerations.get(DataLevel.of(extent.tableId())).get()); } @@ -158,4 +179,14 @@ public void resetMaxSize(long size) { this.queueSize = size; priorityQueues.values().forEach(cjpq -> cjpq.resetMaxSize(this.queueSize)); } + + public void setResourceGroups(Set groups) { + synchronized (allowedGroups) { + allowedGroups.clear(); + allowedGroups.addAll(groups); + } + + priorityQueues.keySet().removeIf(rg -> !allowedGroups.contains(rg)); + // TODO need to cancel futures + } } diff --git a/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java b/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java index fc0d87369ef..79d362f9cd7 100644 --- a/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java @@ -56,6 +56,8 @@ public static void setup() throws Exception { client.securityOperations().changeUserAuthorizations("root", AUTHORIZATIONS); } + // TODO could add a compaction RG to comprehensive IT + // TODO could wait for coordinators? // Wait for 3 managers to have a fate partition assigned to them var srvCtx = getCluster().getServerContext(); Wait.waitFor(() -> { diff --git a/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java new file mode 100644 index 00000000000..55f92fed165 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java @@ -0,0 +1,62 @@ +package org.apache.accumulo.test; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.harness.AccumuloClusterHarness; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.test.compaction.ExternalCompactionTestUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.Test; + +import java.util.SortedSet; +import java.util.TreeSet; + +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP1; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP2; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.MAX_DATA; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.row; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.verify; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.writeData; + +public class MultipleManagerCompactionIT extends AccumuloClusterHarness { + + @Override + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) { + ExternalCompactionTestUtils.configureMiniCluster(cfg, coreSite); + cfg.getClusterServerConfiguration().setNumManagers(3); + } + + @Test + public void test() throws Exception { + String[] names = this.getUniqueNames(2); + try (AccumuloClient client = + Accumulo.newClient().from(getCluster().getClientProperties()).build()) { + + String table1 = names[0]; + createTable(client, table1, "cs1"); + + String table2 = names[1]; + createTable(client, table2, "cs2"); + + writeData(client, table1); + writeData(client, table2); + + compact(client, table1, 2, GROUP1, true); + verify(client, table1, 2); + + SortedSet splits = new TreeSet<>(); + splits.add(new Text(row(MAX_DATA / 2))); + client.tableOperations().addSplits(table2, splits); + + compact(client, table2, 3, GROUP2, true); + verify(client, table2, 3); + + } + } + + + +} From 7909a0cb1c15e2cf7fe6e68731db224ed03e8f33 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 18 Mar 2026 01:35:17 +0000 Subject: [PATCH 04/65] Moves set of shutting down tservers from manager memory to ZK The set of shutting down tservers was causing system fate operations to have to run on the primary manager because this was an in memory set. This caused fate to have different code paths to user vs system fate, this in turn caused problems when trying to distribute compaction coordination. To fix this problem moved the set from an in memory set to a set in zookeeper. The set is managed by fate operations which simplifies the existing code. Only fate operations add and remove from the set and fate keys are used to ensure only one fate operation runs at a time for a tserver instance. The previous in memory set had a lot of code to try to keep it in sync with reality, that is all gone now. There were many bugs with this code in the past. After this change is made fate can be simplified in a follow on commit to remove all specialization for the primary manager. Also the monitor can now directly access this set instead of making an RPC to the manager, will open a follow on issue for this. --- .../org/apache/accumulo/core/Constants.java | 2 + .../apache/accumulo/core/fate/FateKey.java | 43 +++++++- .../server/init/ZooKeeperInitializer.java | 2 + .../org/apache/accumulo/manager/Manager.java | 76 ++++++------- .../manager/ManagerClientServiceHandler.java | 39 +++---- .../tserverOps/BeginTserverShutdown.java | 80 ++++++++++++++ .../manager/tserverOps/ShutdownTServer.java | 75 +++++++------ .../manager/upgrade/Upgrader11to12.java | 11 ++ .../manager/tableOps/ShutdownTServerTest.java | 100 ------------------ 9 files changed, 228 insertions(+), 200 deletions(-) create mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/BeginTserverShutdown.java delete mode 100644 server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java index eb8ba1059eb..d2c556f2c2a 100644 --- a/core/src/main/java/org/apache/accumulo/core/Constants.java +++ b/core/src/main/java/org/apache/accumulo/core/Constants.java @@ -74,6 +74,8 @@ public class Constants { public static final String ZDEAD = "/dead"; public static final String ZDEADTSERVERS = ZDEAD + "/tservers"; + public static final String ZSHUTTING_DOWN_TSERVERS = "/shutting-down-tservers"; + public static final String ZFATE = "/fate"; public static final String ZNEXT_FILE = "/next_file"; diff --git a/core/src/main/java/org/apache/accumulo/core/fate/FateKey.java b/core/src/main/java/org/apache/accumulo/core/fate/FateKey.java index 650dca01af3..19641c000a3 100644 --- a/core/src/main/java/org/apache/accumulo/core/fate/FateKey.java +++ b/core/src/main/java/org/apache/accumulo/core/fate/FateKey.java @@ -27,6 +27,7 @@ import java.util.Optional; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -41,12 +42,14 @@ public class FateKey { private final FateKeyType type; private final Optional keyExtent; private final Optional compactionId; + private final Optional tServerInstance; private final byte[] serialized; private FateKey(FateKeyType type, KeyExtent keyExtent) { this.type = Objects.requireNonNull(type); this.keyExtent = Optional.of(keyExtent); this.compactionId = Optional.empty(); + this.tServerInstance = Optional.empty(); this.serialized = serialize(type, keyExtent); } @@ -54,15 +57,25 @@ private FateKey(FateKeyType type, ExternalCompactionId compactionId) { this.type = Objects.requireNonNull(type); this.keyExtent = Optional.empty(); this.compactionId = Optional.of(compactionId); + this.tServerInstance = Optional.empty(); this.serialized = serialize(type, compactionId); } + private FateKey(FateKeyType type, TServerInstance tServerInstance) { + this.type = Objects.requireNonNull(type); + this.keyExtent = Optional.empty(); + this.compactionId = Optional.empty(); + this.tServerInstance = Optional.of(tServerInstance); + this.serialized = serialize(type, tServerInstance); + } + private FateKey(byte[] serialized) { try (DataInputBuffer buffer = new DataInputBuffer()) { buffer.reset(serialized, serialized.length); this.type = FateKeyType.valueOf(buffer.readUTF()); this.keyExtent = deserializeKeyExtent(type, buffer); this.compactionId = deserializeCompactionId(type, buffer); + this.tServerInstance = deserializeTserverId(type, buffer); this.serialized = serialized; } catch (IOException e) { throw new UncheckedIOException(e); @@ -127,8 +140,12 @@ public static FateKey forMerge(KeyExtent extent) { return new FateKey(FateKeyType.MERGE, extent); } + public static FateKey forShutdown(TServerInstance tServerInstance) { + return new FateKey(FateKeyType.TSERVER_SHUTDOWN, tServerInstance); + } + public enum FateKeyType { - SPLIT, COMPACTION_COMMIT, MERGE + SPLIT, COMPACTION_COMMIT, MERGE, TSERVER_SHUTDOWN } private static byte[] serialize(FateKeyType type, KeyExtent ke) { @@ -155,22 +172,42 @@ private static byte[] serialize(FateKeyType type, ExternalCompactionId compactio } } + private static byte[] serialize(FateKeyType type, TServerInstance tServerInstance) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos)) { + dos.writeUTF(type.toString()); + dos.writeUTF(tServerInstance.getHostPortSession()); + dos.close(); + return baos.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private static Optional deserializeKeyExtent(FateKeyType type, DataInputBuffer buffer) throws IOException { return switch (type) { case SPLIT, MERGE -> Optional.of(KeyExtent.readFrom(buffer)); - case COMPACTION_COMMIT -> Optional.empty(); + case COMPACTION_COMMIT, TSERVER_SHUTDOWN -> Optional.empty(); }; } private static Optional deserializeCompactionId(FateKeyType type, DataInputBuffer buffer) throws IOException { return switch (type) { - case SPLIT, MERGE -> Optional.empty(); + case SPLIT, MERGE, TSERVER_SHUTDOWN -> Optional.empty(); case COMPACTION_COMMIT -> Optional.of(ExternalCompactionId.of(buffer.readUTF())); }; } + private static Optional deserializeTserverId(FateKeyType type, + DataInputBuffer buffer) throws IOException { + return switch (type) { + case SPLIT, MERGE, COMPACTION_COMMIT -> Optional.empty(); + case TSERVER_SHUTDOWN -> Optional.of(new TServerInstance(buffer.readUTF())); + }; + } + @Override public String toString() { var buf = new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE); diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java index 464a6dae5a6..b23c6746803 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java @@ -178,6 +178,8 @@ void initialize(final ServerContext context, final String rootTabletDirName, ZooUtil.NodeExistsPolicy.FAIL); zrwChroot.putPersistentData(Constants.ZMANAGER_ASSISTANT_LOCK, EMPTY_BYTE_ARRAY, ZooUtil.NodeExistsPolicy.FAIL); + zrwChroot.putPersistentData(Constants.ZSHUTTING_DOWN_TSERVERS, EMPTY_BYTE_ARRAY, + ZooUtil.NodeExistsPolicy.FAIL); } /** diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 5ff232765c4..8e639e6ae19 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -58,6 +58,7 @@ import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.cli.ServerOpts; @@ -206,7 +207,6 @@ public class Manager extends AbstractServer private final List watchers = new ArrayList<>(); final Map badServers = Collections.synchronizedMap(new HashMap<>()); - final Set serversToShutdown = Collections.synchronizedSet(new HashSet<>()); final EventCoordinator nextEvent = new EventCoordinator(); RecoveryManager recoveryManager = null; private final ManagerTime timeKeeper; @@ -557,10 +557,6 @@ protected Manager(ServerOpts opts, aconf.getTimeInMillis(Property.MANAGER_RECOVERY_WAL_EXISTENCE_CACHE_TIME); } - public TServerConnection getConnection(TServerInstance server) { - return tserverSet.getConnection(server); - } - void setManagerGoalState(ManagerGoalState state) { try { getContext().getZooSession().asReaderWriter().putPersistentData(Constants.ZMANAGER_GOAL_STATE, @@ -667,6 +663,7 @@ public void run() { } boolean canBalance(DataLevel dataLevel, TServerStatus tServerStatus) { + Set serversToShutdown; if (!badServers.isEmpty()) { log.debug("not balancing {} because the balance information is out-of-date {}", dataLevel, badServers.keySet()); @@ -674,7 +671,7 @@ boolean canBalance(DataLevel dataLevel, TServerStatus tServerStatus) { } else if (getManagerGoalState() == ManagerGoalState.CLEAN_STOP) { log.debug("not balancing {} because the manager is attempting to stop cleanly", dataLevel); return false; - } else if (!serversToShutdown.isEmpty()) { + } else if (!(serversToShutdown = shutdownServers()).isEmpty()) { log.debug("not balancing {} while shutting down servers {}", dataLevel, serversToShutdown); return false; } else { @@ -766,7 +763,6 @@ public void run() { log.debug("stopping {} tablet servers", currentServers.size()); for (TServerInstance server : currentServers) { try { - serversToShutdown.add(server); tserverSet.getConnection(server).fastHalt(primaryManagerLock); } catch (TException e) { // its probably down, and we don't care @@ -1591,38 +1587,35 @@ public void update(LiveTServerSet current, Set deleted, obit.delete(up.getHostPort()); } } - for (TServerInstance dead : deleted) { - String cause = "unexpected failure"; - if (serversToShutdown.contains(dead)) { - cause = "clean shutdown"; // maybe an incorrect assumption + + if (!deleted.isEmpty()) { + // This set is read from zookeeper, so only get it if its actually needed + var serversToShutdown = shutdownServers(); + for (TServerInstance dead : deleted) { + String cause = "unexpected failure"; + if (serversToShutdown.contains(dead)) { + cause = "clean shutdown"; // maybe an incorrect assumption + } + if (!getManagerGoalState().equals(ManagerGoalState.CLEAN_STOP)) { + obit.post(dead.getHostPort(), cause); + } } - if (!getManagerGoalState().equals(ManagerGoalState.CLEAN_STOP)) { - obit.post(dead.getHostPort(), cause); + + Set unexpected = new HashSet<>(deleted); + unexpected.removeAll(serversToShutdown); + if (!unexpected.isEmpty() + && (stillManager() && !getManagerGoalState().equals(ManagerGoalState.CLEAN_STOP))) { + log.warn("Lost servers {}", unexpected); } + badServers.keySet().removeAll(deleted); } - Set unexpected = new HashSet<>(deleted); - unexpected.removeAll(this.serversToShutdown); - if (!unexpected.isEmpty() - && (stillManager() && !getManagerGoalState().equals(ManagerGoalState.CLEAN_STOP))) { - log.warn("Lost servers {}", unexpected); - } - serversToShutdown.removeAll(deleted); - badServers.keySet().removeAll(deleted); // clear out any bad server with the same host/port as a new server synchronized (badServers) { cleanListByHostAndPort(badServers.keySet(), deleted, added); } - synchronized (serversToShutdown) { - cleanListByHostAndPort(serversToShutdown, deleted, added); - } nextEvent.event("There are now %d tablet servers", current.size()); } - - // clear out any servers that are no longer current - // this is needed when we are using a fate operation to shutdown a tserver as it - // will continue to add the server to the serversToShutdown (ACCUMULO-4410) - serversToShutdown.retainAll(current.getCurrentServers()); } static void cleanListByHostAndPort(Collection badServers, @@ -1669,15 +1662,6 @@ public LiveTServersSnapshot tserversSnapshot() { return tserverSet.getSnapshot(); } - // recovers state from the persistent transaction to shutdown a server - public boolean shutdownTServer(TServerInstance server) { - if (serversToShutdown.add(server)) { - nextEvent.event("Tablet Server shutdown requested for %s", server); - return true; - } - return false; - } - public EventCoordinator getEventCoordinator() { return nextEvent; } @@ -1725,12 +1709,8 @@ public ManagerMonitorInfo getManagerMonitorInfo() { result.state = getManagerState(); result.goalState = getManagerGoalState(); result.unassignedTablets = displayUnassigned(); - result.serversShuttingDown = new HashSet<>(); - synchronized (serversToShutdown) { - for (TServerInstance server : serversToShutdown) { - result.serversShuttingDown.add(server.getHostPort()); - } - } + result.serversShuttingDown = + shutdownServers().stream().map(TServerInstance::getHostPort).collect(Collectors.toSet()); DeadServerList obit = new DeadServerList(getContext()); result.deadTabletServers = obit.getList(); return result; @@ -1744,8 +1724,12 @@ public boolean delegationTokensAvailable() { } public Set shutdownServers() { - synchronized (serversToShutdown) { - return Set.copyOf(serversToShutdown); + try { + List children = + getContext().getZooSession().asReader().getChildren(Constants.ZSHUTTING_DOWN_TSERVERS); + return children.stream().map(TServerInstance::new).collect(Collectors.toSet()); + } catch (KeeperException | InterruptedException e) { + throw new RuntimeException(e); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java index f96332dc9d6..d88a94516d6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java @@ -33,7 +33,9 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.Constants; @@ -62,6 +64,7 @@ import org.apache.accumulo.core.fate.FateClient; import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.FateKey; import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.manager.state.tables.TableState; @@ -85,7 +88,7 @@ import org.apache.accumulo.core.securityImpl.thrift.TDelegationTokenConfig; import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.manager.tableOps.FateEnv; -import org.apache.accumulo.manager.tserverOps.ShutdownTServer; +import org.apache.accumulo.manager.tserverOps.BeginTserverShutdown; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.client.ClientServiceHandler; import org.apache.accumulo.server.conf.store.NamespacePropKey; @@ -336,16 +339,16 @@ public void shutdownTabletServer(TInfo info, TCredentials c, String tabletServer } FateClient fate = manager.fateClient(FateInstanceType.META); - FateId fateId = fate.startTransaction(); - String msg = "Shutdown tserver " + tabletServer; + var repo = new TraceRepo<>( + new BeginTserverShutdown(doomed, manager.tserverSet.getResourceGroup(doomed), force)); - fate.seedTransaction(Fate.FateOperation.SHUTDOWN_TSERVER, fateId, - new TraceRepo<>( - new ShutdownTServer(doomed, manager.tserverSet.getResourceGroup(doomed), force)), - false, msg); - fate.waitForCompletion(fateId); - fate.delete(fateId); + CompletableFuture> future; + try (var seeder = fate.beginSeeding()) { + future = seeder.attemptToSeedTransaction(Fate.FateOperation.SHUTDOWN_TSERVER, + FateKey.forShutdown(doomed), repo, true); + } + future.join().ifPresent(fate::waitForCompletion); log.debug("FATE op shutting down " + tabletServer + " finished"); } @@ -360,17 +363,15 @@ public void tabletServerStopping(TInfo tinfo, TCredentials credentials, String t } log.info("Tablet Server {} has reported it's shutting down", tabletServer); var tserver = new TServerInstance(tabletServer); - if (manager.shutdownTServer(tserver)) { - // If there is an exception seeding the fate tx this should cause the RPC to fail which should - // cause the tserver to halt. Because of that not making an attempt to handle failure here. - FateClient fate = manager.fateClient(FateInstanceType.META); - var tid = fate.startTransaction(); - String msg = "Shutdown tserver " + tabletServer; + // If there is an exception seeding the fate tx this should cause the RPC to fail which should + // cause the tserver to halt. Because of that not making an attempt to handle failure here. + FateClient fate = manager.fateClient(FateInstanceType.META); - fate.seedTransaction(Fate.FateOperation.SHUTDOWN_TSERVER, tid, - new TraceRepo<>(new ShutdownTServer(tserver, ResourceGroupId.of(resourceGroup), false)), - true, msg); - } + var repo = new TraceRepo<>( + new BeginTserverShutdown(tserver, ResourceGroupId.of(resourceGroup), false)); + // only seed a new transaction if nothing is running for this tserver + fate.seedTransaction(Fate.FateOperation.SHUTDOWN_TSERVER, FateKey.forShutdown(tserver), repo, + true); } @Override diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/BeginTserverShutdown.java b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/BeginTserverShutdown.java new file mode 100644 index 00000000000..358595ebcac --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/BeginTserverShutdown.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.tserverOps; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.fate.FateId; +import org.apache.accumulo.core.fate.Repo; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil; +import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.manager.tableOps.AbstractFateOperation; +import org.apache.accumulo.manager.tableOps.FateEnv; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.net.HostAndPort; + +public class BeginTserverShutdown extends AbstractFateOperation { + + private static final long serialVersionUID = 1L; + + private static final Logger log = LoggerFactory.getLogger(BeginTserverShutdown.class); + + private final ResourceGroupId resourceGroup; + private final HostAndPort hostAndPort; + private final String serverSession; + private final boolean force; + + public BeginTserverShutdown(TServerInstance server, ResourceGroupId resourceGroup, + boolean force) { + this.hostAndPort = server.getHostAndPort(); + this.resourceGroup = resourceGroup; + this.serverSession = server.getSession(); + this.force = force; + } + + static String createPath(HostAndPort hostPort, String session) { + return Constants.ZSHUTTING_DOWN_TSERVERS + "/" + + new TServerInstance(hostPort, session).getHostPortSession(); + } + + @Override + public Repo call(FateId fateId, FateEnv environment) throws Exception { + if (!force) { + String path = createPath(hostAndPort, serverSession); + // Because these fate operation are seeded with a fate key there should only ever be one fate + // operation running for a tserver instance, so do not need to worry about race conditions + // here + environment.getContext().getZooSession().asReaderWriter().putPersistentData(path, new byte[0], + ZooUtil.NodeExistsPolicy.SKIP); + log.trace("{} created {}", fateId, path); + } + return new ShutdownTServer(hostAndPort, serverSession, resourceGroup, force); + } + + @Override + public void undo(FateId fateId, FateEnv environment) throws Exception { + if (!force) { + String path = createPath(hostAndPort, serverSession); + environment.getContext().getZooSession().asReaderWriter().delete(path); + log.trace("{} removed {}", fateId, path); + } + } +} diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java index 147b8206cca..eb456a8a3c6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java @@ -19,6 +19,7 @@ package org.apache.accumulo.manager.tserverOps; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.accumulo.manager.tserverOps.BeginTserverShutdown.createPath; import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.fate.FateId; @@ -28,10 +29,12 @@ import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.manager.Manager; +import org.apache.accumulo.core.rpc.ThriftUtil; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.tabletserver.thrift.TabletServerClientService; +import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; -import org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection; import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,10 +50,11 @@ public class ShutdownTServer extends AbstractFateOperation { private final String serverSession; private final boolean force; - public ShutdownTServer(TServerInstance server, ResourceGroupId resourceGroup, boolean force) { - this.hostAndPort = server.getHostAndPort(); + public ShutdownTServer(HostAndPort hostAndPort, String serverSession, + ResourceGroupId resourceGroup, boolean force) { + this.hostAndPort = hostAndPort; this.resourceGroup = resourceGroup; - this.serverSession = server.getSession(); + this.serverSession = serverSession; this.force = force; } @@ -62,35 +66,38 @@ public long isReady(FateId fateId, FateEnv env) { return 0; } - // Inform the manager that we want this server to shutdown - Manager manager = (Manager) env; - manager.shutdownTServer(server); - - if (manager.onlineTabletServers().contains(server)) { - TServerConnection connection = manager.getConnection(server); - if (connection != null) { - try { - TabletServerStatus status = connection.getTableMap(false); - if (status.tableMap != null && status.tableMap.isEmpty()) { - log.info("tablet server hosts no tablets {}", server); - connection.halt(manager.getServiceLock()); - log.info("tablet server asked to halt {}", server); - return 0; - } else { - log.info("tablet server {} still has tablets for tables: {}", server, - (status.tableMap == null) ? "null" : status.tableMap.keySet()); - } - } catch (TTransportException ex) { - // expected - } catch (Exception ex) { - log.error("Error talking to tablet server {}: ", server, ex); - } + if (env.onlineTabletServers().contains(server)) { + + TabletServerClientService.Client client = null; - // If the connection was non-null and we could communicate with it - // give the manager some more time to tell it to stop and for the - // tserver to ack the request and stop itself. - return 1000; + try { + client = + ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, hostAndPort, env.getContext()); + TabletServerStatus status = + client.getTabletServerStatus(TraceUtil.traceInfo(), env.getContext().rpcCreds()); + if (status.tableMap != null && status.tableMap.isEmpty()) { + log.info("tablet server hosts no tablets {}", server); + client.halt(TraceUtil.traceInfo(), env.getContext().rpcCreds(), + env.getServiceLock().getLockID().serialize()); + log.info("tablet server asked to halt {}", server); + return 0; + } else { + log.info("tablet server {} still has tablets for tables: {}", server, + (status.tableMap == null) ? "null" : status.tableMap.keySet()); + } + } catch (TTransportException ex) { + // expected + } catch (Exception ex) { + log.error("Error talking to tablet server {}: ", server, ex); + } finally { + ThriftUtil.returnClient(client, env.getContext()); } + + // If the connection was non-null and we could communicate with it + // give the manager some more time to tell it to stop and for the + // tserver to ack the request and stop itself. + return 1000; + } return 0; @@ -108,6 +115,10 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { env.getContext().getServerPaths().createDeadTabletServerPath(resourceGroup, hostAndPort); zoo.putPersistentData(path.toString(), "forced down".getBytes(UTF_8), NodeExistsPolicy.OVERWRITE); + } else { + String path = createPath(hostAndPort, serverSession); + env.getContext().getZooSession().asReaderWriter().delete(path); + log.trace("{} removed {}", fateId, path); } return null; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index a230dc7149a..516d0462da4 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -255,6 +255,8 @@ public void upgradeZookeeper(ServerContext context) { moveTableProperties(context); LOG.info("Add assistant manager node"); addAssistantManager(context); + LOG.info("Adding shutting down tservers node"); + addShuttingDownTservers(context); } @Override @@ -308,6 +310,15 @@ private static void addAssistantManager(ServerContext context) { } } + private static void addShuttingDownTservers(ServerContext context) { + try { + context.getZooSession().asReaderWriter().putPersistentData(Constants.ZSHUTTING_DOWN_TSERVERS, + new byte[0], ZooUtil.NodeExistsPolicy.SKIP); + } catch (KeeperException | InterruptedException e) { + throw new IllegalStateException(e); + } + } + private static void addCompactionsNode(ServerContext context) { try { context.getZooSession().asReaderWriter().putPersistentData(Constants.ZCOMPACTIONS, diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java deleted file mode 100644 index 8dbe53bf122..00000000000 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.manager.tableOps; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.Collections; -import java.util.HashMap; -import java.util.UUID; - -import org.apache.accumulo.core.data.ResourceGroupId; -import org.apache.accumulo.core.fate.FateId; -import org.apache.accumulo.core.fate.FateInstanceType; -import org.apache.accumulo.core.manager.thrift.TableInfo; -import org.apache.accumulo.core.manager.thrift.TabletServerStatus; -import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.manager.Manager; -import org.apache.accumulo.manager.tserverOps.ShutdownTServer; -import org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection; -import org.easymock.EasyMock; -import org.junit.jupiter.api.Test; - -import com.google.common.net.HostAndPort; - -public class ShutdownTServerTest { - - @Test - public void testSingleShutdown() throws Exception { - HostAndPort hap = HostAndPort.fromParts("localhost", 1234); - final TServerInstance tserver = new TServerInstance(hap, "fake"); - final boolean force = false; - - final ShutdownTServer op = new ShutdownTServer(tserver, ResourceGroupId.DEFAULT, force); - - final Manager manager = EasyMock.createMock(Manager.class); - final FateId fateId = FateId.from(FateInstanceType.USER, UUID.randomUUID()); - - final TServerConnection tserverCnxn = EasyMock.createMock(TServerConnection.class); - final TabletServerStatus status = new TabletServerStatus(); - status.tableMap = new HashMap<>(); - // Put in a table info record, don't care what - status.tableMap.put("a_table", new TableInfo()); - - EasyMock.expect(manager.shutdownTServer(tserver)).andReturn(true).once(); - EasyMock.expect(manager.onlineTabletServers()).andReturn(Collections.singleton(tserver)); - EasyMock.expect(manager.getConnection(tserver)).andReturn(tserverCnxn); - EasyMock.expect(tserverCnxn.getTableMap(false)).andReturn(status); - - EasyMock.replay(tserverCnxn, manager); - - // FATE op is not ready - long wait = op.isReady(fateId, manager); - assertTrue(wait > 0, "Expected wait to be greater than 0"); - - EasyMock.verify(tserverCnxn, manager); - - // Reset the mocks - EasyMock.reset(tserverCnxn, manager); - - // reset the table map to the empty set to simulate all tablets unloaded - status.tableMap = new HashMap<>(); - EasyMock.expect(manager.shutdownTServer(tserver)).andReturn(false).once(); - EasyMock.expect(manager.onlineTabletServers()).andReturn(Collections.singleton(tserver)); - EasyMock.expect(manager.getConnection(tserver)).andReturn(tserverCnxn); - EasyMock.expect(tserverCnxn.getTableMap(false)).andReturn(status); - EasyMock.expect(manager.getServiceLock()).andReturn(null); - tserverCnxn.halt(null); - EasyMock.expectLastCall().once(); - - EasyMock.replay(tserverCnxn, manager); - - // FATE op is not ready - wait = op.isReady(fateId, manager); - assertEquals(0, wait, "Expected wait to be 0"); - - var op2 = op.call(fateId, manager); - assertNull(op2, "Expected no follow on step"); - - EasyMock.verify(tserverCnxn, manager); - } - -} From eeac89f2af0c3f15add0b71b5742f33dda196560 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 18 Mar 2026 18:59:11 +0000 Subject: [PATCH 05/65] format code --- .../org/apache/accumulo/manager/upgrade/Upgrader11to12.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index 516d0462da4..f88a9d59c4d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -313,7 +313,7 @@ private static void addAssistantManager(ServerContext context) { private static void addShuttingDownTservers(ServerContext context) { try { context.getZooSession().asReaderWriter().putPersistentData(Constants.ZSHUTTING_DOWN_TSERVERS, - new byte[0], ZooUtil.NodeExistsPolicy.SKIP); + new byte[0], ZooUtil.NodeExistsPolicy.SKIP); } catch (KeeperException | InterruptedException e) { throw new IllegalStateException(e); } From c7eb2199f6f9da6bdacf9c61c8ff13f862e39181 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 18 Mar 2026 23:41:16 +0000 Subject: [PATCH 06/65] Removed specialization for meta fate in the manager After this change meta fate and user fate are both treated mostly the same in the managers. One difference is in assignment, the entire meta fate range is assigned to a single manager. User fate is spread across all managers. But both are assigned out by the primary manager using the same RPCs now. The primary manager used to directly start a meta fate instance. Was able to remove the extension of FateEnv from the manager class in this change, that caused a ripple of test changes. But now there are no longer two different implementations of FateEnv --- .../accumulo/core/fate/FatePartition.java | 4 + .../org/apache/accumulo/manager/Manager.java | 92 ++----------------- .../accumulo/manager/fate/FateManager.java | 90 ++++++++++++------ .../accumulo/manager/fate/FateWorker.java | 81 +++++++++------- .../compact/CompactionDriverTest.java | 13 +-- .../tableOps/merge/MergeTabletsTest.java | 12 +-- .../tableOps/split/UpdateTabletsTest.java | 26 +++--- .../accumulo/test/MultipleManagerFateIT.java | 5 +- .../accumulo/test/ample/TestAmpleUtil.java | 24 ++--- .../test/fate/ManagerRepoIT_SimpleSuite.java | 66 ++++++------- 10 files changed, 198 insertions(+), 215 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/fate/FatePartition.java b/core/src/main/java/org/apache/accumulo/core/fate/FatePartition.java index 33cbbc97240..44907cb788c 100644 --- a/core/src/main/java/org/apache/accumulo/core/fate/FatePartition.java +++ b/core/src/main/java/org/apache/accumulo/core/fate/FatePartition.java @@ -60,4 +60,8 @@ public boolean contains(FateId fateId) { } } + + public FateInstanceType getType() { + return start.getType(); + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 8e639e6ae19..204de1d79d9 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -50,7 +50,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -77,7 +76,6 @@ import org.apache.accumulo.core.fate.FateClient; import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; -import org.apache.accumulo.core.fate.FatePartition; import org.apache.accumulo.core.fate.FateStore; import org.apache.accumulo.core.fate.TraceRepo; import org.apache.accumulo.core.fate.user.UserFateStore; @@ -108,7 +106,6 @@ import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metrics.MetricsInfo; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.trace.TraceUtil; @@ -123,11 +120,9 @@ import org.apache.accumulo.manager.fate.FateManager; import org.apache.accumulo.manager.fate.FateWorker; import org.apache.accumulo.manager.merge.FindMergeableRangeTask; -import org.apache.accumulo.manager.metrics.fate.FateExecutorMetricsProducer; import org.apache.accumulo.manager.metrics.fate.meta.MetaFateMetrics; import org.apache.accumulo.manager.metrics.fate.user.UserFateMetrics; import org.apache.accumulo.manager.recovery.RecoveryManager; -import org.apache.accumulo.manager.split.FileRangeCache; import org.apache.accumulo.manager.split.Splitter; import org.apache.accumulo.manager.state.TableCounts; import org.apache.accumulo.manager.tableOps.FateEnv; @@ -179,10 +174,8 @@ *

* The manager will also coordinate log recoveries and reports general status. */ -// TODO create standalone PrimaryFateEnv class and pull everything into there relatated to -// FateEnv... this will make it much more clear the env is for metadata ops only public class Manager extends AbstractServer - implements LiveTServerSet.Listener, FateEnv, PrimaryManagerThriftService { + implements LiveTServerSet.Listener, PrimaryManagerThriftService { static final Logger log = LoggerFactory.getLogger(Manager.class); @@ -228,8 +221,6 @@ public class Manager extends AbstractServer private final CountDownLatch fateReadyLatch = new CountDownLatch(1); private final AtomicReference>> fateClients = new AtomicReference<>(); - private final AtomicReference>> fateRefs = - new AtomicReference<>(); private volatile FateManager fateManager; static class TServerStatus { @@ -287,7 +278,6 @@ void setTserverStatus(LiveTServersSnapshot snapshot, private final long timeToCacheRecoveryWalExistence; private ExecutorService tableInformationStatusPool = null; - private ThreadPoolExecutor tabletRefreshThreadPool; private final TabletStateStore rootTabletStore; private final TabletStateStore metadataTabletStore; @@ -341,21 +331,15 @@ private void waitForFate() { } /** - * Retrieve the Fate object, blocking until it is ready. This could cause problems if Fate + * Retrieve the FateClient object, blocking until it is ready. This could cause problems if Fate * operations are attempted to be used prior to the Manager being ready for them. If these * operations are triggered by a client side request from a tserver or client, it should be safe * to wait to handle those until Fate is ready, but if it occurs during an upgrade, or some other * time in the Manager before Fate is started, that may result in a deadlock and will need to be * fixed. * - * @return the Fate object, only after the fate components are running and ready + * @return the FateClient object, only after the fate components are running and ready */ - public Fate fate(FateInstanceType type) { - waitForFate(); - var fate = requireNonNull(fateRefs.get(), "fateRefs is not set yet").get(type); - return requireNonNull(fate, () -> "fate type " + type + " is not present"); - } - public FateClient fateClient(FateInstanceType type) { waitForFate(); var client = requireNonNull(fateClients.get(), "fateClients is not set yet").get(type); @@ -498,16 +482,10 @@ int displayUnassigned() { return result; } - @Override public TableManager getTableManager() { return getContext().getTableManager(); } - @Override - public ThreadPoolExecutor getTabletRefreshThreadPool() { - return tabletRefreshThreadPool; - } - public static void main(String[] args) throws Exception { AbstractServer.startServer(new Manager(new ServerOpts(), ServerContext::new, args), log); } @@ -586,17 +564,11 @@ ManagerGoalState getManagerGoalState() { } private Splitter splitter; - private FileRangeCache fileRangeCache; public Splitter getSplitter() { return splitter; } - @Override - public FileRangeCache getFileRangeCache() { - return fileRangeCache; - } - public UpgradeCoordinator.UpgradeStatus getUpgradeStatus() { return upgradeCoordinator.getStatus(); } @@ -605,11 +577,6 @@ public CompactionCoordinator getCompactionCoordinator() { return compactionCoordinator; } - @Override - public void recordCompactionCompletion(ExternalCompactionId ecid) { - getCompactionCoordinator().recordCompletion(ecid); - } - public void hostOndemand(List extents) { extents.forEach(e -> Preconditions.checkArgument(DataLevel.of(e.tableId()) == DataLevel.USER)); @@ -723,14 +690,14 @@ public void run() { case CLEAN_STOP: switch (getManagerState()) { case NORMAL: - fateManager.stop(Duration.ofMinutes(1)); + fateManager.stop(FateInstanceType.USER, Duration.ofMinutes(1)); setManagerState(ManagerState.SAFE_MODE); break; case SAFE_MODE: { // META fate stores its data in Zookeeper and its operations interact with // metadata and root tablets, need to completely shut it down before unloading // metadata and root tablets - fate(FateInstanceType.META).shutdown(1, MINUTES); + fateManager.stop(FateInstanceType.META, Duration.ofMinutes(1)); int count = nonMetaDataTabletsAssignedOrHosted(); log.debug( String.format("There are %d non-metadata tablets assigned or hosted", count)); @@ -959,9 +926,6 @@ private void setupPrimaryMetrics() { getConfiguration().getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL))); metricsInfo.addMetricsProducers(new UserFateMetrics(getContext(), getConfiguration().getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL))); - metricsInfo.addMetricsProducers(new FateExecutorMetricsProducer(getContext(), - fate(FateInstanceType.META).getFateExecutors(), - getConfiguration().getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL))); metricsInfo.addMetricsProducers(this); } @@ -1051,11 +1015,6 @@ public void run() { tableInformationStatusPool = ThreadPools.getServerThreadPools() .createExecutorService(getConfiguration(), Property.MANAGER_STATUS_THREAD_POOL_SIZE, false); - tabletRefreshThreadPool = ThreadPools.getServerThreadPools().getPoolBuilder("Tablet refresh ") - .numCoreThreads(getConfiguration().getCount(Property.MANAGER_TABLET_REFRESH_MINTHREADS)) - .numMaxThreads(getConfiguration().getCount(Property.MANAGER_TABLET_REFRESH_MAXTHREADS)) - .build(); - Thread statusThread = Threads.createCriticalThread("Status Thread", new StatusThread()); statusThread.start(); @@ -1201,7 +1160,6 @@ boolean canSuspendTablets() { this.splitter = new Splitter(this); this.splitter.start(); - this.fileRangeCache = new FileRangeCache(context); setupFate(context); @@ -1289,8 +1247,8 @@ boolean canSuspendTablets() { } log.debug("Shutting down fate."); - fate(FateInstanceType.META).close(); - fateManager.stop(Duration.ZERO); + fateManager.stop(FateInstanceType.USER, Duration.ZERO); + fateManager.stop(FateInstanceType.META, Duration.ZERO); splitter.stop(); @@ -1302,7 +1260,6 @@ boolean canSuspendTablets() { } tableInformationStatusPool.shutdownNow(); - tabletRefreshThreadPool.shutdownNow(); compactionCoordinator.shutdown(); @@ -1350,12 +1307,10 @@ private void setupFate(ServerContext context) { lock -> ServiceLock.isLockHeld(context.getZooCache(), lock); var metaStore = new MetaFateStore(context.getZooSession(), primaryManagerLock.getLockID(), isLockHeld); - var metaInstance = createFateInstance(this, metaStore, context); - // configure this instance to process all data - metaInstance.setPartitions(Set.of(FatePartition.all(FateInstanceType.META))); + var metaFateClient = new FateClient<>(metaStore, TraceRepo::toLogString); var userStore = new UserFateStore(context, SystemTables.FATE.tableName(), managerLock.getLockID(), isLockHeld); - var userFateClient = new FateClient(userStore, TraceRepo::toLogString); + var userFateClient = new FateClient<>(userStore, TraceRepo::toLogString); var metaCleaner = new FateCleaner<>(metaStore, Duration.ofHours(8), this::getSteadyTime); ThreadPools.watchCriticalScheduledTask(context.getScheduledExecutor() @@ -1365,14 +1320,10 @@ private void setupFate(ServerContext context) { .scheduleWithFixedDelay(userCleaner::ageOff, 10, 4 * 60, MINUTES)); if (!fateClients.compareAndSet(null, - Map.of(FateInstanceType.META, metaInstance, FateInstanceType.USER, userFateClient))) { + Map.of(FateInstanceType.META, metaFateClient, FateInstanceType.USER, userFateClient))) { throw new IllegalStateException( "Unexpected previous fateClient reference map already initialized"); } - if (!fateRefs.compareAndSet(null, Map.of(FateInstanceType.META, metaInstance))) { - throw new IllegalStateException( - "Unexpected previous fate reference map already initialized"); - } fateReadyLatch.countDown(); } catch (KeeperException | InterruptedException e) { @@ -1528,11 +1479,6 @@ private void getAssistantManagerLock() throws KeeperException, InterruptedExcept } } - @Override - public ServiceLock getServiceLock() { - return primaryManagerLock; - } - private ServiceLockData getPrimaryManagerLock(final ServiceLockPath zManagerLoc) throws KeeperException, InterruptedException { var zooKeeper = getContext().getZooSession(); @@ -1653,7 +1599,6 @@ public Set onlineTables() { return result; } - @Override public Set onlineTabletServers() { return tserverSet.getSnapshot().getTservers(); } @@ -1666,12 +1611,6 @@ public EventCoordinator getEventCoordinator() { return nextEvent; } - @Override - public EventPublisher getEventPublisher() { - return nextEvent; - } - - @Override public VolumeManager getVolumeManager() { return getContext().getVolumeManager(); } @@ -1738,7 +1677,6 @@ public Set shutdownServers() { * monotonic clock, which will be approximately consistent between different managers or different * runs of the same manager. SteadyTime supports both nanoseconds and milliseconds. */ - @Override public SteadyTime getSteadyTime() { return timeKeeper.getTime(); } @@ -1766,14 +1704,4 @@ public void registerMetrics(MeterRegistry registry) { public ServiceLock getLock() { return managerLock; } - - /** - * Get Threads Pool instance which is used by blocked I/O - * - * @return {@link ExecutorService} - */ - @Override - public ExecutorService getRenamePool() { - return this.renamePool; - } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java index 6b294e0b8ad..9d934913098 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java @@ -32,7 +32,9 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.FatePartition; +import org.apache.accumulo.core.fate.FateStore; import org.apache.accumulo.core.fate.user.UserFateStore; +import org.apache.accumulo.core.fate.zookeeper.MetaFateStore; import org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector; import org.apache.accumulo.core.manager.thrift.FateWorkerService; import org.apache.accumulo.core.metadata.SystemTables; @@ -44,6 +46,7 @@ import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.thrift.TException; +import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,7 +106,8 @@ private void manageAssistants() throws TException, InterruptedException { } Map> currentAssignments = new HashMap<>(); currentPartitions.forEach((k, v) -> currentAssignments.put(k, v.partitions())); - Set desiredParititions = getDesiredPartitions(currentAssignments.size()); + Map> desiredParititions = + getDesiredPartitions(currentAssignments.size()); Map> desired = computeDesiredAssignments(currentAssignments, desiredParititions); @@ -191,7 +195,7 @@ public synchronized void start() { @SuppressFBWarnings(value = "SWL_SLEEP_WITH_LOCK_HELD", justification = "Sleep is okay. Can hold the lock as long as needed, as we are shutting down." + " Don't need or want other operations to run.") - public synchronized void stop(Duration timeout) { + public synchronized void stop(FateInstanceType fateType, Duration timeout) { if (!stop.compareAndSet(false, true)) { return; } @@ -222,7 +226,9 @@ public synchronized void stop(Duration timeout) { var currentPartitions = entry.getValue(); if (!currentPartitions.partitions.isEmpty()) { try { - setPartitions(hostPort, currentPartitions.updateId(), Set.of()); + var copy = new HashSet<>(currentPartitions.partitions); + copy.removeIf(fp -> fp.getType() == fateType); + setPartitions(hostPort, currentPartitions.updateId(), copy); } catch (TException e) { log.warn("Failed to unassign fate partitions {}", hostPort, e); } @@ -232,8 +238,16 @@ public synchronized void stop(Duration timeout) { stableAssignments.set(TreeRangeMap.create()); if (!timer.isExpired()) { - var store = new UserFateStore(context, SystemTables.FATE.tableName(), null, null); - + FateStore store = switch (fateType) { + case USER -> new UserFateStore(context, SystemTables.FATE.tableName(), null, null); + case META -> { + try { + yield new MetaFateStore<>(context.getZooSession(), null, null); + } catch (KeeperException | InterruptedException e) { + throw new IllegalStateException(e); + } + } + }; var reserved = store.getActiveReservations(Set.of(FatePartition.all(FateInstanceType.USER))); while (!reserved.isEmpty() && !timer.isExpired()) { if (log.isTraceEnabled()) { @@ -333,28 +347,41 @@ private boolean setPartitions(HostAndPort address, long updateId, Set> computeDesiredAssignments( Map> currentAssignments, - Set desiredParititions) { + Map> desiredParititions) { - Preconditions.checkArgument(currentAssignments.size() == desiredParititions.size()); Map> desiredAssignments = new HashMap<>(); - var copy = new HashSet<>(desiredParititions); + currentAssignments.keySet().forEach(hp -> { + desiredAssignments.put(hp, new HashSet<>()); + }); + + desiredParititions.forEach((fateType, desiredForType) -> { + // This code can not handle more than one partition per host + Preconditions.checkState(desiredForType.size() <= currentAssignments.size()); + + var added = new HashSet(); + + currentAssignments.forEach((hp, partitions) -> { + var hostAssignments = desiredAssignments.get(hp); + partitions.forEach(partition -> { + if (desiredForType.contains(partition) + && hostAssignments.stream().noneMatch(fp -> fp.getType() == fateType) + && !added.contains(partition)) { + hostAssignments.add(partition); + Preconditions.checkState(added.add(partition)); + } + }); + }); - currentAssignments.forEach((hp, partitions) -> { - if (!partitions.isEmpty()) { - var firstPart = partitions.iterator().next(); - if (copy.contains(firstPart)) { - desiredAssignments.put(hp, Set.of(firstPart)); - copy.remove(firstPart); + var iter = Sets.difference(desiredForType, added).iterator(); + currentAssignments.forEach((hp, partitions) -> { + var hostAssignments = desiredAssignments.get(hp); + if (iter.hasNext() && hostAssignments.stream().noneMatch(fp -> fp.getType() == fateType)) { + hostAssignments.add(iter.next()); } - } - }); + }); - var iter = copy.iterator(); - currentAssignments.forEach((hp, partitions) -> { - if (!desiredAssignments.containsKey(hp)) { - desiredAssignments.put(hp, Set.of(iter.next())); - } + Preconditions.checkState(!iter.hasNext()); }); if (log.isTraceEnabled()) { @@ -363,7 +390,6 @@ private Map> computeDesiredAssignments( log.trace(" desired {} {} {}", hp, parts.size(), parts); }); } - return desiredAssignments; } @@ -371,15 +397,23 @@ private Map> computeDesiredAssignments( * Computes a single partition for each worker such that the partition cover all possible UUIDs * and evenly divide the UUIDs. */ - private Set getDesiredPartitions(int numWorkers) { + private Map> getDesiredPartitions(int numWorkers) { Preconditions.checkArgument(numWorkers >= 0); if (numWorkers == 0) { - return Set.of(); + return Map.of(FateInstanceType.META, Set.of(), FateInstanceType.USER, Set.of()); } // create a single partition per worker that equally divides the space - HashSet desired = new HashSet<>(); + Map> desired = new HashMap<>(); + + // meta fate will never see much activity, so give it a single partition. + desired.put(FateInstanceType.META, + Set.of(new FatePartition(FateId.from(FateInstanceType.META, new UUID(0, 0)), + FateId.from(FateInstanceType.META, new UUID(-1, -1))))); + + Set desiredUser = new HashSet<>(); + // All the shifting is because java does not have unsigned integers. Want to evenly partition // [0,2^64) into numWorker ranges, but can not directly do that. Work w/ 60 bit unsigned // integers to partition the space and then shift over by 4. Used 60 bits instead of 63 so it @@ -392,7 +426,7 @@ private Set getDesiredPartitions(int numWorkers) { UUID startUuid = new UUID(start, 0); UUID endUuid = new UUID(end, 0); - desired.add(new FatePartition(FateId.from(FateInstanceType.USER, startUuid), + desiredUser.add(new FatePartition(FateId.from(FateInstanceType.USER, startUuid), FateId.from(FateInstanceType.USER, endUuid))); } @@ -400,9 +434,11 @@ private Set getDesiredPartitions(int numWorkers) { UUID startUuid = new UUID(start, 0); // last partition has a special end uuid that is all f nibbles. UUID endUuid = new UUID(-1, -1); - desired.add(new FatePartition(FateId.from(FateInstanceType.USER, startUuid), + desiredUser.add(new FatePartition(FateId.from(FateInstanceType.USER, startUuid), FateId.from(FateInstanceType.USER, endUuid))); + desired.put(FateInstanceType.USER, desiredUser); + return desired; } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java index a1409af98ee..5b0261171d0 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java @@ -20,8 +20,11 @@ import static org.apache.accumulo.core.util.LazySingletons.RANDOM; +import java.util.Arrays; import java.util.List; -import java.util.concurrent.TimeUnit; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -31,9 +34,11 @@ import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.fate.Fate; +import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.FatePartition; import org.apache.accumulo.core.fate.FateStore; import org.apache.accumulo.core.fate.user.UserFateStore; +import org.apache.accumulo.core.fate.zookeeper.MetaFateStore; import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.manager.thrift.FateWorkerService; @@ -49,6 +54,7 @@ import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.security.AuditedSecurityOperation; import org.apache.thrift.TException; +import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +67,7 @@ public class FateWorker implements FateWorkerService.Iface { private final AuditedSecurityOperation security; private final LiveTServerSet liveTserverSet; private final FateFactory fateFactory; - private Fate fate; + private final Map> fates = new ConcurrentHashMap<>(); private FateWorkerEnv fateWorkerEnv; public interface FateFactory { @@ -71,17 +77,24 @@ public interface FateFactory { public FateWorker(ServerContext ctx, LiveTServerSet liveTServerSet, FateFactory fateFactory) { this.context = ctx; this.security = ctx.getSecurityOperation(); - this.fate = null; this.liveTserverSet = liveTServerSet; this.fateFactory = fateFactory; + Thread.interrupted() } public synchronized void setLock(ServiceLock lock) { fateWorkerEnv = new FateWorkerEnv(context, lock, liveTserverSet); Predicate isLockHeld = l -> ServiceLock.isLockHeld(context.getZooCache(), l); + try { + MetaFateStore metaStore = + new MetaFateStore<>(context.getZooSession(), lock.getLockID(), isLockHeld); + this.fates.put(FateInstanceType.META, fateFactory.create(fateWorkerEnv, metaStore, context)); + } catch (KeeperException | InterruptedException e) { + throw new IllegalStateException(e); + } UserFateStore store = new UserFateStore<>(context, SystemTables.FATE.tableName(), lock.getLockID(), isLockHeld); - this.fate = fateFactory.create(fateWorkerEnv, store, context); + this.fates.put(FateInstanceType.USER, fateFactory.create(fateWorkerEnv, store, context)); } private Long expectedUpdateId = null; @@ -105,12 +118,8 @@ public TFatePartitions getPartitions(TInfo tinfo, TCredentials credentials) // id expectedUpdateId = updateId; - if (fate == null) { - return new TFatePartitions(updateId, List.of()); - } else { - return new TFatePartitions(updateId, - fate.getPartitions().stream().map(FatePartition::toThrift).toList()); - } + return new TFatePartitions(updateId, fates.values().stream() + .flatMap(fate -> fate.getPartitions().stream()).map(FatePartition::toThrift).toList()); } } @@ -137,16 +146,22 @@ public boolean setPartitions(TInfo tinfo, TCredentials credentials, long updateI synchronized (this) { // The primary manager should not assign any fate partitions until after upgrade is complete. Preconditions.checkState(isUpgradeComplete()); - if (fate != null && expectedUpdateId != null && updateId == expectedUpdateId) { + + if (expectedUpdateId != null && updateId == expectedUpdateId) { // Set to null which makes it so that an update id can only be used once. expectedUpdateId = null; - var desiredSet = desired.stream().map(FatePartition::from).collect(Collectors.toSet()); - var oldPartitions = fate.setPartitions(desiredSet); - log.info("Changed partitions from {} to {}", oldPartitions, desiredSet); + for (var fateType : FateInstanceType.values()) { + var fate = fates.get(fateType); + var desiredSet = desired.stream().map(FatePartition::from) + .filter(fp -> fp.getType() == fateType).collect(Collectors.toSet()); + var oldPartitions = fate.setPartitions(desiredSet); + log.info("Changed partitions for {} from {} to {}", fateType, oldPartitions, desiredSet); + } + return true; } else { - log.debug("Did not change partitions to {} expectedUpdateId:{} updateId:{} fate==null:{}", - desired, expectedUpdateId, updateId, fate == null); + log.debug("Did not change partitions to {} expectedUpdateId:{} updateId:{} fates:{}", + desired, expectedUpdateId, updateId, fates.keySet()); return false; } } @@ -161,28 +176,24 @@ public void seeded(TInfo tinfo, TCredentials credentials, List t SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } - Fate localFate; - synchronized (this) { - localFate = fate; - } + Map> partitions = + tpartitions.stream().map(FatePartition::from) + .collect(Collectors.groupingBy(FatePartition::getType, Collectors.toSet())); - if (localFate != null) { - localFate.seeded(tpartitions.stream().map(FatePartition::from).collect(Collectors.toSet())); - } - } - - public synchronized void stop() { - fate.shutdown(1, TimeUnit.MINUTES); - fate.close(); - fateWorkerEnv.stop(); - fate = null; - fateWorkerEnv = null; + partitions.forEach((fateType, typePartitions) -> { + var fate = fates.get(fateType); + if (fate != null) { + fate.seeded(typePartitions); + } + }); } public synchronized MetricsProducer[] getMetricsProducers() { - Preconditions.checkState(fate != null, "Not started yet"); - return new MetricsProducer[] { - new FateExecutorMetricsProducer(context, fate.getFateExecutors(), context.getConfiguration() - .getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL))}; + Preconditions.checkState(!fates.isEmpty(), "Not started yet"); + return Arrays.stream(FateInstanceType.values()).map(fates::get) + .map(fate -> new FateExecutorMetricsProducer(context, fate.getFateExecutors(), + context.getConfiguration() + .getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL))) + .toArray(MetricsProducer[]::new); } } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java index c95dcd9ad6a..b1ec160734e 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.zookeeper.ZooSession; import org.apache.accumulo.manager.Manager; +import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.manager.tableOps.delete.PreDeleteTable; import org.apache.accumulo.server.ServerContext; import org.apache.zookeeper.data.Stat; @@ -74,24 +75,24 @@ protected boolean isCancelled(FateId fateId, ServerContext context) { } } - private Manager manager; + private FateEnv fateEnv; private ServerContext ctx; private ZooSession zk; @BeforeEach public void setup() { - manager = createMock(Manager.class); + fateEnv = createMock(Manager.class); ctx = createMock(ServerContext.class); zk = createMock(ZooSession.class); expect(ctx.getInstanceID()).andReturn(instance).anyTimes(); expect(ctx.getZooSession()).andReturn(zk).anyTimes(); expect(zk.asReaderWriter()).andReturn(new ZooReaderWriter(zk)).anyTimes(); - expect(manager.getContext()).andReturn(ctx).anyTimes(); + expect(fateEnv.getContext()).andReturn(ctx).anyTimes(); } @AfterEach public void teardown() { - verify(manager, ctx, zk); + verify(fateEnv, ctx, zk); } @Test @@ -107,9 +108,9 @@ public void testTableBeingDeleted() throws Exception { } private void runDriver(CompactionDriver driver, String expectedMessage) { - replay(manager, ctx, zk); + replay(fateEnv, ctx, zk); var e = assertThrows(AcceptableThriftTableOperationException.class, - () -> driver.isReady(FateId.from(FateInstanceType.USER, UUID.randomUUID()), manager)); + () -> driver.isReady(FateId.from(FateInstanceType.USER, UUID.randomUUID()), fateEnv)); assertEquals(e.getTableId(), tableId.toString()); assertEquals(e.getOp(), TableOperation.COMPACT); assertEquals(e.getType(), TableOperationExceptionType.OTHER); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java index 9e3037dbe01..7bd6824f2f6 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java @@ -84,7 +84,7 @@ import org.apache.accumulo.core.metadata.schema.UnSplittableMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; -import org.apache.accumulo.manager.Manager; +import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.gc.AllVolumesDirectory; import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; @@ -425,7 +425,7 @@ private static void testMerge(List inputTablets, TableId tableId end == null ? null : end.getBytes(UTF_8), MergeInfo.Operation.MERGE); MergeTablets mergeTablets = new MergeTablets(mergeInfo); - Manager manager = EasyMock.mock(Manager.class); + FateEnv fateEnv = EasyMock.mock(FateEnv.class); ServerContext context = EasyMock.mock(ServerContext.class); Ample ample = EasyMock.mock(Ample.class); TabletsMetadata.Builder tabletBuilder = EasyMock.mock(TabletsMetadata.Builder.class); @@ -438,7 +438,7 @@ private static void testMerge(List inputTablets, TableId tableId EasyMock.expect(context.getServiceLock()).andReturn(managerLock).anyTimes(); // setup reading the tablets - EasyMock.expect(manager.getContext()).andReturn(context).atLeastOnce(); + EasyMock.expect(fateEnv.getContext()).andReturn(context).atLeastOnce(); EasyMock.expect(context.getAmple()).andReturn(ample).atLeastOnce(); EasyMock.expect(ample.readTablets()).andReturn(tabletBuilder).once(); EasyMock.expect(tabletBuilder.forTable(tableId)).andReturn(tabletBuilder).once(); @@ -478,12 +478,12 @@ private static void testMerge(List inputTablets, TableId tableId ample.putGcFileAndDirCandidates(tableId, dirs); EasyMock.expectLastCall().once(); - EasyMock.replay(manager, context, ample, tabletBuilder, tabletsMetadata, tabletsMutator, + EasyMock.replay(fateEnv, context, ample, tabletBuilder, tabletsMetadata, tabletsMutator, tabletMutator, cr, managerLock); - mergeTablets.call(fateId, manager); + mergeTablets.call(fateId, fateEnv); - EasyMock.verify(manager, context, ample, tabletBuilder, tabletsMetadata, tabletsMutator, + EasyMock.verify(fateEnv, context, ample, tabletBuilder, tabletsMetadata, tabletsMutator, tabletMutator, cr, managerLock); } } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java index 6ceac3d6810..dad40a0cb13 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java @@ -66,8 +66,8 @@ import org.apache.accumulo.core.metadata.schema.UnSplittableMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; -import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.split.FileRangeCache; +import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; import org.apache.hadoop.fs.Path; @@ -233,9 +233,9 @@ public void testManyColumns() throws Exception { String dir1 = "dir1"; String dir2 = "dir2"; - Manager manager = EasyMock.mock(Manager.class); + FateEnv fateEnv = EasyMock.mock(FateEnv.class); ServerContext context = EasyMock.mock(ServerContext.class); - EasyMock.expect(manager.getContext()).andReturn(context).atLeastOnce(); + EasyMock.expect(fateEnv.getContext()).andReturn(context).atLeastOnce(); Ample ample = EasyMock.mock(Ample.class); EasyMock.expect(context.getAmple()).andReturn(ample).atLeastOnce(); FileRangeCache fileRangeCache = EasyMock.mock(FileRangeCache.class); @@ -247,8 +247,8 @@ public void testManyColumns() throws Exception { .andReturn(newFileInfo("d", "f")); EasyMock.expect(fileRangeCache.getCachedFileInfo(tableId, file4)) .andReturn(newFileInfo("d", "j")); - EasyMock.expect(manager.getFileRangeCache()).andReturn(fileRangeCache).atLeastOnce(); - EasyMock.expect(manager.getSteadyTime()).andReturn(SteadyTime.from(100_000, TimeUnit.SECONDS)) + EasyMock.expect(fateEnv.getFileRangeCache()).andReturn(fileRangeCache).atLeastOnce(); + EasyMock.expect(fateEnv.getSteadyTime()).andReturn(SteadyTime.from(100_000, TimeUnit.SECONDS)) .atLeastOnce(); ServiceLock managerLock = EasyMock.mock(ServiceLock.class); @@ -394,7 +394,7 @@ public void testManyColumns() throws Exception { tabletsMutator.close(); EasyMock.expectLastCall().anyTimes(); - EasyMock.replay(manager, context, ample, tabletMeta, fileRangeCache, tabletsMutator, + EasyMock.replay(fateEnv, context, ample, tabletMeta, fileRangeCache, tabletsMutator, tablet1Mutator, tablet2Mutator, tablet3Mutator, cr, compactions); // Now we can actually test the split code that writes the new tablets with a bunch columns in // the original tablet @@ -404,9 +404,9 @@ public void testManyColumns() throws Exception { dirNames.add(dir2); UpdateTablets updateTablets = new UpdateTablets( new SplitInfo(origExtent, TabletMergeabilityUtil.systemDefaultSplits(splits)), dirNames); - updateTablets.call(fateId, manager); + updateTablets.call(fateId, fateEnv); - EasyMock.verify(manager, context, ample, tabletMeta, fileRangeCache, tabletsMutator, + EasyMock.verify(fateEnv, context, ample, tabletMeta, fileRangeCache, tabletsMutator, tablet1Mutator, tablet2Mutator, tablet3Mutator, cr, compactions); } @@ -469,15 +469,15 @@ public void testErrors() throws Exception { private static void testError(KeyExtent origExtent, TabletMetadata tm1, FateId fateId) throws Exception { - Manager manager = EasyMock.mock(Manager.class); + FateEnv fateEnv = EasyMock.mock(FateEnv.class); ServerContext context = EasyMock.mock(ServerContext.class); - EasyMock.expect(manager.getContext()).andReturn(context).atLeastOnce(); + EasyMock.expect(fateEnv.getContext()).andReturn(context).atLeastOnce(); Ample ample = EasyMock.mock(Ample.class); EasyMock.expect(context.getAmple()).andReturn(ample).atLeastOnce(); EasyMock.expect(ample.readTablet(origExtent)).andReturn(tm1); - EasyMock.replay(manager, context, ample); + EasyMock.replay(fateEnv, context, ample); // Now we can actually test the split code that writes the new tablets with a bunch columns in // the original tablet SortedSet splits = new TreeSet<>(List.of(new Text("c"))); @@ -485,8 +485,8 @@ private static void testError(KeyExtent origExtent, TabletMetadata tm1, FateId f dirNames.add("d1"); var updateTablets = new UpdateTablets( new SplitInfo(origExtent, TabletMergeabilityUtil.systemDefaultSplits(splits)), dirNames); - updateTablets.call(fateId, manager); + updateTablets.call(fateId, fateEnv); - EasyMock.verify(manager, context, ample); + EasyMock.verify(fateEnv, context, ample); } } diff --git a/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java b/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java index d0d8bdb5ed8..2367040c4a6 100644 --- a/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MultipleManagerFateIT.java @@ -84,7 +84,6 @@ * */ public class MultipleManagerFateIT extends ConfigurableMacBase { - // A manager that will quickly clean up fate reservations held by dead managers public static class FastFateCleanupManager extends Manager { protected FastFateCleanupManager(ServerOpts opts, String[] args) throws IOException { @@ -111,6 +110,10 @@ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSit cfg.getClusterServerConfiguration().setNumDefaultCompactors(8); // Set this lower so that locks timeout faster cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s"); + // This test could kill a manager after its written a compaction to the metadata table, but + // before it returns it to the compactor via RPC which creates a dead compaction. Need to speed + // up the dead compaction detection to handle this or else the test will hang. + cfg.setProperty(Property.COMPACTION_COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL, "5s"); cfg.setServerClass(ServerType.MANAGER, r -> FastFateCleanupManager.class); super.configure(cfg, hadoopCoreSite); } diff --git a/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleUtil.java b/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleUtil.java index 8b4283f31db..834e9a10bdc 100644 --- a/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleUtil.java +++ b/test/src/main/java/org/apache/accumulo/test/ample/TestAmpleUtil.java @@ -23,29 +23,29 @@ import java.time.Duration; import org.apache.accumulo.core.util.time.SteadyTime; -import org.apache.accumulo.manager.Manager; +import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.test.ample.metadata.TestAmple.TestServerAmpleImpl; import org.easymock.EasyMock; public class TestAmpleUtil { - public static Manager mockWithAmple(ServerContext context, TestServerAmpleImpl ample) { - Manager manager = EasyMock.mock(Manager.class); - EasyMock.expect(manager.getContext()).andReturn(testAmpleServerContext(context, ample)) + public static FateEnv mockWithAmple(ServerContext context, TestServerAmpleImpl ample) { + FateEnv fateEnv = EasyMock.mock(FateEnv.class); + EasyMock.expect(fateEnv.getContext()).andReturn(testAmpleServerContext(context, ample)) .atLeastOnce(); - EasyMock.replay(manager); - return manager; + EasyMock.replay(fateEnv); + return fateEnv; } - public static Manager mockWithAmple(ServerContext context, TestServerAmpleImpl ample, + public static FateEnv mockWithAmple(ServerContext context, TestServerAmpleImpl ample, Duration currentTime) { - Manager manager = EasyMock.mock(Manager.class); - EasyMock.expect(manager.getContext()).andReturn(testAmpleServerContext(context, ample)) + FateEnv fateEnv = EasyMock.mock(FateEnv.class); + EasyMock.expect(fateEnv.getContext()).andReturn(testAmpleServerContext(context, ample)) .atLeastOnce(); - EasyMock.expect(manager.getSteadyTime()).andReturn(SteadyTime.from(currentTime)).anyTimes(); - EasyMock.replay(manager); - return manager; + EasyMock.expect(fateEnv.getSteadyTime()).andReturn(SteadyTime.from(currentTime)).anyTimes(); + EasyMock.replay(fateEnv); + return fateEnv; } } diff --git a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java index 1178f2d8ad8..a36f8704460 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java @@ -75,9 +75,9 @@ import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.harness.SharedMiniClusterBase; -import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.merge.FindMergeableRangeTask.UnmergeableReason; import org.apache.accumulo.manager.tableOps.AbstractFateOperation; +import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.manager.tableOps.compact.CompactionDriver; import org.apache.accumulo.manager.tableOps.merge.DeleteRows; import org.apache.accumulo.manager.tableOps.merge.MergeInfo; @@ -133,7 +133,7 @@ public void testNoWalsMergeRepos(MergeInfo.Operation operation) throws Exception TestServerAmpleImpl testAmple = (TestServerAmpleImpl) TestAmple .create(getCluster().getServerContext(), Map.of(DataLevel.USER, metadataTable)); testAmple.createMetadataFromExisting(client, tableId); - Manager manager = mockWithAmple(getCluster().getServerContext(), testAmple); + FateEnv fateEnv = mockWithAmple(getCluster().getServerContext(), testAmple); // Create a test operation and fate id for testing merge and delete rows // and add operation to test metadata for the tablet @@ -148,15 +148,15 @@ public void testNoWalsMergeRepos(MergeInfo.Operation operation) throws Exception // Build either MergeTablets or DeleteRows repo for testing no WALs, both should check this // condition final MergeInfo mergeInfo = new MergeInfo(tableId, - manager.getContext().getNamespaceId(tableId), null, null, operation); + fateEnv.getContext().getNamespaceId(tableId), null, null, operation); final AbstractFateOperation repo = operation == Operation.MERGE ? new MergeTablets(mergeInfo) : new DeleteRows(mergeInfo); // Also test ReserveTablets isReady() final AbstractFateOperation reserve = new ReserveTablets(mergeInfo); // First, check no errors with the default case - assertEquals(0, reserve.isReady(fateId, manager)); - assertNotNull(repo.call(fateId, manager)); + assertEquals(0, reserve.isReady(fateId, fateEnv)); + assertNotNull(repo.call(fateId, fateEnv)); // Write a WAL to the test metadata and then re-run the repo to check for an error try (TabletsMutator tm = testAmple.mutateTablets()) { @@ -165,10 +165,10 @@ public void testNoWalsMergeRepos(MergeInfo.Operation operation) throws Exception } // Should not be ready due to the presence of a WAL - assertTrue(reserve.isReady(fateId, manager) > 0); + assertTrue(reserve.isReady(fateId, fateEnv) > 0); // Repo should throw an exception due to the WAL existence - var thrown = assertThrows(IllegalStateException.class, () -> repo.call(fateId, manager)); + var thrown = assertThrows(IllegalStateException.class, () -> repo.call(fateId, fateEnv)); assertTrue(thrown.getMessage().contains("has unexpected walogs")); } } @@ -200,57 +200,57 @@ public void testVerifyMergeability() throws Exception { TestServerAmpleImpl testAmple = (TestServerAmpleImpl) TestAmple .create(getCluster().getServerContext(), Map.of(DataLevel.USER, metadataTable)); testAmple.createMetadataFromExisting(client, tableId); - Manager manager = + FateEnv fateEnv = mockWithAmple(getCluster().getServerContext(), testAmple, Duration.ofDays(1)); // Create a test fate id var fateId = FateId.from(FateInstanceType.USER, UUID.randomUUID()); // Tablet c is set to never merge - MergeInfo mergeInfo = new MergeInfo(tableId, manager.getContext().getNamespaceId(tableId), + MergeInfo mergeInfo = new MergeInfo(tableId, fateEnv.getContext().getNamespaceId(tableId), null, new Text("c").getBytes(), Operation.SYSTEM_MERGE); - var repo = new VerifyMergeability(mergeInfo).call(fateId, manager); + var repo = new VerifyMergeability(mergeInfo).call(fateId, fateEnv); assertInstanceOf(UnreserveSystemMerge.class, repo); assertEquals(UnmergeableReason.TABLET_MERGEABILITY, ((UnreserveSystemMerge) repo).getReason()); // Tablets a and b are always merge - mergeInfo = new MergeInfo(tableId, manager.getContext().getNamespaceId(tableId), null, + mergeInfo = new MergeInfo(tableId, fateEnv.getContext().getNamespaceId(tableId), null, new Text("b").getBytes(), Operation.SYSTEM_MERGE); - assertInstanceOf(MergeTablets.class, new VerifyMergeability(mergeInfo).call(fateId, manager)); + assertInstanceOf(MergeTablets.class, new VerifyMergeability(mergeInfo).call(fateId, fateEnv)); - var context = manager.getContext(); + var context = fateEnv.getContext(); // split threshold is 10k so default max merge size is 2500 bytes. // this adds 6 files of 450 each which puts the tablets over teh 2500 threshold addFileMetadata(context, tableId, null, new Text("c"), 3, 450); // Data written to the first two tablets totals 2700 bytes and is too large - repo = new VerifyMergeability(mergeInfo).call(fateId, manager); + repo = new VerifyMergeability(mergeInfo).call(fateId, fateEnv); assertInstanceOf(UnreserveSystemMerge.class, repo); assertEquals(UnmergeableReason.MAX_TOTAL_SIZE, ((UnreserveSystemMerge) repo).getReason()); // Not enough time has passed for Tablet, should be able to merge d and e - mergeInfo = new MergeInfo(tableId, manager.getContext().getNamespaceId(tableId), + mergeInfo = new MergeInfo(tableId, fateEnv.getContext().getNamespaceId(tableId), new Text("c").getBytes(), new Text("e").getBytes(), Operation.SYSTEM_MERGE); - repo = new VerifyMergeability(mergeInfo).call(fateId, manager); + repo = new VerifyMergeability(mergeInfo).call(fateId, fateEnv); assertInstanceOf(UnreserveSystemMerge.class, repo); assertEquals(UnmergeableReason.TABLET_MERGEABILITY, ((UnreserveSystemMerge) repo).getReason()); // update time to 3 days so enough time has passed - manager = mockWithAmple(getCluster().getServerContext(), testAmple, Duration.ofDays(3)); - assertInstanceOf(MergeTablets.class, new VerifyMergeability(mergeInfo).call(fateId, manager)); + fateEnv = mockWithAmple(getCluster().getServerContext(), testAmple, Duration.ofDays(3)); + assertInstanceOf(MergeTablets.class, new VerifyMergeability(mergeInfo).call(fateId, fateEnv)); // last 3 tablets should total 9 files which is < max of 10 - mergeInfo = new MergeInfo(tableId, manager.getContext().getNamespaceId(tableId), + mergeInfo = new MergeInfo(tableId, fateEnv.getContext().getNamespaceId(tableId), new Text("c").getBytes(), null, Operation.SYSTEM_MERGE); addFileMetadata(context, tableId, new Text("c"), null, 3, 10); - assertInstanceOf(MergeTablets.class, new VerifyMergeability(mergeInfo).call(fateId, manager)); + assertInstanceOf(MergeTablets.class, new VerifyMergeability(mergeInfo).call(fateId, fateEnv)); // last 3 tablets should total 12 files which is > max of 10 addFileMetadata(context, tableId, new Text("c"), null, 4, 10); - repo = new VerifyMergeability(mergeInfo).call(fateId, manager); + repo = new VerifyMergeability(mergeInfo).call(fateId, fateEnv); assertInstanceOf(UnreserveSystemMerge.class, repo); assertEquals(UnmergeableReason.MAX_FILE_COUNT, ((UnreserveSystemMerge) repo).getReason()); } @@ -306,7 +306,7 @@ public void testSplitOffline() throws Exception { testAmple.mutateTablet(extent) .putOperation(TabletOperationId.from(TabletOperationType.SPLITTING, fateId)).mutate(); - Manager manager = mockWithAmple(getCluster().getServerContext(), testAmple); + FateEnv fateEnv = mockWithAmple(getCluster().getServerContext(), testAmple); assertEquals(opid, testAmple.readTablet(extent).getOperationId()); @@ -314,7 +314,7 @@ public void testSplitOffline() throws Exception { TabletMergeabilityUtil.systemDefaultSplits(new TreeSet<>(List.of(new Text("sp1")))))); // The repo should delete the opid and throw an exception - assertThrows(ThriftTableOperationException.class, () -> eoRepo.call(fateId, manager)); + assertThrows(ThriftTableOperationException.class, () -> eoRepo.call(fateId, fateEnv)); // the operation id should have been cleaned up before the exception was thrown assertNull(testAmple.readTablet(extent).getOperationId()); @@ -347,11 +347,11 @@ public void testFindSplitsUnsplittable() throws Exception { not(SplitColumnFamily.UNSPLITTABLE_COLUMN)); KeyExtent extent = new KeyExtent(tableId, null, null); - Manager manager = mockWithAmple(getCluster().getServerContext(), testAmple); + FateEnv fateEnv = mockWithAmple(getCluster().getServerContext(), testAmple); FindSplits findSplits = new FindSplits(extent); PreSplit preSplit = (PreSplit) findSplits - .call(FateId.from(FateInstanceType.USER, UUID.randomUUID()), manager); + .call(FateId.from(FateInstanceType.USER, UUID.randomUUID()), fateEnv); // The table should not need splitting assertNull(preSplit); @@ -366,7 +366,7 @@ public void testFindSplitsUnsplittable() throws Exception { findSplits = new FindSplits(extent); preSplit = (PreSplit) findSplits.call(FateId.from(FateInstanceType.USER, UUID.randomUUID()), - manager); + fateEnv); // The table SHOULD now need splitting assertNotNull(preSplit); @@ -408,11 +408,11 @@ public void testFindSplitsDeleteUnsplittable() throws Exception { not(SplitColumnFamily.UNSPLITTABLE_COLUMN)); KeyExtent extent = new KeyExtent(tableId, null, null); - Manager manager = mockWithAmple(getCluster().getServerContext(), testAmple); + FateEnv fateEnv = mockWithAmple(getCluster().getServerContext(), testAmple); FindSplits findSplits = new FindSplits(extent); PreSplit preSplit = (PreSplit) findSplits - .call(FateId.from(FateInstanceType.USER, UUID.randomUUID()), manager); + .call(FateId.from(FateInstanceType.USER, UUID.randomUUID()), fateEnv); // The table should not need splitting assertNull(preSplit); @@ -428,7 +428,7 @@ public void testFindSplitsDeleteUnsplittable() throws Exception { findSplits = new FindSplits(extent); preSplit = (PreSplit) findSplits.call(FateId.from(FateInstanceType.USER, UUID.randomUUID()), - manager); + fateEnv); // The table SHOULD not need splitting assertNull(preSplit); @@ -462,8 +462,8 @@ public void testCompactionDriverCleanup(Pair rangeText) throws Except TestServerAmpleImpl testAmple = (TestServerAmpleImpl) TestAmple .create(getCluster().getServerContext(), Map.of(DataLevel.USER, metadataTable)); testAmple.createMetadataFromExisting(client, tableId); - Manager manager = mockWithAmple(getCluster().getServerContext(), testAmple); - var ctx = manager.getContext(); + FateEnv fateEnv = mockWithAmple(getCluster().getServerContext(), testAmple); + var ctx = fateEnv.getContext(); // Create the CompactionDriver to test with the given range passed into the method final AbstractFateOperation repo = new CompactionDriver(ctx.getNamespaceId(tableId), tableId, @@ -489,7 +489,7 @@ public void testCompactionDriverCleanup(Pair rangeText) throws Except assertEquals(4, extents.size()); // First call undo using the second fateId and verify there's still metadata for the first one - repo.undo(fateId2, manager); + repo.undo(fateId2, fateEnv); try (TabletsMetadata tabletsMetadata = testAmple.readTablets().forTable(tableId).build()) { tabletsMetadata.forEach(tm -> { assertHasCompactionMetadata(fateId1, tm); @@ -499,7 +499,7 @@ public void testCompactionDriverCleanup(Pair rangeText) throws Except // Now call undo on the first fateId which would clean up all the metadata for all the // tablets that overlap with the given range that was provided to the CompactionDriver // during the creation of the repo - repo.undo(fateId1, manager); + repo.undo(fateId1, fateEnv); // First, iterate over only the overlapping tablets and verify that those tablets // were cleaned up and remove any visited tablets from the extents set From d73919c3fb8ea1543eefc39c10f3e68e62afb461 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 20 Mar 2026 19:02:34 +0000 Subject: [PATCH 07/65] fix compile error --- .../java/org/apache/accumulo/manager/fate/FateWorker.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java index 5b0261171d0..f60fc453c0a 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java @@ -68,7 +68,6 @@ public class FateWorker implements FateWorkerService.Iface { private final LiveTServerSet liveTserverSet; private final FateFactory fateFactory; private final Map> fates = new ConcurrentHashMap<>(); - private FateWorkerEnv fateWorkerEnv; public interface FateFactory { Fate create(FateEnv env, FateStore store, ServerContext context); @@ -79,11 +78,10 @@ public FateWorker(ServerContext ctx, LiveTServerSet liveTServerSet, FateFactory this.security = ctx.getSecurityOperation(); this.liveTserverSet = liveTServerSet; this.fateFactory = fateFactory; - Thread.interrupted() } public synchronized void setLock(ServiceLock lock) { - fateWorkerEnv = new FateWorkerEnv(context, lock, liveTserverSet); + FateWorkerEnv fateWorkerEnv = new FateWorkerEnv(context, lock, liveTserverSet); Predicate isLockHeld = l -> ServiceLock.isLockHeld(context.getZooCache(), l); try { MetaFateStore metaStore = From b091f7bacdcf631c243b16b13117e822e97194db Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 20 Mar 2026 19:18:08 +0000 Subject: [PATCH 08/65] fix test --- .../manager/tableOps/compact/CompactionDriverTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java index b1ec160734e..8d407d763dc 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java @@ -38,7 +38,6 @@ import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.zookeeper.ZooSession; -import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.manager.tableOps.delete.PreDeleteTable; import org.apache.accumulo.server.ServerContext; @@ -81,7 +80,7 @@ protected boolean isCancelled(FateId fateId, ServerContext context) { @BeforeEach public void setup() { - fateEnv = createMock(Manager.class); + fateEnv = createMock(FateEnv.class); ctx = createMock(ServerContext.class); zk = createMock(ZooSession.class); expect(ctx.getInstanceID()).andReturn(instance).anyTimes(); From 380e5515d2bb8d921468619f112b8846ce1a40b1 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 20 Mar 2026 23:36:07 +0000 Subject: [PATCH 09/65] Makes fate client available on all managers Before this change a fate client was only available on the primary manager. Now fate clients are avaiable on all managers. The primary manager publishes fate assignment locations in zookeeper. These locations are used by managers to send notifications to other managers when they seed a fate operation. --- .../org/apache/accumulo/core/Constants.java | 2 + .../core/clientImpl/ClientContext.java | 10 +- .../server/init/ZooKeeperInitializer.java | 5 + .../server/manager/FateLocations.java | 107 +++++++++++++ .../org/apache/accumulo/manager/Manager.java | 93 ++++++----- .../accumulo/manager/fate/FateManager.java | 94 ++--------- .../accumulo/manager/fate/FateNotifier.java | 150 ++++++++++++++++++ 7 files changed, 332 insertions(+), 129 deletions(-) create mode 100644 server/base/src/main/java/org/apache/accumulo/server/manager/FateLocations.java create mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/fate/FateNotifier.java diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java index d2c556f2c2a..bb6e49eda45 100644 --- a/core/src/main/java/org/apache/accumulo/core/Constants.java +++ b/core/src/main/java/org/apache/accumulo/core/Constants.java @@ -52,6 +52,8 @@ public class Constants { public static final String ZMANAGER_ASSISTANT_LOCK = ZMANAGERS + "/assistants"; public static final String ZMANAGER_GOAL_STATE = ZMANAGERS + "/goal_state"; public static final String ZMANAGER_TICK = ZMANAGERS + "/tick"; + public static final String ZMANAGER_ASSIGNMENTS = ZMANAGERS + "/assignments"; + public static final String ZMANAGER_FATE_ASSIGNMENTS = ZMANAGER_ASSIGNMENTS + "/fate"; public static final String ZGC = "/gc"; public static final String ZGC_LOCK = ZGC + "/lock"; diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java index 8b65091c491..c9a1fb4d610 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java @@ -37,7 +37,6 @@ import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -1328,15 +1327,12 @@ public void clearTabletLocationCache() { } private static Set createPersistentWatcherPaths() { - Set pathsToWatch = new HashSet<>(); - for (String path : Set.of(Constants.ZCOMPACTORS, Constants.ZDEADTSERVERS, Constants.ZGC_LOCK, + return Set.of(Constants.ZCOMPACTORS, Constants.ZDEADTSERVERS, Constants.ZGC_LOCK, Constants.ZMANAGER_LOCK, Constants.ZMINI_LOCK, Constants.ZMONITOR_LOCK, Constants.ZNAMESPACES, Constants.ZRECOVERY, Constants.ZSSERVERS, Constants.ZTABLES, Constants.ZTSERVERS, Constants.ZUSERS, RootTable.ZROOT_TABLET, Constants.ZTEST_LOCK, - Constants.ZMANAGER_ASSISTANT_LOCK, Constants.ZRESOURCEGROUPS)) { - pathsToWatch.add(path); - } - return pathsToWatch; + Constants.ZMANAGER_ASSISTANT_LOCK, Constants.ZRESOURCEGROUPS, + Constants.ZMANAGER_ASSIGNMENTS); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java index b23c6746803..9d6a59607d3 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java @@ -50,6 +50,7 @@ import org.apache.accumulo.server.conf.store.ResourceGroupPropKey; import org.apache.accumulo.server.conf.store.SystemPropKey; import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.manager.FateLocations; import org.apache.accumulo.server.metadata.RootGcCandidates; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; @@ -180,6 +181,10 @@ void initialize(final ServerContext context, final String rootTabletDirName, ZooUtil.NodeExistsPolicy.FAIL); zrwChroot.putPersistentData(Constants.ZSHUTTING_DOWN_TSERVERS, EMPTY_BYTE_ARRAY, ZooUtil.NodeExistsPolicy.FAIL); + zrwChroot.putPersistentData(Constants.ZMANAGER_ASSIGNMENTS, EMPTY_BYTE_ARRAY, + ZooUtil.NodeExistsPolicy.FAIL); + FateLocations.storeLocations(zrwChroot, Map.of(), ZooUtil.NodeExistsPolicy.FAIL); + } /** diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/FateLocations.java b/server/base/src/main/java/org/apache/accumulo/server/manager/FateLocations.java new file mode 100644 index 00000000000..4a71ff2e085 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/FateLocations.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.manager; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.toUnmodifiableSet; +import static org.apache.accumulo.core.util.LazySingletons.GSON; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.fate.FateId; +import org.apache.accumulo.core.fate.FatePartition; +import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; +import org.apache.accumulo.server.ServerContext; +import org.apache.zookeeper.KeeperException; + +import com.google.common.base.Preconditions; +import com.google.common.net.HostAndPort; +import com.google.common.reflect.TypeToken; + +public class FateLocations { + + private final ServerContext context; + + private long lastUpdateCount; + private Map> lastLocations = null; + + public FateLocations(ServerContext context) { + this.context = context; + } + + public synchronized Map> getLocations() { + + var zooCache = context.getZooCache(); + + if (lastLocations == null || lastUpdateCount != zooCache.getUpdateCount()) { + lastUpdateCount = zooCache.getUpdateCount(); + var json = new String(context.getZooCache().get(Constants.ZMANAGER_FATE_ASSIGNMENTS), UTF_8); + var type = new TypeToken>>>() {}.getType(); + Map>> stringMap = GSON.get().fromJson(json, type); + Map> locations = new HashMap<>(); + stringMap.forEach((hp, parts) -> { + var partsSet = parts.stream().peek(part -> Preconditions.checkArgument(part.size() == 2)) + .map(part -> new FatePartition(FateId.from(part.get(0)), FateId.from(part.get(1)))) + .collect(toUnmodifiableSet()); + locations.put(HostAndPort.fromString(hp), partsSet); + }); + lastLocations = Map.copyOf(locations); + } + + return lastLocations; + } + + private static byte[] serialize(Map> assignments) { + Map>> jsonMap = new HashMap<>(); + assignments.forEach((hp, parts) -> { + var listParts = parts.stream() + .map(part -> List.of(part.start().canonical(), part.end().canonical())).toList(); + jsonMap.put(hp.toString(), listParts); + }); + + var json = GSON.get().toJson(jsonMap); + return json.getBytes(UTF_8); + } + + public static void storeLocations(ZooReaderWriter zoo, + Map> assignments, NodeExistsPolicy nodeExistsPolicy) { + try { + zoo.putPersistentData(Constants.ZMANAGER_FATE_ASSIGNMENTS, serialize(assignments), + nodeExistsPolicy); + } catch (KeeperException | InterruptedException e) { + throw new IllegalStateException("Unable to set fate locations in zookeeper", e); + } + } + + public static void storeLocations(ServerContext context, + Map> assignments) { + try { + context.getZooSession().setData(Constants.ZMANAGER_FATE_ASSIGNMENTS, serialize(assignments), + -1); + } catch (KeeperException | InterruptedException e) { + throw new IllegalStateException("Unable to set fate locations in zookeeper", e); + } + } + +} diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 204de1d79d9..9fb59deb181 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -46,7 +46,6 @@ import java.util.SortedMap; import java.util.UUID; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -54,6 +53,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.function.Supplier; @@ -118,6 +119,7 @@ import org.apache.accumulo.core.zookeeper.ZcStat; import org.apache.accumulo.manager.compaction.coordinator.CompactionCoordinator; import org.apache.accumulo.manager.fate.FateManager; +import org.apache.accumulo.manager.fate.FateNotifier; import org.apache.accumulo.manager.fate.FateWorker; import org.apache.accumulo.manager.merge.FindMergeableRangeTask; import org.apache.accumulo.manager.metrics.fate.meta.MetaFateMetrics; @@ -129,6 +131,7 @@ import org.apache.accumulo.manager.upgrade.UpgradeCoordinator; import org.apache.accumulo.manager.upgrade.UpgradeCoordinator.UpgradeStatus; import org.apache.accumulo.server.AbstractServer; +import org.apache.accumulo.server.AccumuloDataVersion; import org.apache.accumulo.server.PrimaryManagerThriftService; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CompactionConfigStorage; @@ -215,10 +218,6 @@ public class Manager extends AbstractServer private ManagerState state = ManagerState.INITIAL; - // fateReadyLatch and fateRefs go together; when this latch is ready, then the fate references - // should already have been set; ConcurrentHashMap will guarantee that all threads will see - // the initialized fate references after the latch is ready - private final CountDownLatch fateReadyLatch = new CountDownLatch(1); private final AtomicReference>> fateClients = new AtomicReference<>(); private volatile FateManager fateManager; @@ -307,28 +306,7 @@ public boolean stillManager() { return getManagerState() != ManagerState.STOP; } - private void waitForFate() { - try { - // block up to 30 seconds until it's ready; if it's still not ready, introduce some logging - if (!fateReadyLatch.await(30, SECONDS)) { - String msgPrefix = "Unexpected use of fate in thread " + Thread.currentThread().getName() - + " at time " + System.currentTimeMillis(); - // include stack trace so we know where it's coming from, in case we need to troubleshoot it - log.warn("{} blocked until fate starts", msgPrefix, - new IllegalStateException("Attempted fate action before manager finished starting up; " - + "if this doesn't make progress, please report it as a bug to the developers")); - int minutes = 0; - while (!fateReadyLatch.await(5, MINUTES)) { - minutes += 5; - log.warn("{} still blocked after {} minutes; this is getting weird", msgPrefix, minutes); - } - log.debug("{} no longer blocked", msgPrefix); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Thread was interrupted; cannot proceed"); - } - } + private final Lock fateSetupLock = new ReentrantLock(); /** * Retrieve the FateClient object, blocking until it is ready. This could cause problems if Fate @@ -341,7 +319,23 @@ private void waitForFate() { * @return the FateClient object, only after the fate components are running and ready */ public FateClient fateClient(FateInstanceType type) { - waitForFate(); + if (fateClients.get() == null) { + // only want one thread trying to setup fate, lots of threads could call this before its setup + fateSetupLock.lock(); + try { + // check to see if another thread setup fate while we were waiting on the lock + if (fateClients.get() == null) { + // wait for upgrade to be complete + while (AccumuloDataVersion.getCurrentVersion(getContext()) < AccumuloDataVersion.get()) { + log.info("Attempted use of fate before upgrade complete, waiting for upgrade"); + UtilWaitThread.sleep(5000); + } + setupFate(getContext()); + } + } finally { + fateSetupLock.unlock(); + } + } var client = requireNonNull(fateClients.get(), "fateClients is not set yet").get(type); return requireNonNull(client, () -> "fate client type " + type + " is not present"); } @@ -921,7 +915,6 @@ private void setupPrimaryMetrics() { watchers.forEach(watcher -> metricsInfo.addMetricsProducers(watcher.getMetrics())); metricsInfo.addMetricsProducers(requireNonNull(compactionCoordinator)); // ensure fate is completely setup - Preconditions.checkState(fateReadyLatch.getCount() == 0); metricsInfo.addMetricsProducers(new MetaFateMetrics(getContext(), getConfiguration().getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL))); metricsInfo.addMetricsProducers(new UserFateMetrics(getContext(), @@ -1165,7 +1158,7 @@ boolean canSuspendTablets() { fateManager = new FateManager(getContext()); fateManager.start(); - fateClient(FateInstanceType.USER).setSeedingConsumer(fateManager::notifySeeded); + startFateMaintenance(); setupPrimaryMetrics(); @@ -1303,29 +1296,51 @@ protected Fate createFateInstance(FateEnv env, FateStore store private void setupFate(ServerContext context) { try { + Predicate isLockHeld = lock -> ServiceLock.isLockHeld(context.getZooCache(), lock); - var metaStore = new MetaFateStore(context.getZooSession(), - primaryManagerLock.getLockID(), isLockHeld); + var metaStore = + new MetaFateStore(context.getZooSession(), managerLock.getLockID(), isLockHeld); var metaFateClient = new FateClient<>(metaStore, TraceRepo::toLogString); var userStore = new UserFateStore(context, SystemTables.FATE.tableName(), managerLock.getLockID(), isLockHeld); var userFateClient = new FateClient<>(userStore, TraceRepo::toLogString); - var metaCleaner = new FateCleaner<>(metaStore, Duration.ofHours(8), this::getSteadyTime); - ThreadPools.watchCriticalScheduledTask(context.getScheduledExecutor() - .scheduleWithFixedDelay(metaCleaner::ageOff, 10, 4 * 60, MINUTES)); - var userCleaner = new FateCleaner<>(userStore, Duration.ofHours(8), this::getSteadyTime); - ThreadPools.watchCriticalScheduledTask(context.getScheduledExecutor() - .scheduleWithFixedDelay(userCleaner::ageOff, 10, 4 * 60, MINUTES)); + // wire up notifying the correct manager when a fate operation is seeded + FateNotifier fateNotifier = new FateNotifier(context); + fateNotifier.start(); + metaFateClient.setSeedingConsumer(fateNotifier::notifySeeded); + userFateClient.setSeedingConsumer(fateNotifier::notifySeeded); if (!fateClients.compareAndSet(null, Map.of(FateInstanceType.META, metaFateClient, FateInstanceType.USER, userFateClient))) { throw new IllegalStateException( "Unexpected previous fateClient reference map already initialized"); } + } catch (KeeperException | InterruptedException e) { + throw new IllegalStateException("Exception setting up Fate clients", e); + } + } + + /** + * Run fate maintenance task that only run in the primary manager. + */ + private void startFateMaintenance() { + try { + Predicate isLockHeld = + lock -> ServiceLock.isLockHeld(getContext().getZooCache(), lock); + + var metaStore = new MetaFateStore(getContext().getZooSession(), + managerLock.getLockID(), isLockHeld); + var userStore = new UserFateStore(getContext(), SystemTables.FATE.tableName(), + managerLock.getLockID(), isLockHeld); - fateReadyLatch.countDown(); + var metaCleaner = new FateCleaner<>(metaStore, Duration.ofHours(8), this::getSteadyTime); + ThreadPools.watchCriticalScheduledTask(getContext().getScheduledExecutor() + .scheduleWithFixedDelay(metaCleaner::ageOff, 10, 4 * 60, MINUTES)); + var userCleaner = new FateCleaner<>(userStore, Duration.ofHours(8), this::getSteadyTime); + ThreadPools.watchCriticalScheduledTask(getContext().getScheduledExecutor() + .scheduleWithFixedDelay(userCleaner::ageOff, 10, 4 * 60, MINUTES)); } catch (KeeperException | InterruptedException e) { throw new IllegalStateException("Exception setting up FaTE cleanup thread", e); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java index 9d934913098..a5e64497ab1 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java @@ -26,7 +26,6 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.accumulo.core.fate.FateId; @@ -45,16 +44,14 @@ import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.manager.FateLocations; import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; -import com.google.common.collect.Range; -import com.google.common.collect.RangeMap; import com.google.common.collect.Sets; -import com.google.common.collect.TreeRangeMap; import com.google.common.net.HostAndPort; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -80,17 +77,10 @@ public FateManager(ServerContext context) { private final AtomicBoolean stop = new AtomicBoolean(false); - record FateHostPartition(HostAndPort hostPort, FatePartition partition) { - } - - private final AtomicReference> stableAssignments = - new AtomicReference<>(TreeRangeMap.create()); - - private final Map> pendingNotifications = new HashMap<>(); - private void manageAssistants() throws TException, InterruptedException { log.debug("Started Fate Manager"); long stableCount = 0; + long unstableCount = 0; outer: while (!stop.get()) { long sleepTime = Math.min(stableCount * 100, 5_000); @@ -113,18 +103,18 @@ private void manageAssistants() throws TException, InterruptedException { computeDesiredAssignments(currentAssignments, desiredParititions); if (desired.equals(currentAssignments)) { - RangeMap rangeMap = TreeRangeMap.create(); - currentAssignments.forEach((hostAndPort, partitions) -> { - partitions.forEach(partition -> { - rangeMap.put(Range.closed(partition.start(), partition.end()), - new FateHostPartition(hostAndPort, partition)); - }); - }); - stableAssignments.set(rangeMap); + if (stableCount == 0) { + FateLocations.storeLocations(context, currentAssignments); + } stableCount++; + unstableCount = 0; + continue; } else { - stableAssignments.set(TreeRangeMap.create()); + if (unstableCount == 0) { + FateLocations.storeLocations(context, Map.of()); + } stableCount = 0; + unstableCount++; } // are there any workers with extra partitions? If so need to unload those first. @@ -172,11 +162,9 @@ private void manageAssistants() throws TException, InterruptedException { } private Thread assignmentThread = null; - private Thread ntfyThread = null; public synchronized void start() { Preconditions.checkState(assignmentThread == null); - Preconditions.checkState(ntfyThread == null); Preconditions.checkState(!stop.get()); assignmentThread = Threads.createCriticalThread("Fate Manager", () -> { @@ -187,9 +175,6 @@ public synchronized void start() { } }); assignmentThread.start(); - - ntfyThread = Threads.createCriticalThread("Fate Notify", new NotifyTask()); - ntfyThread.start(); } @SuppressFBWarnings(value = "SWL_SLEEP_WITH_LOCK_HELD", @@ -206,9 +191,6 @@ public synchronized void stop(FateInstanceType fateType, Duration timeout) { if (assignmentThread != null) { assignmentThread.join(); } - if (ntfyThread != null) { - ntfyThread.join(); - } } catch (InterruptedException e) { throw new IllegalStateException(e); } @@ -235,8 +217,6 @@ public synchronized void stop(FateInstanceType fateType, Duration timeout) { } } - stableAssignments.set(TreeRangeMap.create()); - if (!timer.isExpired()) { FateStore store = switch (fateType) { case USER -> new UserFateStore(context, SystemTables.FATE.tableName(), null, null); @@ -264,58 +244,6 @@ public synchronized void stop(FateInstanceType fateType, Duration timeout) { } } - /** - * Makes a best effort to notify this fate operation was seeded. - */ - public void notifySeeded(FateId fateId) { - var hostPartition = stableAssignments.get().get(fateId); - if (hostPartition != null) { - synchronized (pendingNotifications) { - pendingNotifications.computeIfAbsent(hostPartition.hostPort(), k -> new HashSet<>()) - .add(hostPartition.partition()); - pendingNotifications.notify(); - } - } - } - - private class NotifyTask implements Runnable { - - @Override - public void run() { - while (!stop.get()) { - try { - Map> copy; - synchronized (pendingNotifications) { - if (pendingNotifications.isEmpty()) { - pendingNotifications.wait(100); - } - copy = Map.copyOf(pendingNotifications); - pendingNotifications.clear(); - } - - for (var entry : copy.entrySet()) { - HostAndPort address = entry.getKey(); - Set partitions = entry.getValue(); - FateWorkerService.Client client = - ThriftUtil.getClient(ThriftClientTypes.FATE_WORKER, address, context); - try { - log.trace("Notifying about seeding {} {}", address, partitions); - client.seeded(TraceUtil.traceInfo(), context.rpcCreds(), - partitions.stream().map(FatePartition::toThrift).toList()); - } finally { - ThriftUtil.returnClient(client, context); - } - } - - } catch (InterruptedException e) { - throw new IllegalStateException(e); - } catch (TException e) { - log.warn("Failed to send notification that fate was seeded", e); - } - } - } - } - /** * Sets the complete set of partitions an assistant manager should work on. It will only succeed * if the update id is valid. The update id avoids race conditions w/ previously queued network diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateNotifier.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateNotifier.java new file mode 100644 index 00000000000..5789832277a --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateNotifier.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.fate; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.accumulo.core.fate.FateId; +import org.apache.accumulo.core.fate.FatePartition; +import org.apache.accumulo.core.manager.thrift.FateWorkerService; +import org.apache.accumulo.core.rpc.ThriftUtil; +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.core.util.threads.Threads; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.manager.FateLocations; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; +import com.google.common.collect.TreeRangeMap; +import com.google.common.net.HostAndPort; + +/** + * Responsible for sending notifications that fate operations were seeded between managers. These + * notifications are best effort, it's ok if they are lost. When lost it means fate operations will + * not be as responsive, but they will still eventually run. These notifications are important for + * interactive use of Accumulo where something like create table run in the shell should be + * responsive. Responsiveness is also important for Accumulo's integration test. + */ +public class FateNotifier { + + private static final Logger log = LoggerFactory.getLogger(FateNotifier.class); + + private final Map> pendingNotifications = new HashMap<>(); + private final ServerContext context; + private final AtomicBoolean stop = new AtomicBoolean(); + private final FateLocations fateLocations; + + private Map> lastLocations; + private RangeMap hostMapping; + + private Thread ntfyThread; + + public FateNotifier(ServerContext context) { + this.context = context; + this.fateLocations = new FateLocations(context); + } + + public synchronized void start() { + Preconditions.checkState(ntfyThread == null); + ntfyThread = Threads.createCriticalThread("Fate Notification Sender", new NotifyTask()); + ntfyThread.start(); + } + + record FateHostPartition(HostAndPort hostPort, FatePartition partition) { + } + + private synchronized RangeMap getHostMapping() { + + if (hostMapping == null || lastLocations != fateLocations.getLocations()) { + lastLocations = fateLocations.getLocations(); + RangeMap rangeMap = TreeRangeMap.create(); + lastLocations.forEach((hostAndPort, partitions) -> { + partitions.forEach(partition -> { + rangeMap.put(Range.closed(partition.start(), partition.end()), + new FateHostPartition(hostAndPort, partition)); + }); + }); + hostMapping = rangeMap; + } + + return hostMapping; + } + + /** + * Makes a best effort to notify the appropriate manager this fate operation was seeded. + */ + public void notifySeeded(FateId fateId) { + var hostPartition = getHostMapping().get(fateId); + if (hostPartition != null) { + synchronized (pendingNotifications) { + pendingNotifications.computeIfAbsent(hostPartition.hostPort(), k -> new HashSet<>()) + .add(hostPartition.partition()); + pendingNotifications.notify(); + } + } + } + + private class NotifyTask implements Runnable { + + @Override + public void run() { + while (!stop.get()) { + try { + Map> copy; + synchronized (pendingNotifications) { + if (pendingNotifications.isEmpty()) { + pendingNotifications.wait(100); + } + copy = Map.copyOf(pendingNotifications); + pendingNotifications.clear(); + } + + for (var entry : copy.entrySet()) { + HostAndPort address = entry.getKey(); + Set partitions = entry.getValue(); + FateWorkerService.Client client = + ThriftUtil.getClient(ThriftClientTypes.FATE_WORKER, address, context); + try { + log.trace("Notifying about seeding {} {}", address, partitions); + client.seeded(TraceUtil.traceInfo(), context.rpcCreds(), + partitions.stream().map(FatePartition::toThrift).toList()); + } finally { + ThriftUtil.returnClient(client, context); + } + } + + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } catch (TException e) { + log.warn("Failed to send notification that fate was seeded", e); + } + } + } + } + +} From 714a8f16aec3070c9f5459ff3a4233bf7af2f90e Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Sat, 21 Mar 2026 22:36:35 +0000 Subject: [PATCH 10/65] fix test --- .../compaction/queue/CompactionJobQueuesTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java index d3c1acd469b..0e96d351408 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java @@ -77,6 +77,8 @@ public void testFullScanHandling() throws Exception { CompactionJobQueues jobQueues = new CompactionJobQueues(1000000); + jobQueues.setResourceGroups(Set.of(cg1, cg2, cg3)); + jobQueues.beginFullScan(DataLevel.USER); jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg1))); @@ -237,6 +239,7 @@ public void testFullScanLevels() throws Exception { var cg1 = ResourceGroupId.of("CG1"); CompactionJobQueues jobQueues = new CompactionJobQueues(1000000); + jobQueues.setResourceGroups(Set.of(cg1)); jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg1))); jobQueues.add(extent2, List.of(newJob((short) 2, 6, cg1))); @@ -276,6 +279,8 @@ public void testAddPollRaceCondition() throws Exception { ResourceGroupId[] groups = Stream.of("G1", "G2", "G3").map(ResourceGroupId::of).toArray(ResourceGroupId[]::new); + jobQueues.setResourceGroups(Set.of(groups)); + var executor = Executors.newFixedThreadPool(groups.length); List> futures = new ArrayList<>(); @@ -340,6 +345,8 @@ public void testGetAsync() throws Exception { var cg1 = ResourceGroupId.of("CG1"); + jobQueues.setResourceGroups(Set.of(cg1)); + var job1 = newJob((short) 1, 5, cg1); var job2 = newJob((short) 2, 6, cg1); var job3 = newJob((short) 3, 7, cg1); @@ -410,6 +417,8 @@ public void testResetSize() throws Exception { var cg1 = ResourceGroupId.of("CG1"); var cg2 = ResourceGroupId.of("CG2"); + jobQueues.setResourceGroups(Set.of(cg1, cg2)); + jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg1))); assertEquals(Set.of(cg1), jobQueues.getQueueIds()); @@ -426,4 +435,6 @@ public void testResetSize() throws Exception { assertEquals(500000, jobQueues.getQueueMaxSize(cg1)); assertEquals(500000, jobQueues.getQueueMaxSize(cg2)); } + + // TODO add test to ensure only the set resource groups can be added to and pulled from } From eb689dbf66abc3a1f60b88303996121e028a0d12 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Mon, 23 Mar 2026 19:36:36 +0000 Subject: [PATCH 11/65] WIP --- .../apache/accumulo/server/util/FindCompactionTmpFiles.java | 3 +++ .../manager/compaction/coordinator/CompactionCoordinator.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java index 06755d1807b..b61d53fc875 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java @@ -127,6 +127,9 @@ public static Set findTempFiles(ServerContext context, String tableId) }); } } + + // TODO also need to remove compactions that are committing in fate + LOG.trace("Final set of compaction tmp files after removing active compactions: {}", matches); return matches; } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 2e41c0a933a..bd8630fc1c1 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -983,6 +983,10 @@ public boolean test(TabletMetadata tabletMetadata) { ecidsForTablet.clear(); ecidsForTablet.addAll(compactions.get(extent)); + // TODO this entire process may be better off as a batch process that periodically looks + // for tmp files to delete. If this fails, it will never try again. This adds a lot of + // work to processing a failed compaction. There is already code for a batch process, its + // just not called all the time. if (!ecidsForTablet.isEmpty()) { final TabletMetadata tm = ctx.getAmple().readTablet(extent, ColumnType.DIR); if (tm != null) { From 37b2383e484443e87ea3f81092d14fa0896d953a Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Mon, 23 Mar 2026 23:32:32 +0000 Subject: [PATCH 12/65] removes CoordinatorSummaryLogger This logger was useful in 2.1. However in 4.0 its redundant with functionality in the monitor and is just extra code to maintain. Removing it also removes a usage of the running cache which may be helpful for #6217 --- .../coordinator/CompactionCoordinator.java | 10 --- .../coordinator/CoordinatorSummaryLogger.java | 88 ------------------- .../compaction/CompactionCoordinatorTest.java | 3 - 3 files changed, 101 deletions(-) delete mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorSummaryLogger.java diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index aee9679cc01..135fa77abac 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -369,7 +369,6 @@ public void run() { } startDeadCompactionDetector(); - startQueueRunningSummaryLogging(); startFailureSummaryLogging(); startInternalStateCleaner(ctx.getScheduledExecutor()); @@ -804,15 +803,6 @@ private void captureFailure(ExternalCompactionId ecid, KeyExtent extent) { failingTables.compute(extent.tableId(), FailureCounts::incrementFailure); } - protected void startQueueRunningSummaryLogging() { - CoordinatorSummaryLogger summaryLogger = - new CoordinatorSummaryLogger(ctx, this.jobQueues, this.RUNNING_CACHE, compactorCounts); - - ScheduledFuture future = ctx.getScheduledExecutor() - .scheduleWithFixedDelay(summaryLogger::logSummary, 0, 1, TimeUnit.MINUTES); - ThreadPools.watchNonCriticalScheduledTask(future); - } - protected void startFailureSummaryLogging() { ScheduledFuture future = ctx.getScheduledExecutor().scheduleWithFixedDelay(this::printStats, 0, 5, TimeUnit.MINUTES); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorSummaryLogger.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorSummaryLogger.java deleted file mode 100644 index 2d71c2d7969..00000000000 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorSummaryLogger.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.manager.compaction.coordinator; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; -import org.apache.accumulo.core.data.ResourceGroupId; -import org.apache.accumulo.core.data.TableId; -import org.apache.accumulo.core.dataImpl.KeyExtent; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; -import org.apache.accumulo.manager.compaction.queue.CompactionJobQueues; -import org.apache.accumulo.server.ServerContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.github.benmanes.caffeine.cache.Cache; - -public class CoordinatorSummaryLogger { - private static final Logger LOG = LoggerFactory.getLogger(CoordinatorSummaryLogger.class); - - private final ServerContext ctx; - private final CompactionJobQueues jobQueues; - private final Map running; - private final Cache compactorCounts; - - public CoordinatorSummaryLogger(ServerContext ctx, CompactionJobQueues jobQueues, - Map running, - Cache compactorCounts) { - this.ctx = ctx; - this.jobQueues = jobQueues; - this.running = running; - this.compactorCounts = compactorCounts; - } - - public void logSummary() { - - final Map perQueueRunningCount = new HashMap<>(); - final Map perTableRunningCount = new HashMap<>(); - - running.values().forEach(rc -> { - TableId tid = KeyExtent.fromThrift(rc.getJob().getExtent()).tableId(); - String tableName = null; - try { - tableName = ctx.getQualifiedTableName(tid); - } catch (TableNotFoundException e) { - tableName = "Unmapped table id: " + tid.canonical(); - } - perQueueRunningCount - .computeIfAbsent(ResourceGroupId.of(rc.getGroupName()), q -> new AtomicLong(0)) - .incrementAndGet(); - perTableRunningCount.computeIfAbsent(tableName, t -> new AtomicLong(0)).incrementAndGet(); - }); - - perQueueRunningCount.forEach((groupId, count) -> { - LOG.info( - "Queue {}: compactors: {}, queued majc (minimum, possibly higher): {}, running majc: {}", - groupId, compactorCounts.asMap().getOrDefault(groupId, 0), - // This map only contains the highest priority for each tserver. So when tservers have - // other priorities that need to compact or have more than one compaction for a - // priority level this count will be lower than the actual number of queued. - jobQueues.getQueuedJobs(groupId), count.get()); - - }); - perTableRunningCount - .forEach((t, count) -> LOG.info("Running compactions for table {}: {}", t, count)); - } - -} diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index a11e345a993..9603d8a53c0 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -118,9 +118,6 @@ protected int countCompactors(ResourceGroupId groupName) { return 3; } - @Override - protected void startQueueRunningSummaryLogging() {} - @Override protected void startFailureSummaryLogging() {} From 1b540d0c913bddf1368efa1933a162cee29052ec Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 24 Mar 2026 01:13:20 +0000 Subject: [PATCH 13/65] passes group and address from compactor for stats Compaction and completion and failure were computing stats in the coordinator that needed the resource group and compactor address. This information was obtained from the running cache. Modified the RPCs to pass this information instead. This removes a usage of the running cache in coordinator which will be helpful for #6217. --- .../thrift/CompactionCoordinatorService.java | 492 +++++++++++++++++- .../main/thrift/compaction-coordinator.thrift | 4 + .../apache/accumulo/compactor/Compactor.java | 6 +- .../coordinator/CompactionCoordinator.java | 32 +- .../compaction/CompactionCoordinatorTest.java | 8 +- 5 files changed, 488 insertions(+), 54 deletions(-) diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index 734d7f9027c..486e6e988e9 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -29,13 +29,13 @@ public class CompactionCoordinatorService { public interface Iface { - public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactorAddress) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; public TNextCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; public void updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, TCompactionStatusUpdate status, long timestamp) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; + public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactorAddress) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; @@ -49,13 +49,13 @@ public interface Iface { public interface AsyncIface { - public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactorAddress, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void updateCompactionStatus(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, TCompactionStatusUpdate status, long timestamp, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactorAddress, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; @@ -90,13 +90,13 @@ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.prot } @Override - public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactorAddress) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { - send_compactionCompleted(tinfo, credentials, externalCompactionId, extent, stats); + send_compactionCompleted(tinfo, credentials, externalCompactionId, extent, stats, groupName, compactorAddress); recv_compactionCompleted(); } - public void send_compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) throws org.apache.thrift.TException + public void send_compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactorAddress) throws org.apache.thrift.TException { compactionCompleted_args args = new compactionCompleted_args(); args.setTinfo(tinfo); @@ -104,6 +104,8 @@ public void send_compactionCompleted(org.apache.accumulo.core.clientImpl.thrift. args.setExternalCompactionId(externalCompactionId); args.setExtent(extent); args.setStats(stats); + args.setGroupName(groupName); + args.setCompactorAddress(compactorAddress); sendBase("compactionCompleted", args); } @@ -186,13 +188,13 @@ public void recv_updateCompactionStatus() throws org.apache.accumulo.core.client } @Override - public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException + public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactorAddress) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { - send_compactionFailed(tinfo, credentials, externalCompactionId, extent, exceptionClassName, failureState); + send_compactionFailed(tinfo, credentials, externalCompactionId, extent, exceptionClassName, failureState, groupName, compactorAddress); recv_compactionFailed(); } - public void send_compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.thrift.TException + public void send_compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactorAddress) throws org.apache.thrift.TException { compactionFailed_args args = new compactionFailed_args(); args.setTinfo(tinfo); @@ -201,6 +203,8 @@ public void send_compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TIn args.setExtent(extent); args.setExceptionClassName(exceptionClassName); args.setFailureState(failureState); + args.setGroupName(groupName); + args.setCompactorAddress(compactorAddress); sendBase("compactionFailed", args); } @@ -343,9 +347,9 @@ public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, } @Override - public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactorAddress, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); - compactionCompleted_call method_call = new compactionCompleted_call(tinfo, credentials, externalCompactionId, extent, stats, resultHandler, this, ___protocolFactory, ___transport); + compactionCompleted_call method_call = new compactionCompleted_call(tinfo, credentials, externalCompactionId, extent, stats, groupName, compactorAddress, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; ___manager.call(method_call); } @@ -356,13 +360,17 @@ public static class compactionCompleted_call extends org.apache.thrift.async.TAs private java.lang.String externalCompactionId; private org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; private org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats; - public compactionCompleted_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + private java.lang.String groupName; + private java.lang.String compactorAddress; + public compactionCompleted_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactorAddress, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { super(client, protocolFactory, transport, resultHandler, false); this.tinfo = tinfo; this.credentials = credentials; this.externalCompactionId = externalCompactionId; this.extent = extent; this.stats = stats; + this.groupName = groupName; + this.compactorAddress = compactorAddress; } @Override @@ -374,6 +382,8 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa args.setExternalCompactionId(externalCompactionId); args.setExtent(extent); args.setStats(stats); + args.setGroupName(groupName); + args.setCompactorAddress(compactorAddress); args.write(prot); prot.writeMessageEnd(); } @@ -486,9 +496,9 @@ public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.Thrift } @Override - public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactorAddress, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); - compactionFailed_call method_call = new compactionFailed_call(tinfo, credentials, externalCompactionId, extent, exceptionClassName, failureState, resultHandler, this, ___protocolFactory, ___transport); + compactionFailed_call method_call = new compactionFailed_call(tinfo, credentials, externalCompactionId, extent, exceptionClassName, failureState, groupName, compactorAddress, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; ___manager.call(method_call); } @@ -500,7 +510,9 @@ public static class compactionFailed_call extends org.apache.thrift.async.TAsync private org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; private java.lang.String exceptionClassName; private TCompactionState failureState; - public compactionFailed_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + private java.lang.String groupName; + private java.lang.String compactorAddress; + public compactionFailed_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactorAddress, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { super(client, protocolFactory, transport, resultHandler, false); this.tinfo = tinfo; this.credentials = credentials; @@ -508,6 +520,8 @@ public compactionFailed_call(org.apache.accumulo.core.clientImpl.thrift.TInfo ti this.extent = extent; this.exceptionClassName = exceptionClassName; this.failureState = failureState; + this.groupName = groupName; + this.compactorAddress = compactorAddress; } @Override @@ -520,6 +534,8 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa args.setExtent(extent); args.setExceptionClassName(exceptionClassName); args.setFailureState(failureState); + args.setGroupName(groupName); + args.setCompactorAddress(compactorAddress); args.write(prot); prot.writeMessageEnd(); } @@ -743,7 +759,7 @@ protected boolean rethrowUnhandledExceptions() { public compactionCompleted_result getResult(I iface, compactionCompleted_args args) throws org.apache.thrift.TException { compactionCompleted_result result = new compactionCompleted_result(); try { - iface.compactionCompleted(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.stats); + iface.compactionCompleted(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.stats, args.groupName, args.compactorAddress); } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { result.sec = sec; } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { @@ -845,7 +861,7 @@ protected boolean rethrowUnhandledExceptions() { public compactionFailed_result getResult(I iface, compactionFailed_args args) throws org.apache.thrift.TException { compactionFailed_result result = new compactionFailed_result(); try { - iface.compactionFailed(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.exceptionClassName, args.failureState); + iface.compactionFailed(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.exceptionClassName, args.failureState, args.groupName, args.compactorAddress); } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { result.sec = sec; } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { @@ -1078,7 +1094,7 @@ protected boolean isOneway() { @Override public void start(I iface, compactionCompleted_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.compactionCompleted(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.stats,resultHandler); + iface.compactionCompleted(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.stats, args.groupName, args.compactorAddress,resultHandler); } } @@ -1301,7 +1317,7 @@ protected boolean isOneway() { @Override public void start(I iface, compactionFailed_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.compactionFailed(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.exceptionClassName, args.failureState,resultHandler); + iface.compactionFailed(args.tinfo, args.credentials, args.externalCompactionId, args.extent, args.exceptionClassName, args.failureState, args.groupName, args.compactorAddress,resultHandler); } } @@ -1580,6 +1596,8 @@ public static class compactionCompleted_args implements org.apache.thrift.TBase< private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); private static final org.apache.thrift.protocol.TField EXTENT_FIELD_DESC = new org.apache.thrift.protocol.TField("extent", org.apache.thrift.protocol.TType.STRUCT, (short)4); private static final org.apache.thrift.protocol.TField STATS_FIELD_DESC = new org.apache.thrift.protocol.TField("stats", org.apache.thrift.protocol.TType.STRUCT, (short)5); + private static final org.apache.thrift.protocol.TField GROUP_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("groupName", org.apache.thrift.protocol.TType.STRING, (short)6); + private static final org.apache.thrift.protocol.TField COMPACTOR_ADDRESS_FIELD_DESC = new org.apache.thrift.protocol.TField("compactorAddress", org.apache.thrift.protocol.TType.STRING, (short)7); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new compactionCompleted_argsStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new compactionCompleted_argsTupleSchemeFactory(); @@ -1589,6 +1607,8 @@ public static class compactionCompleted_args implements org.apache.thrift.TBase< public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats; // required + public @org.apache.thrift.annotation.Nullable java.lang.String groupName; // required + public @org.apache.thrift.annotation.Nullable java.lang.String compactorAddress; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -1596,7 +1616,9 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { CREDENTIALS((short)2, "credentials"), EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"), EXTENT((short)4, "extent"), - STATS((short)5, "stats"); + STATS((short)5, "stats"), + GROUP_NAME((short)6, "groupName"), + COMPACTOR_ADDRESS((short)7, "compactorAddress"); private static final java.util.Map byName = new java.util.HashMap(); @@ -1622,6 +1644,10 @@ public static _Fields findByThriftId(int fieldId) { return EXTENT; case 5: // STATS return STATS; + case 6: // GROUP_NAME + return GROUP_NAME; + case 7: // COMPACTOR_ADDRESS + return COMPACTOR_ADDRESS; default: return null; } @@ -1678,6 +1704,10 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class))); tmpMap.put(_Fields.STATS, new org.apache.thrift.meta_data.FieldMetaData("stats", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats.class))); + tmpMap.put(_Fields.GROUP_NAME, new org.apache.thrift.meta_data.FieldMetaData("groupName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.COMPACTOR_ADDRESS, new org.apache.thrift.meta_data.FieldMetaData("compactorAddress", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compactionCompleted_args.class, metaDataMap); } @@ -1690,7 +1720,9 @@ public compactionCompleted_args( org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, - org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats) + org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, + java.lang.String groupName, + java.lang.String compactorAddress) { this(); this.tinfo = tinfo; @@ -1698,6 +1730,8 @@ public compactionCompleted_args( this.externalCompactionId = externalCompactionId; this.extent = extent; this.stats = stats; + this.groupName = groupName; + this.compactorAddress = compactorAddress; } /** @@ -1719,6 +1753,12 @@ public compactionCompleted_args(compactionCompleted_args other) { if (other.isSetStats()) { this.stats = new org.apache.accumulo.core.tabletserver.thrift.TCompactionStats(other.stats); } + if (other.isSetGroupName()) { + this.groupName = other.groupName; + } + if (other.isSetCompactorAddress()) { + this.compactorAddress = other.compactorAddress; + } } @Override @@ -1733,6 +1773,8 @@ public void clear() { this.externalCompactionId = null; this.extent = null; this.stats = null; + this.groupName = null; + this.compactorAddress = null; } @org.apache.thrift.annotation.Nullable @@ -1860,6 +1902,56 @@ public void setStatsIsSet(boolean value) { } } + @org.apache.thrift.annotation.Nullable + public java.lang.String getGroupName() { + return this.groupName; + } + + public compactionCompleted_args setGroupName(@org.apache.thrift.annotation.Nullable java.lang.String groupName) { + this.groupName = groupName; + return this; + } + + public void unsetGroupName() { + this.groupName = null; + } + + /** Returns true if field groupName is set (has been assigned a value) and false otherwise */ + public boolean isSetGroupName() { + return this.groupName != null; + } + + public void setGroupNameIsSet(boolean value) { + if (!value) { + this.groupName = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getCompactorAddress() { + return this.compactorAddress; + } + + public compactionCompleted_args setCompactorAddress(@org.apache.thrift.annotation.Nullable java.lang.String compactorAddress) { + this.compactorAddress = compactorAddress; + return this; + } + + public void unsetCompactorAddress() { + this.compactorAddress = null; + } + + /** Returns true if field compactorAddress is set (has been assigned a value) and false otherwise */ + public boolean isSetCompactorAddress() { + return this.compactorAddress != null; + } + + public void setCompactorAddressIsSet(boolean value) { + if (!value) { + this.compactorAddress = null; + } + } + @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -1903,6 +1995,22 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; + case GROUP_NAME: + if (value == null) { + unsetGroupName(); + } else { + setGroupName((java.lang.String)value); + } + break; + + case COMPACTOR_ADDRESS: + if (value == null) { + unsetCompactorAddress(); + } else { + setCompactorAddress((java.lang.String)value); + } + break; + } } @@ -1925,6 +2033,12 @@ public java.lang.Object getFieldValue(_Fields field) { case STATS: return getStats(); + case GROUP_NAME: + return getGroupName(); + + case COMPACTOR_ADDRESS: + return getCompactorAddress(); + } throw new java.lang.IllegalStateException(); } @@ -1947,6 +2061,10 @@ public boolean isSet(_Fields field) { return isSetExtent(); case STATS: return isSetStats(); + case GROUP_NAME: + return isSetGroupName(); + case COMPACTOR_ADDRESS: + return isSetCompactorAddress(); } throw new java.lang.IllegalStateException(); } @@ -2009,6 +2127,24 @@ public boolean equals(compactionCompleted_args that) { return false; } + boolean this_present_groupName = true && this.isSetGroupName(); + boolean that_present_groupName = true && that.isSetGroupName(); + if (this_present_groupName || that_present_groupName) { + if (!(this_present_groupName && that_present_groupName)) + return false; + if (!this.groupName.equals(that.groupName)) + return false; + } + + boolean this_present_compactorAddress = true && this.isSetCompactorAddress(); + boolean that_present_compactorAddress = true && that.isSetCompactorAddress(); + if (this_present_compactorAddress || that_present_compactorAddress) { + if (!(this_present_compactorAddress && that_present_compactorAddress)) + return false; + if (!this.compactorAddress.equals(that.compactorAddress)) + return false; + } + return true; } @@ -2036,6 +2172,14 @@ public int hashCode() { if (isSetStats()) hashCode = hashCode * 8191 + stats.hashCode(); + hashCode = hashCode * 8191 + ((isSetGroupName()) ? 131071 : 524287); + if (isSetGroupName()) + hashCode = hashCode * 8191 + groupName.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCompactorAddress()) ? 131071 : 524287); + if (isSetCompactorAddress()) + hashCode = hashCode * 8191 + compactorAddress.hashCode(); + return hashCode; } @@ -2097,6 +2241,26 @@ public int compareTo(compactionCompleted_args other) { return lastComparison; } } + lastComparison = java.lang.Boolean.compare(isSetGroupName(), other.isSetGroupName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGroupName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groupName, other.groupName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCompactorAddress(), other.isSetCompactorAddress()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCompactorAddress()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compactorAddress, other.compactorAddress); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -2160,6 +2324,22 @@ public java.lang.String toString() { sb.append(this.stats); } first = false; + if (!first) sb.append(", "); + sb.append("groupName:"); + if (this.groupName == null) { + sb.append("null"); + } else { + sb.append(this.groupName); + } + first = false; + if (!first) sb.append(", "); + sb.append("compactorAddress:"); + if (this.compactorAddress == null) { + sb.append("null"); + } else { + sb.append(this.compactorAddress); + } + first = false; sb.append(")"); return sb.toString(); } @@ -2261,6 +2441,22 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, compactionCompleted org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 6: // GROUP_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.groupName = iprot.readString(); + struct.setGroupNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 7: // COMPACTOR_ADDRESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.compactorAddress = iprot.readString(); + struct.setCompactorAddressIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -2302,6 +2498,16 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, compactionComplete struct.stats.write(oprot); oprot.writeFieldEnd(); } + if (struct.groupName != null) { + oprot.writeFieldBegin(GROUP_NAME_FIELD_DESC); + oprot.writeString(struct.groupName); + oprot.writeFieldEnd(); + } + if (struct.compactorAddress != null) { + oprot.writeFieldBegin(COMPACTOR_ADDRESS_FIELD_DESC); + oprot.writeString(struct.compactorAddress); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -2336,7 +2542,13 @@ public void write(org.apache.thrift.protocol.TProtocol prot, compactionCompleted if (struct.isSetStats()) { optionals.set(4); } - oprot.writeBitSet(optionals, 5); + if (struct.isSetGroupName()) { + optionals.set(5); + } + if (struct.isSetCompactorAddress()) { + optionals.set(6); + } + oprot.writeBitSet(optionals, 7); if (struct.isSetTinfo()) { struct.tinfo.write(oprot); } @@ -2352,12 +2564,18 @@ public void write(org.apache.thrift.protocol.TProtocol prot, compactionCompleted if (struct.isSetStats()) { struct.stats.write(oprot); } + if (struct.isSetGroupName()) { + oprot.writeString(struct.groupName); + } + if (struct.isSetCompactorAddress()) { + oprot.writeString(struct.compactorAddress); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, compactionCompleted_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(5); + java.util.BitSet incoming = iprot.readBitSet(7); if (incoming.get(0)) { struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); struct.tinfo.read(iprot); @@ -2382,6 +2600,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, compactionCompleted_ struct.stats.read(iprot); struct.setStatsIsSet(true); } + if (incoming.get(5)) { + struct.groupName = iprot.readString(); + struct.setGroupNameIsSet(true); + } + if (incoming.get(6)) { + struct.compactorAddress = iprot.readString(); + struct.setCompactorAddressIsSet(true); + } } } @@ -5589,6 +5815,8 @@ public static class compactionFailed_args implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -5643,6 +5875,10 @@ public static _Fields findByThriftId(int fieldId) { return EXCEPTION_CLASS_NAME; case 6: // FAILURE_STATE return FAILURE_STATE; + case 7: // GROUP_NAME + return GROUP_NAME; + case 8: // COMPACTOR_ADDRESS + return COMPACTOR_ADDRESS; default: return null; } @@ -5701,6 +5937,10 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.FAILURE_STATE, new org.apache.thrift.meta_data.FieldMetaData("failureState", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TCompactionState.class))); + tmpMap.put(_Fields.GROUP_NAME, new org.apache.thrift.meta_data.FieldMetaData("groupName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.COMPACTOR_ADDRESS, new org.apache.thrift.meta_data.FieldMetaData("compactorAddress", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(compactionFailed_args.class, metaDataMap); } @@ -5714,7 +5954,9 @@ public compactionFailed_args( java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, - TCompactionState failureState) + TCompactionState failureState, + java.lang.String groupName, + java.lang.String compactorAddress) { this(); this.tinfo = tinfo; @@ -5723,6 +5965,8 @@ public compactionFailed_args( this.extent = extent; this.exceptionClassName = exceptionClassName; this.failureState = failureState; + this.groupName = groupName; + this.compactorAddress = compactorAddress; } /** @@ -5747,6 +5991,12 @@ public compactionFailed_args(compactionFailed_args other) { if (other.isSetFailureState()) { this.failureState = other.failureState; } + if (other.isSetGroupName()) { + this.groupName = other.groupName; + } + if (other.isSetCompactorAddress()) { + this.compactorAddress = other.compactorAddress; + } } @Override @@ -5762,6 +6012,8 @@ public void clear() { this.extent = null; this.exceptionClassName = null; this.failureState = null; + this.groupName = null; + this.compactorAddress = null; } @org.apache.thrift.annotation.Nullable @@ -5922,6 +6174,56 @@ public void setFailureStateIsSet(boolean value) { } } + @org.apache.thrift.annotation.Nullable + public java.lang.String getGroupName() { + return this.groupName; + } + + public compactionFailed_args setGroupName(@org.apache.thrift.annotation.Nullable java.lang.String groupName) { + this.groupName = groupName; + return this; + } + + public void unsetGroupName() { + this.groupName = null; + } + + /** Returns true if field groupName is set (has been assigned a value) and false otherwise */ + public boolean isSetGroupName() { + return this.groupName != null; + } + + public void setGroupNameIsSet(boolean value) { + if (!value) { + this.groupName = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getCompactorAddress() { + return this.compactorAddress; + } + + public compactionFailed_args setCompactorAddress(@org.apache.thrift.annotation.Nullable java.lang.String compactorAddress) { + this.compactorAddress = compactorAddress; + return this; + } + + public void unsetCompactorAddress() { + this.compactorAddress = null; + } + + /** Returns true if field compactorAddress is set (has been assigned a value) and false otherwise */ + public boolean isSetCompactorAddress() { + return this.compactorAddress != null; + } + + public void setCompactorAddressIsSet(boolean value) { + if (!value) { + this.compactorAddress = null; + } + } + @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -5973,6 +6275,22 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; + case GROUP_NAME: + if (value == null) { + unsetGroupName(); + } else { + setGroupName((java.lang.String)value); + } + break; + + case COMPACTOR_ADDRESS: + if (value == null) { + unsetCompactorAddress(); + } else { + setCompactorAddress((java.lang.String)value); + } + break; + } } @@ -5998,6 +6316,12 @@ public java.lang.Object getFieldValue(_Fields field) { case FAILURE_STATE: return getFailureState(); + case GROUP_NAME: + return getGroupName(); + + case COMPACTOR_ADDRESS: + return getCompactorAddress(); + } throw new java.lang.IllegalStateException(); } @@ -6022,6 +6346,10 @@ public boolean isSet(_Fields field) { return isSetExceptionClassName(); case FAILURE_STATE: return isSetFailureState(); + case GROUP_NAME: + return isSetGroupName(); + case COMPACTOR_ADDRESS: + return isSetCompactorAddress(); } throw new java.lang.IllegalStateException(); } @@ -6093,6 +6421,24 @@ public boolean equals(compactionFailed_args that) { return false; } + boolean this_present_groupName = true && this.isSetGroupName(); + boolean that_present_groupName = true && that.isSetGroupName(); + if (this_present_groupName || that_present_groupName) { + if (!(this_present_groupName && that_present_groupName)) + return false; + if (!this.groupName.equals(that.groupName)) + return false; + } + + boolean this_present_compactorAddress = true && this.isSetCompactorAddress(); + boolean that_present_compactorAddress = true && that.isSetCompactorAddress(); + if (this_present_compactorAddress || that_present_compactorAddress) { + if (!(this_present_compactorAddress && that_present_compactorAddress)) + return false; + if (!this.compactorAddress.equals(that.compactorAddress)) + return false; + } + return true; } @@ -6124,6 +6470,14 @@ public int hashCode() { if (isSetFailureState()) hashCode = hashCode * 8191 + failureState.getValue(); + hashCode = hashCode * 8191 + ((isSetGroupName()) ? 131071 : 524287); + if (isSetGroupName()) + hashCode = hashCode * 8191 + groupName.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCompactorAddress()) ? 131071 : 524287); + if (isSetCompactorAddress()) + hashCode = hashCode * 8191 + compactorAddress.hashCode(); + return hashCode; } @@ -6195,6 +6549,26 @@ public int compareTo(compactionFailed_args other) { return lastComparison; } } + lastComparison = java.lang.Boolean.compare(isSetGroupName(), other.isSetGroupName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGroupName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groupName, other.groupName); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCompactorAddress(), other.isSetCompactorAddress()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCompactorAddress()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compactorAddress, other.compactorAddress); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -6266,6 +6640,22 @@ public java.lang.String toString() { sb.append(this.failureState); } first = false; + if (!first) sb.append(", "); + sb.append("groupName:"); + if (this.groupName == null) { + sb.append("null"); + } else { + sb.append(this.groupName); + } + first = false; + if (!first) sb.append(", "); + sb.append("compactorAddress:"); + if (this.compactorAddress == null) { + sb.append("null"); + } else { + sb.append(this.compactorAddress); + } + first = false; sb.append(")"); return sb.toString(); } @@ -6371,6 +6761,22 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, compactionFailed_ar org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 7: // GROUP_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.groupName = iprot.readString(); + struct.setGroupNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 8: // COMPACTOR_ADDRESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.compactorAddress = iprot.readString(); + struct.setCompactorAddressIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -6417,6 +6823,16 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, compactionFailed_a oprot.writeI32(struct.failureState.getValue()); oprot.writeFieldEnd(); } + if (struct.groupName != null) { + oprot.writeFieldBegin(GROUP_NAME_FIELD_DESC); + oprot.writeString(struct.groupName); + oprot.writeFieldEnd(); + } + if (struct.compactorAddress != null) { + oprot.writeFieldBegin(COMPACTOR_ADDRESS_FIELD_DESC); + oprot.writeString(struct.compactorAddress); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -6454,7 +6870,13 @@ public void write(org.apache.thrift.protocol.TProtocol prot, compactionFailed_ar if (struct.isSetFailureState()) { optionals.set(5); } - oprot.writeBitSet(optionals, 6); + if (struct.isSetGroupName()) { + optionals.set(6); + } + if (struct.isSetCompactorAddress()) { + optionals.set(7); + } + oprot.writeBitSet(optionals, 8); if (struct.isSetTinfo()) { struct.tinfo.write(oprot); } @@ -6473,12 +6895,18 @@ public void write(org.apache.thrift.protocol.TProtocol prot, compactionFailed_ar if (struct.isSetFailureState()) { oprot.writeI32(struct.failureState.getValue()); } + if (struct.isSetGroupName()) { + oprot.writeString(struct.groupName); + } + if (struct.isSetCompactorAddress()) { + oprot.writeString(struct.compactorAddress); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, compactionFailed_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(6); + java.util.BitSet incoming = iprot.readBitSet(8); if (incoming.get(0)) { struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); struct.tinfo.read(iprot); @@ -6506,6 +6934,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, compactionFailed_arg struct.failureState = org.apache.accumulo.core.compaction.thrift.TCompactionState.findByValue(iprot.readI32()); struct.setFailureStateIsSet(true); } + if (incoming.get(6)) { + struct.groupName = iprot.readString(); + struct.setGroupNameIsSet(true); + } + if (incoming.get(7)) { + struct.compactorAddress = iprot.readString(); + struct.setCompactorAddressIsSet(true); + } } } diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index d583978d20f..e7f7cdc3bb3 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -84,6 +84,8 @@ service CompactionCoordinatorService { 3:string externalCompactionId 4:data.TKeyExtent extent 5:tabletserver.TCompactionStats stats + 6:string groupName + 7:string compactorAddress )throws( 1:client.ThriftSecurityException sec 2:client.ThriftNotActiveServiceException tnase @@ -127,6 +129,8 @@ service CompactionCoordinatorService { 4:data.TKeyExtent extent 5:string exceptionClassName 6:TCompactionState failureState + 7:string groupName + 8:string compactorAddress )throws( 1:client.ThriftSecurityException sec 2:client.ThriftNotActiveServiceException tnase diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 82301f98172..a0e333ab8d5 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -508,7 +508,8 @@ protected void updateCompactionFailed(TExternalCompactionJob job, TCompactionSta Client coordinatorClient = getCoordinatorClient(); try { coordinatorClient.compactionFailed(TraceUtil.traceInfo(), getContext().rpcCreds(), - job.getExternalCompactionId(), job.extent, message, why); + job.getExternalCompactionId(), job.extent, message, why, + getResourceGroup().canonical(), getAdvertiseAddress().toString()); return ""; } finally { ThriftUtil.returnClient(coordinatorClient, getContext()); @@ -531,7 +532,8 @@ protected void updateCompactionCompleted(TExternalCompactionJob job, TCompaction Client coordinatorClient = getCoordinatorClient(); try { coordinatorClient.compactionCompleted(TraceUtil.traceInfo(), getContext().rpcCreds(), - job.getExternalCompactionId(), job.extent, stats); + job.getExternalCompactionId(), job.extent, stats, getResourceGroup().canonical(), + getAdvertiseAddress().toString()); return ""; } finally { ThriftUtil.returnClient(coordinatorClient, getContext()); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index aee9679cc01..034331d5a2f 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -726,8 +726,8 @@ private Optional getCompactionConfig(ResolvedCompactionJob rcJ */ @Override public void compactionCompleted(TInfo tinfo, TCredentials credentials, - String externalCompactionId, TKeyExtent textent, TCompactionStats stats) - throws ThriftSecurityException { + String externalCompactionId, TKeyExtent textent, TCompactionStats stats, String groupName, + String compactorAddress) throws ThriftSecurityException { // do not expect users to call this directly, expect other tservers to call this method if (!security.canPerformSystemActions(credentials)) { throw new AccumuloSecurityException(credentials.getPrincipal(), @@ -742,7 +742,7 @@ public void compactionCompleted(TInfo tinfo, TCredentials credentials, LOG.info("Compaction completed, id: {}, stats: {}, extent: {}", externalCompactionId, stats, extent); final var ecid = ExternalCompactionId.of(externalCompactionId); - captureSuccess(ecid, extent); + captureSuccess(ResourceGroupId.of(groupName), compactorAddress, extent); var tabletMeta = ctx.getAmple().readTablet(extent, ECOMP, SELECTED, LOCATION, FILES, COMPACTED, OPID); @@ -773,8 +773,8 @@ public void compactionCompleted(TInfo tinfo, TCredentials credentials, @Override public void compactionFailed(TInfo tinfo, TCredentials credentials, String externalCompactionId, - TKeyExtent extent, String exceptionMessage, TCompactionState failureState) - throws ThriftSecurityException { + TKeyExtent extent, String exceptionMessage, TCompactionState failureState, String groupName, + String compactorAddress) throws ThriftSecurityException { // do not expect users to call this directly, expect other tservers to call this method if (!security.canPerformSystemActions(credentials)) { throw new AccumuloSecurityException(credentials.getPrincipal(), @@ -789,18 +789,14 @@ public void compactionFailed(TInfo tinfo, TCredentials credentials, String exter externalCompactionId, fromThriftExtent, exceptionMessage); final var ecid = ExternalCompactionId.of(externalCompactionId); if (failureState == TCompactionState.FAILED) { - captureFailure(ecid, fromThriftExtent); + captureFailure(ResourceGroupId.of(groupName), compactorAddress, fromThriftExtent); } compactionsFailed(Map.of(ecid, KeyExtent.fromThrift(extent))); } - private void captureFailure(ExternalCompactionId ecid, KeyExtent extent) { - var tec = RUNNING_CACHE.get(ecid); - if (tec != null) { - failingQueues.compute(ResourceGroupId.of(tec.getGroupName()), - FailureCounts::incrementFailure); - failingCompactors.compute(tec.getCompactor(), FailureCounts::incrementFailure); - } + private void captureFailure(ResourceGroupId group, String compactorAddress, KeyExtent extent) { + failingQueues.compute(group, FailureCounts::incrementFailure); + failingCompactors.compute(compactorAddress, FailureCounts::incrementFailure); failingTables.compute(extent.tableId(), FailureCounts::incrementFailure); } @@ -853,13 +849,9 @@ private void printStats() { printStats("Compactor", failingCompactors, true); } - private void captureSuccess(ExternalCompactionId ecid, KeyExtent extent) { - var tec = RUNNING_CACHE.get(ecid); - if (tec != null) { - failingQueues.compute(ResourceGroupId.of(tec.getGroupName()), - FailureCounts::incrementSuccess); - failingCompactors.compute(tec.getCompactor(), FailureCounts::incrementSuccess); - } + private void captureSuccess(ResourceGroupId groupId, String compactorAddress, KeyExtent extent) { + failingQueues.compute(groupId, FailureCounts::incrementSuccess); + failingCompactors.compute(compactorAddress, FailureCounts::incrementSuccess); failingTables.compute(extent.tableId(), FailureCounts::incrementSuccess); } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index a11e345a993..9a62c1a3249 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -142,13 +142,13 @@ protected void startConfigMonitor(ScheduledThreadPoolExecutor schedExecutor) {} @Override public void compactionCompleted(TInfo tinfo, TCredentials credentials, - String externalCompactionId, TKeyExtent textent, TCompactionStats stats) - throws ThriftSecurityException {} + String externalCompactionId, TKeyExtent textent, TCompactionStats stats, String groupName, + String compactorAddress) throws ThriftSecurityException {} @Override public void compactionFailed(TInfo tinfo, TCredentials credentials, String externalCompactionId, - TKeyExtent extent, String exceptionClassName, TCompactionState failureState) - throws ThriftSecurityException {} + TKeyExtent extent, String exceptionClassName, TCompactionState failureState, + String groupName, String compactorAddress) throws ThriftSecurityException {} void setMetadataCompactionIds(Set mci) { metadataCompactionIds = mci; From 5e06811804679144fbf9f8d66c257e0796faaa7e Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 24 Mar 2026 20:44:21 +0000 Subject: [PATCH 14/65] Removes the completed set from the coordinator This change is made in support of #6217. Successfully ran all ITs that were changed. --- .../thrift/CompactionCoordinatorService.java | 1275 ----------------- .../main/thrift/compaction-coordinator.thrift | 11 - .../coordinator/CompactionCoordinator.java | 30 - .../compaction/ExternalCompaction2ITBase.java | 15 +- .../compaction/ExternalCompaction4_IT.java | 7 +- .../ExternalCompactionTestUtils.java | 54 +- .../compaction/ExternalCompaction_3_IT.java | 5 +- .../test/compaction/SplitCancelsMajCIT.java | 5 +- 8 files changed, 25 insertions(+), 1377 deletions(-) diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index 734d7f9027c..008869646ce 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -39,8 +39,6 @@ public interface Iface { public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException; @@ -59,8 +57,6 @@ public interface AsyncIface { public void getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; @@ -248,37 +244,6 @@ public TExternalCompactionMap recv_getRunningCompactions() throws org.apache.acc throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRunningCompactions failed: unknown result"); } - @Override - public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException - { - send_getCompletedCompactions(tinfo, credentials); - return recv_getCompletedCompactions(); - } - - public void send_getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.thrift.TException - { - getCompletedCompactions_args args = new getCompletedCompactions_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - sendBase("getCompletedCompactions", args); - } - - public TExternalCompactionMap recv_getCompletedCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException - { - getCompletedCompactions_result result = new getCompletedCompactions_result(); - receiveBase(result, "getCompletedCompactions"); - if (result.isSetSuccess()) { - return result.success; - } - if (result.sec != null) { - throw result.sec; - } - if (result.tnase != null) { - throw result.tnase; - } - throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getCompletedCompactions failed: unknown result"); - } - @Override public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { @@ -574,44 +539,6 @@ public TExternalCompactionMap getResult() throws org.apache.accumulo.core.client } } - @Override - public void getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - checkReady(); - getCompletedCompactions_call method_call = new getCompletedCompactions_call(tinfo, credentials, resultHandler, this, ___protocolFactory, ___transport); - this.___currentMethod = method_call; - ___manager.call(method_call); - } - - public static class getCompletedCompactions_call extends org.apache.thrift.async.TAsyncMethodCall { - private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; - private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; - public getCompletedCompactions_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { - super(client, protocolFactory, transport, resultHandler, false); - this.tinfo = tinfo; - this.credentials = credentials; - } - - @Override - public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { - prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getCompletedCompactions", org.apache.thrift.protocol.TMessageType.CALL, 0)); - getCompletedCompactions_args args = new getCompletedCompactions_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.write(prot); - prot.writeMessageEnd(); - } - - @Override - public TExternalCompactionMap getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { - if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { - throw new java.lang.IllegalStateException("Method call not finished!"); - } - org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); - org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); - return (new Client(prot)).recv_getCompletedCompactions(); - } - } - @Override public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); @@ -713,7 +640,6 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { - public getCompletedCompactions() { - super("getCompletedCompactions"); - } - - @Override - public getCompletedCompactions_args getEmptyArgsInstance() { - return new getCompletedCompactions_args(); - } - - @Override - protected boolean isOneway() { - return false; - } - - @Override - protected boolean rethrowUnhandledExceptions() { - return false; - } - - @Override - public getCompletedCompactions_result getResult(I iface, getCompletedCompactions_args args) throws org.apache.thrift.TException { - getCompletedCompactions_result result = new getCompletedCompactions_result(); - try { - result.success = iface.getCompletedCompactions(args.tinfo, args.credentials); - } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { - result.sec = sec; - } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - result.tnase = tnase; - } - return result; - } - } - public static class cancel extends org.apache.thrift.ProcessFunction { public cancel() { super("cancel"); @@ -1002,7 +894,6 @@ protected AsyncProcessor(I iface, java.util.Map extends org.apache.thrift.AsyncProcessFunction { - public getCompletedCompactions() { - super("getCompletedCompactions"); - } - - @Override - public getCompletedCompactions_args getEmptyArgsInstance() { - return new getCompletedCompactions_args(); - } - - @Override - public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { - final org.apache.thrift.AsyncProcessFunction fcall = this; - return new org.apache.thrift.async.AsyncMethodCallback() { - @Override - public void onComplete(TExternalCompactionMap o) { - getCompletedCompactions_result result = new getCompletedCompactions_result(); - result.success = o; - try { - fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); - } catch (org.apache.thrift.transport.TTransportException e) { - _LOGGER.error("TTransportException writing to internal frame buffer", e); - fb.close(); - } catch (java.lang.Exception e) { - _LOGGER.error("Exception writing to internal frame buffer", e); - onError(e); - } - } - @Override - public void onError(java.lang.Exception e) { - byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; - org.apache.thrift.TSerializable msg; - getCompletedCompactions_result result = new getCompletedCompactions_result(); - if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { - result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; - result.setSecIsSet(true); - msg = result; - } else if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) { - result.tnase = (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) e; - result.setTnaseIsSet(true); - msg = result; - } else if (e instanceof org.apache.thrift.transport.TTransportException) { - _LOGGER.error("TTransportException inside handler", e); - fb.close(); - return; - } else if (e instanceof org.apache.thrift.TApplicationException) { - _LOGGER.error("TApplicationException inside handler", e); - msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; - msg = (org.apache.thrift.TApplicationException)e; - } else { - _LOGGER.error("Exception inside handler", e); - msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; - msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); - } - try { - fcall.sendResponse(fb,msg,msgType,seqid); - } catch (java.lang.Exception ex) { - _LOGGER.error("Exception writing to internal frame buffer", ex); - fb.close(); - } - } - }; - } - - @Override - protected boolean isOneway() { - return false; - } - - @Override - public void start(I iface, getCompletedCompactions_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.getCompletedCompactions(args.tinfo, args.credentials,resultHandler); - } - } - public static class cancel extends org.apache.thrift.AsyncProcessFunction { public cancel() { super("cancel"); @@ -8092,1097 +7908,6 @@ private static S scheme(org.apache. } } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class getCompletedCompactions_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getCompletedCompactions_args"); - - private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getCompletedCompactions_argsStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getCompletedCompactions_argsTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - TINFO((short)1, "tinfo"), - CREDENTIALS((short)2, "credentials"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // TINFO - return TINFO; - case 2: // CREDENTIALS - return CREDENTIALS; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); - tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getCompletedCompactions_args.class, metaDataMap); - } - - public getCompletedCompactions_args() { - } - - public getCompletedCompactions_args( - org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, - org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) - { - this(); - this.tinfo = tinfo; - this.credentials = credentials; - } - - /** - * Performs a deep copy on other. - */ - public getCompletedCompactions_args(getCompletedCompactions_args other) { - if (other.isSetTinfo()) { - this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); - } - if (other.isSetCredentials()) { - this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); - } - } - - @Override - public getCompletedCompactions_args deepCopy() { - return new getCompletedCompactions_args(this); - } - - @Override - public void clear() { - this.tinfo = null; - this.credentials = null; - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { - return this.tinfo; - } - - public getCompletedCompactions_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { - this.tinfo = tinfo; - return this; - } - - public void unsetTinfo() { - this.tinfo = null; - } - - /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ - public boolean isSetTinfo() { - return this.tinfo != null; - } - - public void setTinfoIsSet(boolean value) { - if (!value) { - this.tinfo = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { - return this.credentials; - } - - public getCompletedCompactions_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { - this.credentials = credentials; - return this; - } - - public void unsetCredentials() { - this.credentials = null; - } - - /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ - public boolean isSetCredentials() { - return this.credentials != null; - } - - public void setCredentialsIsSet(boolean value) { - if (!value) { - this.credentials = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case TINFO: - if (value == null) { - unsetTinfo(); - } else { - setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); - } - break; - - case CREDENTIALS: - if (value == null) { - unsetCredentials(); - } else { - setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case TINFO: - return getTinfo(); - - case CREDENTIALS: - return getCredentials(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case TINFO: - return isSetTinfo(); - case CREDENTIALS: - return isSetCredentials(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof getCompletedCompactions_args) - return this.equals((getCompletedCompactions_args)that); - return false; - } - - public boolean equals(getCompletedCompactions_args that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_tinfo = true && this.isSetTinfo(); - boolean that_present_tinfo = true && that.isSetTinfo(); - if (this_present_tinfo || that_present_tinfo) { - if (!(this_present_tinfo && that_present_tinfo)) - return false; - if (!this.tinfo.equals(that.tinfo)) - return false; - } - - boolean this_present_credentials = true && this.isSetCredentials(); - boolean that_present_credentials = true && that.isSetCredentials(); - if (this_present_credentials || that_present_credentials) { - if (!(this_present_credentials && that_present_credentials)) - return false; - if (!this.credentials.equals(that.credentials)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); - if (isSetTinfo()) - hashCode = hashCode * 8191 + tinfo.hashCode(); - - hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); - if (isSetCredentials()) - hashCode = hashCode * 8191 + credentials.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(getCompletedCompactions_args other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTinfo()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCredentials()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("getCompletedCompactions_args("); - boolean first = true; - - sb.append("tinfo:"); - if (this.tinfo == null) { - sb.append("null"); - } else { - sb.append(this.tinfo); - } - first = false; - if (!first) sb.append(", "); - sb.append("credentials:"); - if (this.credentials == null) { - sb.append("null"); - } else { - sb.append(this.credentials); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (tinfo != null) { - tinfo.validate(); - } - if (credentials != null) { - credentials.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class getCompletedCompactions_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getCompletedCompactions_argsStandardScheme getScheme() { - return new getCompletedCompactions_argsStandardScheme(); - } - } - - private static class getCompletedCompactions_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, getCompletedCompactions_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // TINFO - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // CREDENTIALS - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, getCompletedCompactions_args struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.tinfo != null) { - oprot.writeFieldBegin(TINFO_FIELD_DESC); - struct.tinfo.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.credentials != null) { - oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); - struct.credentials.write(oprot); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class getCompletedCompactions_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getCompletedCompactions_argsTupleScheme getScheme() { - return new getCompletedCompactions_argsTupleScheme(); - } - } - - private static class getCompletedCompactions_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, getCompletedCompactions_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetTinfo()) { - optionals.set(0); - } - if (struct.isSetCredentials()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); - if (struct.isSetTinfo()) { - struct.tinfo.write(oprot); - } - if (struct.isSetCredentials()) { - struct.credentials.write(oprot); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, getCompletedCompactions_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); - if (incoming.get(0)) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } - if (incoming.get(1)) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class getCompletedCompactions_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getCompletedCompactions_result"); - - private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); - private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getCompletedCompactions_resultStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getCompletedCompactions_resultTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable TExternalCompactionMap success; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - SUCCESS((short)0, "success"), - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 0: // SUCCESS - return SUCCESS; - case 1: // SEC - return SEC; - case 2: // TNASE - return TNASE; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExternalCompactionMap.class))); - tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getCompletedCompactions_result.class, metaDataMap); - } - - public getCompletedCompactions_result() { - } - - public getCompletedCompactions_result( - TExternalCompactionMap success, - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) - { - this(); - this.success = success; - this.sec = sec; - this.tnase = tnase; - } - - /** - * Performs a deep copy on other. - */ - public getCompletedCompactions_result(getCompletedCompactions_result other) { - if (other.isSetSuccess()) { - this.success = new TExternalCompactionMap(other.success); - } - if (other.isSetSec()) { - this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); - } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } - } - - @Override - public getCompletedCompactions_result deepCopy() { - return new getCompletedCompactions_result(this); - } - - @Override - public void clear() { - this.success = null; - this.sec = null; - this.tnase = null; - } - - @org.apache.thrift.annotation.Nullable - public TExternalCompactionMap getSuccess() { - return this.success; - } - - public getCompletedCompactions_result setSuccess(@org.apache.thrift.annotation.Nullable TExternalCompactionMap success) { - this.success = success; - return this; - } - - public void unsetSuccess() { - this.success = null; - } - - /** Returns true if field success is set (has been assigned a value) and false otherwise */ - public boolean isSetSuccess() { - return this.success != null; - } - - public void setSuccessIsSet(boolean value) { - if (!value) { - this.success = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { - return this.sec; - } - - public getCompletedCompactions_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { - this.sec = sec; - return this; - } - - public void unsetSec() { - this.sec = null; - } - - /** Returns true if field sec is set (has been assigned a value) and false otherwise */ - public boolean isSetSec() { - return this.sec != null; - } - - public void setSecIsSet(boolean value) { - if (!value) { - this.sec = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public getCompletedCompactions_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case SUCCESS: - if (value == null) { - unsetSuccess(); - } else { - setSuccess((TExternalCompactionMap)value); - } - break; - - case SEC: - if (value == null) { - unsetSec(); - } else { - setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); - } - break; - - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case SUCCESS: - return getSuccess(); - - case SEC: - return getSec(); - - case TNASE: - return getTnase(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case SUCCESS: - return isSetSuccess(); - case SEC: - return isSetSec(); - case TNASE: - return isSetTnase(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof getCompletedCompactions_result) - return this.equals((getCompletedCompactions_result)that); - return false; - } - - public boolean equals(getCompletedCompactions_result that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_success = true && this.isSetSuccess(); - boolean that_present_success = true && that.isSetSuccess(); - if (this_present_success || that_present_success) { - if (!(this_present_success && that_present_success)) - return false; - if (!this.success.equals(that.success)) - return false; - } - - boolean this_present_sec = true && this.isSetSec(); - boolean that_present_sec = true && that.isSetSec(); - if (this_present_sec || that_present_sec) { - if (!(this_present_sec && that_present_sec)) - return false; - if (!this.sec.equals(that.sec)) - return false; - } - - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetSuccess()) ? 131071 : 524287); - if (isSetSuccess()) - hashCode = hashCode * 8191 + success.hashCode(); - - hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); - if (isSetSec()) - hashCode = hashCode * 8191 + sec.hashCode(); - - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(getCompletedCompactions_result other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetSuccess(), other.isSetSuccess()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetSuccess()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetSec()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("getCompletedCompactions_result("); - boolean first = true; - - sb.append("success:"); - if (this.success == null) { - sb.append("null"); - } else { - sb.append(this.success); - } - first = false; - if (!first) sb.append(", "); - sb.append("sec:"); - if (this.sec == null) { - sb.append("null"); - } else { - sb.append(this.sec); - } - first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (success != null) { - success.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class getCompletedCompactions_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getCompletedCompactions_resultStandardScheme getScheme() { - return new getCompletedCompactions_resultStandardScheme(); - } - } - - private static class getCompletedCompactions_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, getCompletedCompactions_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 0: // SUCCESS - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.success = new TExternalCompactionMap(); - struct.success.read(iprot); - struct.setSuccessIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 1: // SEC - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); - struct.sec.read(iprot); - struct.setSecIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, getCompletedCompactions_result struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.success != null) { - oprot.writeFieldBegin(SUCCESS_FIELD_DESC); - struct.success.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.sec != null) { - oprot.writeFieldBegin(SEC_FIELD_DESC); - struct.sec.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class getCompletedCompactions_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getCompletedCompactions_resultTupleScheme getScheme() { - return new getCompletedCompactions_resultTupleScheme(); - } - } - - private static class getCompletedCompactions_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, getCompletedCompactions_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetSuccess()) { - optionals.set(0); - } - if (struct.isSetSec()) { - optionals.set(1); - } - if (struct.isSetTnase()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); - if (struct.isSetSuccess()) { - struct.success.write(oprot); - } - if (struct.isSetSec()) { - struct.sec.write(oprot); - } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, getCompletedCompactions_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); - if (incoming.get(0)) { - struct.success = new TExternalCompactionMap(); - struct.success.read(iprot); - struct.setSuccessIsSet(true); - } - if (incoming.get(1)) { - struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); - struct.sec.read(iprot); - struct.setSecIsSet(true); - } - if (incoming.get(2)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) public static class cancel_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("cancel_args"); diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index d583978d20f..bd50382dab2 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -143,17 +143,6 @@ service CompactionCoordinatorService { 2:client.ThriftNotActiveServiceException tnase ) - /* - * Called by the Monitor to get progress information - */ - TExternalCompactionMap getCompletedCompactions( - 1:client.TInfo tinfo - 2:security.TCredentials credentials - )throws( - 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase - ) - void cancel( 1:client.TInfo tinfo 2:security.TCredentials credentials diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index aee9679cc01..80f80d4002b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -217,7 +217,6 @@ static FailureCounts incrementSuccess(Object key, FailureCounts counts) { // Exposed for tests protected final CountDownLatch shutdown = new CountDownLatch(1); - private final Cache completed; private final LoadingCache compactionConfigCache; private final Cache tabletDirCache; private final DeadCompactionDetector deadCompactionDetector; @@ -247,9 +246,6 @@ public CompactionCoordinator(Manager manager, this.fateClients = fateClients; - completed = ctx.getCaches().createNewBuilder(CacheName.COMPACTIONS_COMPLETED, true) - .maximumSize(200).expireAfterWrite(10, TimeUnit.MINUTES).build(); - CacheLoader loader = fateId -> CompactionConfigStorage.getConfig(ctx, fateId); @@ -1014,9 +1010,6 @@ public void recordCompletion(TInfo tinfo, TCredentials credentials, String exter public void recordCompletion(ExternalCompactionId ecid) { var tec = RUNNING_CACHE.remove(ecid); - if (tec != null) { - completed.put(ecid, tec); - } } protected Set readExternalCompactionIds() { @@ -1052,29 +1045,6 @@ public TExternalCompactionMap getRunningCompactions(TInfo tinfo, TCredentials cr return result; } - /** - * Return information about recently completed compactions - * - * @param tinfo trace info - * @param credentials tcredentials object - * @return map of ECID to TExternalCompaction objects - * @throws ThriftSecurityException permission error - */ - @Override - public TExternalCompactionMap getCompletedCompactions(TInfo tinfo, TCredentials credentials) - throws ThriftSecurityException { - // do not expect users to call this directly, expect other tservers to call this method - if (!security.canPerformSystemActions(credentials)) { - throw new AccumuloSecurityException(credentials.getPrincipal(), - SecurityErrorCode.PERMISSION_DENIED).asThriftException(); - } - final TExternalCompactionMap result = new TExternalCompactionMap(); - completed.asMap().forEach((ecid, tec) -> { - result.putToCompactions(ecid.canonical(), tec); - }); - return result; - } - @Override public void cancel(TInfo tinfo, TCredentials credentials, String externalCompactionId) throws TException { diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java index 9622cf05668..ad7b19325bb 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java @@ -23,8 +23,8 @@ import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP4; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.MAX_DATA; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; -import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionCompleted; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionRunning; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionsNoLongerRunning; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.countTablets; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.row; @@ -50,7 +50,6 @@ import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.clientImpl.TableOperationsImpl; -import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.metadata.SystemTables; @@ -122,8 +121,7 @@ public void testSplitCancelsExternalCompaction() throws Exception { client.tableOperations().addSplits(table1, splits); - confirmCompactionCompleted(getCluster().getServerContext(), ecids, - TCompactionState.CANCELLED); + confirmCompactionsNoLongerRunning(getCluster().getServerContext(), ecids); // ensure compaction ids were deleted by split operation from metadata table try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets() @@ -192,8 +190,7 @@ public void testUserCompactionCancellation() throws Exception { assertNotNull(e); assertEquals(TableOperationsImpl.COMPACTION_CANCELED_MSG, e.getMessage()); - confirmCompactionCompleted(getCluster().getServerContext(), ecids, - TCompactionState.CANCELLED); + confirmCompactionsNoLongerRunning(getCluster().getServerContext(), ecids); // ensure the canceled compaction deletes any tablet metadata related to the compaction while (countTablets(getCluster().getServerContext(), table1, @@ -236,8 +233,7 @@ public void testDeleteTableCancelsUserExternalCompaction() throws Exception { client.tableOperations().delete(table1); - confirmCompactionCompleted(getCluster().getServerContext(), ecids, - TCompactionState.CANCELLED); + confirmCompactionsNoLongerRunning(getCluster().getServerContext(), ecids); // Ensure compaction did not write anything to metadata table after delete table try (var scanner = client.createScanner(SystemTables.METADATA.tableName())) { @@ -281,6 +277,7 @@ public void testDeleteTableCancelsExternalCompaction() throws Exception { Thread t = new Thread(r); t.start(); latch.await(); + assertNull(error.get()); // Wait for the compaction to start by waiting for 1 external compaction column Set ecids = ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(ctx, tid); @@ -293,7 +290,7 @@ public void testDeleteTableCancelsExternalCompaction() throws Exception { LoggerFactory.getLogger(getClass()).debug("Table deleted."); - confirmCompactionCompleted(ctx, ecids, TCompactionState.CANCELLED); + confirmCompactionsNoLongerRunning(ctx, ecids); LoggerFactory.getLogger(getClass()).debug("Confirmed compaction cancelled."); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java index cce42c26f41..2d57c94e607 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java @@ -20,8 +20,8 @@ import static org.apache.accumulo.core.conf.Property.TABLE_FILE_MAX; import static org.apache.accumulo.core.conf.Property.TABLE_MAJC_RATIO; -import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionCompleted; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionRunning; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionsNoLongerRunning; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.writeData; @@ -39,7 +39,6 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.user.AgeOffFilter; @@ -173,9 +172,7 @@ public void testDeleteTableCancelsSleepingExternalCompaction() throws Exception client.tableOperations().delete(table1); - confirmCompactionCompleted(getCluster().getServerContext(), ecids, - TCompactionState.CANCELLED); - + confirmCompactionsNoLongerRunning(getCluster().getServerContext(), ecids); } } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java index 457abf588f6..fab985bced0 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java @@ -18,8 +18,8 @@ */ package org.apache.accumulo.test.compaction; +import static java.util.concurrent.TimeUnit.MINUTES; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -162,7 +162,7 @@ public static void writeData(AccumuloClient client, String table1, int rows) } } - client.tableOperations().flush(table1); + client.tableOperations().flush(table1, null, null, true); } public static void writeData(AccumuloClient client, String table1) @@ -265,19 +265,6 @@ public static TExternalCompactionMap getRunningCompactions(ClientContext context } } - private static TExternalCompactionMap getCompletedCompactions(ClientContext context, - Optional coordinatorHost) throws Exception { - CompactionCoordinatorService.Client client = - ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost.orElseThrow(), context); - try { - TExternalCompactionMap completed = - client.getCompletedCompactions(TraceUtil.traceInfo(), context.rpcCreds()); - return completed; - } finally { - ThriftUtil.returnClient(client, context); - } - } - public static TCompactionState getLastState(TExternalCompaction status) { ArrayList timestamps = new ArrayList<>(status.getUpdates().size()); status.getUpdates().keySet().forEach(k -> timestamps.add(k)); @@ -349,33 +336,18 @@ public static int confirmCompactionRunning(ServerContext ctx, Set ecids, - TCompactionState expectedState) throws Exception { - Optional coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(ctx); - if (coordinatorHost.isEmpty()) { - throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper"); - } - - // The running compaction should be removed - TExternalCompactionMap running = - ExternalCompactionTestUtils.getRunningCompactions(ctx, coordinatorHost); - while (running.getCompactions() != null && running.getCompactions().keySet().stream() - .anyMatch((e) -> ecids.contains(ExternalCompactionId.of(e)))) { - running = ExternalCompactionTestUtils.getRunningCompactions(ctx, coordinatorHost); - } - // The compaction should be in the completed list with the expected state - TExternalCompactionMap completed = - ExternalCompactionTestUtils.getCompletedCompactions(ctx, coordinatorHost); - while (completed.getCompactions() == null) { - UtilWaitThread.sleep(50); - completed = ExternalCompactionTestUtils.getCompletedCompactions(ctx, coordinatorHost); - } - for (ExternalCompactionId e : ecids) { - TExternalCompaction tec = completed.getCompactions().get(e.canonical()); - assertNotNull(tec); - assertEquals(expectedState, ExternalCompactionTestUtils.getLastState(tec)); - } + /** + * Waits for compactions to no longer be running on compactors + */ + public static void confirmCompactionsNoLongerRunning(ServerContext ctx, + Set ecids) { + Wait.waitFor(() -> { + Set running = new HashSet<>(); + ExternalCompactionUtil.getCompactionsRunningOnCompactors(ctx, + tec -> running.add(ExternalCompactionId.of(tec.getJob().getExternalCompactionId()))); + return Collections.disjoint(running, ecids); + }, MINUTES.toMillis(5)); } public static void assertNoCompactionMetadata(ServerContext ctx, String tableName) { diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java index 9c855ba390d..840fab3148c 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java @@ -20,7 +20,7 @@ import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP2; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; -import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionCompleted; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionsNoLongerRunning; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getRunningCompactions; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids; @@ -137,8 +137,7 @@ public void testMergeCancelsExternalCompaction() throws Exception { Text end = md.get(1).getEndRow(); client.tableOperations().merge(table1, start, end); - confirmCompactionCompleted(getCluster().getServerContext(), ecids, - TCompactionState.CANCELLED); + confirmCompactionsNoLongerRunning(getCluster().getServerContext(), ecids); // ensure compaction ids were deleted by merge operation from metadata table try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets() diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java index e7ee6a19bbe..b3e7b2553fc 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java @@ -19,6 +19,7 @@ package org.apache.accumulo.test.compaction; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionsNoLongerRunning; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -34,7 +35,6 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.TableId; @@ -119,8 +119,7 @@ public void test() throws Exception { partitionKeys.add(new Text("10")); c.tableOperations().addSplits(tableName, partitionKeys); - ExternalCompactionTestUtils.confirmCompactionCompleted(getCluster().getServerContext(), - compactionIds, TCompactionState.CANCELLED); + confirmCompactionsNoLongerRunning(getCluster().getServerContext(), compactionIds); thread.join(); // wait for the restarted compaction From 36b0fc1b175d2eb710ea604eb719cad20f9a11c9 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 24 Mar 2026 21:56:04 +0000 Subject: [PATCH 15/65] removes coordinator RPC to obtain running cache Modified code that used this RPC to get the same information directly from compactors instead. Making this change in support of #6217 --- .../thrift/CompactionCoordinatorService.java | 1275 ----------------- .../main/thrift/compaction-coordinator.thrift | 11 - .../accumulo/server/util/ListCompactions.java | 19 +- .../coordinator/CompactionCoordinator.java | 24 - .../accumulo/test/ListCompactionsIT.java | 21 +- .../ExternalCompactionProgressIT.java | 13 +- .../ExternalCompactionTestUtils.java | 39 +- .../compaction/ExternalCompaction_3_IT.java | 9 +- .../test/functional/GracefulShutdownIT.java | 8 +- 9 files changed, 33 insertions(+), 1386 deletions(-) diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index 734d7f9027c..3e44223b74d 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -37,8 +37,6 @@ public interface Iface { public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; @@ -57,8 +55,6 @@ public interface AsyncIface { public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; @@ -217,37 +213,6 @@ public void recv_compactionFailed() throws org.apache.accumulo.core.clientImpl.t return; } - @Override - public TExternalCompactionMap getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException - { - send_getRunningCompactions(tinfo, credentials); - return recv_getRunningCompactions(); - } - - public void send_getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.thrift.TException - { - getRunningCompactions_args args = new getRunningCompactions_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - sendBase("getRunningCompactions", args); - } - - public TExternalCompactionMap recv_getRunningCompactions() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException - { - getRunningCompactions_result result = new getRunningCompactions_result(); - receiveBase(result, "getRunningCompactions"); - if (result.isSetSuccess()) { - return result.success; - } - if (result.sec != null) { - throw result.sec; - } - if (result.tnase != null) { - throw result.tnase; - } - throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getRunningCompactions failed: unknown result"); - } - @Override public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { @@ -536,44 +501,6 @@ public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.Thrift } } - @Override - public void getRunningCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - checkReady(); - getRunningCompactions_call method_call = new getRunningCompactions_call(tinfo, credentials, resultHandler, this, ___protocolFactory, ___transport); - this.___currentMethod = method_call; - ___manager.call(method_call); - } - - public static class getRunningCompactions_call extends org.apache.thrift.async.TAsyncMethodCall { - private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; - private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; - public getRunningCompactions_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { - super(client, protocolFactory, transport, resultHandler, false); - this.tinfo = tinfo; - this.credentials = credentials; - } - - @Override - public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { - prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getRunningCompactions", org.apache.thrift.protocol.TMessageType.CALL, 0)); - getRunningCompactions_args args = new getRunningCompactions_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.write(prot); - prot.writeMessageEnd(); - } - - @Override - public TExternalCompactionMap getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { - if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { - throw new java.lang.IllegalStateException("Method call not finished!"); - } - org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); - org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); - return (new Client(prot)).recv_getRunningCompactions(); - } - } - @Override public void getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); @@ -712,7 +639,6 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { - public getRunningCompactions() { - super("getRunningCompactions"); - } - - @Override - public getRunningCompactions_args getEmptyArgsInstance() { - return new getRunningCompactions_args(); - } - - @Override - protected boolean isOneway() { - return false; - } - - @Override - protected boolean rethrowUnhandledExceptions() { - return false; - } - - @Override - public getRunningCompactions_result getResult(I iface, getRunningCompactions_args args) throws org.apache.thrift.TException { - getRunningCompactions_result result = new getRunningCompactions_result(); - try { - result.success = iface.getRunningCompactions(args.tinfo, args.credentials); - } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { - result.sec = sec; - } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - result.tnase = tnase; - } - return result; - } - } - public static class getCompletedCompactions extends org.apache.thrift.ProcessFunction { public getCompletedCompactions() { super("getCompletedCompactions"); @@ -1001,7 +893,6 @@ protected AsyncProcessor(I iface, java.util.Map extends org.apache.thrift.AsyncProcessFunction { - public getRunningCompactions() { - super("getRunningCompactions"); - } - - @Override - public getRunningCompactions_args getEmptyArgsInstance() { - return new getRunningCompactions_args(); - } - - @Override - public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { - final org.apache.thrift.AsyncProcessFunction fcall = this; - return new org.apache.thrift.async.AsyncMethodCallback() { - @Override - public void onComplete(TExternalCompactionMap o) { - getRunningCompactions_result result = new getRunningCompactions_result(); - result.success = o; - try { - fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); - } catch (org.apache.thrift.transport.TTransportException e) { - _LOGGER.error("TTransportException writing to internal frame buffer", e); - fb.close(); - } catch (java.lang.Exception e) { - _LOGGER.error("Exception writing to internal frame buffer", e); - onError(e); - } - } - @Override - public void onError(java.lang.Exception e) { - byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; - org.apache.thrift.TSerializable msg; - getRunningCompactions_result result = new getRunningCompactions_result(); - if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { - result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; - result.setSecIsSet(true); - msg = result; - } else if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) { - result.tnase = (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) e; - result.setTnaseIsSet(true); - msg = result; - } else if (e instanceof org.apache.thrift.transport.TTransportException) { - _LOGGER.error("TTransportException inside handler", e); - fb.close(); - return; - } else if (e instanceof org.apache.thrift.TApplicationException) { - _LOGGER.error("TApplicationException inside handler", e); - msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; - msg = (org.apache.thrift.TApplicationException)e; - } else { - _LOGGER.error("Exception inside handler", e); - msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; - msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); - } - try { - fcall.sendResponse(fb,msg,msgType,seqid); - } catch (java.lang.Exception ex) { - _LOGGER.error("Exception writing to internal frame buffer", ex); - fb.close(); - } - } - }; - } - - @Override - protected boolean isOneway() { - return false; - } - - @Override - public void start(I iface, getRunningCompactions_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.getRunningCompactions(args.tinfo, args.credentials,resultHandler); - } - } - public static class getCompletedCompactions extends org.apache.thrift.AsyncProcessFunction { public getCompletedCompactions() { super("getCompletedCompactions"); @@ -7001,1097 +6817,6 @@ private static S scheme(org.apache. } } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class getRunningCompactions_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRunningCompactions_args"); - - private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getRunningCompactions_argsStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getRunningCompactions_argsTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - TINFO((short)1, "tinfo"), - CREDENTIALS((short)2, "credentials"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // TINFO - return TINFO; - case 2: // CREDENTIALS - return CREDENTIALS; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); - tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRunningCompactions_args.class, metaDataMap); - } - - public getRunningCompactions_args() { - } - - public getRunningCompactions_args( - org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, - org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) - { - this(); - this.tinfo = tinfo; - this.credentials = credentials; - } - - /** - * Performs a deep copy on other. - */ - public getRunningCompactions_args(getRunningCompactions_args other) { - if (other.isSetTinfo()) { - this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); - } - if (other.isSetCredentials()) { - this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); - } - } - - @Override - public getRunningCompactions_args deepCopy() { - return new getRunningCompactions_args(this); - } - - @Override - public void clear() { - this.tinfo = null; - this.credentials = null; - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { - return this.tinfo; - } - - public getRunningCompactions_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { - this.tinfo = tinfo; - return this; - } - - public void unsetTinfo() { - this.tinfo = null; - } - - /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ - public boolean isSetTinfo() { - return this.tinfo != null; - } - - public void setTinfoIsSet(boolean value) { - if (!value) { - this.tinfo = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { - return this.credentials; - } - - public getRunningCompactions_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { - this.credentials = credentials; - return this; - } - - public void unsetCredentials() { - this.credentials = null; - } - - /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ - public boolean isSetCredentials() { - return this.credentials != null; - } - - public void setCredentialsIsSet(boolean value) { - if (!value) { - this.credentials = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case TINFO: - if (value == null) { - unsetTinfo(); - } else { - setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); - } - break; - - case CREDENTIALS: - if (value == null) { - unsetCredentials(); - } else { - setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case TINFO: - return getTinfo(); - - case CREDENTIALS: - return getCredentials(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case TINFO: - return isSetTinfo(); - case CREDENTIALS: - return isSetCredentials(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof getRunningCompactions_args) - return this.equals((getRunningCompactions_args)that); - return false; - } - - public boolean equals(getRunningCompactions_args that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_tinfo = true && this.isSetTinfo(); - boolean that_present_tinfo = true && that.isSetTinfo(); - if (this_present_tinfo || that_present_tinfo) { - if (!(this_present_tinfo && that_present_tinfo)) - return false; - if (!this.tinfo.equals(that.tinfo)) - return false; - } - - boolean this_present_credentials = true && this.isSetCredentials(); - boolean that_present_credentials = true && that.isSetCredentials(); - if (this_present_credentials || that_present_credentials) { - if (!(this_present_credentials && that_present_credentials)) - return false; - if (!this.credentials.equals(that.credentials)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); - if (isSetTinfo()) - hashCode = hashCode * 8191 + tinfo.hashCode(); - - hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); - if (isSetCredentials()) - hashCode = hashCode * 8191 + credentials.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(getRunningCompactions_args other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTinfo()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCredentials()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("getRunningCompactions_args("); - boolean first = true; - - sb.append("tinfo:"); - if (this.tinfo == null) { - sb.append("null"); - } else { - sb.append(this.tinfo); - } - first = false; - if (!first) sb.append(", "); - sb.append("credentials:"); - if (this.credentials == null) { - sb.append("null"); - } else { - sb.append(this.credentials); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (tinfo != null) { - tinfo.validate(); - } - if (credentials != null) { - credentials.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class getRunningCompactions_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getRunningCompactions_argsStandardScheme getScheme() { - return new getRunningCompactions_argsStandardScheme(); - } - } - - private static class getRunningCompactions_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, getRunningCompactions_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // TINFO - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // CREDENTIALS - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, getRunningCompactions_args struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.tinfo != null) { - oprot.writeFieldBegin(TINFO_FIELD_DESC); - struct.tinfo.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.credentials != null) { - oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); - struct.credentials.write(oprot); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class getRunningCompactions_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getRunningCompactions_argsTupleScheme getScheme() { - return new getRunningCompactions_argsTupleScheme(); - } - } - - private static class getRunningCompactions_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, getRunningCompactions_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetTinfo()) { - optionals.set(0); - } - if (struct.isSetCredentials()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); - if (struct.isSetTinfo()) { - struct.tinfo.write(oprot); - } - if (struct.isSetCredentials()) { - struct.credentials.write(oprot); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, getRunningCompactions_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); - if (incoming.get(0)) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } - if (incoming.get(1)) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class getRunningCompactions_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getRunningCompactions_result"); - - private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); - private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getRunningCompactions_resultStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getRunningCompactions_resultTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable TExternalCompactionMap success; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - SUCCESS((short)0, "success"), - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 0: // SUCCESS - return SUCCESS; - case 1: // SEC - return SEC; - case 2: // TNASE - return TNASE; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExternalCompactionMap.class))); - tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getRunningCompactions_result.class, metaDataMap); - } - - public getRunningCompactions_result() { - } - - public getRunningCompactions_result( - TExternalCompactionMap success, - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) - { - this(); - this.success = success; - this.sec = sec; - this.tnase = tnase; - } - - /** - * Performs a deep copy on other. - */ - public getRunningCompactions_result(getRunningCompactions_result other) { - if (other.isSetSuccess()) { - this.success = new TExternalCompactionMap(other.success); - } - if (other.isSetSec()) { - this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); - } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } - } - - @Override - public getRunningCompactions_result deepCopy() { - return new getRunningCompactions_result(this); - } - - @Override - public void clear() { - this.success = null; - this.sec = null; - this.tnase = null; - } - - @org.apache.thrift.annotation.Nullable - public TExternalCompactionMap getSuccess() { - return this.success; - } - - public getRunningCompactions_result setSuccess(@org.apache.thrift.annotation.Nullable TExternalCompactionMap success) { - this.success = success; - return this; - } - - public void unsetSuccess() { - this.success = null; - } - - /** Returns true if field success is set (has been assigned a value) and false otherwise */ - public boolean isSetSuccess() { - return this.success != null; - } - - public void setSuccessIsSet(boolean value) { - if (!value) { - this.success = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { - return this.sec; - } - - public getRunningCompactions_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { - this.sec = sec; - return this; - } - - public void unsetSec() { - this.sec = null; - } - - /** Returns true if field sec is set (has been assigned a value) and false otherwise */ - public boolean isSetSec() { - return this.sec != null; - } - - public void setSecIsSet(boolean value) { - if (!value) { - this.sec = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public getRunningCompactions_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case SUCCESS: - if (value == null) { - unsetSuccess(); - } else { - setSuccess((TExternalCompactionMap)value); - } - break; - - case SEC: - if (value == null) { - unsetSec(); - } else { - setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); - } - break; - - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case SUCCESS: - return getSuccess(); - - case SEC: - return getSec(); - - case TNASE: - return getTnase(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case SUCCESS: - return isSetSuccess(); - case SEC: - return isSetSec(); - case TNASE: - return isSetTnase(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof getRunningCompactions_result) - return this.equals((getRunningCompactions_result)that); - return false; - } - - public boolean equals(getRunningCompactions_result that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_success = true && this.isSetSuccess(); - boolean that_present_success = true && that.isSetSuccess(); - if (this_present_success || that_present_success) { - if (!(this_present_success && that_present_success)) - return false; - if (!this.success.equals(that.success)) - return false; - } - - boolean this_present_sec = true && this.isSetSec(); - boolean that_present_sec = true && that.isSetSec(); - if (this_present_sec || that_present_sec) { - if (!(this_present_sec && that_present_sec)) - return false; - if (!this.sec.equals(that.sec)) - return false; - } - - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetSuccess()) ? 131071 : 524287); - if (isSetSuccess()) - hashCode = hashCode * 8191 + success.hashCode(); - - hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); - if (isSetSec()) - hashCode = hashCode * 8191 + sec.hashCode(); - - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(getRunningCompactions_result other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetSuccess(), other.isSetSuccess()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetSuccess()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetSec()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("getRunningCompactions_result("); - boolean first = true; - - sb.append("success:"); - if (this.success == null) { - sb.append("null"); - } else { - sb.append(this.success); - } - first = false; - if (!first) sb.append(", "); - sb.append("sec:"); - if (this.sec == null) { - sb.append("null"); - } else { - sb.append(this.sec); - } - first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (success != null) { - success.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class getRunningCompactions_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getRunningCompactions_resultStandardScheme getScheme() { - return new getRunningCompactions_resultStandardScheme(); - } - } - - private static class getRunningCompactions_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, getRunningCompactions_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 0: // SUCCESS - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.success = new TExternalCompactionMap(); - struct.success.read(iprot); - struct.setSuccessIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 1: // SEC - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); - struct.sec.read(iprot); - struct.setSecIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, getRunningCompactions_result struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.success != null) { - oprot.writeFieldBegin(SUCCESS_FIELD_DESC); - struct.success.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.sec != null) { - oprot.writeFieldBegin(SEC_FIELD_DESC); - struct.sec.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class getRunningCompactions_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public getRunningCompactions_resultTupleScheme getScheme() { - return new getRunningCompactions_resultTupleScheme(); - } - } - - private static class getRunningCompactions_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, getRunningCompactions_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetSuccess()) { - optionals.set(0); - } - if (struct.isSetSec()) { - optionals.set(1); - } - if (struct.isSetTnase()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); - if (struct.isSetSuccess()) { - struct.success.write(oprot); - } - if (struct.isSetSec()) { - struct.sec.write(oprot); - } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, getRunningCompactions_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); - if (incoming.get(0)) { - struct.success = new TExternalCompactionMap(); - struct.success.read(iprot); - struct.setSuccessIsSet(true); - } - if (incoming.get(1)) { - struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); - struct.sec.read(iprot); - struct.setSecIsSet(true); - } - if (incoming.get(2)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) public static class getCompletedCompactions_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getCompletedCompactions_args"); diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index d583978d20f..31999683b93 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -132,17 +132,6 @@ service CompactionCoordinatorService { 2:client.ThriftNotActiveServiceException tnase ) - /* - * Called by the Monitor to get progress information - */ - TExternalCompactionMap getRunningCompactions( - 1:client.TInfo tinfo - 2:security.TCredentials credentials - )throws( - 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase - ) - /* * Called by the Monitor to get progress information */ diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListCompactions.java b/server/base/src/main/java/org/apache/accumulo/server/util/ListCompactions.java index 5d9547f860b..635339f842c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListCompactions.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListCompactions.java @@ -21,17 +21,14 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.apache.accumulo.core.cli.ServerOpts; import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; -import org.apache.accumulo.core.compaction.thrift.TExternalCompactionMap; import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.tabletserver.thrift.TCompactionKind; -import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.core.util.compaction.RunningCompactionInfo; import org.apache.accumulo.server.ServerContext; @@ -190,25 +187,15 @@ protected List getRunningCompactions(ServerContext con coordinatorClient = ExternalCompactionUtil.getCoordinatorClient(context); // Fetch running compactions as a list and convert to a map - TExternalCompactionMap running = - coordinatorClient.getRunningCompactions(TraceUtil.traceInfo(), context.rpcCreds()); - List results = new ArrayList<>(); + ExternalCompactionUtil.getCompactionsRunningOnCompactors(context, + tec -> results.add(new RunningCompactionSummary(tec, details))); - if (running == null || running.getCompactions() == null - || running.getCompactions().isEmpty()) { + if (results.isEmpty()) { System.out.println("No running compactions found."); return results; } - for (Map.Entry entry : running.getCompactions().entrySet()) { - TExternalCompaction ec = entry.getValue(); - if (ec == null) { - continue; - } - var summary = new RunningCompactionSummary(ec, details); - results.add(summary); - } return results; } catch (Exception e) { throw new IllegalStateException("Unable to get running compactions.", e); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index aee9679cc01..dcda3f8f846 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -1028,30 +1028,6 @@ protected Set readExternalCompactionIds() { } } - /** - * Return information about running compactions - * - * @param tinfo trace info - * @param credentials tcredentials object - * @return map of ECID to TExternalCompaction objects - * @throws ThriftSecurityException permission error - */ - @Override - public TExternalCompactionMap getRunningCompactions(TInfo tinfo, TCredentials credentials) - throws ThriftSecurityException { - // do not expect users to call this directly, expect other tservers to call this method - if (!security.canPerformSystemActions(credentials)) { - throw new AccumuloSecurityException(credentials.getPrincipal(), - SecurityErrorCode.PERMISSION_DENIED).asThriftException(); - } - - final TExternalCompactionMap result = new TExternalCompactionMap(); - RUNNING_CACHE.forEach((ecid, tec) -> { - result.putToCompactions(ecid.canonical(), tec); - }); - return result; - } - /** * Return information about recently completed compactions * diff --git a/test/src/main/java/org/apache/accumulo/test/ListCompactionsIT.java b/test/src/main/java/org/apache/accumulo/test/ListCompactionsIT.java index 6c57190e5ca..56dcc135aeb 100644 --- a/test/src/main/java/org/apache/accumulo/test/ListCompactionsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ListCompactionsIT.java @@ -31,14 +31,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.compaction.thrift.TExternalCompactionMap; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; -import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.harness.MiniClusterConfigurationCallback; import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; @@ -52,7 +49,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import com.google.common.net.HostAndPort; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; @@ -100,16 +96,13 @@ public void testListRunningCompactions() throws Exception { writeData(client, tableName); compact(client, tableName, 2, GROUP7, false); - Optional coordinatorHost = - ExternalCompactionUtil.findCompactionCoordinator(getCluster().getServerContext()); - // wait for the compaction to start - TExternalCompactionMap expected = ExternalCompactionTestUtils - .getRunningCompactions(getCluster().getServerContext(), coordinatorHost); - while (expected == null || expected.getCompactionsSize() == 0) { + var expected = + ExternalCompactionTestUtils.getRunningCompactions(getCluster().getServerContext()); + while (expected.isEmpty()) { Thread.sleep(1000); - expected = ExternalCompactionTestUtils - .getRunningCompactions(getCluster().getServerContext(), coordinatorHost); + expected = + ExternalCompactionTestUtils.getRunningCompactions(getCluster().getServerContext()); } final List running = @@ -117,8 +110,8 @@ public void testListRunningCompactions() throws Exception { final Map compactionsByEcid = new HashMap<>(); running.forEach(rcs -> compactionsByEcid.put(rcs.getEcid(), rcs)); - assertEquals(expected.getCompactionsSize(), compactionsByEcid.size()); - expected.getCompactions().values().forEach(tec -> { + assertEquals(expected.size(), compactionsByEcid.size()); + expected.values().forEach(tec -> { RunningCompactionSummary rcs = compactionsByEcid.get(tec.job.getExternalCompactionId()); assertNotNull(rcs); assertEquals(tec.getJob().getExternalCompactionId(), rcs.getEcid()); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java index 073406c1477..531fd30751b 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java @@ -37,7 +37,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -49,13 +48,11 @@ import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; -import org.apache.accumulo.core.compaction.thrift.TExternalCompactionMap; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.iterators.IteratorUtil; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; -import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.core.util.compaction.RunningCompactionInfo; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.harness.AccumuloClusterHarness; @@ -67,7 +64,6 @@ import org.apache.accumulo.test.util.Wait; import org.apache.hadoop.conf.Configuration; import org.apache.thrift.TException; -import org.apache.thrift.transport.TTransportException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -75,8 +71,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.net.HostAndPort; - /** * Tests that external compactions report progress from start to finish. To prevent flaky test * failures, we only measure progress in quarter segments: STARTED, QUARTER, HALF, THREE_QUARTERS. @@ -327,13 +321,8 @@ public Thread startChecker() { */ private void checkRunning() throws TException { ServerContext ctx = getCluster().getServerContext(); - Optional coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(ctx); - if (coordinatorHost.isEmpty()) { - throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper"); - } - TExternalCompactionMap ecList = getRunningCompactions(ctx, coordinatorHost); - Map ecMap = ecList.getCompactions(); + Map ecMap = getRunningCompactions(ctx); if (ecMap != null) { ecMap.forEach((ecid, ec) -> { // returns null if it's a new mapping diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java index 457abf588f6..ae18692b32e 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -79,7 +80,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.io.Text; -import org.apache.thrift.TException; import org.apache.thrift.transport.TTransportException; import com.beust.jcommander.internal.Maps; @@ -252,16 +252,14 @@ public static void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuratio coreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); } - public static TExternalCompactionMap getRunningCompactions(ClientContext context, - Optional coordinatorHost) throws TException { - CompactionCoordinatorService.Client client = - ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost.orElseThrow(), context); + public static Map getRunningCompactions(ClientContext context) { + Map running = new HashMap<>(); try { - TExternalCompactionMap running = - client.getRunningCompactions(TraceUtil.traceInfo(), context.rpcCreds()); + ExternalCompactionUtil.getCompactionsRunningOnCompactors(context, + tec -> running.put(tec.getJob().getExternalCompactionId(), tec)); return running; - } finally { - ThriftUtil.returnClient(client, context); + } catch (InterruptedException e) { + throw new IllegalStateException(e); } } @@ -331,15 +329,12 @@ public static int confirmCompactionRunning(ServerContext ctx, Set ecids.contains(ExternalCompactionId.of(e)))) { - running = ExternalCompactionTestUtils.getRunningCompactions(ctx, coordinatorHost); + var running = ExternalCompactionTestUtils.getRunningCompactions(ctx); + while (running.keySet().stream().anyMatch((e) -> ecids.contains(ExternalCompactionId.of(e)))) { + running = ExternalCompactionTestUtils.getRunningCompactions(ctx); } // The compaction should be in the completed list with the expected state TExternalCompactionMap completed = diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java index 9c855ba390d..ed963511a20 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java @@ -45,7 +45,6 @@ import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate; import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; -import org.apache.accumulo.core.compaction.thrift.TExternalCompactionMap; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; @@ -232,8 +231,8 @@ private Map getRunningCompactionInfo final Map results = new HashMap<>(); while (results.isEmpty()) { - TExternalCompactionMap running = null; - while (running == null || running.getCompactions() == null) { + Map running = null; + while (running == null || running.isEmpty()) { try { Optional coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(ctx); @@ -241,14 +240,14 @@ private Map getRunningCompactionInfo throw new TTransportException( "Unable to get CompactionCoordinator address from ZooKeeper"); } - running = getRunningCompactions(ctx, coordinatorHost); + running = getRunningCompactions(ctx); } catch (TException t) { running = null; Thread.sleep(2000); } } for (ExternalCompactionId ecid : ecids) { - final TExternalCompaction tec = running.getCompactions().get(ecid.canonical()); + final TExternalCompaction tec = running.get(ecid.canonical()); if (tec != null && tec.getUpdatesSize() > 0) { // When the coordinator restarts it inserts a message into the updates. If this // is the last message, then don't insert this into the results. We want to get diff --git a/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java b/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java index 289b2c06a5c..338a75fcbc8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/GracefulShutdownIT.java @@ -239,13 +239,9 @@ public void testGracefulShutdown() throws Exception { client.instanceOperations().getServers(ServerId.Type.MANAGER); assertNotNull(newManagerLocations); assertEquals(1, newManagerLocations.size()); - final HostAndPort newManagerAddress = - HostAndPort.fromString(newManagerLocations.iterator().next().toHostPortString()); - assertEquals(0, ExternalCompactionTestUtils - .getRunningCompactions(ctx, Optional.of(newManagerAddress)).getCompactionsSize()); + assertEquals(0, ExternalCompactionTestUtils.getRunningCompactions(ctx).size()); client.tableOperations().compact(tableName, cc); - Wait.waitFor(() -> ExternalCompactionTestUtils - .getRunningCompactions(ctx, Optional.of(newManagerAddress)).getCompactionsSize() > 0); + Wait.waitFor(() -> !ExternalCompactionTestUtils.getRunningCompactions(ctx).isEmpty()); StopServers.signalGracefulShutdown(ctx, compactorAddress); Wait.waitFor(() -> { control.refreshProcesses(ServerType.COMPACTOR); From 88db7a3ff0e7ef3259474a489372d51e674053ce Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 24 Mar 2026 22:26:06 +0000 Subject: [PATCH 16/65] Removes coordinator cancel RPC Modified the command that calls this to instead reach out directly to the compactor. This change is made in support of #6217, it removes a useage of the running cahce and it simplifies the coordinators RPCs. --- .../thrift/CompactionCoordinatorService.java | 1271 ----------------- .../main/thrift/compaction-coordinator.thrift | 9 - .../server/util/CancelCompaction.java | 37 +- .../coordinator/CompactionCoordinator.java | 24 - 4 files changed, 28 insertions(+), 1313 deletions(-) diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index 734d7f9027c..19bcece45ff 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -41,8 +41,6 @@ public interface Iface { public TExternalCompactionMap getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException; } @@ -61,8 +59,6 @@ public interface AsyncIface { public void getCompletedCompactions(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; } @@ -279,35 +275,6 @@ public TExternalCompactionMap recv_getCompletedCompactions() throws org.apache.a throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getCompletedCompactions failed: unknown result"); } - @Override - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException - { - send_cancel(tinfo, credentials, externalCompactionId); - recv_cancel(); - } - - public void send_cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException - { - cancel_args args = new cancel_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.setExternalCompactionId(externalCompactionId); - sendBase("cancel", args); - } - - public void recv_cancel() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException - { - cancel_result result = new cancel_result(); - receiveBase(result, "cancel"); - if (result.sec != null) { - throw result.sec; - } - if (result.tnase != null) { - throw result.tnase; - } - return; - } - @Override public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException { @@ -612,48 +579,6 @@ public TExternalCompactionMap getResult() throws org.apache.accumulo.core.client } } - @Override - public void cancel(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - checkReady(); - cancel_call method_call = new cancel_call(tinfo, credentials, externalCompactionId, resultHandler, this, ___protocolFactory, ___transport); - this.___currentMethod = method_call; - ___manager.call(method_call); - } - - public static class cancel_call extends org.apache.thrift.async.TAsyncMethodCall { - private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; - private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; - private java.lang.String externalCompactionId; - public cancel_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { - super(client, protocolFactory, transport, resultHandler, false); - this.tinfo = tinfo; - this.credentials = credentials; - this.externalCompactionId = externalCompactionId; - } - - @Override - public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { - prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("cancel", org.apache.thrift.protocol.TMessageType.CALL, 0)); - cancel_args args = new cancel_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.setExternalCompactionId(externalCompactionId); - args.write(prot); - prot.writeMessageEnd(); - } - - @Override - public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException { - if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { - throw new java.lang.IllegalStateException("Method call not finished!"); - } - org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); - org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); - (new Client(prot)).recv_cancel(); - return null; - } - } - @Override public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); @@ -714,7 +639,6 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { - public cancel() { - super("cancel"); - } - - @Override - public cancel_args getEmptyArgsInstance() { - return new cancel_args(); - } - - @Override - protected boolean isOneway() { - return false; - } - - @Override - protected boolean rethrowUnhandledExceptions() { - return false; - } - - @Override - public cancel_result getResult(I iface, cancel_args args) throws org.apache.thrift.TException { - cancel_result result = new cancel_result(); - try { - iface.cancel(args.tinfo, args.credentials, args.externalCompactionId); - } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { - result.sec = sec; - } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - result.tnase = tnase; - } - return result; - } - } - public static class recordCompletion extends org.apache.thrift.ProcessFunction { public recordCompletion() { super("recordCompletion"); @@ -1003,7 +893,6 @@ protected AsyncProcessor(I iface, java.util.Map extends org.apache.thrift.AsyncProcessFunction { - public cancel() { - super("cancel"); - } - - @Override - public cancel_args getEmptyArgsInstance() { - return new cancel_args(); - } - - @Override - public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { - final org.apache.thrift.AsyncProcessFunction fcall = this; - return new org.apache.thrift.async.AsyncMethodCallback() { - @Override - public void onComplete(Void o) { - cancel_result result = new cancel_result(); - try { - fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); - } catch (org.apache.thrift.transport.TTransportException e) { - _LOGGER.error("TTransportException writing to internal frame buffer", e); - fb.close(); - } catch (java.lang.Exception e) { - _LOGGER.error("Exception writing to internal frame buffer", e); - onError(e); - } - } - @Override - public void onError(java.lang.Exception e) { - byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; - org.apache.thrift.TSerializable msg; - cancel_result result = new cancel_result(); - if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { - result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; - result.setSecIsSet(true); - msg = result; - } else if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) { - result.tnase = (org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException) e; - result.setTnaseIsSet(true); - msg = result; - } else if (e instanceof org.apache.thrift.transport.TTransportException) { - _LOGGER.error("TTransportException inside handler", e); - fb.close(); - return; - } else if (e instanceof org.apache.thrift.TApplicationException) { - _LOGGER.error("TApplicationException inside handler", e); - msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; - msg = (org.apache.thrift.TApplicationException)e; - } else { - _LOGGER.error("Exception inside handler", e); - msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; - msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); - } - try { - fcall.sendResponse(fb,msg,msgType,seqid); - } catch (java.lang.Exception ex) { - _LOGGER.error("Exception writing to internal frame buffer", ex); - fb.close(); - } - } - }; - } - - @Override - protected boolean isOneway() { - return false; - } - - @Override - public void start(I iface, cancel_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.cancel(args.tinfo, args.credentials, args.externalCompactionId,resultHandler); - } - } - public static class recordCompletion extends org.apache.thrift.AsyncProcessFunction { public recordCompletion() { super("recordCompletion"); @@ -9183,1092 +8998,6 @@ private static S scheme(org.apache. } } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class cancel_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("cancel_args"); - - private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); - private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new cancel_argsStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new cancel_argsTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required - public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - TINFO((short)1, "tinfo"), - CREDENTIALS((short)2, "credentials"), - EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // TINFO - return TINFO; - case 2: // CREDENTIALS - return CREDENTIALS; - case 3: // EXTERNAL_COMPACTION_ID - return EXTERNAL_COMPACTION_ID; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); - tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); - tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("externalCompactionId", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(cancel_args.class, metaDataMap); - } - - public cancel_args() { - } - - public cancel_args( - org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, - org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, - java.lang.String externalCompactionId) - { - this(); - this.tinfo = tinfo; - this.credentials = credentials; - this.externalCompactionId = externalCompactionId; - } - - /** - * Performs a deep copy on other. - */ - public cancel_args(cancel_args other) { - if (other.isSetTinfo()) { - this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); - } - if (other.isSetCredentials()) { - this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); - } - if (other.isSetExternalCompactionId()) { - this.externalCompactionId = other.externalCompactionId; - } - } - - @Override - public cancel_args deepCopy() { - return new cancel_args(this); - } - - @Override - public void clear() { - this.tinfo = null; - this.credentials = null; - this.externalCompactionId = null; - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { - return this.tinfo; - } - - public cancel_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { - this.tinfo = tinfo; - return this; - } - - public void unsetTinfo() { - this.tinfo = null; - } - - /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ - public boolean isSetTinfo() { - return this.tinfo != null; - } - - public void setTinfoIsSet(boolean value) { - if (!value) { - this.tinfo = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { - return this.credentials; - } - - public cancel_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { - this.credentials = credentials; - return this; - } - - public void unsetCredentials() { - this.credentials = null; - } - - /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ - public boolean isSetCredentials() { - return this.credentials != null; - } - - public void setCredentialsIsSet(boolean value) { - if (!value) { - this.credentials = null; - } - } - - @org.apache.thrift.annotation.Nullable - public java.lang.String getExternalCompactionId() { - return this.externalCompactionId; - } - - public cancel_args setExternalCompactionId(@org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId) { - this.externalCompactionId = externalCompactionId; - return this; - } - - public void unsetExternalCompactionId() { - this.externalCompactionId = null; - } - - /** Returns true if field externalCompactionId is set (has been assigned a value) and false otherwise */ - public boolean isSetExternalCompactionId() { - return this.externalCompactionId != null; - } - - public void setExternalCompactionIdIsSet(boolean value) { - if (!value) { - this.externalCompactionId = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case TINFO: - if (value == null) { - unsetTinfo(); - } else { - setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); - } - break; - - case CREDENTIALS: - if (value == null) { - unsetCredentials(); - } else { - setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); - } - break; - - case EXTERNAL_COMPACTION_ID: - if (value == null) { - unsetExternalCompactionId(); - } else { - setExternalCompactionId((java.lang.String)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case TINFO: - return getTinfo(); - - case CREDENTIALS: - return getCredentials(); - - case EXTERNAL_COMPACTION_ID: - return getExternalCompactionId(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case TINFO: - return isSetTinfo(); - case CREDENTIALS: - return isSetCredentials(); - case EXTERNAL_COMPACTION_ID: - return isSetExternalCompactionId(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof cancel_args) - return this.equals((cancel_args)that); - return false; - } - - public boolean equals(cancel_args that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_tinfo = true && this.isSetTinfo(); - boolean that_present_tinfo = true && that.isSetTinfo(); - if (this_present_tinfo || that_present_tinfo) { - if (!(this_present_tinfo && that_present_tinfo)) - return false; - if (!this.tinfo.equals(that.tinfo)) - return false; - } - - boolean this_present_credentials = true && this.isSetCredentials(); - boolean that_present_credentials = true && that.isSetCredentials(); - if (this_present_credentials || that_present_credentials) { - if (!(this_present_credentials && that_present_credentials)) - return false; - if (!this.credentials.equals(that.credentials)) - return false; - } - - boolean this_present_externalCompactionId = true && this.isSetExternalCompactionId(); - boolean that_present_externalCompactionId = true && that.isSetExternalCompactionId(); - if (this_present_externalCompactionId || that_present_externalCompactionId) { - if (!(this_present_externalCompactionId && that_present_externalCompactionId)) - return false; - if (!this.externalCompactionId.equals(that.externalCompactionId)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); - if (isSetTinfo()) - hashCode = hashCode * 8191 + tinfo.hashCode(); - - hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); - if (isSetCredentials()) - hashCode = hashCode * 8191 + credentials.hashCode(); - - hashCode = hashCode * 8191 + ((isSetExternalCompactionId()) ? 131071 : 524287); - if (isSetExternalCompactionId()) - hashCode = hashCode * 8191 + externalCompactionId.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(cancel_args other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTinfo()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCredentials()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetExternalCompactionId(), other.isSetExternalCompactionId()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetExternalCompactionId()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.externalCompactionId, other.externalCompactionId); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("cancel_args("); - boolean first = true; - - sb.append("tinfo:"); - if (this.tinfo == null) { - sb.append("null"); - } else { - sb.append(this.tinfo); - } - first = false; - if (!first) sb.append(", "); - sb.append("credentials:"); - if (this.credentials == null) { - sb.append("null"); - } else { - sb.append(this.credentials); - } - first = false; - if (!first) sb.append(", "); - sb.append("externalCompactionId:"); - if (this.externalCompactionId == null) { - sb.append("null"); - } else { - sb.append(this.externalCompactionId); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (tinfo != null) { - tinfo.validate(); - } - if (credentials != null) { - credentials.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class cancel_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public cancel_argsStandardScheme getScheme() { - return new cancel_argsStandardScheme(); - } - } - - private static class cancel_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, cancel_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // TINFO - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // CREDENTIALS - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 3: // EXTERNAL_COMPACTION_ID - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.externalCompactionId = iprot.readString(); - struct.setExternalCompactionIdIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, cancel_args struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.tinfo != null) { - oprot.writeFieldBegin(TINFO_FIELD_DESC); - struct.tinfo.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.credentials != null) { - oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); - struct.credentials.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.externalCompactionId != null) { - oprot.writeFieldBegin(EXTERNAL_COMPACTION_ID_FIELD_DESC); - oprot.writeString(struct.externalCompactionId); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class cancel_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public cancel_argsTupleScheme getScheme() { - return new cancel_argsTupleScheme(); - } - } - - private static class cancel_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, cancel_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetTinfo()) { - optionals.set(0); - } - if (struct.isSetCredentials()) { - optionals.set(1); - } - if (struct.isSetExternalCompactionId()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); - if (struct.isSetTinfo()) { - struct.tinfo.write(oprot); - } - if (struct.isSetCredentials()) { - struct.credentials.write(oprot); - } - if (struct.isSetExternalCompactionId()) { - oprot.writeString(struct.externalCompactionId); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, cancel_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); - if (incoming.get(0)) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } - if (incoming.get(1)) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } - if (incoming.get(2)) { - struct.externalCompactionId = iprot.readString(); - struct.setExternalCompactionIdIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class cancel_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("cancel_result"); - - private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField TNASE_FIELD_DESC = new org.apache.thrift.protocol.TField("tnase", org.apache.thrift.protocol.TType.STRUCT, (short)2); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new cancel_resultStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new cancel_resultTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - SEC((short)1, "sec"), - TNASE((short)2, "tnase"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // SEC - return SEC; - case 2: // TNASE - return TNASE; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); - tmpMap.put(_Fields.TNASE, new org.apache.thrift.meta_data.FieldMetaData("tnase", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException.class))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(cancel_result.class, metaDataMap); - } - - public cancel_result() { - } - - public cancel_result( - org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec, - org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) - { - this(); - this.sec = sec; - this.tnase = tnase; - } - - /** - * Performs a deep copy on other. - */ - public cancel_result(cancel_result other) { - if (other.isSetSec()) { - this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); - } - if (other.isSetTnase()) { - this.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(other.tnase); - } - } - - @Override - public cancel_result deepCopy() { - return new cancel_result(this); - } - - @Override - public void clear() { - this.sec = null; - this.tnase = null; - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { - return this.sec; - } - - public cancel_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { - this.sec = sec; - return this; - } - - public void unsetSec() { - this.sec = null; - } - - /** Returns true if field sec is set (has been assigned a value) and false otherwise */ - public boolean isSetSec() { - return this.sec != null; - } - - public void setSecIsSet(boolean value) { - if (!value) { - this.sec = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException getTnase() { - return this.tnase; - } - - public cancel_result setTnase(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException tnase) { - this.tnase = tnase; - return this; - } - - public void unsetTnase() { - this.tnase = null; - } - - /** Returns true if field tnase is set (has been assigned a value) and false otherwise */ - public boolean isSetTnase() { - return this.tnase != null; - } - - public void setTnaseIsSet(boolean value) { - if (!value) { - this.tnase = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case SEC: - if (value == null) { - unsetSec(); - } else { - setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); - } - break; - - case TNASE: - if (value == null) { - unsetTnase(); - } else { - setTnase((org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case SEC: - return getSec(); - - case TNASE: - return getTnase(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case SEC: - return isSetSec(); - case TNASE: - return isSetTnase(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof cancel_result) - return this.equals((cancel_result)that); - return false; - } - - public boolean equals(cancel_result that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_sec = true && this.isSetSec(); - boolean that_present_sec = true && that.isSetSec(); - if (this_present_sec || that_present_sec) { - if (!(this_present_sec && that_present_sec)) - return false; - if (!this.sec.equals(that.sec)) - return false; - } - - boolean this_present_tnase = true && this.isSetTnase(); - boolean that_present_tnase = true && that.isSetTnase(); - if (this_present_tnase || that_present_tnase) { - if (!(this_present_tnase && that_present_tnase)) - return false; - if (!this.tnase.equals(that.tnase)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); - if (isSetSec()) - hashCode = hashCode * 8191 + sec.hashCode(); - - hashCode = hashCode * 8191 + ((isSetTnase()) ? 131071 : 524287); - if (isSetTnase()) - hashCode = hashCode * 8191 + tnase.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(cancel_result other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetSec()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetTnase(), other.isSetTnase()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTnase()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tnase, other.tnase); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("cancel_result("); - boolean first = true; - - sb.append("sec:"); - if (this.sec == null) { - sb.append("null"); - } else { - sb.append(this.sec); - } - first = false; - if (!first) sb.append(", "); - sb.append("tnase:"); - if (this.tnase == null) { - sb.append("null"); - } else { - sb.append(this.tnase); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class cancel_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public cancel_resultStandardScheme getScheme() { - return new cancel_resultStandardScheme(); - } - } - - private static class cancel_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, cancel_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // SEC - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); - struct.sec.read(iprot); - struct.setSecIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // TNASE - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, cancel_result struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.sec != null) { - oprot.writeFieldBegin(SEC_FIELD_DESC); - struct.sec.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.tnase != null) { - oprot.writeFieldBegin(TNASE_FIELD_DESC); - struct.tnase.write(oprot); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class cancel_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public cancel_resultTupleScheme getScheme() { - return new cancel_resultTupleScheme(); - } - } - - private static class cancel_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, cancel_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetSec()) { - optionals.set(0); - } - if (struct.isSetTnase()) { - optionals.set(1); - } - oprot.writeBitSet(optionals, 2); - if (struct.isSetSec()) { - struct.sec.write(oprot); - } - if (struct.isSetTnase()) { - struct.tnase.write(oprot); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, cancel_result struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(2); - if (incoming.get(0)) { - struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); - struct.sec.read(iprot); - struct.setSecIsSet(true); - } - if (incoming.get(1)) { - struct.tnase = new org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException(); - struct.tnase.read(iprot); - struct.setTnaseIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) public static class recordCompletion_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("recordCompletion_args"); diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index d583978d20f..cd57197fd0d 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -154,15 +154,6 @@ service CompactionCoordinatorService { 2:client.ThriftNotActiveServiceException tnase ) - void cancel( - 1:client.TInfo tinfo - 2:security.TCredentials credentials - 3:string externalCompactionId - )throws( - 1:client.ThriftSecurityException sec - 2:client.ThriftNotActiveServiceException tnase - ) - oneway void recordCompletion( 1:client.TInfo tinfo 2:security.TCredentials credentials diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/CancelCompaction.java b/server/base/src/main/java/org/apache/accumulo/server/util/CancelCompaction.java index 546b4867545..ad1a2240a23 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/CancelCompaction.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/CancelCompaction.java @@ -18,11 +18,16 @@ */ package org.apache.accumulo.server.util; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP; + +import java.util.Map; +import java.util.Optional; + import org.apache.accumulo.core.cli.ServerOpts; import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.CompactionMetadata; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; -import org.apache.accumulo.core.rpc.ThriftUtil; -import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.util.CancelCompaction.CancelCommandOpts; @@ -33,6 +38,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.google.auto.service.AutoService; +import com.google.common.net.HostAndPort; @AutoService(KeywordExecutable.class) public class CancelCompaction extends ServerKeywordExecutable { @@ -63,15 +69,28 @@ public CommandGroup commandGroup() { protected void cancelCompaction(ServerContext context, String ecid) { CompactionCoordinatorService.Client coordinatorClient = null; - ecid = ExternalCompactionId.from(ecid).canonical(); - try { - coordinatorClient = ExternalCompactionUtil.getCoordinatorClient(context); - coordinatorClient.cancel(TraceUtil.traceInfo(), context.rpcCreds(), ecid); - System.out.println("Cancel sent to coordinator for " + ecid); + + try (var tablets = + context.getAmple().readTablets().forLevel(Ample.DataLevel.USER).fetch(ECOMP).build()) { + var cid = ExternalCompactionId.from(ecid); + System.out.println("Looking for " + ecid + " in metadata table"); + Optional> ecomp = + tablets.stream().flatMap(tm -> tm.getExternalCompactions().entrySet().stream()) + .filter(e -> e.getKey().equals(cid)).findFirst(); + + if (ecomp.isPresent()) { + var entry = ecomp.orElseThrow(); + System.out.println( + "Asking compactor " + entry.getValue().getCompactorId() + " to cancel " + ecid); + ExternalCompactionUtil.cancelCompaction(context, + HostAndPort.fromString(entry.getValue().getCompactorId()), entry.getKey().canonical()); + System.out + .println("Asked compactor " + entry.getValue().getCompactorId() + " to cancel " + ecid); + } else { + System.out.println("No compaction found for " + ecid); + } } catch (Exception e) { throw new IllegalStateException("Exception calling cancel compaction for " + ecid, e); - } finally { - ThriftUtil.returnClient(coordinatorClient, context); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 135369fe7ac..6cab8310c2d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -60,16 +60,12 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableDeletedException; -import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.compaction.CompactableFile; import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode; import org.apache.accumulo.core.clientImpl.thrift.TInfo; -import org.apache.accumulo.core.clientImpl.thrift.TableOperation; -import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType; import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; -import org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException; import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate; @@ -78,7 +74,6 @@ import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; @@ -1047,25 +1042,6 @@ public TExternalCompactionMap getCompletedCompactions(TInfo tinfo, TCredentials return result; } - @Override - public void cancel(TInfo tinfo, TCredentials credentials, String externalCompactionId) - throws TException { - var runningCompaction = RUNNING_CACHE.get(ExternalCompactionId.of(externalCompactionId)); - var extent = KeyExtent.fromThrift(runningCompaction.getJob().getExtent()); - try { - NamespaceId nsId = this.ctx.getNamespaceId(extent.tableId()); - if (!security.canCompact(credentials, extent.tableId(), nsId)) { - throw new AccumuloSecurityException(credentials.getPrincipal(), - SecurityErrorCode.PERMISSION_DENIED).asThriftException(); - } - } catch (TableNotFoundException e) { - throw new ThriftTableOperationException(extent.tableId().canonical(), null, - TableOperation.COMPACT_CANCEL, TableOperationExceptionType.NOTFOUND, e.getMessage()); - } - - cancelCompactionOnCompactor(runningCompaction.getCompactor(), externalCompactionId); - } - /* Method exists to be called from test */ public CompactionJobQueues getJobQueues() { return jobQueues; From e4fdfb8bed9b114fe205d61b58d84baccdb1fc9e Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Mon, 23 Mar 2026 11:43:39 -0700 Subject: [PATCH 17/65] fixes resource group IT (#6233) Resource group IT had two test that used the same resource group names and could see each others property changes. One of the test was recently and changed and this caused the other test to break. Modified the test to use unique names. --- .../test/conf/ResourceGroupConfigIT.java | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/conf/ResourceGroupConfigIT.java b/test/src/main/java/org/apache/accumulo/test/conf/ResourceGroupConfigIT.java index 4448c0dfed3..cc418195fe6 100644 --- a/test/src/main/java/org/apache/accumulo/test/conf/ResourceGroupConfigIT.java +++ b/test/src/main/java/org/apache/accumulo/test/conf/ResourceGroupConfigIT.java @@ -65,6 +65,7 @@ import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -96,6 +97,16 @@ public static void teardown() { SharedMiniClusterBase.stopMiniCluster(); } + @BeforeEach + public void beforeTest() throws Exception { + var rgops = getCluster().getServerContext().resourceGroupOperations(); + for (var rgid : rgops.list()) { + if (!rgid.equals(ResourceGroupId.DEFAULT)) { + rgops.remove(rgid); + } + } + } + @Test public void testProperties() throws Exception { @@ -321,9 +332,9 @@ public void testPermissions() throws Exception { @Test public void testMultipleProperties() throws Exception { - final String FIRST = "FIRST"; - final String SECOND = "SECOND"; - final String THIRD = "THIRD"; + final String FIRST = "FIRST_MP"; + final String SECOND = "SECOND_MP"; + final String THIRD = "THIRD_MP"; final ResourceGroupId first = ResourceGroupId.of(FIRST); final ResourceGroupId second = ResourceGroupId.of(SECOND); @@ -356,6 +367,10 @@ public void testMultipleProperties() throws Exception { rgops.create(second); rgops.create(third); + assertEquals(Map.of(), rgops.getProperties(first)); + assertEquals(Map.of(), rgops.getProperties(second)); + assertEquals(Map.of(), rgops.getProperties(third)); + rgops.modifyProperties(first, (map) -> map.putAll(firstProps)); rgops.modifyProperties(second, (map) -> map.putAll(secondProps)); rgops.modifyProperties(third, (map) -> map.putAll(thirdProps)); @@ -381,15 +396,18 @@ public void testMultipleProperties() throws Exception { assertEquals(defaultProps, rgops.getProperties(ResourceGroupId.DEFAULT)); assertThrows(ResourceGroupNotFoundException.class, ()->rgops.getProperties(third)); assertEquals(Set.of(ResourceGroupId.DEFAULT, first, second), rgops.list()); + + rgops.remove(first); + rgops.remove(second); } } @Test public void testMultipleConfigurations() throws Exception { - final String FIRST = "FIRST"; - final String SECOND = "SECOND"; - final String THIRD = "THIRD"; + final String FIRST = "FIRST_MC"; + final String SECOND = "SECOND_MC"; + final String THIRD = "THIRD_MC"; final ResourceGroupId first = ResourceGroupId.of(FIRST); final ResourceGroupId second = ResourceGroupId.of(SECOND); @@ -436,6 +454,10 @@ public void testMultipleConfigurations() throws Exception { rgops.create(second); rgops.create(third); + assertEquals(Map.of(), rgops.getProperties(first)); + assertEquals(Map.of(), rgops.getProperties(second)); + assertEquals(Map.of(), rgops.getProperties(third)); + getCluster().getConfig().getClusterServerConfiguration().setNumDefaultCompactors(1); getCluster().getConfig().getClusterServerConfiguration().addCompactorResourceGroup(FIRST, 1); getCluster().getConfig().getClusterServerConfiguration().addCompactorResourceGroup(SECOND, 1); @@ -497,6 +519,11 @@ public void testMultipleConfigurations() throws Exception { Wait.waitFor(() -> cc.getServerPaths() .getTabletServer(ResourceGroupPredicate.exact(third), AddressSelector.all(), true).size() == 0); + + rgops.remove(first); + rgops.remove(second); + rgops.remove(third); + } } From 131b1b605f66209fa5d2072351c754525bb970e3 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Tue, 24 Mar 2026 17:40:44 -0400 Subject: [PATCH 18/65] Compute compaction related suggestions in Monitor (#6239) Modified SystemInformation.finish to compute compaction related problems after all of the metrics have been gathered. Additionally centralized some duplicated code related to processing the value of the FMetric. --- .../coordinator/CompactionCoordinator.java | 46 --------------- .../compaction/CompactionCoordinatorTest.java | 4 -- .../monitor/next/SystemInformation.java | 56 ++++++++++++++++--- .../next/serializers/FMetricSerializer.java | 8 ++- .../serializers/MetricResponseSerializer.java | 11 +--- .../monitor/next/sservers/ScanServerView.java | 15 +---- 6 files changed, 56 insertions(+), 84 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index bd8630fc1c1..29d3d0cee1a 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -209,9 +209,6 @@ static FailureCounts incrementSuccess(Object key, FailureCounts counts) { protected final Map RUNNING_CACHE = new ConcurrentHashMap<>(); - /* Map of group name to last time compactor called to get a compaction job */ - private final Map TIME_COMPACTOR_LAST_CHECKED = new ConcurrentHashMap<>(); - private final ServerContext ctx; private final AuditedSecurityOperation security; private final CompactionJobQueues jobQueues; @@ -384,30 +381,6 @@ public void run() { LOG.info("Shutting down"); } - private Map> getIdleCompactors(Set runningCompactors) { - - final Map> allCompactors = new HashMap<>(); - runningCompactors.forEach((csi) -> allCompactors - .computeIfAbsent(csi.getResourceGroup().canonical(), (k) -> new HashSet<>()) - .add(HostAndPort.fromParts(csi.getHost(), csi.getPort()))); - - final Set emptyQueues = new HashSet<>(); - - // Remove all of the compactors that are running a compaction - RUNNING_CACHE.values().forEach(tec -> { - Set busyCompactors = allCompactors.get(tec.getGroupName()); - if (busyCompactors != null - && busyCompactors.remove(HostAndPort.fromString(tec.getCompactor()))) { - if (busyCompactors.isEmpty()) { - emptyQueues.add(tec.getGroupName()); - } - } - }); - // Remove entries with empty queues - emptyQueues.forEach(e -> allCompactors.remove(e)); - return allCompactors; - } - protected void startDeadCompactionDetector() { deadCompactionDetector.start(); } @@ -440,7 +413,6 @@ public TNextCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials } ResourceGroupId groupId = ResourceGroupId.of(groupName); LOG.trace("getCompactionJob called for group {} by compactor {}", groupId, compactorAddress); - TIME_COMPACTOR_LAST_CHECKED.put(groupId, System.currentTimeMillis()); TExternalCompactionJob result = null; @@ -1335,10 +1307,6 @@ public void cleanUpInternalState() { } }); - final Set trackedGroups = Set.copyOf(TIME_COMPACTOR_LAST_CHECKED.keySet()); - TIME_COMPACTOR_LAST_CHECKED.keySet().retainAll(groupsInConfiguration); - LOG.debug("No longer tracking compactor check-in times for groups: {}", - Sets.difference(trackedGroups, TIME_COMPACTOR_LAST_CHECKED.keySet())); } final Set runningCompactors = getRunningCompactors(); @@ -1367,19 +1335,5 @@ public void cleanUpInternalState() { compactorsWithNoGroups); } - final long now = System.currentTimeMillis(); - final long warningTime = getMissingCompactorWarningTime(); - Map> idleCompactors = getIdleCompactors(runningCompactors); - for (ResourceGroupId groupName : groupsInConfiguration) { - long lastCheckTime = - TIME_COMPACTOR_LAST_CHECKED.getOrDefault(groupName, coordinatorStartTime); - if ((now - lastCheckTime) > warningTime && jobQueues.getQueuedJobs(groupName) > 0 - && idleCompactors.containsKey(groupName.canonical())) { - LOG.warn( - "The group {} has queued jobs and {} idle compactors, however none have checked in " - + "with coordinator for {}ms", - groupName, idleCompactors.get(groupName.canonical()).size(), warningTime); - } - } } } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index a11e345a993..881cfa51ce3 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -39,7 +39,6 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.clientImpl.thrift.TInfo; @@ -370,11 +369,8 @@ public void testGetCompactionJobNoJobs() throws Exception { @Test public void testCleanUpRunning() throws Exception { TExternalCompaction ext1 = createMock(TExternalCompaction.class); - expect(ext1.getGroupName()).andReturn(Constants.DEFAULT_RESOURCE_GROUP_NAME).atLeastOnce(); TExternalCompaction ext2 = createMock(TExternalCompaction.class); - expect(ext2.getGroupName()).andReturn(Constants.DEFAULT_RESOURCE_GROUP_NAME).atLeastOnce(); TExternalCompaction ext3 = createMock(TExternalCompaction.class); - expect(ext3.getGroupName()).andReturn(Constants.DEFAULT_RESOURCE_GROUP_NAME).atLeastOnce(); replay(ext1, ext2, ext3); TestCoordinator coordinator = new TestCoordinator(manager, new ArrayList<>()); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index 36bf500a99e..27b22d17006 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -27,6 +27,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -50,6 +51,7 @@ import org.apache.accumulo.core.dataImpl.TabletIdImpl; import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.TabletState; +import org.apache.accumulo.core.metrics.Metric; import org.apache.accumulo.core.metrics.flatbuffers.FMetric; import org.apache.accumulo.core.metrics.flatbuffers.FTag; import org.apache.accumulo.core.process.thrift.MetricResponse; @@ -463,24 +465,18 @@ private void updateAggregates(final MetricResponse response, break; } } - double value = fm.dvalue(); - if (value == 0.0) { - value = fm.ivalue(); - if (value == 0.0) { - value = fm.lvalue(); - } - } + Number value = getMetricValue(fm); final Id id = new Id(name, (statisticTag == null) ? Tags.empty() : Tags.of(statisticTag.key(), statisticTag.value()), null, null, Type.valueOf(fm.type())); total .computeIfAbsent(id, (k) -> new CumulativeDistributionSummary(id, Clock.SYSTEM, DSC, 1.0, false)) - .record(value); + .record(value.doubleValue()); rgMetrics .computeIfAbsent(id, (k) -> new CumulativeDistributionSummary(id, Clock.SYSTEM, DSC, 1.0, false)) - .record(value); + .record(value.doubleValue()); }); } @@ -587,6 +583,36 @@ public void finish() { + " group " + balancerRG + ", but there are no TabletServers."); } } + for (String rg : getResourceGroups()) { + Set rgCompactors = getCompactorResourceGroupServers(rg); + List metrics = queueMetrics.get(rg); + Optional queued = metrics.stream() + .filter(fm -> fm.name().equals(Metric.COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED.getName())) + .findFirst(); + if (queued.isPresent()) { + Number numQueued = getMetricValue(queued.orElseThrow()); + if (numQueued.longValue() > 0) { + if (rgCompactors == null || rgCompactors.size() == 0) { + suggestions.add("Compactor group " + rg + " has " + numQueued.longValue() + + " queued compactions but no running compactors"); + } else { + // Check for idle compactors. + Map rgMetrics = + getCompactorResourceGroupMetricSummary(rg); + Optional> idleMetric = rgMetrics.entrySet() + .stream().filter(e -> e.getKey().getName().equals(Metric.SERVER_IDLE.getName())) + .findFirst(); + if (idleMetric.isPresent()) { + var metric = idleMetric.orElseThrow().getValue(); + if (metric.max() == 1.0D) { + suggestions.add("Compactor group " + rg + " has queued jobs and idle compactors."); + } + } + + } + } + } + } Set scanServers = new HashSet<>(); sservers.values().forEach(scanServers::addAll); int problemScanServerCount = (int) problemHosts.stream() @@ -692,4 +718,16 @@ public ScanServerView getScanServerView() { return this.scanServerView; } + public static Number getMetricValue(FMetric metric) { + if (metric.ivalue() != 0) { + return metric.ivalue(); + } + if (metric.lvalue() != 0L) { + return metric.lvalue(); + } + if (metric.dvalue() != 0.0d) { + return metric.dvalue(); + } + return 0; + } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/FMetricSerializer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/FMetricSerializer.java index 51823415d74..01219713dd7 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/FMetricSerializer.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/FMetricSerializer.java @@ -47,6 +47,12 @@ public void serialize(FMetric value, JsonGenerator gen, SerializerProvider seria gen.writeEndObject(); } gen.writeEndArray(); + serializeValueField(gen, fm); + gen.writeEndObject(); + + } + + public static void serializeValueField(JsonGenerator gen, FMetric fm) throws IOException { // Write the number as the value (preserve negatives) if (fm.ivalue() != 0) { gen.writeNumberField("value", fm.ivalue()); @@ -57,8 +63,6 @@ public void serialize(FMetric value, JsonGenerator gen, SerializerProvider seria } else { gen.writeNumberField("value", 0); } - gen.writeEndObject(); - } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/MetricResponseSerializer.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/MetricResponseSerializer.java index a058e054f77..54ce5f8aded 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/MetricResponseSerializer.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/serializers/MetricResponseSerializer.java @@ -54,16 +54,7 @@ public void serialize(MetricResponse value, JsonGenerator gen, SerializerProvide gen.writeEndObject(); } gen.writeEndArray(); - // Write the number as the value (preserve negatives) - if (fm.ivalue() != 0) { - gen.writeNumberField("value", fm.ivalue()); - } else if (fm.lvalue() != 0L) { - gen.writeNumberField("value", fm.lvalue()); - } else if (fm.dvalue() != 0.0d) { - gen.writeNumberField("value", fm.dvalue()); - } else { - gen.writeNumberField("value", 0); - } + FMetricSerializer.serializeValueField(gen, fm); gen.writeEndObject(); } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/sservers/ScanServerView.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/sservers/ScanServerView.java index a212424877d..05ee018d82f 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/sservers/ScanServerView.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/sservers/ScanServerView.java @@ -31,6 +31,7 @@ import org.apache.accumulo.core.metrics.flatbuffers.FMetric; import org.apache.accumulo.core.metrics.flatbuffers.FTag; import org.apache.accumulo.core.process.thrift.MetricResponse; +import org.apache.accumulo.monitor.next.SystemInformation; import org.apache.accumulo.server.metrics.MetricResponseWrapper; /** @@ -145,7 +146,7 @@ private static Map metricValuesByName(MetricResponse response) { var metricStatistic = extractStatistic(metric); if (metricStatistic == null || metricStatistic.equals("value") || metricStatistic.equals("count")) { - values.putIfAbsent(metric.name(), metricNumericValue(metric)); + values.putIfAbsent(metric.name(), SystemInformation.getMetricValue(metric)); } } return values; @@ -168,16 +169,4 @@ private static String normalizeStatistic(String statistic) { return statistic.toLowerCase(); } - private static Number metricNumericValue(FMetric metric) { - if (metric.ivalue() != 0) { - return metric.ivalue(); - } - if (metric.lvalue() != 0L) { - return metric.lvalue(); - } - if (metric.dvalue() != 0.0d) { - return metric.dvalue(); - } - return 0; - } } From 6f1a4a45fd713f9daf95339af1e6978526287c54 Mon Sep 17 00:00:00 2001 From: Christopher Tubbs Date: Tue, 24 Mar 2026 18:09:37 -0400 Subject: [PATCH 19/65] Trivial updates to release candidate script * Update script to reduce differences with other Accumulo repos --- src/build/create-release-candidate.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/build/create-release-candidate.sh b/src/build/create-release-candidate.sh index d9ad0548cad..2fa4c14ace6 100755 --- a/src/build/create-release-candidate.sh +++ b/src/build/create-release-candidate.sh @@ -23,8 +23,8 @@ scriptname=$(basename "$0") projectroot="$(git rev-parse --show-toplevel)" || exit 1 cd "$projectroot" || exit 1 export tlpName=accumulo -export projName="$tlpName" -export projNameLong="Apache ${projName^}" +export projName="$(xmllint --shell assemble/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="artifactId"]/text()' | grep content= | cut -f2 -d=)" +export projNameLong="$(xmllint --shell assemble/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="name"]/text()' | grep content= | cut -f2 -d=)" export stagingRepoPrefix="https://repository.apache.org/content/repositories/orgapache$tlpName" export srcQualifier="src" export relTestingUrl="https://$tlpName.apache.org/contributor/verifying-release" @@ -163,8 +163,9 @@ createEmail() { echo echo " Remember, $(red DO NOT PUSH) the $(red "$tag") tag until after the vote" echo " passes and the tag is re-made with a gpg signature using:" - echo " $(red "git tag -f -s -m '$projNameLong $ver' $tag") \\" - echo " $(red "$commit")" + echo " $(red "git tag -f -s -m '$projNameLong $ver'") \\" + echo " $(red "$tag") \\" + echo " $(red "$commit")" echo yellow "IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!! IMPORTANT!!" echo @@ -204,8 +205,9 @@ Branch: $(green "$branch") If this vote passes, a gpg-signed tag will be created using: - $(green "git tag -f -s -m '$projNameLong $ver' $tag") \\ - $(green "$commit") + $(green "git tag -f -s -m '$projNameLong $ver'") \\ + $(green "$tag") \\ + $(green "$commit") Staging repo: $(green "$stagingRepoPrefix-$stagingrepo") Source (official release artifact): $(green "$stagingRepoPrefix-$stagingrepo/org/apache/$tlpName/$projName/$ver/$projName-$ver-$srcQualifier.tar.gz") @@ -227,6 +229,8 @@ SHA512 ($(green "$projName-$ver-bin.tar.gz")) = $(yellow "$binSha") Release notes (in progress) can be found at: $(green "https://$tlpName.apache.org/release/$projName-$ver") +Issues and pull requests related to this release can be found at: $(green "https://github.com/apache/$projName/issues?q=milestone%3A$ver") + Release testing instructions: $relTestingUrl Please vote one of: From 00758f47275fac9f45dafe4d7353683ae50a005c Mon Sep 17 00:00:00 2001 From: Christopher Tubbs Date: Tue, 24 Mar 2026 18:19:22 -0400 Subject: [PATCH 20/65] Fix shellcheck --- src/build/create-release-candidate.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/build/create-release-candidate.sh b/src/build/create-release-candidate.sh index 2fa4c14ace6..3a0dd41ae13 100755 --- a/src/build/create-release-candidate.sh +++ b/src/build/create-release-candidate.sh @@ -23,8 +23,10 @@ scriptname=$(basename "$0") projectroot="$(git rev-parse --show-toplevel)" || exit 1 cd "$projectroot" || exit 1 export tlpName=accumulo -export projName="$(xmllint --shell assemble/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="artifactId"]/text()' | grep content= | cut -f2 -d=)" -export projNameLong="$(xmllint --shell assemble/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="name"]/text()' | grep content= | cut -f2 -d=)" +projName="$(xmllint --shell assemble/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="artifactId"]/text()' | grep content= | cut -f2 -d=)" +export projName +projNameLong="$(xmllint --shell assemble/pom.xml <<<'xpath /*[local-name()="project"]/*[local-name()="name"]/text()' | grep content= | cut -f2 -d=)" +export projNameLong export stagingRepoPrefix="https://repository.apache.org/content/repositories/orgapache$tlpName" export srcQualifier="src" export relTestingUrl="https://$tlpName.apache.org/contributor/verifying-release" From 4db7b62e56690facdfe62b0a9d293bf31cfc60eb Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 24 Mar 2026 23:19:35 +0000 Subject: [PATCH 21/65] removes unused method --- .../manager/compaction/coordinator/CompactionCoordinator.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index f7cf83bf236..420d7c34405 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -378,10 +378,6 @@ protected long getMissingCompactorWarningTime() { return this.ctx.getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME) * 3; } - public long getNumRunningCompactions() { - return RUNNING_CACHE.size(); - } - /** * Return the next compaction job from the queue to a Compactor * From 3ccccad56c1396e3e03f8577617b7390b6d0564e Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 25 Mar 2026 00:07:16 +0000 Subject: [PATCH 22/65] Removes running cache from the coordinator --- .../thrift/CompactionCoordinatorService.java | 728 ------------------ .../compaction/thrift/CompactorService.java | 36 +- .../thrift/TExternalCompactionMap.java | 471 ----------- .../main/thrift/compaction-coordinator.thrift | 11 - .../org/apache/accumulo/manager/Manager.java | 6 - .../coordinator/CompactionCoordinator.java | 141 +--- .../coordinator/commit/PutGcCandidates.java | 2 - .../coordinator/commit/RefreshTablet.java | 3 - .../accumulo/manager/fate/FateWorkerEnv.java | 23 - .../accumulo/manager/tableOps/FateEnv.java | 3 - .../compaction/CompactionCoordinatorTest.java | 105 --- 11 files changed, 21 insertions(+), 1508 deletions(-) delete mode 100644 core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index cf40536be89..4f446373ac9 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -37,8 +37,6 @@ public interface Iface { public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactor) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException; - } public interface AsyncIface { @@ -51,8 +49,6 @@ public interface AsyncIface { public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -209,21 +205,6 @@ public void recv_compactionFailed() throws org.apache.accumulo.core.clientImpl.t return; } - @Override - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException - { - send_recordCompletion(tinfo, credentials, externalCompactionId); - } - - public void send_recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException - { - recordCompletion_args args = new recordCompletion_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.setExternalCompactionId(externalCompactionId); - sendBaseOneway("recordCompletion", args); - } - } public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -449,47 +430,6 @@ public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.Thrift } } - @Override - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - checkReady(); - recordCompletion_call method_call = new recordCompletion_call(tinfo, credentials, externalCompactionId, resultHandler, this, ___protocolFactory, ___transport); - this.___currentMethod = method_call; - ___manager.call(method_call); - } - - public static class recordCompletion_call extends org.apache.thrift.async.TAsyncMethodCall { - private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; - private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; - private java.lang.String externalCompactionId; - public recordCompletion_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { - super(client, protocolFactory, transport, resultHandler, true); - this.tinfo = tinfo; - this.credentials = credentials; - this.externalCompactionId = externalCompactionId; - } - - @Override - public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { - prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("recordCompletion", org.apache.thrift.protocol.TMessageType.ONEWAY, 0)); - recordCompletion_args args = new recordCompletion_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.setExternalCompactionId(externalCompactionId); - args.write(prot); - prot.writeMessageEnd(); - } - - @Override - public Void getResult() throws org.apache.thrift.TException { - if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { - throw new java.lang.IllegalStateException("Method call not finished!"); - } - org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); - org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); - return null; - } - } - } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -507,7 +447,6 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { - public recordCompletion() { - super("recordCompletion"); - } - - @Override - public recordCompletion_args getEmptyArgsInstance() { - return new recordCompletion_args(); - } - - @Override - protected boolean isOneway() { - return true; - } - - @Override - protected boolean rethrowUnhandledExceptions() { - return false; - } - - @Override - public org.apache.thrift.TBase getResult(I iface, recordCompletion_args args) throws org.apache.thrift.TException { - iface.recordCompletion(args.tinfo, args.credentials, args.externalCompactionId); - return null; - } - } - } public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { @@ -691,7 +603,6 @@ protected AsyncProcessor(I iface, java.util.Map extends org.apache.thrift.AsyncProcessFunction { - public recordCompletion() { - super("recordCompletion"); - } - - @Override - public recordCompletion_args getEmptyArgsInstance() { - return new recordCompletion_args(); - } - - @Override - public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { - final org.apache.thrift.AsyncProcessFunction fcall = this; - return new org.apache.thrift.async.AsyncMethodCallback() { - @Override - public void onComplete(Void o) { - } - @Override - public void onError(java.lang.Exception e) { - if (e instanceof org.apache.thrift.transport.TTransportException) { - _LOGGER.error("TTransportException inside handler", e); - fb.close(); - } else { - _LOGGER.error("Exception inside oneway handler", e); - } - } - }; - } - - @Override - protected boolean isOneway() { - return true; - } - - @Override - public void start(I iface, recordCompletion_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.recordCompletion(args.tinfo, args.credentials, args.externalCompactionId,resultHandler); - } - } - } @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) @@ -6884,604 +6755,5 @@ private static S scheme(org.apache. } } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class recordCompletion_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("recordCompletion_args"); - - private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); - private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new recordCompletion_argsStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new recordCompletion_argsTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required - public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - TINFO((short)1, "tinfo"), - CREDENTIALS((short)2, "credentials"), - EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // TINFO - return TINFO; - case 2: // CREDENTIALS - return CREDENTIALS; - case 3: // EXTERNAL_COMPACTION_ID - return EXTERNAL_COMPACTION_ID; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); - tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); - tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("externalCompactionId", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(recordCompletion_args.class, metaDataMap); - } - - public recordCompletion_args() { - } - - public recordCompletion_args( - org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, - org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, - java.lang.String externalCompactionId) - { - this(); - this.tinfo = tinfo; - this.credentials = credentials; - this.externalCompactionId = externalCompactionId; - } - - /** - * Performs a deep copy on other. - */ - public recordCompletion_args(recordCompletion_args other) { - if (other.isSetTinfo()) { - this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); - } - if (other.isSetCredentials()) { - this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); - } - if (other.isSetExternalCompactionId()) { - this.externalCompactionId = other.externalCompactionId; - } - } - - @Override - public recordCompletion_args deepCopy() { - return new recordCompletion_args(this); - } - - @Override - public void clear() { - this.tinfo = null; - this.credentials = null; - this.externalCompactionId = null; - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { - return this.tinfo; - } - - public recordCompletion_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { - this.tinfo = tinfo; - return this; - } - - public void unsetTinfo() { - this.tinfo = null; - } - - /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ - public boolean isSetTinfo() { - return this.tinfo != null; - } - - public void setTinfoIsSet(boolean value) { - if (!value) { - this.tinfo = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { - return this.credentials; - } - - public recordCompletion_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { - this.credentials = credentials; - return this; - } - - public void unsetCredentials() { - this.credentials = null; - } - - /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ - public boolean isSetCredentials() { - return this.credentials != null; - } - - public void setCredentialsIsSet(boolean value) { - if (!value) { - this.credentials = null; - } - } - - @org.apache.thrift.annotation.Nullable - public java.lang.String getExternalCompactionId() { - return this.externalCompactionId; - } - - public recordCompletion_args setExternalCompactionId(@org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId) { - this.externalCompactionId = externalCompactionId; - return this; - } - - public void unsetExternalCompactionId() { - this.externalCompactionId = null; - } - - /** Returns true if field externalCompactionId is set (has been assigned a value) and false otherwise */ - public boolean isSetExternalCompactionId() { - return this.externalCompactionId != null; - } - - public void setExternalCompactionIdIsSet(boolean value) { - if (!value) { - this.externalCompactionId = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case TINFO: - if (value == null) { - unsetTinfo(); - } else { - setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); - } - break; - - case CREDENTIALS: - if (value == null) { - unsetCredentials(); - } else { - setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); - } - break; - - case EXTERNAL_COMPACTION_ID: - if (value == null) { - unsetExternalCompactionId(); - } else { - setExternalCompactionId((java.lang.String)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case TINFO: - return getTinfo(); - - case CREDENTIALS: - return getCredentials(); - - case EXTERNAL_COMPACTION_ID: - return getExternalCompactionId(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case TINFO: - return isSetTinfo(); - case CREDENTIALS: - return isSetCredentials(); - case EXTERNAL_COMPACTION_ID: - return isSetExternalCompactionId(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof recordCompletion_args) - return this.equals((recordCompletion_args)that); - return false; - } - - public boolean equals(recordCompletion_args that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_tinfo = true && this.isSetTinfo(); - boolean that_present_tinfo = true && that.isSetTinfo(); - if (this_present_tinfo || that_present_tinfo) { - if (!(this_present_tinfo && that_present_tinfo)) - return false; - if (!this.tinfo.equals(that.tinfo)) - return false; - } - - boolean this_present_credentials = true && this.isSetCredentials(); - boolean that_present_credentials = true && that.isSetCredentials(); - if (this_present_credentials || that_present_credentials) { - if (!(this_present_credentials && that_present_credentials)) - return false; - if (!this.credentials.equals(that.credentials)) - return false; - } - - boolean this_present_externalCompactionId = true && this.isSetExternalCompactionId(); - boolean that_present_externalCompactionId = true && that.isSetExternalCompactionId(); - if (this_present_externalCompactionId || that_present_externalCompactionId) { - if (!(this_present_externalCompactionId && that_present_externalCompactionId)) - return false; - if (!this.externalCompactionId.equals(that.externalCompactionId)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); - if (isSetTinfo()) - hashCode = hashCode * 8191 + tinfo.hashCode(); - - hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); - if (isSetCredentials()) - hashCode = hashCode * 8191 + credentials.hashCode(); - - hashCode = hashCode * 8191 + ((isSetExternalCompactionId()) ? 131071 : 524287); - if (isSetExternalCompactionId()) - hashCode = hashCode * 8191 + externalCompactionId.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(recordCompletion_args other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTinfo()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCredentials()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetExternalCompactionId(), other.isSetExternalCompactionId()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetExternalCompactionId()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.externalCompactionId, other.externalCompactionId); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("recordCompletion_args("); - boolean first = true; - - sb.append("tinfo:"); - if (this.tinfo == null) { - sb.append("null"); - } else { - sb.append(this.tinfo); - } - first = false; - if (!first) sb.append(", "); - sb.append("credentials:"); - if (this.credentials == null) { - sb.append("null"); - } else { - sb.append(this.credentials); - } - first = false; - if (!first) sb.append(", "); - sb.append("externalCompactionId:"); - if (this.externalCompactionId == null) { - sb.append("null"); - } else { - sb.append(this.externalCompactionId); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (tinfo != null) { - tinfo.validate(); - } - if (credentials != null) { - credentials.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class recordCompletion_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public recordCompletion_argsStandardScheme getScheme() { - return new recordCompletion_argsStandardScheme(); - } - } - - private static class recordCompletion_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, recordCompletion_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // TINFO - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // CREDENTIALS - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 3: // EXTERNAL_COMPACTION_ID - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.externalCompactionId = iprot.readString(); - struct.setExternalCompactionIdIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, recordCompletion_args struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.tinfo != null) { - oprot.writeFieldBegin(TINFO_FIELD_DESC); - struct.tinfo.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.credentials != null) { - oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); - struct.credentials.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.externalCompactionId != null) { - oprot.writeFieldBegin(EXTERNAL_COMPACTION_ID_FIELD_DESC); - oprot.writeString(struct.externalCompactionId); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class recordCompletion_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public recordCompletion_argsTupleScheme getScheme() { - return new recordCompletion_argsTupleScheme(); - } - } - - private static class recordCompletion_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, recordCompletion_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetTinfo()) { - optionals.set(0); - } - if (struct.isSetCredentials()) { - optionals.set(1); - } - if (struct.isSetExternalCompactionId()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); - if (struct.isSetTinfo()) { - struct.tinfo.write(oprot); - } - if (struct.isSetCredentials()) { - struct.credentials.write(oprot); - } - if (struct.isSetExternalCompactionId()) { - oprot.writeString(struct.externalCompactionId); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, recordCompletion_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); - if (incoming.get(0)) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } - if (incoming.get(1)) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } - if (incoming.get(2)) { - struct.externalCompactionId = iprot.readString(); - struct.setExternalCompactionIdIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - private static void unusedMethod() {} } diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java index d9d867d8d2d..c59d19d98f6 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java @@ -3668,14 +3668,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveCompaction case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list28 = iprot.readListBegin(); - struct.success = new java.util.ArrayList(_list28.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem29; - for (int _i30 = 0; _i30 < _list28.size; ++_i30) + org.apache.thrift.protocol.TList _list18 = iprot.readListBegin(); + struct.success = new java.util.ArrayList(_list18.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem19; + for (int _i20 = 0; _i20 < _list18.size; ++_i20) { - _elem29 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem29.read(iprot); - struct.success.add(_elem29); + _elem19 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem19.read(iprot); + struct.success.add(_elem19); } iprot.readListEnd(); } @@ -3713,9 +3713,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveCompactio oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter31 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter21 : struct.success) { - _iter31.write(oprot); + _iter21.write(oprot); } oprot.writeListEnd(); } @@ -3755,9 +3755,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveCompaction if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter32 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter22 : struct.success) { - _iter32.write(oprot); + _iter22.write(oprot); } } } @@ -3772,14 +3772,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveCompactions java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list33 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.success = new java.util.ArrayList(_list33.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem34; - for (int _i35 = 0; _i35 < _list33.size; ++_i35) + org.apache.thrift.protocol.TList _list23 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.success = new java.util.ArrayList(_list23.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem24; + for (int _i25 = 0; _i25 < _list23.size; ++_i25) { - _elem34 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem34.read(iprot); - struct.success.add(_elem34); + _elem24 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem24.read(iprot); + struct.success.add(_elem24); } } struct.setSuccessIsSet(true); diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java deleted file mode 100644 index a5b3d74a2b9..00000000000 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java +++ /dev/null @@ -1,471 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Autogenerated by Thrift Compiler (0.17.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -package org.apache.accumulo.core.compaction.thrift; - -@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -public class TExternalCompactionMap implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TExternalCompactionMap"); - - private static final org.apache.thrift.protocol.TField COMPACTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compactions", org.apache.thrift.protocol.TType.MAP, (short)1); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new TExternalCompactionMapStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new TExternalCompactionMapTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable java.util.Map compactions; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - COMPACTIONS((short)1, "compactions"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // COMPACTIONS - return COMPACTIONS; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.COMPACTIONS, new org.apache.thrift.meta_data.FieldMetaData("compactions", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExternalCompaction.class)))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TExternalCompactionMap.class, metaDataMap); - } - - public TExternalCompactionMap() { - } - - public TExternalCompactionMap( - java.util.Map compactions) - { - this(); - this.compactions = compactions; - } - - /** - * Performs a deep copy on other. - */ - public TExternalCompactionMap(TExternalCompactionMap other) { - if (other.isSetCompactions()) { - java.util.Map __this__compactions = new java.util.HashMap(other.compactions.size()); - for (java.util.Map.Entry other_element : other.compactions.entrySet()) { - - java.lang.String other_element_key = other_element.getKey(); - TExternalCompaction other_element_value = other_element.getValue(); - - java.lang.String __this__compactions_copy_key = other_element_key; - - TExternalCompaction __this__compactions_copy_value = new TExternalCompaction(other_element_value); - - __this__compactions.put(__this__compactions_copy_key, __this__compactions_copy_value); - } - this.compactions = __this__compactions; - } - } - - @Override - public TExternalCompactionMap deepCopy() { - return new TExternalCompactionMap(this); - } - - @Override - public void clear() { - this.compactions = null; - } - - public int getCompactionsSize() { - return (this.compactions == null) ? 0 : this.compactions.size(); - } - - public void putToCompactions(java.lang.String key, TExternalCompaction val) { - if (this.compactions == null) { - this.compactions = new java.util.HashMap(); - } - this.compactions.put(key, val); - } - - @org.apache.thrift.annotation.Nullable - public java.util.Map getCompactions() { - return this.compactions; - } - - public TExternalCompactionMap setCompactions(@org.apache.thrift.annotation.Nullable java.util.Map compactions) { - this.compactions = compactions; - return this; - } - - public void unsetCompactions() { - this.compactions = null; - } - - /** Returns true if field compactions is set (has been assigned a value) and false otherwise */ - public boolean isSetCompactions() { - return this.compactions != null; - } - - public void setCompactionsIsSet(boolean value) { - if (!value) { - this.compactions = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case COMPACTIONS: - if (value == null) { - unsetCompactions(); - } else { - setCompactions((java.util.Map)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case COMPACTIONS: - return getCompactions(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case COMPACTIONS: - return isSetCompactions(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof TExternalCompactionMap) - return this.equals((TExternalCompactionMap)that); - return false; - } - - public boolean equals(TExternalCompactionMap that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_compactions = true && this.isSetCompactions(); - boolean that_present_compactions = true && that.isSetCompactions(); - if (this_present_compactions || that_present_compactions) { - if (!(this_present_compactions && that_present_compactions)) - return false; - if (!this.compactions.equals(that.compactions)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetCompactions()) ? 131071 : 524287); - if (isSetCompactions()) - hashCode = hashCode * 8191 + compactions.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(TExternalCompactionMap other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetCompactions(), other.isSetCompactions()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCompactions()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compactions, other.compactions); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("TExternalCompactionMap("); - boolean first = true; - - sb.append("compactions:"); - if (this.compactions == null) { - sb.append("null"); - } else { - sb.append(this.compactions); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class TExternalCompactionMapStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public TExternalCompactionMapStandardScheme getScheme() { - return new TExternalCompactionMapStandardScheme(); - } - } - - private static class TExternalCompactionMapStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // COMPACTIONS - if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { - { - org.apache.thrift.protocol.TMap _map18 = iprot.readMapBegin(); - struct.compactions = new java.util.HashMap(2*_map18.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key19; - @org.apache.thrift.annotation.Nullable TExternalCompaction _val20; - for (int _i21 = 0; _i21 < _map18.size; ++_i21) - { - _key19 = iprot.readString(); - _val20 = new TExternalCompaction(); - _val20.read(iprot); - struct.compactions.put(_key19, _val20); - } - iprot.readMapEnd(); - } - struct.setCompactionsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.compactions != null) { - oprot.writeFieldBegin(COMPACTIONS_FIELD_DESC); - { - oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.compactions.size())); - for (java.util.Map.Entry _iter22 : struct.compactions.entrySet()) - { - oprot.writeString(_iter22.getKey()); - _iter22.getValue().write(oprot); - } - oprot.writeMapEnd(); - } - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class TExternalCompactionMapTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public TExternalCompactionMapTupleScheme getScheme() { - return new TExternalCompactionMapTupleScheme(); - } - } - - private static class TExternalCompactionMapTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetCompactions()) { - optionals.set(0); - } - oprot.writeBitSet(optionals, 1); - if (struct.isSetCompactions()) { - { - oprot.writeI32(struct.compactions.size()); - for (java.util.Map.Entry _iter23 : struct.compactions.entrySet()) - { - oprot.writeString(_iter23.getKey()); - _iter23.getValue().write(oprot); - } - } - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(1); - if (incoming.get(0)) { - { - org.apache.thrift.protocol.TMap _map24 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT); - struct.compactions = new java.util.HashMap(2*_map24.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key25; - @org.apache.thrift.annotation.Nullable TExternalCompaction _val26; - for (int _i27 = 0; _i27 < _map24.size; ++_i27) - { - _key25 = iprot.readString(); - _val26 = new TExternalCompaction(); - _val26.read(iprot); - struct.compactions.put(_key25, _val26); - } - } - struct.setCompactionsIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - private static void unusedMethod() {} -} - diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index 29e4dde868d..8effd88aff5 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -60,10 +60,6 @@ struct TExternalCompactionList { 1:list compactions } -struct TExternalCompactionMap { - 1:map compactions -} - struct TNextCompactionJob { 1:tabletserver.TExternalCompactionJob job // The total number of compactors servicing the queue this job was requested for @@ -135,13 +131,6 @@ service CompactionCoordinatorService { 1:client.ThriftSecurityException sec 2:client.ThriftNotActiveServiceException tnase ) - - - oneway void recordCompletion( - 1:client.TInfo tinfo - 2:security.TCredentials credentials - 3:string externalCompactionId - ) } service CompactorService { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 5ff232765c4..60e09f3dd89 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -107,7 +107,6 @@ import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metrics.MetricsInfo; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.trace.TraceUtil; @@ -609,11 +608,6 @@ public CompactionCoordinator getCompactionCoordinator() { return compactionCoordinator; } - @Override - public void recordCompactionCompletion(ExternalCompactionId ecid) { - getCompactionCoordinator().recordCompletion(ecid); - } - public void hostOndemand(List extents) { extents.forEach(e -> Preconditions.checkArgument(DataLevel.of(e.tableId()) == DataLevel.USER)); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 0ff67d48501..efa149fc7ce 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -27,7 +27,6 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED; -import static org.apache.accumulo.core.util.threads.ThreadPoolNames.COMPACTOR_RUNNING_COMPACTIONS_POOL; import java.io.FileNotFoundException; import java.io.IOException; @@ -69,7 +68,6 @@ import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate; -import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -135,7 +133,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -191,16 +188,6 @@ static FailureCounts incrementSuccess(Object key, FailureCounts counts) { public static final String RESTART_UPDATE_MSG = "Coordinator restarted, compaction found in progress"; - /* - * Map of compactionId to TExternalCompaction. This is an informational cache of what external - * compactions may be running. Its possible it may contain external compactions that are not - * actually running. It may not contain compactions that are actually running. The metadata table - * is the most authoritative source of what external compactions are currently running, but it - * does not have the stats that this map has. - */ - protected final Map RUNNING_CACHE = - new ConcurrentHashMap<>(); - private final ServerContext ctx; private final AuditedSecurityOperation security; private final CompactionJobQueues jobQueues; @@ -331,30 +318,6 @@ public void run() { startConfigMonitor(ctx.getScheduledExecutor()); startCompactorZKCleaner(ctx.getScheduledExecutor()); - // On a re-start of the coordinator it's possible that external compactions are in-progress. - // Attempt to get the running compactions on the compactors and then resolve which tserver - // the external compaction came from to re-populate the RUNNING collection. - LOG.info("Checking for running external compactions"); - // On re-start contact the running Compactors to try and seed the list of running compactions - try { - List running = getCompactionsRunningOnCompactors(); - if (running.isEmpty()) { - LOG.info("No running external compactions found"); - } else { - LOG.info("Found {} running external compactions", running.size()); - running.forEach(tec -> { - TCompactionStatusUpdate update = new TCompactionStatusUpdate(); - update.setState(TCompactionState.IN_PROGRESS); - update.setMessage(RESTART_UPDATE_MSG); - tec.putToUpdates(coordinatorStartTime, update); - RUNNING_CACHE.put(ExternalCompactionId.of(tec.getJob().getExternalCompactionId()), tec); - }); - } - } catch (InterruptedException e) { - throw new IllegalStateException( - "Thread interrupted while retrieving running compactions from compactors", e); - } - startDeadCompactionDetector(); startFailureSummaryLogging(); startInternalStateCleaner(ctx.getScheduledExecutor()); @@ -372,14 +335,6 @@ protected void startDeadCompactionDetector() { deadCompactionDetector.start(); } - protected long getMissingCompactorWarningTime() { - return this.ctx.getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME) * 3; - } - - public long getNumRunningCompactions() { - return RUNNING_CACHE.size(); - } - /** * Return the next compaction job from the queue to a Compactor * @@ -425,13 +380,6 @@ public TNextCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials if (ecm != null) { result = createThriftJob(externalCompactionId, ecm, rcJob, compactionConfig); - // It is possible that by the time this added that the the compactor that made this request - // is dead. In this cases the compaction is not actually running. - TExternalCompaction tec = new TExternalCompaction(); - tec.setCompactor(compactorAddress); - tec.setGroupName(groupName); - tec.setJob(result); - RUNNING_CACHE.put(ExternalCompactionId.of(result.getExternalCompactionId()), tec); TabletLogger.compacting(rcJob.getExtent(), rcJob.getSelectedFateId(), cid, compactorAddress, rcJob, ecm.getCompactTmpName()); break; @@ -913,8 +861,6 @@ public boolean test(TabletMetadata tabletMetadata) { } }); } - - compactions.values().forEach(ecids -> ecids.forEach(this::recordCompletion)); } /** @@ -936,28 +882,9 @@ public void updateCompactionStatus(TInfo tinfo, TCredentials credentials, throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } + // TODO could remove this thrift method LOG.debug("Compaction status update, id: {}, timestamp: {}, update: {}", externalCompactionId, timestamp, update); - final TExternalCompaction tec = - RUNNING_CACHE.get(ExternalCompactionId.of(externalCompactionId)); - if (null != tec) { - if (update.getState() == TCompactionState.STARTED) { - tec.setStartTime(timestamp); - } - tec.putToUpdates(timestamp, update); - } - } - - @Override - public void recordCompletion(TInfo tinfo, TCredentials credentials, String externalCompactionId) - throws TException { - if (security.canPerformSystemActions(credentials)) { - recordCompletion(ExternalCompactionId.of(externalCompactionId)); - } - } - - public void recordCompletion(ExternalCompactionId ecid) { - var tec = RUNNING_CACHE.remove(ecid); } protected Set readExternalCompactionIds() { @@ -974,35 +901,11 @@ public CompactionJobQueues getJobQueues() { return jobQueues; } - /* Method exists to be overridden in test to hide static method */ - protected List getCompactionsRunningOnCompactors() - throws InterruptedException { - int numCompactors = this.ctx.instanceOperations().getServers(ServerId.Type.COMPACTOR).size(); - final ExecutorService executor = - ThreadPools.getServerThreadPools().getPoolBuilder(COMPACTOR_RUNNING_COMPACTIONS_POOL) - .numCoreThreads(numCompactors / 10).build(); - try { - List running = new ArrayList<>(); - @SuppressWarnings("unused") - List failures = ExternalCompactionUtil.getCompactionsRunningOnCompactors(this.ctx, - executor, (t) -> running.add(t)); - return running; - } finally { - executor.shutdownNow(); - } - } - /* Method exists to be overridden in test to hide static method */ protected Set getRunningCompactors() { return ctx.instanceOperations().getServers(ServerId.Type.COMPACTOR); } - /* Method exists to be overridden in test to hide static method */ - protected void cancelCompactionOnCompactor(String address, String externalCompactionId) { - HostAndPort hostPort = HostAndPort.fromString(address); - ExternalCompactionUtil.cancelCompaction(this.ctx, hostPort, externalCompactionId); - } - private void deleteEmpty(ZooReaderWriter zoorw, String path) throws KeeperException, InterruptedException { try { @@ -1082,13 +985,8 @@ public void cleanUpInternalState() { // This method does the following: // - // 1. Removes entries from RUNNING_CACHE and LONG_RUNNING_COMPACTIONS_BY_RG that are not really - // running - // 2. Cancels running compactions for groups that are not in the current configuration - // 3. Remove groups not in configuration from TIME_COMPACTOR_LAST_CHECKED - // 4. Log groups with no compactors - // 5. Log compactors with no groups - // 6. Log groups with compactors and queued jos that have not checked in + // 1. Log groups with no compactors + // 2. Log compactors with no groups var config = ctx.getConfiguration(); ThreadPools.resizePool(reservationPools.get(DataLevel.ROOT), config, @@ -1098,23 +996,10 @@ public void cleanUpInternalState() { ThreadPools.resizePool(reservationPools.get(DataLevel.USER), config, Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_USER); - // grab a snapshot of the ids in the set before reading the metadata table. This is done to - // avoid removing things that are added while reading the metadata. - final Set idsSnapshot = Set.copyOf(RUNNING_CACHE.keySet()); - // grab the ids that are listed as running in the metadata table. It important that this is done // after getting the snapshot. final Set idsInMetadata = readExternalCompactionIds(); LOG.trace("Current ECIDs in metadata: {}", idsInMetadata.size()); - LOG.trace("Current ECIDs in running cache: {}", idsSnapshot.size()); - - final Set idsToRemove = Sets.difference(idsSnapshot, idsInMetadata); - - // remove ids that are in the running set but not in the metadata table - idsToRemove.forEach(this::recordCompletion); - if (idsToRemove.size() > 0) { - LOG.debug("Removed stale entries from RUNNING_CACHE : {}", idsToRemove); - } // Get the set of groups being referenced in the current configuration Set groupsInConfiguration = null; @@ -1127,26 +1012,6 @@ public void cleanUpInternalState() { return; } - // Compaction jobs are created in the TabletGroupWatcher and added to the Coordinator - // via the addJobs method which adds the job to the CompactionJobQueues object. - final Set groupsWithJobs = jobQueues.getQueueIds(); - - final Set jobGroupsNotInConfiguration = - Sets.difference(groupsWithJobs, groupsInConfiguration); - - if (jobGroupsNotInConfiguration != null && !jobGroupsNotInConfiguration.isEmpty()) { - RUNNING_CACHE.values().forEach(tec -> { - if (jobGroupsNotInConfiguration.contains(ResourceGroupId.of(tec.getGroupName()))) { - LOG.warn( - "External compaction {} running in group {} on compactor {}," - + " but group not found in current configuration. Failing compaction...", - tec.getJob().getExternalCompactionId(), tec.getGroupName(), tec.getCompactor()); - cancelCompactionOnCompactor(tec.getCompactor(), tec.getJob().getExternalCompactionId()); - } - }); - - } - final Set runningCompactors = getRunningCompactors(); final Set runningCompactorGroups = new HashSet<>(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java index 3933a625048..89cdedab8ec 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java @@ -20,7 +20,6 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; @@ -41,7 +40,6 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { env.getContext().getAmple().putGcCandidates(commitData.getTableId(), commitData.getJobFiles()); if (refreshLocation == null) { - env.recordCompactionCompletion(ExternalCompactionId.of(commitData.ecid)); return null; } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java index d3be75c0128..cdcf649e02b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java @@ -27,7 +27,6 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; @@ -63,8 +62,6 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { executorService.shutdownNow(); } - env.recordCompactionCompletion(ExternalCompactionId.of(compactionId)); - return null; } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java index e7909dd0a71..01cf2baea20 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java @@ -28,18 +28,15 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; import org.apache.accumulo.core.trace.TraceUtil; -import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.core.util.time.SteadyTime; @@ -176,26 +173,6 @@ public EventPublisher getEventPublisher() { return eventHandler; } - @Override - public void recordCompactionCompletion(ExternalCompactionId ecid) { - var coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(getContext()); - if (coordinatorHost.isPresent()) { - CompactionCoordinatorService.Client client = null; - try { - client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost.orElseThrow(), - getContext()); - client.recordCompletion(TraceUtil.traceInfo(), getContext().rpcCreds(), ecid.canonical()); - log.trace("Sent compaction completion {} {}", coordinatorHost, ecid); - } catch (TException te) { - log.trace("Failed to send compaction completion {} {}", coordinatorHost, ecid, te); - } finally { - ThriftUtil.returnClient(client, getContext()); - } - } else { - log.trace("No coordinator found, dropping compaction completion for {}", ecid); - } - } - @Override public Set onlineTabletServers() { return liveTServerSet.getSnapshot().getTservers(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java index f64b3851ae6..9eece1208fd 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java @@ -23,7 +23,6 @@ import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.manager.EventPublisher; import org.apache.accumulo.manager.split.FileRangeCache; @@ -36,8 +35,6 @@ public interface FateEnv { EventPublisher getEventPublisher(); - void recordCompactionCompletion(ExternalCompactionId ecid); - Set onlineTabletServers(); TableManager getTableManager(); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index 3aa3f848d69..bd8add9eee5 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -18,7 +18,6 @@ */ package org.apache.accumulo.manager.compaction; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; @@ -32,7 +31,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -84,14 +82,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; -import com.google.common.net.HostAndPort; - public class CompactionCoordinatorTest { private static final ResourceGroupId GROUP_ID = ResourceGroupId.of("R2DQ"); - private final HostAndPort tserverAddr = HostAndPort.fromParts("192.168.1.1", 9090); - public MetricsInfo getMockMetrics() { MetricsInfo metricsInfo = createMock(MetricsInfo.class); metricsInfo.addMetricsProducers(anyObject()); @@ -150,29 +144,10 @@ void setMetadataCompactionIds(Set mci) { metadataCompactionIds = mci; } - @Override - protected Set readExternalCompactionIds() { - if (metadataCompactionIds == null) { - return RUNNING_CACHE.keySet(); - } else { - return metadataCompactionIds; - } - } - - public Map getRunning() { - return RUNNING_CACHE; - } - public void resetInternals() { - getRunning().clear(); metadataCompactionIds = null; } - @Override - protected List getCompactionsRunningOnCompactors() { - return runningCompactions; - } - @Override protected Set getRunningCompactors() { return Set.of(); @@ -206,10 +181,6 @@ protected TExternalCompactionJob createThriftJob(String externalCompactionId, .toThrift(), Map.of()); } - - @Override - protected void cancelCompactionOnCompactor(String address, String externalCompactionId) {} - } private TableId tableId; @@ -259,50 +230,10 @@ public void verifyMocks() { public void testCoordinatorColdStart() throws Exception { var coordinator = new TestCoordinator(manager, new ArrayList<>()); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); coordinator.run(); coordinator.shutdown(); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); - } - - @Test - public void testCoordinatorRestartOneRunningCompaction() throws Exception { - List runningCompactions = new ArrayList<>(); - ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID()); - - TExternalCompactionJob job = createMock(TExternalCompactionJob.class); - expect(job.getExternalCompactionId()).andReturn(eci.toString()).atLeastOnce(); - TKeyExtent extent = new TKeyExtent(); - extent.setTable("1".getBytes(UTF_8)); - - TExternalCompaction current = new TExternalCompaction(); - current.setCompactor(tserverAddr.toString()); - current.setGroupName(GROUP_ID.canonical()); - current.setJob(job); - - runningCompactions.add(current); - - replay(job); - - var coordinator = new TestCoordinator(manager, runningCompactions); - coordinator.resetInternals(); - assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); - coordinator.run(); - coordinator.shutdown(); - assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(1, coordinator.getRunning().size()); - - Map running = coordinator.getRunning(); - Entry ecomp = running.entrySet().iterator().next(); - assertEquals(eci, ecomp.getKey()); - TExternalCompaction tec = ecomp.getValue(); - assertEquals(GROUP_ID, ResourceGroupId.of(tec.getGroupName())); - assertEquals(tserverAddr.toString(), tec.getCompactor()); - - verify(job); } @Test @@ -318,14 +249,12 @@ public void testGetCompactionJob() throws Exception { var coordinator = new TestCoordinator(manager, new ArrayList<>()); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); // Use coordinator.run() to populate the internal data structures. This is tested in a different // test. coordinator.run(); coordinator.shutdown(); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); // Add a job to the job queue CompactionJob job = @@ -344,12 +273,6 @@ public void testGetCompactionJob() throws Exception { assertEquals(ke, KeyExtent.fromThrift(createdJob.getExtent())); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(1, coordinator.getRunning().size()); - Entry entry = - coordinator.getRunning().entrySet().iterator().next(); - assertEquals(eci.toString(), entry.getKey().toString()); - assertEquals("localhost:10241", entry.getValue().getCompactor()); - assertEquals(eci.toString(), entry.getValue().getJob().getExternalCompactionId()); verify(tm); } @@ -362,32 +285,4 @@ public void testGetCompactionJobNoJobs() throws Exception { assertEquals(3, nextJob.getCompactorCount()); assertNull(nextJob.getJob().getExternalCompactionId()); } - - @Test - public void testCleanUpRunning() throws Exception { - TExternalCompaction ext1 = createMock(TExternalCompaction.class); - TExternalCompaction ext2 = createMock(TExternalCompaction.class); - TExternalCompaction ext3 = createMock(TExternalCompaction.class); - replay(ext1, ext2, ext3); - - TestCoordinator coordinator = new TestCoordinator(manager, new ArrayList<>()); - - var ecid1 = ExternalCompactionId.generate(UUID.randomUUID()); - var ecid2 = ExternalCompactionId.generate(UUID.randomUUID()); - var ecid3 = ExternalCompactionId.generate(UUID.randomUUID()); - - coordinator.getRunning().put(ecid1, ext1); - coordinator.getRunning().put(ecid2, ext2); - coordinator.getRunning().put(ecid3, ext3); - coordinator.cleanUpInternalState(); - - assertEquals(Set.of(ecid1, ecid2, ecid3), coordinator.getRunning().keySet()); - - coordinator.setMetadataCompactionIds(Set.of(ecid1, ecid2)); - coordinator.cleanUpInternalState(); - - assertEquals(Set.of(ecid1, ecid2), coordinator.getRunning().keySet()); - - verify(ext1, ext2, ext3); - } } From b67fb2db2ccb3069a965d73e2bc2bae5b30db414 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 00:25:32 +0000 Subject: [PATCH 23/65] WIP --- .../thrift/CompactionCoordinatorService.java | 1708 +++++++++++++++-- .../compaction/thrift/CompactorService.java | 36 +- .../main/thrift/compaction-coordinator.thrift | 12 +- .../org/apache/accumulo/manager/Manager.java | 2 - .../compaction/CompactionJobClient.java | 15 +- .../coordinator/CompactionCoordinator.java | 39 +- .../coordinator/CoordinatorManager.java | 144 +- .../compaction/queue/CompactionJobQueues.java | 8 +- .../accumulo/manager/fate/FateManager.java | 29 +- .../manager/multi/ManagerAssignment.java | 92 + .../queue/CompactionJobQueuesTest.java | 10 +- .../manager/multi/ManagerAssignmentTest.java | 121 ++ .../test/MultipleManagerCompactionIT.java | 50 +- .../src/main/resources/log4j2-test.properties | 3 + 14 files changed, 1987 insertions(+), 282 deletions(-) create mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java create mode 100644 server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index fd5ab8dec55..6aa7dc826ff 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -41,7 +41,9 @@ public interface Iface { public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; + public java.util.Set getResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; + + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, java.util.Set groups) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; } @@ -59,7 +61,9 @@ public interface AsyncIface { public void endFullJobScan(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String dataLevel, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void getResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, org.apache.thrift.async.AsyncMethodCallback> resultHandler) throws org.apache.thrift.TException; + + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; } @@ -245,17 +249,47 @@ public void recv_endFullJobScan() throws org.apache.accumulo.core.clientImpl.thr } @Override - public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + public java.util.Set getResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + { + send_getResourceGroups(tinfo, credentials, updateId); + return recv_getResourceGroups(); + } + + public void send_getResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId) throws org.apache.thrift.TException + { + getResourceGroups_args args = new getResourceGroups_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setUpdateId(updateId); + sendBase("getResourceGroups", args); + } + + public java.util.Set recv_getResourceGroups() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + { + getResourceGroups_result result = new getResourceGroups_result(); + receiveBase(result, "getResourceGroups"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.sec != null) { + throw result.sec; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getResourceGroups failed: unknown result"); + } + + @Override + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, java.util.Set groups) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { - send_setResourceGroups(tinfo, credentials, groups); + send_setResourceGroups(tinfo, credentials, updateId, groups); recv_setResourceGroups(); } - public void send_setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups) throws org.apache.thrift.TException + public void send_setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, java.util.Set groups) throws org.apache.thrift.TException { setResourceGroups_args args = new setResourceGroups_args(); args.setTinfo(tinfo); args.setCredentials(credentials); + args.setUpdateId(updateId); args.setGroups(groups); sendBase("setResourceGroups", args); } @@ -573,9 +607,50 @@ public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.Thrift } @Override - public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + public void getResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, org.apache.thrift.async.AsyncMethodCallback> resultHandler) throws org.apache.thrift.TException { + checkReady(); + getResourceGroups_call method_call = new getResourceGroups_call(tinfo, credentials, updateId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class getResourceGroups_call extends org.apache.thrift.async.TAsyncMethodCall> { + private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; + private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; + private long updateId; + public getResourceGroups_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, org.apache.thrift.async.AsyncMethodCallback> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tinfo = tinfo; + this.credentials = credentials; + this.updateId = updateId; + } + + @Override + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getResourceGroups", org.apache.thrift.protocol.TMessageType.CALL, 0)); + getResourceGroups_args args = new getResourceGroups_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setUpdateId(updateId); + args.write(prot); + prot.writeMessageEnd(); + } + + @Override + public java.util.Set getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new java.lang.IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getResourceGroups(); + } + } + + @Override + public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); - setResourceGroups_call method_call = new setResourceGroups_call(tinfo, credentials, groups, resultHandler, this, ___protocolFactory, ___transport); + setResourceGroups_call method_call = new setResourceGroups_call(tinfo, credentials, updateId, groups, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; ___manager.call(method_call); } @@ -583,11 +658,13 @@ public void setResourceGroups(org.apache.accumulo.core.clientImpl.thrift.TInfo t public static class setResourceGroups_call extends org.apache.thrift.async.TAsyncMethodCall { private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; + private long updateId; private java.util.Set groups; - public setResourceGroups_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + public setResourceGroups_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, long updateId, java.util.Set groups, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { super(client, protocolFactory, transport, resultHandler, false); this.tinfo = tinfo; this.credentials = credentials; + this.updateId = updateId; this.groups = groups; } @@ -597,6 +674,7 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa setResourceGroups_args args = new setResourceGroups_args(); args.setTinfo(tinfo); args.setCredentials(credentials); + args.setUpdateId(updateId); args.setGroups(groups); args.write(prot); prot.writeMessageEnd(); @@ -633,6 +711,7 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { + public getResourceGroups() { + super("getResourceGroups"); + } + + @Override + public getResourceGroups_args getEmptyArgsInstance() { + return new getResourceGroups_args(); + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + protected boolean rethrowUnhandledExceptions() { + return false; + } + + @Override + public getResourceGroups_result getResult(I iface, getResourceGroups_args args) throws org.apache.thrift.TException { + getResourceGroups_result result = new getResourceGroups_result(); + try { + result.success = iface.getResourceGroups(args.tinfo, args.credentials, args.updateId); + } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + result.sec = sec; + } + return result; + } + } + public static class setResourceGroups extends org.apache.thrift.ProcessFunction { public setResourceGroups() { super("setResourceGroups"); @@ -848,7 +959,7 @@ protected boolean rethrowUnhandledExceptions() { public setResourceGroups_result getResult(I iface, setResourceGroups_args args) throws org.apache.thrift.TException { setResourceGroups_result result = new setResourceGroups_result(); try { - iface.setResourceGroups(args.tinfo, args.credentials, args.groups); + iface.setResourceGroups(args.tinfo, args.credentials, args.updateId, args.groups); } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { result.sec = sec; } @@ -875,6 +986,7 @@ protected AsyncProcessor(I iface, java.util.Map extends org.apache.thrift.AsyncProcessFunction> { + public getResourceGroups() { + super("getResourceGroups"); + } + + @Override + public getResourceGroups_args getEmptyArgsInstance() { + return new getResourceGroups_args(); + } + + @Override + public org.apache.thrift.async.AsyncMethodCallback> getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new org.apache.thrift.async.AsyncMethodCallback>() { + @Override + public void onComplete(java.util.Set o) { + getResourceGroups_result result = new getResourceGroups_result(); + result.success = o; + try { + fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + } catch (org.apache.thrift.transport.TTransportException e) { + _LOGGER.error("TTransportException writing to internal frame buffer", e); + fb.close(); + } catch (java.lang.Exception e) { + _LOGGER.error("Exception writing to internal frame buffer", e); + onError(e); + } + } + @Override + public void onError(java.lang.Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TSerializable msg; + getResourceGroups_result result = new getResourceGroups_result(); + if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { + result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; + result.setSecIsSet(true); + msg = result; + } else if (e instanceof org.apache.thrift.transport.TTransportException) { + _LOGGER.error("TTransportException inside handler", e); + fb.close(); + return; + } else if (e instanceof org.apache.thrift.TApplicationException) { + _LOGGER.error("TApplicationException inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TApplicationException)e; + } else { + _LOGGER.error("Exception inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + } catch (java.lang.Exception ex) { + _LOGGER.error("Exception writing to internal frame buffer", ex); + fb.close(); + } + } + }; + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + public void start(I iface, getResourceGroups_args args, org.apache.thrift.async.AsyncMethodCallback> resultHandler) throws org.apache.thrift.TException { + iface.getResourceGroups(args.tinfo, args.credentials, args.updateId,resultHandler); + } + } + public static class setResourceGroups extends org.apache.thrift.AsyncProcessFunction { public setResourceGroups() { super("setResourceGroups"); @@ -1336,7 +1519,7 @@ protected boolean isOneway() { @Override public void start(I iface, setResourceGroups_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.setResourceGroups(args.tinfo, args.credentials, args.groups,resultHandler); + iface.setResourceGroups(args.tinfo, args.credentials, args.updateId, args.groups,resultHandler); } } @@ -8188,25 +8371,25 @@ private static S scheme(org.apache. } @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class setResourceGroups_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("setResourceGroups_args"); + public static class getResourceGroups_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getResourceGroups_args"); private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); - private static final org.apache.thrift.protocol.TField GROUPS_FIELD_DESC = new org.apache.thrift.protocol.TField("groups", org.apache.thrift.protocol.TType.SET, (short)3); + private static final org.apache.thrift.protocol.TField UPDATE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("updateId", org.apache.thrift.protocol.TType.I64, (short)3); - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new setResourceGroups_argsStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new setResourceGroups_argsTupleSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getResourceGroups_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getResourceGroups_argsTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required - public @org.apache.thrift.annotation.Nullable java.util.Set groups; // required + public long updateId; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { TINFO((short)1, "tinfo"), CREDENTIALS((short)2, "credentials"), - GROUPS((short)3, "groups"); + UPDATE_ID((short)3, "updateId"); private static final java.util.Map byName = new java.util.HashMap(); @@ -8226,8 +8409,8 @@ public static _Fields findByThriftId(int fieldId) { return TINFO; case 2: // CREDENTIALS return CREDENTIALS; - case 3: // GROUPS - return GROUPS; + case 3: // UPDATE_ID + return UPDATE_ID; default: return null; } @@ -8271,6 +8454,8 @@ public java.lang.String getFieldName() { } // isset id assignments + private static final int __UPDATEID_ISSET_ID = 0; + private byte __isset_bitfield = 0; public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -8278,53 +8463,52 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); - tmpMap.put(_Fields.GROUPS, new org.apache.thrift.meta_data.FieldMetaData("groups", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.UPDATE_ID, new org.apache.thrift.meta_data.FieldMetaData("updateId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(setResourceGroups_args.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getResourceGroups_args.class, metaDataMap); } - public setResourceGroups_args() { + public getResourceGroups_args() { } - public setResourceGroups_args( + public getResourceGroups_args( org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, - java.util.Set groups) + long updateId) { this(); this.tinfo = tinfo; this.credentials = credentials; - this.groups = groups; + this.updateId = updateId; + setUpdateIdIsSet(true); } /** * Performs a deep copy on other. */ - public setResourceGroups_args(setResourceGroups_args other) { + public getResourceGroups_args(getResourceGroups_args other) { + __isset_bitfield = other.__isset_bitfield; if (other.isSetTinfo()) { this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); } if (other.isSetCredentials()) { this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); } - if (other.isSetGroups()) { - java.util.Set __this__groups = new java.util.HashSet(other.groups); - this.groups = __this__groups; - } + this.updateId = other.updateId; } @Override - public setResourceGroups_args deepCopy() { - return new setResourceGroups_args(this); + public getResourceGroups_args deepCopy() { + return new getResourceGroups_args(this); } @Override public void clear() { this.tinfo = null; this.credentials = null; - this.groups = null; + setUpdateIdIsSet(false); + this.updateId = 0; } @org.apache.thrift.annotation.Nullable @@ -8332,7 +8516,7 @@ public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { return this.tinfo; } - public setResourceGroups_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { + public getResourceGroups_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { this.tinfo = tinfo; return this; } @@ -8357,7 +8541,7 @@ public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials( return this.credentials; } - public setResourceGroups_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { + public getResourceGroups_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { this.credentials = credentials; return this; } @@ -8377,45 +8561,27 @@ public void setCredentialsIsSet(boolean value) { } } - public int getGroupsSize() { - return (this.groups == null) ? 0 : this.groups.size(); - } - - @org.apache.thrift.annotation.Nullable - public java.util.Iterator getGroupsIterator() { - return (this.groups == null) ? null : this.groups.iterator(); - } - - public void addToGroups(java.lang.String elem) { - if (this.groups == null) { - this.groups = new java.util.HashSet(); - } - this.groups.add(elem); - } - - @org.apache.thrift.annotation.Nullable - public java.util.Set getGroups() { - return this.groups; + public long getUpdateId() { + return this.updateId; } - public setResourceGroups_args setGroups(@org.apache.thrift.annotation.Nullable java.util.Set groups) { - this.groups = groups; + public getResourceGroups_args setUpdateId(long updateId) { + this.updateId = updateId; + setUpdateIdIsSet(true); return this; } - public void unsetGroups() { - this.groups = null; + public void unsetUpdateId() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __UPDATEID_ISSET_ID); } - /** Returns true if field groups is set (has been assigned a value) and false otherwise */ - public boolean isSetGroups() { - return this.groups != null; + /** Returns true if field updateId is set (has been assigned a value) and false otherwise */ + public boolean isSetUpdateId() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __UPDATEID_ISSET_ID); } - public void setGroupsIsSet(boolean value) { - if (!value) { - this.groups = null; - } + public void setUpdateIdIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __UPDATEID_ISSET_ID, value); } @Override @@ -8437,11 +8603,11 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case GROUPS: + case UPDATE_ID: if (value == null) { - unsetGroups(); + unsetUpdateId(); } else { - setGroups((java.util.Set)value); + setUpdateId((java.lang.Long)value); } break; @@ -8458,8 +8624,8 @@ public java.lang.Object getFieldValue(_Fields field) { case CREDENTIALS: return getCredentials(); - case GROUPS: - return getGroups(); + case UPDATE_ID: + return getUpdateId(); } throw new java.lang.IllegalStateException(); @@ -8477,20 +8643,20 @@ public boolean isSet(_Fields field) { return isSetTinfo(); case CREDENTIALS: return isSetCredentials(); - case GROUPS: - return isSetGroups(); + case UPDATE_ID: + return isSetUpdateId(); } throw new java.lang.IllegalStateException(); } @Override public boolean equals(java.lang.Object that) { - if (that instanceof setResourceGroups_args) - return this.equals((setResourceGroups_args)that); + if (that instanceof getResourceGroups_args) + return this.equals((getResourceGroups_args)that); return false; } - public boolean equals(setResourceGroups_args that) { + public boolean equals(getResourceGroups_args that) { if (that == null) return false; if (this == that) @@ -8514,12 +8680,12 @@ public boolean equals(setResourceGroups_args that) { return false; } - boolean this_present_groups = true && this.isSetGroups(); - boolean that_present_groups = true && that.isSetGroups(); - if (this_present_groups || that_present_groups) { - if (!(this_present_groups && that_present_groups)) + boolean this_present_updateId = true; + boolean that_present_updateId = true; + if (this_present_updateId || that_present_updateId) { + if (!(this_present_updateId && that_present_updateId)) return false; - if (!this.groups.equals(that.groups)) + if (this.updateId != that.updateId) return false; } @@ -8538,15 +8704,13 @@ public int hashCode() { if (isSetCredentials()) hashCode = hashCode * 8191 + credentials.hashCode(); - hashCode = hashCode * 8191 + ((isSetGroups()) ? 131071 : 524287); - if (isSetGroups()) - hashCode = hashCode * 8191 + groups.hashCode(); + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(updateId); return hashCode; } @Override - public int compareTo(setResourceGroups_args other) { + public int compareTo(getResourceGroups_args other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } @@ -8573,12 +8737,12 @@ public int compareTo(setResourceGroups_args other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetGroups(), other.isSetGroups()); + lastComparison = java.lang.Boolean.compare(isSetUpdateId(), other.isSetUpdateId()); if (lastComparison != 0) { return lastComparison; } - if (isSetGroups()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groups, other.groups); + if (isSetUpdateId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.updateId, other.updateId); if (lastComparison != 0) { return lastComparison; } @@ -8604,7 +8768,7 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("setResourceGroups_args("); + java.lang.StringBuilder sb = new java.lang.StringBuilder("getResourceGroups_args("); boolean first = true; sb.append("tinfo:"); @@ -8623,12 +8787,8 @@ public java.lang.String toString() { } first = false; if (!first) sb.append(", "); - sb.append("groups:"); - if (this.groups == null) { - sb.append("null"); - } else { - sb.append(this.groups); - } + sb.append("updateId:"); + sb.append(this.updateId); first = false; sb.append(")"); return sb.toString(); @@ -8655,23 +8815,25 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); } } - private static class setResourceGroups_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + private static class getResourceGroups_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { @Override - public setResourceGroups_argsStandardScheme getScheme() { - return new setResourceGroups_argsStandardScheme(); + public getResourceGroups_argsStandardScheme getScheme() { + return new getResourceGroups_argsStandardScheme(); } } - private static class setResourceGroups_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + private static class getResourceGroups_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, setResourceGroups_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, getResourceGroups_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -8699,20 +8861,10 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, setResourceGroups_a org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 3: // GROUPS - if (schemeField.type == org.apache.thrift.protocol.TType.SET) { - { - org.apache.thrift.protocol.TSet _set34 = iprot.readSetBegin(); - struct.groups = new java.util.HashSet(2*_set34.size); - @org.apache.thrift.annotation.Nullable java.lang.String _elem35; - for (int _i36 = 0; _i36 < _set34.size; ++_i36) - { - _elem35 = iprot.readString(); - struct.groups.add(_elem35); - } - iprot.readSetEnd(); - } - struct.setGroupsIsSet(true); + case 3: // UPDATE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.updateId = iprot.readI64(); + struct.setUpdateIdIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -8729,7 +8881,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, setResourceGroups_a } @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, setResourceGroups_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, getResourceGroups_args struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); @@ -8743,35 +8895,26 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, setResourceGroups_ struct.credentials.write(oprot); oprot.writeFieldEnd(); } - if (struct.groups != null) { - oprot.writeFieldBegin(GROUPS_FIELD_DESC); - { - oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRING, struct.groups.size())); - for (java.lang.String _iter37 : struct.groups) - { - oprot.writeString(_iter37); - } - oprot.writeSetEnd(); - } - oprot.writeFieldEnd(); - } + oprot.writeFieldBegin(UPDATE_ID_FIELD_DESC); + oprot.writeI64(struct.updateId); + oprot.writeFieldEnd(); oprot.writeFieldStop(); oprot.writeStructEnd(); } } - private static class setResourceGroups_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + private static class getResourceGroups_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { @Override - public setResourceGroups_argsTupleScheme getScheme() { - return new setResourceGroups_argsTupleScheme(); + public getResourceGroups_argsTupleScheme getScheme() { + return new getResourceGroups_argsTupleScheme(); } } - private static class setResourceGroups_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + private static class getResourceGroups_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, getResourceGroups_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; java.util.BitSet optionals = new java.util.BitSet(); if (struct.isSetTinfo()) { @@ -8780,7 +8923,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_a if (struct.isSetCredentials()) { optionals.set(1); } - if (struct.isSetGroups()) { + if (struct.isSetUpdateId()) { optionals.set(2); } oprot.writeBitSet(optionals, 3); @@ -8790,13 +8933,1278 @@ public void write(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_a if (struct.isSetCredentials()) { struct.credentials.write(oprot); } - if (struct.isSetGroups()) { - { - oprot.writeI32(struct.groups.size()); - for (java.lang.String _iter38 : struct.groups) - { - oprot.writeString(_iter38); - } + if (struct.isSetUpdateId()) { + oprot.writeI64(struct.updateId); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getResourceGroups_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } + if (incoming.get(1)) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } + if (incoming.get(2)) { + struct.updateId = iprot.readI64(); + struct.setUpdateIdIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class getResourceGroups_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getResourceGroups_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.SET, (short)0); + private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getResourceGroups_resultStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getResourceGroups_resultTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable java.util.Set success; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + SEC((short)1, "sec"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // SEC + return SEC; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getResourceGroups_result.class, metaDataMap); + } + + public getResourceGroups_result() { + } + + public getResourceGroups_result( + java.util.Set success, + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) + { + this(); + this.success = success; + this.sec = sec; + } + + /** + * Performs a deep copy on other. + */ + public getResourceGroups_result(getResourceGroups_result other) { + if (other.isSetSuccess()) { + java.util.Set __this__success = new java.util.HashSet(other.success); + this.success = __this__success; + } + if (other.isSetSec()) { + this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); + } + } + + @Override + public getResourceGroups_result deepCopy() { + return new getResourceGroups_result(this); + } + + @Override + public void clear() { + this.success = null; + this.sec = null; + } + + public int getSuccessSize() { + return (this.success == null) ? 0 : this.success.size(); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Iterator getSuccessIterator() { + return (this.success == null) ? null : this.success.iterator(); + } + + public void addToSuccess(java.lang.String elem) { + if (this.success == null) { + this.success = new java.util.HashSet(); + } + this.success.add(elem); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Set getSuccess() { + return this.success; + } + + public getResourceGroups_result setSuccess(@org.apache.thrift.annotation.Nullable java.util.Set success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { + return this.sec; + } + + public getResourceGroups_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + this.sec = sec; + return this; + } + + public void unsetSec() { + this.sec = null; + } + + /** Returns true if field sec is set (has been assigned a value) and false otherwise */ + public boolean isSetSec() { + return this.sec != null; + } + + public void setSecIsSet(boolean value) { + if (!value) { + this.sec = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((java.util.Set)value); + } + break; + + case SEC: + if (value == null) { + unsetSec(); + } else { + setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case SEC: + return getSec(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case SEC: + return isSetSec(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof getResourceGroups_result) + return this.equals((getResourceGroups_result)that); + return false; + } + + public boolean equals(getResourceGroups_result that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_sec = true && this.isSetSec(); + boolean that_present_sec = true && that.isSetSec(); + if (this_present_sec || that_present_sec) { + if (!(this_present_sec && that_present_sec)) + return false; + if (!this.sec.equals(that.sec)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetSuccess()) ? 131071 : 524287); + if (isSetSuccess()) + hashCode = hashCode * 8191 + success.hashCode(); + + hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); + if (isSetSec()) + hashCode = hashCode * 8191 + sec.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(getResourceGroups_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetSuccess(), other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSec()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("getResourceGroups_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("sec:"); + if (this.sec == null) { + sb.append("null"); + } else { + sb.append(this.sec); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getResourceGroups_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public getResourceGroups_resultStandardScheme getScheme() { + return new getResourceGroups_resultStandardScheme(); + } + } + + private static class getResourceGroups_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, getResourceGroups_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set34 = iprot.readSetBegin(); + struct.success = new java.util.HashSet(2*_set34.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem35; + for (int _i36 = 0; _i36 < _set34.size; ++_i36) + { + _elem35 = iprot.readString(); + struct.success.add(_elem35); + } + iprot.readSetEnd(); + } + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // SEC + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, getResourceGroups_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRING, struct.success.size())); + for (java.lang.String _iter37 : struct.success) + { + oprot.writeString(_iter37); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + if (struct.sec != null) { + oprot.writeFieldBegin(SEC_FIELD_DESC); + struct.sec.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getResourceGroups_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public getResourceGroups_resultTupleScheme getScheme() { + return new getResourceGroups_resultTupleScheme(); + } + } + + private static class getResourceGroups_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getResourceGroups_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetSec()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + { + oprot.writeI32(struct.success.size()); + for (java.lang.String _iter38 : struct.success) + { + oprot.writeString(_iter38); + } + } + } + if (struct.isSetSec()) { + struct.sec.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getResourceGroups_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + { + org.apache.thrift.protocol.TSet _set39 = iprot.readSetBegin(org.apache.thrift.protocol.TType.STRING); + struct.success = new java.util.HashSet(2*_set39.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem40; + for (int _i41 = 0; _i41 < _set39.size; ++_i41) + { + _elem40 = iprot.readString(); + struct.success.add(_elem40); + } + } + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class setResourceGroups_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("setResourceGroups_args"); + + private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField UPDATE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("updateId", org.apache.thrift.protocol.TType.I64, (short)3); + private static final org.apache.thrift.protocol.TField GROUPS_FIELD_DESC = new org.apache.thrift.protocol.TField("groups", org.apache.thrift.protocol.TType.SET, (short)4); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new setResourceGroups_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new setResourceGroups_argsTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required + public long updateId; // required + public @org.apache.thrift.annotation.Nullable java.util.Set groups; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TINFO((short)1, "tinfo"), + CREDENTIALS((short)2, "credentials"), + UPDATE_ID((short)3, "updateId"), + GROUPS((short)4, "groups"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TINFO + return TINFO; + case 2: // CREDENTIALS + return CREDENTIALS; + case 3: // UPDATE_ID + return UPDATE_ID; + case 4: // GROUPS + return GROUPS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __UPDATEID_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); + tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); + tmpMap.put(_Fields.UPDATE_ID, new org.apache.thrift.meta_data.FieldMetaData("updateId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.GROUPS, new org.apache.thrift.meta_data.FieldMetaData("groups", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.SetMetaData(org.apache.thrift.protocol.TType.SET, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(setResourceGroups_args.class, metaDataMap); + } + + public setResourceGroups_args() { + } + + public setResourceGroups_args( + org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, + org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, + long updateId, + java.util.Set groups) + { + this(); + this.tinfo = tinfo; + this.credentials = credentials; + this.updateId = updateId; + setUpdateIdIsSet(true); + this.groups = groups; + } + + /** + * Performs a deep copy on other. + */ + public setResourceGroups_args(setResourceGroups_args other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetTinfo()) { + this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); + } + if (other.isSetCredentials()) { + this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); + } + this.updateId = other.updateId; + if (other.isSetGroups()) { + java.util.Set __this__groups = new java.util.HashSet(other.groups); + this.groups = __this__groups; + } + } + + @Override + public setResourceGroups_args deepCopy() { + return new setResourceGroups_args(this); + } + + @Override + public void clear() { + this.tinfo = null; + this.credentials = null; + setUpdateIdIsSet(false); + this.updateId = 0; + this.groups = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { + return this.tinfo; + } + + public setResourceGroups_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { + this.tinfo = tinfo; + return this; + } + + public void unsetTinfo() { + this.tinfo = null; + } + + /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ + public boolean isSetTinfo() { + return this.tinfo != null; + } + + public void setTinfoIsSet(boolean value) { + if (!value) { + this.tinfo = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { + return this.credentials; + } + + public setResourceGroups_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { + this.credentials = credentials; + return this; + } + + public void unsetCredentials() { + this.credentials = null; + } + + /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ + public boolean isSetCredentials() { + return this.credentials != null; + } + + public void setCredentialsIsSet(boolean value) { + if (!value) { + this.credentials = null; + } + } + + public long getUpdateId() { + return this.updateId; + } + + public setResourceGroups_args setUpdateId(long updateId) { + this.updateId = updateId; + setUpdateIdIsSet(true); + return this; + } + + public void unsetUpdateId() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __UPDATEID_ISSET_ID); + } + + /** Returns true if field updateId is set (has been assigned a value) and false otherwise */ + public boolean isSetUpdateId() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __UPDATEID_ISSET_ID); + } + + public void setUpdateIdIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __UPDATEID_ISSET_ID, value); + } + + public int getGroupsSize() { + return (this.groups == null) ? 0 : this.groups.size(); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Iterator getGroupsIterator() { + return (this.groups == null) ? null : this.groups.iterator(); + } + + public void addToGroups(java.lang.String elem) { + if (this.groups == null) { + this.groups = new java.util.HashSet(); + } + this.groups.add(elem); + } + + @org.apache.thrift.annotation.Nullable + public java.util.Set getGroups() { + return this.groups; + } + + public setResourceGroups_args setGroups(@org.apache.thrift.annotation.Nullable java.util.Set groups) { + this.groups = groups; + return this; + } + + public void unsetGroups() { + this.groups = null; + } + + /** Returns true if field groups is set (has been assigned a value) and false otherwise */ + public boolean isSetGroups() { + return this.groups != null; + } + + public void setGroupsIsSet(boolean value) { + if (!value) { + this.groups = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case TINFO: + if (value == null) { + unsetTinfo(); + } else { + setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); + } + break; + + case CREDENTIALS: + if (value == null) { + unsetCredentials(); + } else { + setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); + } + break; + + case UPDATE_ID: + if (value == null) { + unsetUpdateId(); + } else { + setUpdateId((java.lang.Long)value); + } + break; + + case GROUPS: + if (value == null) { + unsetGroups(); + } else { + setGroups((java.util.Set)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case TINFO: + return getTinfo(); + + case CREDENTIALS: + return getCredentials(); + + case UPDATE_ID: + return getUpdateId(); + + case GROUPS: + return getGroups(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case TINFO: + return isSetTinfo(); + case CREDENTIALS: + return isSetCredentials(); + case UPDATE_ID: + return isSetUpdateId(); + case GROUPS: + return isSetGroups(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof setResourceGroups_args) + return this.equals((setResourceGroups_args)that); + return false; + } + + public boolean equals(setResourceGroups_args that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_tinfo = true && this.isSetTinfo(); + boolean that_present_tinfo = true && that.isSetTinfo(); + if (this_present_tinfo || that_present_tinfo) { + if (!(this_present_tinfo && that_present_tinfo)) + return false; + if (!this.tinfo.equals(that.tinfo)) + return false; + } + + boolean this_present_credentials = true && this.isSetCredentials(); + boolean that_present_credentials = true && that.isSetCredentials(); + if (this_present_credentials || that_present_credentials) { + if (!(this_present_credentials && that_present_credentials)) + return false; + if (!this.credentials.equals(that.credentials)) + return false; + } + + boolean this_present_updateId = true; + boolean that_present_updateId = true; + if (this_present_updateId || that_present_updateId) { + if (!(this_present_updateId && that_present_updateId)) + return false; + if (this.updateId != that.updateId) + return false; + } + + boolean this_present_groups = true && this.isSetGroups(); + boolean that_present_groups = true && that.isSetGroups(); + if (this_present_groups || that_present_groups) { + if (!(this_present_groups && that_present_groups)) + return false; + if (!this.groups.equals(that.groups)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); + if (isSetTinfo()) + hashCode = hashCode * 8191 + tinfo.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); + if (isSetCredentials()) + hashCode = hashCode * 8191 + credentials.hashCode(); + + hashCode = hashCode * 8191 + org.apache.thrift.TBaseHelper.hashCode(updateId); + + hashCode = hashCode * 8191 + ((isSetGroups()) ? 131071 : 524287); + if (isSetGroups()) + hashCode = hashCode * 8191 + groups.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(setResourceGroups_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTinfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCredentials()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetUpdateId(), other.isSetUpdateId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetUpdateId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.updateId, other.updateId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetGroups(), other.isSetGroups()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGroups()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groups, other.groups); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("setResourceGroups_args("); + boolean first = true; + + sb.append("tinfo:"); + if (this.tinfo == null) { + sb.append("null"); + } else { + sb.append(this.tinfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("credentials:"); + if (this.credentials == null) { + sb.append("null"); + } else { + sb.append(this.credentials); + } + first = false; + if (!first) sb.append(", "); + sb.append("updateId:"); + sb.append(this.updateId); + first = false; + if (!first) sb.append(", "); + sb.append("groups:"); + if (this.groups == null) { + sb.append("null"); + } else { + sb.append(this.groups); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (tinfo != null) { + tinfo.validate(); + } + if (credentials != null) { + credentials.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class setResourceGroups_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public setResourceGroups_argsStandardScheme getScheme() { + return new setResourceGroups_argsStandardScheme(); + } + } + + private static class setResourceGroups_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, setResourceGroups_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TINFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // CREDENTIALS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // UPDATE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.I64) { + struct.updateId = iprot.readI64(); + struct.setUpdateIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // GROUPS + if (schemeField.type == org.apache.thrift.protocol.TType.SET) { + { + org.apache.thrift.protocol.TSet _set42 = iprot.readSetBegin(); + struct.groups = new java.util.HashSet(2*_set42.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem43; + for (int _i44 = 0; _i44 < _set42.size; ++_i44) + { + _elem43 = iprot.readString(); + struct.groups.add(_elem43); + } + iprot.readSetEnd(); + } + struct.setGroupsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, setResourceGroups_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tinfo != null) { + oprot.writeFieldBegin(TINFO_FIELD_DESC); + struct.tinfo.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.credentials != null) { + oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); + struct.credentials.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(UPDATE_ID_FIELD_DESC); + oprot.writeI64(struct.updateId); + oprot.writeFieldEnd(); + if (struct.groups != null) { + oprot.writeFieldBegin(GROUPS_FIELD_DESC); + { + oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRING, struct.groups.size())); + for (java.lang.String _iter45 : struct.groups) + { + oprot.writeString(_iter45); + } + oprot.writeSetEnd(); + } + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class setResourceGroups_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public setResourceGroups_argsTupleScheme getScheme() { + return new setResourceGroups_argsTupleScheme(); + } + } + + private static class setResourceGroups_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetTinfo()) { + optionals.set(0); + } + if (struct.isSetCredentials()) { + optionals.set(1); + } + if (struct.isSetUpdateId()) { + optionals.set(2); + } + if (struct.isSetGroups()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetTinfo()) { + struct.tinfo.write(oprot); + } + if (struct.isSetCredentials()) { + struct.credentials.write(oprot); + } + if (struct.isSetUpdateId()) { + oprot.writeI64(struct.updateId); + } + if (struct.isSetGroups()) { + { + oprot.writeI32(struct.groups.size()); + for (java.lang.String _iter46 : struct.groups) + { + oprot.writeString(_iter46); + } } } } @@ -8804,7 +10212,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_a @Override public void read(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); + java.util.BitSet incoming = iprot.readBitSet(4); if (incoming.get(0)) { struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); struct.tinfo.read(iprot); @@ -8816,14 +10224,18 @@ public void read(org.apache.thrift.protocol.TProtocol prot, setResourceGroups_ar struct.setCredentialsIsSet(true); } if (incoming.get(2)) { + struct.updateId = iprot.readI64(); + struct.setUpdateIdIsSet(true); + } + if (incoming.get(3)) { { - org.apache.thrift.protocol.TSet _set39 = iprot.readSetBegin(org.apache.thrift.protocol.TType.STRING); - struct.groups = new java.util.HashSet(2*_set39.size); - @org.apache.thrift.annotation.Nullable java.lang.String _elem40; - for (int _i41 = 0; _i41 < _set39.size; ++_i41) + org.apache.thrift.protocol.TSet _set47 = iprot.readSetBegin(org.apache.thrift.protocol.TType.STRING); + struct.groups = new java.util.HashSet(2*_set47.size); + @org.apache.thrift.annotation.Nullable java.lang.String _elem48; + for (int _i49 = 0; _i49 < _set47.size; ++_i49) { - _elem40 = iprot.readString(); - struct.groups.add(_elem40); + _elem48 = iprot.readString(); + struct.groups.add(_elem48); } } struct.setGroupsIsSet(true); diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java index 2fee49858bb..ba4f02996f6 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java @@ -3668,14 +3668,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveCompaction case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list42 = iprot.readListBegin(); - struct.success = new java.util.ArrayList(_list42.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem43; - for (int _i44 = 0; _i44 < _list42.size; ++_i44) + org.apache.thrift.protocol.TList _list50 = iprot.readListBegin(); + struct.success = new java.util.ArrayList(_list50.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem51; + for (int _i52 = 0; _i52 < _list50.size; ++_i52) { - _elem43 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem43.read(iprot); - struct.success.add(_elem43); + _elem51 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem51.read(iprot); + struct.success.add(_elem51); } iprot.readListEnd(); } @@ -3713,9 +3713,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveCompactio oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter45 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter53 : struct.success) { - _iter45.write(oprot); + _iter53.write(oprot); } oprot.writeListEnd(); } @@ -3755,9 +3755,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveCompaction if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter46 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter54 : struct.success) { - _iter46.write(oprot); + _iter54.write(oprot); } } } @@ -3772,14 +3772,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveCompactions java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list47 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.success = new java.util.ArrayList(_list47.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem48; - for (int _i49 = 0; _i49 < _list47.size; ++_i49) + org.apache.thrift.protocol.TList _list55 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.success = new java.util.ArrayList(_list55.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem56; + for (int _i57 = 0; _i57 < _list55.size; ++_i57) { - _elem48 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem48.read(iprot); - struct.success.add(_elem48); + _elem56 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem56.read(iprot); + struct.success.add(_elem56); } } struct.setSuccessIsSet(true); diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index 670d0e439ea..973255c0513 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -78,7 +78,6 @@ struct TResolvedCompactionJob { 9:bool overlapsSelectedFiles } - exception UnknownCompactionIdException {} service CompactionCoordinatorService { @@ -149,10 +148,19 @@ service CompactionCoordinatorService { 1:client.ThriftSecurityException sec ) + set getResourceGroups( + 1:client.TInfo tinfo + 2:security.TCredentials credentials + 3:i64 updateId + )throws( + 1:client.ThriftSecurityException sec + ) + void setResourceGroups( 1:client.TInfo tinfo 2:security.TCredentials credentials - 3:set groups + 3:i64 updateId + 4:set groups )throws( 1:client.ThriftSecurityException sec ) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 072a997f579..e5550b3d971 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -1155,8 +1155,6 @@ boolean canSuspendTablets() { this.splitter = new Splitter(this); this.splitter.start(); - setupFate(context); - fateManager = new FateManager(getContext()); fateManager.start(); startFateMaintenance(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java index 61a124000ca..e2eda1fdd1d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java @@ -59,6 +59,7 @@ record CoordinatorConnection(CompactionCoordinatorService.Client client, private final Map coordinatorLocations; private final Map coordinatorConnections = new HashMap<>(); + private final Map ignoredCounts = new HashMap<>(); private static final int BUFFER_SIZE = 1000; @@ -69,6 +70,8 @@ public CompactionJobClient(ServerContext context, Ample.DataLevel dataLevel, boo this.coordinatorLocations = context.getCoordinatorLocations(true); + log.trace("Using coordinator locations {}", coordinatorLocations); + var uniqueHosts = new HashSet<>(coordinatorLocations.values()); for (var hostPort : uniqueHosts) { try { @@ -92,12 +95,14 @@ public void addJobs(TabletMetadata tabletMetadata, Collection job var resolvedJob = new ResolvedCompactionJob(job, tabletMetadata); var hostPort = coordinatorLocations.get(resolvedJob.getGroup()); if (hostPort == null) { - log.debug("Ignoring job, no coordinator found {}", job.getGroup()); + log.trace("Ignoring job, no coordinator found {}", job.getGroup()); + ignoredCounts.merge(job.getGroup(), 1L, Long::sum); continue; } var coordinator = coordinatorConnections.get(hostPort); if (coordinator == null) { - log.debug("Ignoring job, no connection found {}", job.getGroup()); + log.trace("Ignoring job, no connection found {}", job.getGroup()); + ignoredCounts.merge(job.getGroup(), 1L, Long::sum); continue; } @@ -118,7 +123,7 @@ public void addJobs(TabletMetadata tabletMetadata, Collection job private void sendJobs(CoordinatorConnection coordinator) throws TException { List thriftJobs = new ArrayList<>(coordinator.jobBuffer.size()); for (var job : coordinator.jobBuffer) { - log.debug("Sending job {} {} {}", coordinator.address, job.getGroup(), job.getExtent()); + log.trace("Sending job {} {} {}", coordinator.address, job.getGroup(), job.getExtent()); thriftJobs.add(job.toThrift()); } coordinator.client.addJobs(TraceUtil.traceInfo(), context.rpcCreds(), thriftJobs); @@ -140,5 +145,9 @@ public void close() { ThriftUtil.returnClient(coordinator.client, context); } })); + + if (!ignoredCounts.isEmpty()) { + log.debug("Ignored compaction job counts {}", ignoredCounts); + } } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 45970815724..f870b35bb3d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -70,6 +70,7 @@ import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; import org.apache.accumulo.core.compaction.thrift.TResolvedCompactionJob; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.AbstractId; import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; @@ -609,8 +610,15 @@ public void endFullJobScan(TInfo tinfo, TCredentials credentials, String dataLev jobQueues.endFullScan(DataLevel.valueOf(dataLevel)); } + private static class UpdateId { + long updateId; + boolean set; + } + + private final UpdateId expectedUpdateId = new UpdateId(); + @Override - public void setResourceGroups(TInfo tinfo, TCredentials credentials, Set groups) + public Set getResourceGroups(TInfo tinfo, TCredentials credentials, long updateId) throws ThriftSecurityException, TException { if (!security.canPerformSystemActions(credentials)) { // TODO does not seem like the correct exception, also this code snippet was copied. @@ -618,10 +626,35 @@ public void setResourceGroups(TInfo tinfo, TCredentials credentials, Set SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } + synchronized (expectedUpdateId) { + // invalidate any outstanding updates and set a new one time use update id + expectedUpdateId.updateId = updateId; + expectedUpdateId.set = true; + return jobQueues.getAllowedGroups().stream().map(AbstractId::canonical).collect(toSet()); + } + } + + @Override + public void setResourceGroups(TInfo tinfo, TCredentials credentials, long updateId, + Set groups) throws ThriftSecurityException, TException { + if (!security.canPerformSystemActions(credentials)) { + // TODO does not seem like the correct exception, also this code snippet was copied. + throw new AccumuloSecurityException(credentials.getPrincipal(), + SecurityErrorCode.PERMISSION_DENIED).asThriftException(); + } + // TODO validate that upgrade is complete like FateWorker does - jobQueues.setResourceGroups(groups.stream().map(ResourceGroupId::of).collect(toSet())); - LOG.debug("Set resource groups to {}", groups); + // only allow an update id to be used once + synchronized (expectedUpdateId) { + if (expectedUpdateId.updateId == updateId && expectedUpdateId.set) { + jobQueues.setAllowedGroups(groups.stream().map(ResourceGroupId::of).collect(toSet())); + LOG.debug("Set allowed resource groups to {}", groups); + expectedUpdateId.set = false; + } else { + LOG.debug("Did not set resource groups because update id did not match"); + } + } } // TODO remove diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index c0d39c10b65..14e24691649 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -19,9 +19,10 @@ package org.apache.accumulo.manager.compaction.coordinator; import static java.util.stream.Collectors.toSet; +import static org.apache.accumulo.manager.multi.ManagerAssignment.computeAssignments; +import static org.apache.accumulo.server.compaction.CompactionPluginUtils.getConfiguredCompactionResourceGroups; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -33,10 +34,10 @@ import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.core.util.LazySingletons; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.compaction.CompactionPluginUtils; import org.apache.accumulo.server.compaction.CoordinatorLocations; import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; @@ -64,88 +65,89 @@ public CoordinatorManager(ServerContext context) { private void managerCoordinators() { while (true) { + try { + long updateId = LazySingletons.RANDOM.get().nextLong(); + var current = getCurrentAssignments(updateId); + + log.trace("Current assignments {}", current); + + var configuredGroups = getConfiguredCompactionResourceGroups(context).stream() + .map(ResourceGroupId::of).collect(toSet()); - var assistants = context.getServerPaths() - .getAssistantManagers(ServiceLockPaths.AddressSelector.all(), true); - if (!assistants.isEmpty()) { - var compactorGroups = CompactionPluginUtils.getConfiguredCompactionResourceGroups(context) - .stream().map(ResourceGroupId::of).collect(toSet()); - - int minGroupsPerAssistant = compactorGroups.size() / assistants.size(); - int maxGroupsPerAssistant = - minGroupsPerAssistant + Math.min(compactorGroups.size() % assistants.size(), 1); - - Map currentLocations = - context.getCoordinatorLocations(false); - - // group by coordinator - Map> groupsPerCompactor = new HashMap<>(); - currentLocations.forEach((rg, hp) -> { - groupsPerCompactor.computeIfAbsent(hp, hp2 -> new HashSet<>()).add(rg); - }); - - boolean needsUpdates = !currentLocations.keySet().containsAll(compactorGroups); - // Look for any compactors that are not correct - for (var groups : groupsPerCompactor.values()) { - if (groups.size() < minGroupsPerAssistant || groups.size() > maxGroupsPerAssistant - || !compactorGroups.containsAll(groups)) { - needsUpdates = true; - } - } - - if (needsUpdates) { - // TODO this does not try to keep current assignemnts - // TODO this should ask coordinators what they currently have instead of assuming ZK is - // correct, on failure ZK could be out of sync... otherwise always set RG on - // coordinators to ensure they match ZK - // TODO combine/share code w/ fate for assignments, that is why the todos above are not - // done, this code is temporary hack and it has problems. - Map> updates = new HashMap<>(); - assistants.forEach( - slp -> updates.put(HostAndPort.fromString(slp.getServer()), new HashSet<>())); - - var iter = compactorGroups.iterator(); - updates.values().forEach(groups -> { - while (groups.size() < minGroupsPerAssistant) { - groups.add(iter.next()); - } - }); - - updates.values().forEach(groups -> { - while (iter.hasNext() && groups.size() < maxGroupsPerAssistant) { - groups.add(iter.next()); - } - }); - - updates.forEach(this::setResourceGroups); - Map newLocations = new HashMap<>(); - updates.forEach((hp, groups) -> { - groups.forEach(group -> newLocations.put(group, hp)); - }); - - CoordinatorLocations.setLocations(context.getZooSession().asReaderWriter(), - newLocations, NodeExistsPolicy.OVERWRITE); - log.debug("Set locations {}", newLocations); - } + log.trace("Configured groups {}", configuredGroups); + + var desired = computeAssignments(current, configuredGroups); + + log.trace("Desired assignments {}", desired); + if (!current.equals(desired)) { + setAssignments(updateId, desired); } - // TODO better exception handling - } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException - | InterruptedException | KeeperException e) { - log.warn("Failed to manage coordinators", e); + + updateZookeeper(desired); + } catch (ReflectiveOperationException | InterruptedException | KeeperException e) { + // TODO + throw new RuntimeException(e); } - // TODO not good for test + // TODO not good for tests UtilWaitThread.sleep(5000); } } - private void setResourceGroups(HostAndPort hp, Set groups) { + private void updateZookeeper(Map> desired) + throws InterruptedException, KeeperException { + // transform desired set to look like what is in zookeeper + HashMap transformed = new HashMap<>(); + desired.forEach((hp, rgs) -> { + rgs.forEach(rg -> transformed.put(rg, hp)); + }); + + var assignmentsInZk = context.getCoordinatorLocations(true); + if (!transformed.equals(assignmentsInZk)) { + CoordinatorLocations.setLocations(context.getZooSession().asReaderWriter(), transformed, + NodeExistsPolicy.OVERWRITE); + log.debug("Set new coordinator locations {}", desired); + + } + } + + private Map> getCurrentAssignments(long updateId) { + var assistants = + context.getServerPaths().getAssistantManagers(ServiceLockPaths.AddressSelector.all(), true); + + Map> assignments = new HashMap<>(); + + for (var assistant : assistants) { + CompactionCoordinatorService.Client client = null; + try { + var hp = HostAndPort.fromString(assistant.getServer()); + client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, hp, context); + var groups = client.getResourceGroups(TraceUtil.traceInfo(), context.rpcCreds(), updateId); + assignments.put(hp, groups.stream().map(ResourceGroupId::of).collect(toSet())); + } catch (TException e) { + // TODO + throw new RuntimeException(e); + } finally { + ThriftUtil.returnClient(client, context); + } + } + + return assignments; + } + + private void setAssignments(long updateId, Map> assignments) { + assignments.forEach((hp, groups) -> { + setResourceGroups(hp, groups, updateId); + }); + } + + private void setResourceGroups(HostAndPort hp, Set groups, long updateId) { CompactionCoordinatorService.Client client = null; try { client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, hp, context); - client.setResourceGroups(TraceUtil.traceInfo(), context.rpcCreds(), + client.setResourceGroups(TraceUtil.traceInfo(), context.rpcCreds(), updateId, groups.stream().map(AbstractId::canonical).collect(toSet())); } catch (TException e) { // TODO diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java index d17e57838b1..61bcb385052 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java @@ -180,7 +180,13 @@ public void resetMaxSize(long size) { priorityQueues.values().forEach(cjpq -> cjpq.resetMaxSize(this.queueSize)); } - public void setResourceGroups(Set groups) { + public Set getAllowedGroups() { + synchronized (allowedGroups) { + return Set.copyOf(allowedGroups); + } + } + + public void setAllowedGroups(Set groups) { synchronized (allowedGroups) { allowedGroups.clear(); allowedGroups.addAll(groups); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java index a5e64497ab1..f990fc0247f 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java @@ -42,6 +42,7 @@ import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.CountDownTimer; import org.apache.accumulo.core.util.threads.Threads; +import org.apache.accumulo.manager.multi.ManagerAssignment; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.manager.FateLocations; @@ -284,32 +285,8 @@ private Map> computeDesiredAssignments( }); desiredParititions.forEach((fateType, desiredForType) -> { - // This code can not handle more than one partition per host - Preconditions.checkState(desiredForType.size() <= currentAssignments.size()); - - var added = new HashSet(); - - currentAssignments.forEach((hp, partitions) -> { - var hostAssignments = desiredAssignments.get(hp); - partitions.forEach(partition -> { - if (desiredForType.contains(partition) - && hostAssignments.stream().noneMatch(fp -> fp.getType() == fateType) - && !added.contains(partition)) { - hostAssignments.add(partition); - Preconditions.checkState(added.add(partition)); - } - }); - }); - - var iter = Sets.difference(desiredForType, added).iterator(); - currentAssignments.forEach((hp, partitions) -> { - var hostAssignments = desiredAssignments.get(hp); - if (iter.hasNext() && hostAssignments.stream().noneMatch(fp -> fp.getType() == fateType)) { - hostAssignments.add(iter.next()); - } - }); - - Preconditions.checkState(!iter.hasNext()); + var assignments = ManagerAssignment.computeAssignments(currentAssignments, desiredForType); + assignments.forEach((hp, parts) -> desiredAssignments.get(hp).addAll(parts)); }); if (log.isTraceEnabled()) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java b/server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java new file mode 100644 index 00000000000..419878b15a8 --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.multi; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.Sets; +import com.google.common.net.HostAndPort; + +public abstract class ManagerAssignment { + + /** + * Generic utility code to assign partitions to host. It evenly spread the partitions across host + * and tried to keep partition already on a host in place if possible. + */ + public static Map> computeAssignments(Map> current, + Set desiredPartitions) { + if (current.isEmpty()) { + return Map.of(); + } + + int minAssignments = desiredPartitions.size() / current.size(); + int maxAssignments = minAssignments + Math.min(desiredPartitions.size() % current.size(), 1); + int hostThatCanHaveMax = desiredPartitions.size() % current.size(); + + System.out.printf("minA:%d maxA:%d htchm:%d\n", minAssignments, maxAssignments, + hostThatCanHaveMax); + + Map> assignments = new HashMap<>(); + Set assigned = new HashSet<>(); + + Set hostThatHaveMax = new HashSet<>(); + // Make a pass through to keep anything in place that can be kept + for (var entry : current.entrySet()) { + HostAndPort hp = entry.getKey(); + Set currParts = entry.getValue(); + var hostAssignments = new HashSet(); + var iter = + Sets.intersection(desiredPartitions, Sets.difference(currParts, assigned)).iterator(); + + int canAssign = hostThatHaveMax.size() < hostThatCanHaveMax ? maxAssignments : minAssignments; + + while (iter.hasNext() && hostAssignments.size() < canAssign) { + hostAssignments.add(iter.next()); + } + + if (hostAssignments.size() >= maxAssignments) { + hostThatHaveMax.add(hp); + } + + assignments.put(hp, hostAssignments); + assigned.addAll(hostAssignments); + } + + // Make a second pass to get every partition assigned + for (var hp : current.keySet()) { + var hostAssignments = assignments.get(hp); + int canAssign = hostThatHaveMax.size() < hostThatCanHaveMax ? maxAssignments : minAssignments; + var iter = Sets.difference(desiredPartitions, assigned).iterator(); + while (iter.hasNext() && hostAssignments.size() < canAssign) { + hostAssignments.add(iter.next()); + } + if (hostAssignments.size() >= maxAssignments) { + hostThatHaveMax.add(hp); + } + assigned.addAll(hostAssignments); + } + + System.out.println("hostThatHaveMax:" + hostThatHaveMax); + + return assignments; + } +} diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java index 0e96d351408..0efb1a6237d 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java @@ -77,7 +77,7 @@ public void testFullScanHandling() throws Exception { CompactionJobQueues jobQueues = new CompactionJobQueues(1000000); - jobQueues.setResourceGroups(Set.of(cg1, cg2, cg3)); + jobQueues.setAllowedGroups(Set.of(cg1, cg2, cg3)); jobQueues.beginFullScan(DataLevel.USER); @@ -239,7 +239,7 @@ public void testFullScanLevels() throws Exception { var cg1 = ResourceGroupId.of("CG1"); CompactionJobQueues jobQueues = new CompactionJobQueues(1000000); - jobQueues.setResourceGroups(Set.of(cg1)); + jobQueues.setAllowedGroups(Set.of(cg1)); jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg1))); jobQueues.add(extent2, List.of(newJob((short) 2, 6, cg1))); @@ -279,7 +279,7 @@ public void testAddPollRaceCondition() throws Exception { ResourceGroupId[] groups = Stream.of("G1", "G2", "G3").map(ResourceGroupId::of).toArray(ResourceGroupId[]::new); - jobQueues.setResourceGroups(Set.of(groups)); + jobQueues.setAllowedGroups(Set.of(groups)); var executor = Executors.newFixedThreadPool(groups.length); @@ -345,7 +345,7 @@ public void testGetAsync() throws Exception { var cg1 = ResourceGroupId.of("CG1"); - jobQueues.setResourceGroups(Set.of(cg1)); + jobQueues.setAllowedGroups(Set.of(cg1)); var job1 = newJob((short) 1, 5, cg1); var job2 = newJob((short) 2, 6, cg1); @@ -417,7 +417,7 @@ public void testResetSize() throws Exception { var cg1 = ResourceGroupId.of("CG1"); var cg2 = ResourceGroupId.of("CG2"); - jobQueues.setResourceGroups(Set.of(cg1, cg2)); + jobQueues.setAllowedGroups(Set.of(cg1, cg2)); jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg1))); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java new file mode 100644 index 00000000000..154d9b474c9 --- /dev/null +++ b/server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.multi; + +import static org.apache.accumulo.manager.multi.ManagerAssignment.computeAssignments; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.util.Sets; +import org.junit.jupiter.api.Test; + +import com.google.common.net.HostAndPort; + +public class ManagerAssignmentTest { + @Test + public void testComputeAssignments() { + assertEquals(Map.of(), computeAssignments(Map.of(), Set.of("a", "b"))); + + var hp1 = HostAndPort.fromParts("127.0.0.1", 5000); + var hp2 = HostAndPort.fromParts("127.0.0.1", 5001); + var hp3 = HostAndPort.fromParts("127.0.0.1", 5002); + var hp4 = HostAndPort.fromParts("127.0.0.1", 5003); + + runTest(Map.of(hp1, Set.of()), Set.of("a"), 1, 1, 1); + runTest(Map.of(hp1, Set.of()), Set.of("a", "b"), 2, 2, 1); + + Map> threeEmptyHost = + Map.of(hp1, Set.of(), hp2, Set.of(), hp3, Set.of()); + runTest(threeEmptyHost, Set.of("a"), 0, 1, 1); + runTest(threeEmptyHost, Set.of("a", "b"), 0, 1, 2); + runTest(threeEmptyHost, Set.of("a", "b", "c"), 1, 1, 3); + runTest(threeEmptyHost, Set.of("a", "b", "c", "d"), 1, 2, 1); + runTest(threeEmptyHost, Set.of("a", "b", "c", "d", "e"), 1, 2, 2); + runTest(threeEmptyHost, Set.of("a", "b", "c", "d", "e", "f"), 2, 2, 3); + runTest(threeEmptyHost, Set.of("a", "b", "c", "d", "e", "f", "g"), 2, 3, 1); + + Map> fourEmptyHost = + Map.of(hp1, Set.of(), hp2, Set.of(), hp3, Set.of(), hp4, Set.of()); + runTest(fourEmptyHost, Set.of("a"), 0, 1, 1); + runTest(fourEmptyHost, Set.of("a", "b"), 0, 1, 2); + runTest(fourEmptyHost, Set.of("a", "b", "c"), 0, 1, 3); + runTest(fourEmptyHost, Set.of("a", "b", "c", "d"), 1, 1, 4); + runTest(fourEmptyHost, Set.of("a", "b", "c", "d", "e"), 1, 2, 1); + + var assignments = + runTest(Map.of(hp1, Set.of("a", "z"), hp2, Set.of("b", "c", "x"), hp3, Set.of("d", "e")), + Set.of("a", "b", "c", "d", "e", "f"), 2, 2, 3); + assertEquals(Map.of(hp1, Set.of("a", "f"), hp2, Set.of("b", "c"), hp3, Set.of("d", "e")), + assignments); + + assignments = + runTest(Map.of(hp1, Set.of("a", "b", "c"), hp2, Set.of("d", "x"), hp3, Set.of("e")), + Set.of("a", "b", "c", "d", "e", "f"), 2, 2, 3); + assertEquals(2, Sets.intersection(assignments.get(hp1), Set.of("a", "b", "c")).size()); + assertTrue(assignments.get(hp2).contains("d")); + assertTrue(assignments.get(hp3).contains("e")); + + assignments = + runTest(Map.of(hp1, Set.of("a", "b", "c"), hp2, Set.of("d", "e", "f", "g"), hp3, Set.of()), + Set.of("a", "b", "c", "d", "e", "f", "g"), 2, 3, 1); + int iss1 = Sets.intersection(assignments.get(hp1), Set.of("a", "b", "c")).size(); + int iss2 = Sets.intersection(assignments.get(hp2), Set.of("d", "e", "f", "g")).size(); + assertTrue((iss1 == 2 && iss2 == 3) || (iss1 == 3 && iss2 == 2)); + } + + @Test + public void testBug() { + var hp1 = HostAndPort.fromParts("127.0.0.1", 5000); + var hp2 = HostAndPort.fromParts("127.0.0.1", 5001); + var hp3 = HostAndPort.fromParts("127.0.0.1", 5002); + var hp4 = HostAndPort.fromParts("127.0.0.1", 5003); + var hp5 = HostAndPort.fromParts("127.0.0.1", 5004); + + var assignments = runTest( + Map.of(hp1, Set.of("a", "b", "c"), hp2, Set.of("d", "e", "f"), hp3, Set.of("g", "h", "i"), + hp4, Set.of(), hp5, Set.of()), + Set.of("a", "b", "c", "d", "e", "f", "g", "h", "i"), 1, 2, 4); + assertEquals(2, Sets.intersection(assignments.get(hp1), Set.of("a", "b", "c")).size()); + assertEquals(2, Sets.intersection(assignments.get(hp2), Set.of("d", "e", "f")).size()); + assertEquals(2, Sets.intersection(assignments.get(hp3), Set.of("g", "h", "i")).size()); + } + + private Map> runTest(Map> currentAssignments, + Set parts, int expectedMin, int expectedMax, int expectedHostWithMax) { + var assignments = computeAssignments(currentAssignments, parts); + assertEquals(currentAssignments.keySet(), assignments.keySet()); + HashSet partsCopy = new HashSet<>(parts); + + // ensure every partition is assigned and only assigned once + assignments.values().forEach(hostAssignments -> { + assertTrue(partsCopy.containsAll(hostAssignments), () -> partsCopy + " " + hostAssignments); + partsCopy.removeAll(hostAssignments); + assertTrue(hostAssignments.size() == expectedMin || hostAssignments.size() == expectedMax); + }); + + assertEquals(expectedHostWithMax, + assignments.values().stream().mapToInt(Set::size).filter(s -> s == expectedMax).count()); + + return assignments; + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java index 15c66b66e8e..eac08708a39 100644 --- a/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java @@ -32,19 +32,28 @@ import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; -import org.apache.accumulo.harness.AccumuloClusterHarness; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.test.compaction.ExternalCompactionTestUtils; +import org.apache.accumulo.test.functional.ConfigurableMacBase; +import org.apache.accumulo.test.util.Wait; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; -public class MultipleManagerCompactionIT extends AccumuloClusterHarness { +public class MultipleManagerCompactionIT extends ConfigurableMacBase { @Override - public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) { + public void configure(MiniAccumuloConfigImpl cfg, Configuration coreSite) { ExternalCompactionTestUtils.configureMiniCluster(cfg, coreSite); cfg.getClusterServerConfiguration().setNumManagers(3); + // This test could kill a manager after its written a compaction to the metadata table, but + // before it returns it to the compactor via RPC which creates a dead compaction. Need to speed + // up the dead compaction detection to handle this or else the test will hang. + cfg.setProperty(Property.COMPACTION_COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL, "5s"); + // Speed up time to detect a dead manager process by lowering ZK timeout + cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s"); } @Test @@ -53,6 +62,10 @@ public void test() throws Exception { try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) { + // wait for three coordinator locations to show up in zookeeper + Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() + .distinct().count() == 3); + String table1 = names[0]; createTable(client, table1, "cs1"); @@ -72,6 +85,37 @@ public void test() throws Exception { compact(client, table2, 3, GROUP2, true); verify(client, table2, 3); + getCluster().getConfig().getClusterServerConfiguration().setNumManagers(5); + getCluster().getClusterControl().start(ServerType.MANAGER); + + // wait for five coordinator locations to show up in zookeeper + Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() + .distinct().count() == 5); + + compact(client, table1, 3, GROUP1, true); + verify(client, table1, 6); + + compact(client, table2, 5, GROUP2, true); + verify(client, table2, 15); + + // kill three managers + for (var proc : getCluster().getProcesses().get(ServerType.MANAGER).stream().limit(3) + .toList()) { + getCluster().getClusterControl().killProcess(ServerType.MANAGER, proc); + } + + // TODO check that all compactors RGs are present in coordinator locations + + // wait for three coordinator locations to show up in zookeeper + Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() + .distinct().count() == 2, 120_000); // TODO adjust ZK timeout + + compact(client, table1, 5, GROUP1, true); + verify(client, table1, 30); + + compact(client, table2, 2, GROUP2, true); + verify(client, table2, 30); + } } diff --git a/test/src/main/resources/log4j2-test.properties b/test/src/main/resources/log4j2-test.properties index f75e6ec8f57..8eb1ce5f422 100644 --- a/test/src/main/resources/log4j2-test.properties +++ b/test/src/main/resources/log4j2-test.properties @@ -163,5 +163,8 @@ logger.45.level = trace logger.46.name = org.apache.accumulo.core.spi.scan logger.46.level = info +logger.47.name = org.apache.accumulo.manager.tableOps.compact +logger.47.level = trace + rootLogger.level = debug rootLogger.appenderRef.console.ref = STDOUT From 483908674784d87cd8a8f3d79aab272edfa4aeae Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 14:09:31 +0000 Subject: [PATCH 24/65] WIP --- .../accumulo/test/MultipleManagerCompactionIT.java | 9 ++++++--- .../test/compaction/ExternalCompactionTestUtils.java | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java index eac08708a39..005a554236e 100644 --- a/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java @@ -23,9 +23,11 @@ import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.MAX_DATA; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable; +import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getExpectedGroups; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.row; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.verify; import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.writeData; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.SortedSet; import java.util.TreeSet; @@ -65,6 +67,7 @@ public void test() throws Exception { // wait for three coordinator locations to show up in zookeeper Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() .distinct().count() == 3); + assertEquals(getExpectedGroups(), getServerContext().getCoordinatorLocations(true).keySet()); String table1 = names[0]; createTable(client, table1, "cs1"); @@ -91,6 +94,7 @@ public void test() throws Exception { // wait for five coordinator locations to show up in zookeeper Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() .distinct().count() == 5); + assertEquals(getExpectedGroups(), getServerContext().getCoordinatorLocations(true).keySet()); compact(client, table1, 3, GROUP1, true); verify(client, table1, 6); @@ -104,11 +108,10 @@ public void test() throws Exception { getCluster().getClusterControl().killProcess(ServerType.MANAGER, proc); } - // TODO check that all compactors RGs are present in coordinator locations - // wait for three coordinator locations to show up in zookeeper Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() - .distinct().count() == 2, 120_000); // TODO adjust ZK timeout + .distinct().count() == 2, 120_000); + assertEquals(getExpectedGroups(), getServerContext().getCoordinatorLocations(true).keySet()); compact(client, table1, 5, GROUP1, true); verify(client, table1, 30); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java index 07d3ea60fb8..e2afa74bbb1 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java @@ -57,6 +57,7 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; @@ -247,6 +248,12 @@ public static void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuratio coreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); } + public static Set getExpectedGroups() { + return Set.of(ResourceGroupId.DEFAULT, ResourceGroupId.of(GROUP1), ResourceGroupId.of(GROUP2), + ResourceGroupId.of(GROUP3), ResourceGroupId.of(GROUP4), ResourceGroupId.of(GROUP5), + ResourceGroupId.of(GROUP6), ResourceGroupId.of(GROUP7), ResourceGroupId.of(GROUP8)); + } + public static Map getRunningCompactions(ClientContext context) { Map running = new HashMap<>(); try { From 6ba11063527d771c71ab0e604b4f1d316c6a5ae0 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 15:20:56 +0000 Subject: [PATCH 25/65] WIP --- .../queue/CompactionJobPriorityQueue.java | 11 +++++ .../compaction/queue/CompactionJobQueues.java | 44 +++++++++++-------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java index f0b0bec8c6d..6dd2f245cba 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java @@ -369,6 +369,17 @@ public synchronized void clearIfInactive(Duration duration) { } } + public synchronized void clear() { + futures.forEach(future -> { + future.complete(null); + }); + futures.clear(); + + jobQueue.clear(); + tabletJobs.clear(); + jobAges.clear(); + } + public synchronized void resetMaxSize(long size) { Preconditions.checkArgument(size > 0); long oldSize = maxSize.getAndSet(size); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java index 61bcb385052..db18daf51b1 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java @@ -18,16 +18,17 @@ */ package org.apache.accumulo.manager.compaction.queue; +import static java.util.Objects.requireNonNull; + import java.util.Collection; import java.util.Collections; import java.util.EnumMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentHashMap.KeySetView; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.accumulo.core.data.ResourceGroupId; @@ -52,7 +53,8 @@ public class CompactionJobQueues { private volatile long queueSize; - private final Set allowedGroups = Collections.synchronizedSet(new HashSet<>()); + private final AtomicReference> allowedGroups = + new AtomicReference<>(Set.of()); private final Map currentGenerations; @@ -94,7 +96,7 @@ public void add(KeyExtent extent, Collection jobs) { } } - public KeySetView getQueueIds() { + public Set getQueueIds() { return priorityQueues.keySet(); } @@ -103,12 +105,12 @@ public CompactionJobPriorityQueue getQueue(ResourceGroupId groupId) { } public long getQueueMaxSize(ResourceGroupId groupId) { - var prioQ = priorityQueues.get(groupId); + var prioQ = getQueue(groupId); return prioQ == null ? 0 : prioQ.getMaxSize(); } public long getQueuedJobs(ResourceGroupId groupId) { - var prioQ = priorityQueues.get(groupId); + var prioQ = getQueue(groupId); return prioQ == null ? 0 : prioQ.getQueuedJobs(); } @@ -125,13 +127,15 @@ public long getQueuedJobCount() { } private CompactionJobPriorityQueue getOrCreateQueue(ResourceGroupId groupId) { - return priorityQueues.computeIfAbsent(groupId, gid -> { - if (allowedGroups.contains(gid)) { + var pq = priorityQueues.computeIfAbsent(groupId, gid -> { + if (allowedGroups.get().contains(gid)) { return new CompactionJobPriorityQueue(gid, queueSize, ResolvedCompactionJob.WEIGHER); } else { return null; } }); + + return pq; } /** @@ -150,7 +154,7 @@ public CompletableFuture getAsync(ResourceGroupId groupId) { } public CompactionJob poll(ResourceGroupId groupId) { - var prioQ = priorityQueues.get(groupId); + var prioQ = getQueue(groupId); if (prioQ == null) { return null; } @@ -181,18 +185,20 @@ public void resetMaxSize(long size) { } public Set getAllowedGroups() { - synchronized (allowedGroups) { - return Set.copyOf(allowedGroups); - } + return allowedGroups.get(); } - public void setAllowedGroups(Set groups) { - synchronized (allowedGroups) { - allowedGroups.clear(); - allowedGroups.addAll(groups); - } + public synchronized void setAllowedGroups(Set groups) { + var snapshot = Set.copyOf(requireNonNull(groups)); + + allowedGroups.set(snapshot); + + priorityQueues.forEach((rg, pq) -> { + if (!snapshot.contains(rg)) { + pq.clear(); + } + }); - priorityQueues.keySet().removeIf(rg -> !allowedGroups.contains(rg)); - // TODO need to cancel futures + priorityQueues.keySet().removeIf(rg -> !snapshot.contains(rg)); } } From 0ceea2daf14d8ee83d821daf6ac5d4bd7b95b102 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 15:37:32 +0000 Subject: [PATCH 26/65] Removes getAsync from compaction jobs queues Compaction job queues had a getAsync method that were not used. This was causing complexity w/ #6217, so removing them. Can add them back later if needed. --- .../queue/CompactionJobPriorityQueue.java | 55 +------------ .../compaction/queue/CompactionJobQueues.java | 13 --- .../queue/CompactionJobPriorityQueueTest.java | 35 -------- .../queue/CompactionJobQueuesTest.java | 79 ------------------- 4 files changed, 1 insertion(+), 181 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java index f0b0bec8c6d..2171de3c3d9 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java @@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkState; import java.time.Duration; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -31,7 +30,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -129,8 +127,6 @@ public MetaJob(CompactionJob job, KeyExtent extent) { private final AtomicLong maxSize; private final AtomicLong rejectedJobs; private final AtomicLong dequeuedJobs; - private final ArrayDeque> futures; - private long futuresAdded = 0; private final Map jobAges; private final Supplier jobQueueStats; private final AtomicReference> jobQueueTimer; @@ -159,7 +155,6 @@ public CompactionJobPriorityQueue(ResourceGroupId groupId, long maxSize, this.groupId = groupId; this.rejectedJobs = new AtomicLong(0); this.dequeuedJobs = new AtomicLong(0); - this.futures = new ArrayDeque<>(); this.jobAges = new ConcurrentHashMap<>(); this.jobQueueStats = Suppliers.memoizeWithExpiration( () -> new CompactionJobPriorityQueueStats(jobAges), 5, TimeUnit.SECONDS); @@ -198,25 +193,7 @@ public synchronized int add(KeyExtent extent, Collection jobs, lo HashSet newEntries = new HashSet<>(jobs.size()); int jobsAdded = 0; - outer: for (CompactionJob job : jobs) { - var future = futures.poll(); - while (future != null) { - // its expected that if futures are present then the queue is empty, if this is not true - // then there is a bug - Preconditions.checkState(jobQueue.isEmpty()); - // Queue should be empty so jobAges should be empty - Preconditions.checkState(jobAges.isEmpty()); - if (future.complete(job)) { - // successfully completed a future with this job, so do not need to queue the job - jobsAdded++; - // Record a time of 0 as job as we were able to complete immediately and there - // were no jobs waiting - jobQueueTimer.get().ifPresent(jqt -> jqt.record(Duration.ZERO)); - continue outer; - } // else the future was canceled or timed out so could not complete it - future = futures.poll(); - } - + for (CompactionJob job : jobs) { CjpqKey cjqpKey = addJobToQueue(extent, job); if (cjqpKey != null) { checkState(newEntries.add(cjqpKey)); @@ -289,36 +266,6 @@ public synchronized CompactionJob poll() { return first == null ? null : first.getValue().job; } - public synchronized CompletableFuture getAsync() { - var job = poll(); - if (job != null) { - return CompletableFuture.completedFuture(job); - } - - // There is currently nothing in the queue, so create an uncompleted future and queue it up to - // be completed when something does arrive. - CompletableFuture future = new CompletableFuture<>(); - futures.add(future); - futuresAdded++; - // Handle the case where nothing is ever being added to this queue and futures are constantly - // being obtained and cancelled. If nothing is done these canceled futures would just keep - // building up in memory. The following code periodically checks to see if there are canceled - // futures to remove. - if (futuresAdded % FUTURE_CHECK_THRESHOLD == 0 - && futures.size() >= 2 * FUTURE_CHECK_THRESHOLD) { - futures.removeIf(CompletableFuture::isDone); - // It is not expected that the future we just created would be done, if it were it would have - // been removed. - Preconditions.checkState(!future.isDone()); - } - return future; - } - - @VisibleForTesting - synchronized int futuresSize() { - return futures.size(); - } - // exists for tests synchronized CompactionJob peek() { var firstEntry = jobQueue.firstEntry(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java index e87c231bed9..983befa047b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.EnumMap; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap.KeySetView; import java.util.concurrent.atomic.AtomicLong; @@ -120,18 +119,6 @@ public long getQueuedJobCount() { return count; } - /** - * Asynchronously get a compaction job from the queue. If the queue currently has jobs then a - * completed future will be returned containing the highest priority job in the queue. If the - * queue is currently empty, then an uncompleted future will be returned and later when something - * is added to the queue the future will be completed. - */ - public CompletableFuture getAsync(ResourceGroupId groupId) { - var pq = priorityQueues.computeIfAbsent(groupId, - gid -> new CompactionJobPriorityQueue(gid, queueSize, ResolvedCompactionJob.WEIGHER)); - return pq.getAsync(); - } - public CompactionJob poll(ResourceGroupId groupId) { var prioQ = priorityQueues.get(groupId); if (prioQ == null) { diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueueTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueueTest.java index 7fc20955f5d..5b49e814320 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueueTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueueTest.java @@ -23,12 +23,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.CompletableFuture; import org.apache.accumulo.core.client.admin.compaction.CompactableFile; import org.apache.accumulo.core.data.ResourceGroupId; @@ -293,39 +291,6 @@ public void test() { assertEquals(0, stats.getAvgAge().toMillis()); } - /** - * Test to ensure that canceled futures do not build up in memory. - */ - @Test - public void testAsyncCancelCleanup() { - CompactionJobPriorityQueue queue = new CompactionJobPriorityQueue(GROUP, 100, mj -> 1); - - List> futures = new ArrayList<>(); - - int maxFuturesSize = 0; - - // Add 11 below so that cadence of clearing differs from the internal check cadence - final int CANCEL_THRESHOLD = CompactionJobPriorityQueue.FUTURE_CHECK_THRESHOLD / 10 + 11; - final int ITERATIONS = CompactionJobPriorityQueue.FUTURE_CHECK_THRESHOLD * 20; - - for (int x = 0; x < ITERATIONS; x++) { - futures.add(queue.getAsync()); - - maxFuturesSize = Math.max(maxFuturesSize, queue.futuresSize()); - - if (futures.size() >= CANCEL_THRESHOLD) { - futures.forEach(f -> f.cancel(true)); - futures.clear(); - } - } - - maxFuturesSize = Math.max(maxFuturesSize, queue.futuresSize()); - - assertTrue(maxFuturesSize - < 2 * (CompactionJobPriorityQueue.FUTURE_CHECK_THRESHOLD + CANCEL_THRESHOLD)); - assertTrue(maxFuturesSize > 2 * CompactionJobPriorityQueue.FUTURE_CHECK_THRESHOLD); - } - @Test public void testResetMaxSize() { TreeSet expected = new TreeSet<>(CompactionJobPrioritizer.JOB_COMPARATOR); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java index d3c1acd469b..d34cc2666d3 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java @@ -19,11 +19,7 @@ package org.apache.accumulo.manager.compaction.queue; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.URI; import java.net.URISyntaxException; @@ -31,11 +27,8 @@ import java.util.Collection; import java.util.List; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; @@ -328,78 +321,6 @@ public void testAddPollRaceCondition() throws Exception { assertEquals(numToAdd, totalSeen); } - @Test - public void testGetAsync() throws Exception { - CompactionJobQueues jobQueues = new CompactionJobQueues(1000000); - - var tid = TableId.of("1"); - var extent1 = new KeyExtent(tid, new Text("z"), new Text("q")); - var extent2 = new KeyExtent(tid, new Text("q"), new Text("l")); - var extent3 = new KeyExtent(tid, new Text("l"), new Text("c")); - var extent4 = new KeyExtent(tid, new Text("c"), new Text("a")); - - var cg1 = ResourceGroupId.of("CG1"); - - var job1 = newJob((short) 1, 5, cg1); - var job2 = newJob((short) 2, 6, cg1); - var job3 = newJob((short) 3, 7, cg1); - var job4 = newJob((short) 4, 8, cg1); - - var future1 = jobQueues.getAsync(cg1); - var future2 = jobQueues.getAsync(cg1); - - assertFalse(future1.isDone()); - assertFalse(future2.isDone()); - - jobQueues.add(extent1, List.of(job1)); - jobQueues.add(extent2, List.of(job2)); - // Futures were immediately completed so nothing should be queued - assertTrue(jobQueues.getQueue(cg1).getJobAges().isEmpty()); - - jobQueues.add(extent3, List.of(job3)); - jobQueues.add(extent4, List.of(job4)); - // No futures available, so jobAges should exist for 2 tablets - assertEquals(2, jobQueues.getQueue(cg1).getJobAges().size()); - - var future3 = jobQueues.getAsync(cg1); - var future4 = jobQueues.getAsync(cg1); - - // Should be back to 0 size after futures complete - assertTrue(jobQueues.getQueue(cg1).getJobAges().isEmpty()); - - assertTrue(future1.isDone()); - assertTrue(future2.isDone()); - assertTrue(future3.isDone()); - assertTrue(future4.isDone()); - - assertEquals(job1, future1.get()); - assertEquals(job2, future2.get()); - assertEquals(job4, future3.get()); - assertEquals(job3, future4.get()); - - // test cancelling a future and having a future timeout - var future5 = jobQueues.getAsync(cg1); - assertFalse(future5.isDone()); - future5.cancel(false); - var future6 = jobQueues.getAsync(cg1); - assertFalse(future6.isDone()); - future6.orTimeout(10, TimeUnit.MILLISECONDS); - // Wait for future6 to timeout to make sure future7 will - // receive the job when added to the queue - var ex = assertThrows(ExecutionException.class, future6::get); - assertInstanceOf(TimeoutException.class, ex.getCause()); - var future7 = jobQueues.getAsync(cg1); - assertFalse(future7.isDone()); - // since future5 was canceled and future6 timed out, this addition should go to future7 - var job5 = newJob((short) 1, 5, cg1); - jobQueues.add(extent1, List.of(job5)); - assertTrue(future7.isDone()); - assertEquals(job5, future7.get()); - assertTrue(future5.isDone()); - assertTrue(future6.isCompletedExceptionally()); - assertTrue(future6.isDone()); - } - @Test public void testResetSize() throws Exception { CompactionJobQueues jobQueues = new CompactionJobQueues(1000000); From f99275c1a3b417e6c2fea945fbef25f524f78d67 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 16:02:54 +0000 Subject: [PATCH 27/65] WIP --- .../coordinator/CompactionCoordinator.java | 18 +++++--- .../accumulo/manager/fate/FateWorker.java | 18 ++------ .../manager/upgrade/UpgradeCheck.java | 46 +++++++++++++++++++ 3 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCheck.java diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index f870b35bb3d..e17da0b4d56 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -118,6 +118,7 @@ import org.apache.accumulo.manager.compaction.queue.CompactionJobQueues; import org.apache.accumulo.manager.compaction.queue.ResolvedCompactionJob; import org.apache.accumulo.manager.tableOps.FateEnv; +import org.apache.accumulo.manager.upgrade.UpgradeCheck; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CompactionConfigStorage; import org.apache.accumulo.server.compaction.CompactionPluginUtils; @@ -202,6 +203,8 @@ static FailureCounts incrementSuccess(Object key, FailureCounts counts) { private final Map reservationPools; private final Set activeCompactorReservationRequest = ConcurrentHashMap.newKeySet(); + private final UpgradeCheck upgradeCheck; + public CompactionCoordinator(Manager manager, Function> fateClients) { this.ctx = manager.getContext(); @@ -251,7 +254,8 @@ public CompactionCoordinator(Manager manager, compactorCounts = ctx.getCaches().createNewBuilder(CacheName.COMPACTOR_COUNTS, false) .expireAfterWrite(2, TimeUnit.MINUTES).build(this::countCompactors); - // At this point the manager does not have its lock so no actions should be taken yet + + upgradeCheck = new UpgradeCheck(ctx); } protected int countCompactors(ResourceGroupId groupName) { @@ -586,16 +590,15 @@ public void beginFullJobScan(TInfo tinfo, TCredentials credentials, String dataL public void addJobs(TInfo tinfo, TCredentials credentials, List jobs) throws TException { if (!security.canPerformSystemActions(credentials)) { - // this is a oneway method so throwing an exception is not useful - // TODO maybe log? + LOG.warn("Thrift call attempted to add job and did not have proper access. {}", + credentials.getPrincipal()); return; } for (var tjob : jobs) { var job = ResolvedCompactionJob.fromThrift(tjob); - // TODO remove or improve - LOG.debug("Adding compaction job {} {}", job.getGroup(), job.getExtent()); - // TODO maybe no longer need to pass a list + LOG.trace("Adding compaction job {} {} {} {}", job.getGroup(), job.getPriority(), + job.getExtent(), job.getJobFiles().size()); jobQueues.add(job.getExtent(), List.of(job)); } } @@ -643,7 +646,8 @@ public void setResourceGroups(TInfo tinfo, TCredentials credentials, long update SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } - // TODO validate that upgrade is complete like FateWorker does + // Do not expect this to be called before upgrade is complete + Preconditions.checkState(upgradeCheck.isUpgradeComplete()); // only allow an update id to be used once synchronized (expectedUpdateId) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java index f60fc453c0a..22449504283 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorker.java @@ -49,7 +49,7 @@ import org.apache.accumulo.core.securityImpl.thrift.TCredentials; import org.apache.accumulo.manager.metrics.fate.FateExecutorMetricsProducer; import org.apache.accumulo.manager.tableOps.FateEnv; -import org.apache.accumulo.server.AccumuloDataVersion; +import org.apache.accumulo.manager.upgrade.UpgradeCheck; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.security.AuditedSecurityOperation; @@ -68,6 +68,7 @@ public class FateWorker implements FateWorkerService.Iface { private final LiveTServerSet liveTserverSet; private final FateFactory fateFactory; private final Map> fates = new ConcurrentHashMap<>(); + private final UpgradeCheck upgradeCheck; public interface FateFactory { Fate create(FateEnv env, FateStore store, ServerContext context); @@ -78,6 +79,7 @@ public FateWorker(ServerContext ctx, LiveTServerSet liveTServerSet, FateFactory this.security = ctx.getSecurityOperation(); this.liveTserverSet = liveTServerSet; this.fateFactory = fateFactory; + this.upgradeCheck = new UpgradeCheck(ctx); } public synchronized void setLock(ServiceLock lock) { @@ -121,18 +123,6 @@ public TFatePartitions getPartitions(TInfo tinfo, TCredentials credentials) } } - private boolean upgradeComplete = false; - - // Checks in persistent storage if upgrade is complete. Once it sees its complete remembers this - // and stops checking. - private synchronized boolean isUpgradeComplete() { - if (!upgradeComplete) { - upgradeComplete = AccumuloDataVersion.getCurrentVersion(context) >= AccumuloDataVersion.get(); - } - - return upgradeComplete; - } - @Override public boolean setPartitions(TInfo tinfo, TCredentials credentials, long updateId, List desired) throws ThriftSecurityException { @@ -143,7 +133,7 @@ public boolean setPartitions(TInfo tinfo, TCredentials credentials, long updateI synchronized (this) { // The primary manager should not assign any fate partitions until after upgrade is complete. - Preconditions.checkState(isUpgradeComplete()); + Preconditions.checkState(upgradeCheck.isUpgradeComplete()); if (expectedUpdateId != null && updateId == expectedUpdateId) { // Set to null which makes it so that an update id can only be used once. diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCheck.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCheck.java new file mode 100644 index 00000000000..cd0ef1e3a46 --- /dev/null +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCheck.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.manager.upgrade; + +import org.apache.accumulo.server.AccumuloDataVersion; +import org.apache.accumulo.server.ServerContext; + +/** + * Checks persistent data to see if upgrade is complete and remembers if it is. + */ +public class UpgradeCheck { + + private final ServerContext context; + + private boolean upgradeComplete = false; + + public UpgradeCheck(ServerContext context) { + this.context = context; + } + + // Checks in persistent storage if upgrade is complete. Once it sees its complete remembers this + // and stops checking. + public synchronized boolean isUpgradeComplete() { + if (!upgradeComplete) { + upgradeComplete = AccumuloDataVersion.getCurrentVersion(context) >= AccumuloDataVersion.get(); + } + + return upgradeComplete; + } +} From 3c959398002600ea434b2d25283d58c4bb830152 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 17:35:36 +0000 Subject: [PATCH 28/65] WIP --- .../org/apache/accumulo/manager/Manager.java | 6 +- .../coordinator/CompactionCoordinator.java | 110 +++--------------- .../coordinator/CoordinatorManager.java | 70 ++++++++++- .../coordinator/DeadCompactionDetector.java | 7 +- .../compaction/CompactionCoordinatorTest.java | 9 +- 5 files changed, 93 insertions(+), 109 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index e5550b3d971..711616cbc13 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -941,6 +941,7 @@ public void run() { PrimaryManagerThriftServiceWrapper.service(ManagerClientService.Iface.class, ManagerClientService.Processor::new, new ManagerClientServiceHandler(this), this); compactionCoordinator = new CompactionCoordinator(this, this::fateClient); + compactionCoordinator.start(); // This is not wrapped w/ HighlyAvailableServiceWrapper because it can be run by any manager. FateWorker fateWorker = new FateWorker(context, tserverSet, this::createFateInstance); @@ -1146,10 +1147,7 @@ boolean canSuspendTablets() { balanceManager.startBackGroundTask(); Threads.createCriticalThread("ScanServer Cleanup Thread", new ScanServerZKCleaner()).start(); - // Don't call start the CompactionCoordinator until we have tservers and upgrade is complete. - compactionCoordinator.start(); - - coordinatorManager = new CoordinatorManager(context); + coordinatorManager = new CoordinatorManager(context, this::fateClient); coordinatorManager.start(); this.splitter = new Splitter(this); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index e17da0b4d56..8c6aee8b356 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -32,7 +32,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; -import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -80,7 +79,6 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.FateKey; -import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.iteratorsImpl.system.SystemIteratorUtil; import org.apache.accumulo.core.logging.ConditionalLogger.ConditionalLogAction; import org.apache.accumulo.core.logging.TabletLogger; @@ -114,7 +112,6 @@ import org.apache.accumulo.manager.compaction.coordinator.commit.CommitCompaction; import org.apache.accumulo.manager.compaction.coordinator.commit.CompactionCommitData; import org.apache.accumulo.manager.compaction.coordinator.commit.RenameCompactionFile; -import org.apache.accumulo.manager.compaction.queue.CompactionJobPriorityQueue; import org.apache.accumulo.manager.compaction.queue.CompactionJobQueues; import org.apache.accumulo.manager.compaction.queue.ResolvedCompactionJob; import org.apache.accumulo.manager.tableOps.FateEnv; @@ -128,7 +125,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.thrift.TException; -import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -191,7 +187,6 @@ static FailureCounts incrementSuccess(Object key, FailureCounts counts) { private final LoadingCache compactionConfigCache; private final Cache tabletDirCache; - private final DeadCompactionDetector deadCompactionDetector; private final QueueMetrics queueMetrics; private final Manager manager; @@ -237,9 +232,6 @@ public CompactionCoordinator(Manager manager, tabletDirCache = ctx.getCaches().createNewBuilder(CacheName.COMPACTION_DIR_CACHE, true) .maximumWeight(10485760L).weigher(weigher).build(); - deadCompactionDetector = - new DeadCompactionDetector(this.ctx, this, ctx.getScheduledExecutor(), fateClients); - var rootReservationPool = ThreadPools.getServerThreadPools().createExecutorService( ctx.getConfiguration(), Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_ROOT, true); @@ -284,18 +276,6 @@ public void shutdown() { } } - protected void startCompactorZKCleaner(ScheduledThreadPoolExecutor schedExecutor) { - ScheduledFuture future = schedExecutor - .scheduleWithFixedDelay(this::cleanUpEmptyCompactorPathInZK, 0, 5, TimeUnit.MINUTES); - ThreadPools.watchNonCriticalScheduledTask(future); - } - - protected void startInternalStateCleaner(ScheduledThreadPoolExecutor schedExecutor) { - ScheduledFuture future = - schedExecutor.scheduleWithFixedDelay(this::resizeThreadPools, 0, 5, TimeUnit.MINUTES); - ThreadPools.watchNonCriticalScheduledTask(future); - } - protected void startConfigMonitor(ScheduledThreadPoolExecutor schedExecutor) { ScheduledFuture future = schedExecutor.scheduleWithFixedDelay(this::checkForConfigChanges, 0, 1, TimeUnit.MINUTES); @@ -306,6 +286,15 @@ private void checkForConfigChanges() { long jobQueueMaxSize = ctx.getConfiguration().getAsBytes(Property.MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE); jobQueues.resetMaxSize(jobQueueMaxSize); + + var config = ctx.getConfiguration(); + ThreadPools.resizePool(reservationPools.get(DataLevel.ROOT), config, + Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_ROOT); + ThreadPools.resizePool(reservationPools.get(DataLevel.METADATA), config, + Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_META); + ThreadPools.resizePool(reservationPools.get(DataLevel.USER), config, + Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_USER); + } @Override @@ -313,11 +302,8 @@ public void run() { this.coordinatorStartTime = System.currentTimeMillis(); startConfigMonitor(ctx.getScheduledExecutor()); - startCompactorZKCleaner(ctx.getScheduledExecutor()); - startDeadCompactionDetector(); startFailureSummaryLogging(); - startInternalStateCleaner(ctx.getScheduledExecutor()); try { shutdown.await(); @@ -328,10 +314,6 @@ public void run() { LOG.info("Shutting down"); } - protected void startDeadCompactionDetector() { - deadCompactionDetector.start(); - } - /** * Return the next compaction job from the queue to a Compactor * @@ -748,7 +730,7 @@ public void compactionCompleted(TInfo tinfo, TCredentials credentials, LOG.debug("Not committing compaction {} for {} because of table state {}", ecid, extent, tableState); // cleanup metadata table and files related to the compaction - compactionsFailed(Map.of(ecid, extent)); + compactionsFailed(ctx, Map.of(ecid, extent)); return; } @@ -783,7 +765,7 @@ public void compactionFailed(TInfo tinfo, TCredentials credentials, String exter if (failureState == TCompactionState.FAILED) { captureFailure(ResourceGroupId.of(groupName), compactorAddress, fromThriftExtent); } - compactionsFailed(Map.of(ecid, KeyExtent.fromThrift(extent))); + compactionsFailed(ctx, Map.of(ecid, KeyExtent.fromThrift(extent))); } private void captureFailure(ResourceGroupId group, String compactorAddress, KeyExtent extent) { @@ -838,7 +820,8 @@ private void captureSuccess(ResourceGroupId groupId, String compactorAddress, Ke failingTables.compute(extent.tableId(), FailureCounts::incrementSuccess); } - void compactionsFailed(Map compactions) { + static void compactionsFailed(ServerContext ctx, + Map compactions) { // Need to process each level by itself because the conditional tablet mutator does not support // mutating multiple data levels at the same time. Also the conditional tablet mutator does not // support submitting multiple mutations for a single tablet, so need to group by extent. @@ -852,10 +835,11 @@ void compactionsFailed(Map compactions) { }); groupedCompactions - .forEach((dataLevel, levelCompactions) -> compactionFailedForLevel(levelCompactions)); + .forEach((dataLevel, levelCompactions) -> compactionFailedForLevel(ctx, levelCompactions)); } - void compactionFailedForLevel(Map> compactions) { + static void compactionFailedForLevel(ServerContext ctx, + Map> compactions) { try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) { compactions.forEach((extent, ecids) -> { @@ -942,7 +926,8 @@ public boolean test(TabletMetadata tabletMetadata) { } else { // TabletMetadata does not exist for the extent. This could be due to a merge or // split operation. Use the utility to find tmp files at the table level - deadCompactionDetector.addTableId(extent.tableId()); + // TODO + // deadCompactionDetector.addTableId(extent.tableId()); } } } @@ -954,63 +939,4 @@ public boolean test(TabletMetadata tabletMetadata) { public CompactionJobQueues getJobQueues() { return jobQueues; } - - private void deleteEmpty(ZooReaderWriter zoorw, String path) - throws KeeperException, InterruptedException { - try { - LOG.debug("Deleting empty ZK node {}", path); - zoorw.delete(path); - } catch (KeeperException.NotEmptyException e) { - LOG.debug("Failed to delete {} its not empty, likely an expected race condition.", path); - } - } - - private void cleanUpEmptyCompactorPathInZK() { - - final var zoorw = this.ctx.getZooSession().asReaderWriter(); - - try { - var groups = zoorw.getChildren(Constants.ZCOMPACTORS); - - for (String group : groups) { - final String qpath = Constants.ZCOMPACTORS + "/" + group; - final ResourceGroupId cgid = ResourceGroupId.of(group); - final var compactors = zoorw.getChildren(qpath); - - if (compactors.isEmpty()) { - deleteEmpty(zoorw, qpath); - // Group has no compactors, we can clear its - // associated priority queue of jobs - CompactionJobPriorityQueue queue = getJobQueues().getQueue(cgid); - if (queue != null) { - queue.clearIfInactive(Duration.ofMinutes(10)); - } - } else { - for (String compactor : compactors) { - String cpath = Constants.ZCOMPACTORS + "/" + group + "/" + compactor; - var lockNodes = - zoorw.getChildren(Constants.ZCOMPACTORS + "/" + group + "/" + compactor); - if (lockNodes.isEmpty()) { - deleteEmpty(zoorw, cpath); - } - } - } - } - } catch (KeeperException | RuntimeException e) { - LOG.warn("Failed to clean up compactors", e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException(e); - } - } - - public void resizeThreadPools() { - var config = ctx.getConfiguration(); - ThreadPools.resizePool(reservationPools.get(DataLevel.ROOT), config, - Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_ROOT); - ThreadPools.resizePool(reservationPools.get(DataLevel.METADATA), config, - Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_META); - ThreadPools.resizePool(reservationPools.get(DataLevel.USER), config, - Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_USER); - } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index 14e24691649..309ba1b88b3 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -25,10 +25,18 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.data.AbstractId; import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.fate.FateClient; +import org.apache.accumulo.core.fate.FateInstanceType; +import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.core.lock.ServiceLockPaths; import org.apache.accumulo.core.rpc.ThriftUtil; @@ -36,7 +44,9 @@ import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.LazySingletons; import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; +import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CoordinatorLocations; import org.apache.thrift.TException; @@ -57,12 +67,17 @@ public class CoordinatorManager { private final ServerContext context; - public CoordinatorManager(ServerContext context) { + public CoordinatorManager(ServerContext context, + Function> fateClients) { this.context = context; + deadCompactionDetector = + new DeadCompactionDetector(context, context.getScheduledExecutor(), fateClients); } private Thread assignmentThread = null; + private final DeadCompactionDetector deadCompactionDetector; + private void managerCoordinators() { while (true) { @@ -160,6 +175,10 @@ private void setResourceGroups(HostAndPort hp, Set groups, long public synchronized void start() { Preconditions.checkState(assignmentThread == null); + startCompactorZKCleaner(context.getScheduledExecutor()); + + deadCompactionDetector.start(); + assignmentThread = Threads.createCriticalThread("Coordinator Manager", () -> { try { managerCoordinators(); @@ -170,5 +189,54 @@ public synchronized void start() { assignmentThread.start(); } + protected void startCompactorZKCleaner(ScheduledThreadPoolExecutor schedExecutor) { + ScheduledFuture future = schedExecutor + .scheduleWithFixedDelay(this::cleanUpEmptyCompactorPathInZK, 0, 5, TimeUnit.MINUTES); + ThreadPools.watchNonCriticalScheduledTask(future); + } + + private void deleteEmpty(ZooReaderWriter zoorw, String path) + throws KeeperException, InterruptedException { + try { + log.debug("Deleting empty ZK node {}", path); + zoorw.delete(path); + } catch (KeeperException.NotEmptyException e) { + log.debug("Failed to delete {} its not empty, likely an expected race condition.", path); + } + } + + private void cleanUpEmptyCompactorPathInZK() { + + final var zoorw = this.context.getZooSession().asReaderWriter(); + + try { + var groups = zoorw.getChildren(Constants.ZCOMPACTORS); + + for (String group : groups) { + final String qpath = Constants.ZCOMPACTORS + "/" + group; + final ResourceGroupId cgid = ResourceGroupId.of(group); + final var compactors = zoorw.getChildren(qpath); + + if (compactors.isEmpty()) { + deleteEmpty(zoorw, qpath); + } else { + for (String compactor : compactors) { + String cpath = Constants.ZCOMPACTORS + "/" + group + "/" + compactor; + var lockNodes = + zoorw.getChildren(Constants.ZCOMPACTORS + "/" + group + "/" + compactor); + if (lockNodes.isEmpty()) { + deleteEmpty(zoorw, cpath); + } + } + } + } + } catch (KeeperException | RuntimeException e) { + log.warn("Failed to clean up compactors", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + // TODO stop() } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java index ce04296a615..dbd9bc68e28 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java @@ -59,17 +59,14 @@ public class DeadCompactionDetector { private static final Logger log = LoggerFactory.getLogger(DeadCompactionDetector.class); private final ServerContext context; - private final CompactionCoordinator coordinator; private final ScheduledThreadPoolExecutor schedExecutor; private final ConcurrentHashMap deadCompactions; private final Set tablesWithUnreferencedTmpFiles = new HashSet<>(); private final Function> fateClients; - public DeadCompactionDetector(ServerContext context, CompactionCoordinator coordinator, - ScheduledThreadPoolExecutor stpe, + public DeadCompactionDetector(ServerContext context, ScheduledThreadPoolExecutor stpe, Function> fateClients) { this.context = context; - this.coordinator = coordinator; this.schedExecutor = stpe; this.deadCompactions = new ConcurrentHashMap<>(); this.fateClients = fateClients; @@ -227,7 +224,7 @@ private void detectDeadCompactions() { tabletCompactions.forEach((ecid, extent) -> { log.warn("Compaction believed to be dead, failing it: id: {}, extent: {}", ecid, extent); }); - coordinator.compactionsFailed(tabletCompactions); + CompactionCoordinator.compactionsFailed(context, tabletCompactions); this.deadCompactions.keySet().removeAll(toFail); } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index fcb8d9afb13..dd034a84c5d 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -113,16 +113,11 @@ protected int countCompactors(ResourceGroupId groupName) { @Override protected void startFailureSummaryLogging() {} - @Override - protected void startDeadCompactionDetector() {} - - @Override - protected void startCompactorZKCleaner(ScheduledThreadPoolExecutor schedExecutor) {} - - @Override + // TODO remove this, used to override protected void startInternalStateCleaner(ScheduledThreadPoolExecutor schedExecutor) { // This is called from CompactionCoordinator.run(). Counting down // the latch will exit the run method + // TODO this.shutdown.countDown(); } From b42a0078d5f2c0e723a79bd9bbb024bfd824cfe0 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 17:52:47 +0000 Subject: [PATCH 29/65] WIP --- .../compaction/CompactionCoordinatorTest.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index dd034a84c5d..7812ee97dda 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -113,14 +113,6 @@ protected int countCompactors(ResourceGroupId groupName) { @Override protected void startFailureSummaryLogging() {} - // TODO remove this, used to override - protected void startInternalStateCleaner(ScheduledThreadPoolExecutor schedExecutor) { - // This is called from CompactionCoordinator.run(). Counting down - // the latch will exit the run method - // TODO - this.shutdown.countDown(); - } - @Override protected void startConfigMonitor(ScheduledThreadPoolExecutor schedExecutor) {} @@ -219,7 +211,6 @@ public void verifyMocks() { public void testCoordinatorColdStart() throws Exception { var coordinator = new TestCoordinator(manager, new ArrayList<>()); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - coordinator.run(); coordinator.shutdown(); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); @@ -238,14 +229,12 @@ public void testGetCompactionJob() throws Exception { var coordinator = new TestCoordinator(manager, new ArrayList<>()); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - // Use coordinator.run() to populate the internal data structures. This is tested in a different - // test. - coordinator.run(); coordinator.shutdown(); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); // Add a job to the job queue + coordinator.getJobQueues().setAllowedGroups(Set.of(GROUP_ID)); CompactionJob job = new CompactionJobImpl((short) 1, GROUP_ID, Collections.emptyList(), CompactionKind.SYSTEM); coordinator.addJobs(tm, Collections.singleton(job)); From 0c331e629a41cd38090d670900c8382cac50f76d Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 27 Mar 2026 18:06:41 +0000 Subject: [PATCH 30/65] WIP --- .../queue/CompactionJobQueuesTest.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java index 081d3e08d39..80930d3f45e 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueuesTest.java @@ -355,5 +355,50 @@ public void testResetSize() throws Exception { assertEquals(500000, jobQueues.getQueueMaxSize(cg2)); } - // TODO add test to ensure only the set resource groups can be added to and pulled from + @Test + public void testDisallowedGroup() throws Exception { + CompactionJobQueues jobQueues = new CompactionJobQueues(1000000); + + var tid = TableId.of("1"); + var extent1 = new KeyExtent(tid, new Text("z"), new Text("q")); + var extent2 = new KeyExtent(tid, new Text("m"), new Text("c")); + + var cg1 = ResourceGroupId.of("CG1"); + var cg2 = ResourceGroupId.of("CG2"); + + jobQueues.setAllowedGroups(Set.of(cg1)); + + jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg1))); + jobQueues.add(extent2, List.of(newJob((short) 2, 3, cg1))); + jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg2))); + + assertEquals(Set.of(cg1), jobQueues.getQueueIds()); + + assertEquals(2, jobQueues.getQueuedJobs(cg1)); + assertEquals(0, jobQueues.getQueuedJobs(cg2)); + + assertEquals(2, jobQueues.poll(cg1).getPriority()); + assertNull(jobQueues.poll(cg2)); + assertEquals(1, jobQueues.getQueuedJobs(cg1)); + assertEquals(0, jobQueues.getQueuedJobs(cg2)); + + // this should clear out the single job for cg1 + jobQueues.setAllowedGroups(Set.of(cg2)); + assertEquals(0, jobQueues.getQueuedJobs(cg1)); + assertEquals(0, jobQueues.getQueuedJobs(cg2)); + assertNull(jobQueues.poll(cg1)); + assertNull(jobQueues.poll(cg2)); + + // should only add job for cg2 now + jobQueues.add(extent1, List.of(newJob((short) 1, 5, cg1))); + jobQueues.add(extent1, List.of(newJob((short) 7, 5, cg2))); + + assertEquals(0, jobQueues.getQueuedJobs(cg1)); + assertEquals(1, jobQueues.getQueuedJobs(cg2)); + assertNull(jobQueues.poll(cg1)); + assertEquals(7, jobQueues.poll(cg2).getPriority()); + assertNull(jobQueues.poll(cg2)); + assertEquals(0, jobQueues.getQueuedJobs(cg1)); + assertEquals(0, jobQueues.getQueuedJobs(cg2)); + } } From 2b6cbb64c67930933561a30615870d958cc76fe2 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 31 Mar 2026 22:47:41 +0000 Subject: [PATCH 31/65] Removes in memory set from dead compaction detector Removed an in memory set of tables ids in the dead compaction detectors that contained table ids that may have compaction tmp files that needed cleanup. This set would be hard to maintain in multiple managers. Also the set could lose track of tables if the process died. Replaced the in memory set with a set in the metadata table. This set is directly populated by the split and merge fate operations, so there is no chance of losing track of things when a process dies. Also this set is more narrow and allows looking for tmp files to cleanup in single tablets dirs rather than scanning an entire tables dir. Also made a change to the order in which tmp files are deleted for failed compactions. They used to be deleted after the metadata for the compaction was cleaned up, this could lead to losing track of the cleanup if the process died after deleting the metadata but before deleting the tmp file. Now the tmp files are deleted before the metadata entry, so should no longer lose track in process death. This change is needed by #6217 --- .../accumulo/core/metadata/schema/Ample.java | 21 ++++ .../core/metadata/schema/MetadataSchema.java | 32 ++++++ .../compaction/ExternalCompactionUtil.java | 3 +- .../metadata/RemovedCompactionStoreImpl.java | 108 ++++++++++++++++++ .../server/metadata/ServerAmpleImpl.java | 5 + .../server/util/FindCompactionTmpFiles.java | 42 ++++++- .../coordinator/CompactionCoordinator.java | 80 +++---------- .../coordinator/DeadCompactionDetector.java | 75 +++++------- .../manager/tableOps/merge/MergeTablets.java | 17 +++ .../manager/tableOps/split/UpdateTablets.java | 10 ++ .../tableOps/merge/MergeTabletsTest.java | 39 ++++++- .../tableOps/split/UpdateTabletsTest.java | 26 +++++ .../compaction/ExternalCompaction_3_IT.java | 10 +- 13 files changed, 350 insertions(+), 118 deletions(-) create mode 100644 server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java index 4d06f3b1aab..1d874dccf1b 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java @@ -25,6 +25,7 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Stream; import org.apache.accumulo.core.client.ConditionalWriter; import org.apache.accumulo.core.client.admin.TabletAvailability; @@ -669,4 +670,24 @@ default void removeBulkLoadInProgressFlag(String path) { default ScanServerRefStore scanServerRefs() { throw new UnsupportedOperationException(); } + + record RemovedCompaction(ExternalCompactionId id, TableId table, String dir) { + }; + + /** + * Tracks compactions that were removed from the metadata table but may still be running on + * compactors. The tmp files associated with these compactions can eventually be removed when the + * compaction is no longer running. + */ + interface RemovedCompactionStore { + Stream list(); + + void add(Collection removedCompactions); + + void delete(Collection removedCompactions); + } + + default RemovedCompactionStore removedCompactions() { + throw new UnsupportedOperationException(); + } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java index 39e1c35e3b0..bb1309583e0 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java @@ -547,4 +547,36 @@ public static String getRowPrefix() { } + /** + * Holds information about compactions that were deleted from tablets metadata by split or merge + * operations. These may have tmp files that need to be cleaned up. + */ + public static class RemovedCompactionSection { + private static final Section section = + new Section(RESERVED_PREFIX + "rcomp", true, RESERVED_PREFIX + "rcomq", false); + + public static Range getRange() { + return section.getRange(); + } + + public static String getRowPrefix() { + return section.getRowPrefix(); + } + + public static Ample.RemovedCompaction decodeRow(String row) { + String[] fields = row.split("#"); + Preconditions.checkArgument(fields.length == 4); + Preconditions.checkArgument(getRowPrefix().equals(fields[0])); + return new Ample.RemovedCompaction(ExternalCompactionId.from(fields[1]), + TableId.of(fields[2]), fields[3]); + } + + public static String encodeRow(Ample.RemovedCompaction rc) { + // put the compaction id first in the row because its uuid will spread out nicely and avoid + // hot spotting + return getRowPrefix() + "#" + rc.id().canonical() + "#" + rc.table().canonical() + "#" + + rc.dir(); + + } + } } diff --git a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java index 149c12e12be..4e842f3681a 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java @@ -24,7 +24,6 @@ import static org.apache.accumulo.core.util.threads.ThreadPoolNames.COMPACTOR_RUNNING_COMPACTION_IDS_POOL; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -262,7 +261,7 @@ public static List getCompactionsRunningOnCompactors(ClientContext con } } - public static Collection + public static Set getCompactionIdsRunningOnCompactors(ClientContext context) { final ExecutorService executor = ThreadPools.getServerThreadPools() .getPoolBuilder(COMPACTOR_RUNNING_COMPACTION_IDS_POOL).numCoreThreads(16).build(); diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java new file mode 100644 index 00000000000..ceb405e13ea --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.metadata; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.metadata.SystemTables; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.RemovedCompactionSection; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.server.ServerContext; + +import com.google.common.base.Preconditions; + +public class RemovedCompactionStoreImpl implements Ample.RemovedCompactionStore { + private final ServerContext context; + + public RemovedCompactionStoreImpl(ServerContext context) { + this.context = context; + } + + private Stream createStream(String tableName) { + Scanner scanner = null; + try { + scanner = context.createScanner(tableName, Authorizations.EMPTY); + } catch (TableNotFoundException e) { + throw new IllegalStateException(e); + } + scanner.setRange(RemovedCompactionSection.getRange()); + return scanner.stream().map(e -> e.getKey().getRowData().toString()) + .map(RemovedCompactionSection::decodeRow).onClose(scanner::close); + } + + @Override + public Stream list() { + return Stream.concat(createStream(SystemTables.ROOT.tableName()), + createStream(SystemTables.METADATA.tableName())); + } + + private void write(Collection removedCompactions, + Function converter) { + Map> byLevel = removedCompactions.stream() + .collect(Collectors.groupingBy(rc -> Ample.DataLevel.of(rc.table()))); + // Do not expect the root to split or merge so it should never have this data + Preconditions.checkArgument(!byLevel.containsKey(Ample.DataLevel.ROOT)); + byLevel.forEach((dl, removed) -> { + try (var writer = context.createBatchWriter(dl.metaTable())) { + for (var rc : removed) { + writer.addMutation(converter.apply(rc)); + } + } catch (TableNotFoundException | MutationsRejectedException e) { + throw new IllegalStateException(e); + } + }); + } + + @Override + public void add(Collection removedCompactions) { + if (removedCompactions.isEmpty()) { + return; + } + + write(removedCompactions, rc -> { + Mutation m = new Mutation(RemovedCompactionSection.encodeRow(rc)); + m.put("", "", ""); + return m; + }); + + } + + @Override + public void delete(Collection removedCompactions) { + if (removedCompactions.isEmpty()) { + return; + } + + write(removedCompactions, rc -> { + Mutation m = new Mutation(RemovedCompactionSection.encodeRow(rc)); + m.putDelete("", ""); + return m; + }); + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java index a4eb470c664..35fc63fe3ff 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java @@ -281,6 +281,11 @@ public ScanServerRefStore scanServerRefs() { return scanServerRefStore; } + @Override + public RemovedCompactionStore removedCompactions() { + return new RemovedCompactionStoreImpl(getContext()); + } + @VisibleForTesting protected ServerContext getContext() { return context; diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java index 06755d1807b..c27a65639a4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java @@ -18,11 +18,13 @@ */ package org.apache.accumulo.server.util; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; @@ -32,7 +34,9 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.cli.ServerOpts; +import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; @@ -46,6 +50,7 @@ import org.apache.accumulo.start.spi.CommandGroups; import org.apache.accumulo.start.spi.KeywordExecutable; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,6 +132,7 @@ public static Set findTempFiles(ServerContext context, String tableId) }); } } + LOG.trace("Final set of compaction tmp files after removing active compactions: {}", matches); return matches; } @@ -141,7 +147,8 @@ public static DeleteStats deleteTempFiles(ServerContext context, Set files throws InterruptedException { final ExecutorService delSvc = Executors.newFixedThreadPool(8); - final List> futures = new ArrayList<>(filesToDelete.size()); + // use a linked list to make removal from the middle of the list quick + final List> futures = new LinkedList<>(); final DeleteStats stats = new DeleteStats(); filesToDelete.forEach(p -> { @@ -190,6 +197,39 @@ public static DeleteStats deleteTempFiles(ServerContext context, Set files return stats; } + // Finds any tmp files matching the given compaction ids in table dir and deletes them. + public static void deleteTmpFiles(ServerContext ctx, TableId tableId, String dirName, + Set ecidsForTablet) { + final Collection vols = ctx.getVolumeManager().getVolumes(); + for (Volume vol : vols) { + try { + final String volPath = vol.getBasePath() + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + + tableId.canonical() + Path.SEPARATOR + dirName; + final FileSystem fs = vol.getFileSystem(); + for (ExternalCompactionId ecid : ecidsForTablet) { + final String fileSuffix = "_tmp_" + ecid.canonical(); + FileStatus[] files = null; + try { + files = fs.listStatus(new Path(volPath), (path) -> path.getName().endsWith(fileSuffix)); + } catch (FileNotFoundException e) { + LOG.trace("Failed to list tablet dir {}", volPath, e); + } + if (files != null) { + for (FileStatus file : files) { + if (!fs.delete(file.getPath(), false)) { + LOG.warn("Unable to delete ecid tmp file: {}: ", file.getPath()); + } else { + LOG.debug("Deleted ecid tmp file: {}", file.getPath()); + } + } + } + } + } catch (IOException e) { + LOG.error("Exception deleting compaction tmp files for table: {}", tableId, e); + } + } + } + public FindCompactionTmpFiles() { super(new FindOpts()); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 51685c166cd..2e4f8b52b6e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -93,8 +93,6 @@ import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; -import org.apache.accumulo.core.metadata.schema.TabletsMetadata; -import org.apache.accumulo.core.metadata.schema.filters.HasExternalCompactionsFilter; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.securityImpl.thrift.TCredentials; import org.apache.accumulo.core.spi.compaction.CompactionJob; @@ -108,7 +106,6 @@ import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; -import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.manager.compaction.coordinator.commit.CommitCompaction; import org.apache.accumulo.manager.compaction.coordinator.commit.CompactionCommitData; @@ -122,8 +119,8 @@ import org.apache.accumulo.server.compaction.CompactionPluginUtils; import org.apache.accumulo.server.security.AuditedSecurityOperation; import org.apache.accumulo.server.tablets.TabletNameGenerator; +import org.apache.accumulo.server.util.FindCompactionTmpFiles; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -765,6 +762,21 @@ void compactionsFailed(Map compactions) { void compactionFailedForLevel(Map> compactions) { + // CompactionFailed is called from the Compactor when either a compaction fails or is cancelled + // and it's called from the DeadCompactionDetector. Remove compaction tmp files from the tablet + // directory that have a corresponding ecid in the name. Must delete any tmp files before + // removing compaction entry from metadata table. This ensures that in the event of process + // death that the dead compaction will be detected in the future and the files removed then. + try (var tablets = ctx.getAmple().readTablets() + .forTablets(compactions.keySet(), Optional.empty()).fetch(ColumnType.DIR).build()) { + for (TabletMetadata tm : tablets) { + var extent = tm.getExtent(); + var ecidsForTablet = compactions.get(extent); + FindCompactionTmpFiles.deleteTmpFiles(ctx, extent.tableId(), tm.getDirName(), + ecidsForTablet); + } + } + try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) { compactions.forEach((extent, ecids) -> { try { @@ -791,78 +803,18 @@ public boolean test(TabletMetadata tabletMetadata) { } }); - final List ecidsForTablet = new ArrayList<>(); tabletsMutator.process().forEach((extent, result) -> { if (result.getStatus() != Ample.ConditionalResult.Status.ACCEPTED) { - // this should try again later when the dead compaction detector runs, lets log it in case // its a persistent problem if (LOG.isDebugEnabled()) { LOG.debug("Unable to remove failed compaction {} {}", extent, compactions.get(extent)); } - } else { - // compactionFailed is called from the Compactor when either a compaction fails or - // is cancelled and it's called from the DeadCompactionDetector. This block is - // entered when the conditional mutator above successfully deletes an ecid from - // the tablet metadata. Remove compaction tmp files from the tablet directory - // that have a corresponding ecid in the name. - - ecidsForTablet.clear(); - ecidsForTablet.addAll(compactions.get(extent)); - - if (!ecidsForTablet.isEmpty()) { - final TabletMetadata tm = ctx.getAmple().readTablet(extent, ColumnType.DIR); - if (tm != null) { - final Collection vols = ctx.getVolumeManager().getVolumes(); - for (Volume vol : vols) { - try { - final String volPath = - vol.getBasePath() + Constants.HDFS_TABLES_DIR + Path.SEPARATOR - + extent.tableId().canonical() + Path.SEPARATOR + tm.getDirName(); - final FileSystem fs = vol.getFileSystem(); - for (ExternalCompactionId ecid : ecidsForTablet) { - final String fileSuffix = "_tmp_" + ecid.canonical(); - FileStatus[] files = null; - try { - files = fs.listStatus(new Path(volPath), - (path) -> path.getName().endsWith(fileSuffix)); - } catch (FileNotFoundException e) { - LOG.trace("Failed to list tablet dir {}", volPath, e); - } - if (files != null) { - for (FileStatus file : files) { - if (!fs.delete(file.getPath(), false)) { - LOG.warn("Unable to delete ecid tmp file: {}: ", file.getPath()); - } else { - LOG.debug("Deleted ecid tmp file: {}", file.getPath()); - } - } - } - } - } catch (IOException e) { - LOG.error("Exception deleting compaction tmp files for tablet: {}", extent, e); - } - } - } else { - // TabletMetadata does not exist for the extent. This could be due to a merge or - // split operation. Use the utility to find tmp files at the table level - deadCompactionDetector.addTableId(extent.tableId()); - } - } } }); } } - protected Set readExternalCompactionIds() { - try (TabletsMetadata tabletsMetadata = - this.ctx.getAmple().readTablets().forLevel(Ample.DataLevel.USER) - .filter(new HasExternalCompactionsFilter()).fetch(ECOMP).build()) { - return tabletsMetadata.stream().flatMap(tm -> tm.getExternalCompactions().keySet().stream()) - .collect(Collectors.toSet()); - } - } - /* Method exists to be called from test */ public CompactionJobQueues getJobQueues() { return jobQueues; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java index ce04296a615..89610550971 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java @@ -18,11 +18,12 @@ */ package org.apache.accumulo.manager.compaction.coordinator; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; @@ -33,11 +34,11 @@ import java.util.stream.Stream; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.FateClient; import org.apache.accumulo.core.fate.FateInstanceType; import org.apache.accumulo.core.fate.FateKey; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metadata.schema.TabletMetadata; @@ -49,8 +50,6 @@ import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.util.FindCompactionTmpFiles; -import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats; -import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,7 +61,6 @@ public class DeadCompactionDetector { private final CompactionCoordinator coordinator; private final ScheduledThreadPoolExecutor schedExecutor; private final ConcurrentHashMap deadCompactions; - private final Set tablesWithUnreferencedTmpFiles = new HashSet<>(); private final Function> fateClients; public DeadCompactionDetector(ServerContext context, CompactionCoordinator coordinator, @@ -75,12 +73,6 @@ public DeadCompactionDetector(ServerContext context, CompactionCoordinator coord this.fateClients = fateClients; } - public void addTableId(TableId tableWithUnreferencedTmpFiles) { - synchronized (tablesWithUnreferencedTmpFiles) { - tablesWithUnreferencedTmpFiles.add(tableWithUnreferencedTmpFiles); - } - } - private void detectDeadCompactions() { /* @@ -162,6 +154,31 @@ private void detectDeadCompactions() { }); } + // Get the list of compaction entires that were removed from the metadata table by a split or + // merge operation. Must get this data before getting the running set of compactions. + List removedCompactions; + try (Stream listing = context.getAmple().removedCompactions().list()) { + removedCompactions = listing.collect(Collectors.toCollection(ArrayList::new)); + } + + // Must get the set of running compactions after reading compaction ids from the metadata table + Set running = null; + if (!removedCompactions.isEmpty() || !tabletCompactions.isEmpty()) { + running = ExternalCompactionUtil.getCompactionIdsRunningOnCompactors(context); + } + + // Delete any tmp files related to compaction metadata entries that were removed by split or + // merge and are no longer running. + if (!removedCompactions.isEmpty()) { + var runningSet = Objects.requireNonNull(running); + removedCompactions.removeIf(rc -> runningSet.contains(rc.id())); + removedCompactions.forEach(rc -> { + log.trace("attempting to delete tmp files for removed compaction {}", rc); + FindCompactionTmpFiles.deleteTmpFiles(context, rc.table(), rc.dir(), Set.of(rc.id())); + }); + context.getAmple().removedCompactions().delete(removedCompactions); + } + if (tabletCompactions.isEmpty()) { // Clear out dead compactions, tservers don't think anything is running log.trace("Clearing the dead compaction map, no tablets have compactions running"); @@ -183,9 +200,6 @@ private void detectDeadCompactions() { // In order for this overall algorithm to be correct and avoid race conditions, the compactor // must return ids covering the time period from before reservation until after commit. If the // ids do not cover this time period then legitimate running compactions could be canceled. - Collection running = - ExternalCompactionUtil.getCompactionIdsRunningOnCompactors(context); - running.forEach(ecid -> { if (tabletCompactions.remove(ecid) != null) { log.debug("Ignoring compaction {} that is running on a compactor", ecid); @@ -230,37 +244,6 @@ private void detectDeadCompactions() { coordinator.compactionsFailed(tabletCompactions); this.deadCompactions.keySet().removeAll(toFail); } - - // Find and delete compaction tmp files that are unreferenced - if (!tablesWithUnreferencedTmpFiles.isEmpty()) { - - Set copy = new HashSet<>(); - synchronized (tablesWithUnreferencedTmpFiles) { - copy.addAll(tablesWithUnreferencedTmpFiles); - tablesWithUnreferencedTmpFiles.clear(); - } - - log.debug("Tables that may have unreferenced compaction tmp files: {}", copy); - for (TableId tid : copy) { - try { - final Set matches = FindCompactionTmpFiles.findTempFiles(context, tid.canonical()); - log.debug("Found the following compaction tmp files for table {}:", tid); - matches.forEach(p -> log.debug("{}", p)); - - if (!matches.isEmpty()) { - log.debug("Deleting compaction tmp files for table {}...", tid); - DeleteStats stats = FindCompactionTmpFiles.deleteTempFiles(context, matches); - log.debug( - "Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}", - tid, stats.success, stats.failure, stats.error); - } - } catch (InterruptedException e) { - log.error("Interrupted while finding compaction tmp files for table: {}", tid.canonical(), - e); - } - } - } - } public void start() { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java index 8e8241fb92d..5ec0954e146 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java @@ -75,6 +75,7 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { Map newFiles = new HashMap<>(); TabletMetadata firstTabletMeta = null; TabletMetadata lastTabletMeta = null; + List removedCompactions = new ArrayList<>(); try (var tabletsMetadata = env.getContext().getAmple().readTablets().forTable(range.tableId()) .overlapping(range.prevEndRow(), range.endRow()).build()) { @@ -141,6 +142,19 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { dirs.clear(); } } + + // These compaction metadata entries will be deleted, queue up removal of the tmp file once + // the compaction is no longer running + tabletMeta.getExternalCompactions().keySet().stream() + .map(ecid -> new Ample.RemovedCompaction(ecid, tabletMeta.getExtent().tableId(), + tabletMeta.getDirName())) + .forEach(removedCompactions::add); + if (removedCompactions.size() > 1000 && tabletsSeen > 1) { + removedCompactions + .forEach(rc -> log.trace("{} adding removed compaction {}", fateId, rc)); + env.getContext().getAmple().removedCompactions().add(removedCompactions); + removedCompactions.clear(); + } } if (tabletsSeen == 1) { @@ -154,6 +168,9 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { lastTabletMeta); } + removedCompactions.forEach(rc -> log.trace("{} adding removed compaction {}", fateId, rc)); + env.getContext().getAmple().removedCompactions().add(removedCompactions); + log.info("{} merge low tablet {}", fateId, firstTabletMeta.getExtent()); log.info("{} merge high tablet {}", fateId, lastTabletMeta.getExtent()); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java index c12386a3721..7f387f64de8 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java @@ -266,6 +266,16 @@ private void addNewTablets(FateId fateId, FateEnv env, TabletMetadata tabletMeta private void updateExistingTablet(FateId fateId, ServerContext ctx, TabletMetadata tabletMetadata, TabletOperationId opid, NavigableMap newTablets, Map> newTabletsFiles) { + + // queue up the tmp files related to these compaction metadata entries to be eventually deleted + // once the compaction is no longer running + var removedCompactions = tabletMetadata.getExternalCompactions().keySet().stream() + .map(ecid -> new Ample.RemovedCompaction(ecid, tabletMetadata.getExtent().tableId(), + tabletMetadata.getDirName())) + .toList(); + removedCompactions.forEach(rc -> log.trace("{} adding removed compaction {}", fateId, rc)); + ctx.getAmple().removedCompactions().add(removedCompactions); + try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) { var newExtent = newTablets.navigableKeySet().last(); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java index 9e3037dbe01..416a76defed 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java @@ -44,11 +44,15 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.UNSPLITTABLE; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED; import static org.apache.accumulo.manager.tableOps.split.UpdateTabletsTest.newSTF; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.util.ArrayList; +import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.OptionalLong; @@ -56,6 +60,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.stream.Stream; import org.apache.accumulo.core.client.admin.TabletAvailability; import org.apache.accumulo.core.data.NamespaceId; @@ -202,9 +207,12 @@ public void testManyColumns() throws Exception { EasyMock.expect(lastTabletMeta.getUnSplittable()).andReturn(unsplittableMeta).atLeastOnce(); EasyMock.expect(lastTabletMeta.getTabletMergeability()).andReturn(mergeability).atLeastOnce(); EasyMock.expect(lastTabletMeta.getMigration()).andReturn(migration).atLeastOnce(); + EasyMock.expect(lastTabletMeta.getDirName()).andReturn("td3").atLeastOnce(); EasyMock.replay(lastTabletMeta, compactions); + Set removedCompactions = new HashSet<>(); + testMerge(List.of(tablet1, tablet2, lastTabletMeta), tableId, null, null, tabletMutator -> { EasyMock.expect(tabletMutator.putTime(MetadataTime.parse("L30"))).andReturn(tabletMutator) .once(); @@ -243,9 +251,13 @@ public void testManyColumns() throws Exception { .andReturn(tabletMutator).once(); EasyMock.expect(tabletMutator.deleteMigration()).andReturn(tabletMutator); - }); + }, removedCompactions::add); EasyMock.verify(lastTabletMeta, compactions); + + assertEquals(Set.of(new Ample.RemovedCompaction(cid1, tableId, "td3"), + new Ample.RemovedCompaction(cid2, tableId, "td3"), + new Ample.RemovedCompaction(cid3, tableId, "td3")), removedCompactions); } @Test @@ -420,6 +432,12 @@ public void testTime() throws Exception { private static void testMerge(List inputTablets, TableId tableId, String start, String end, Consumer expectationsSetter) throws Exception { + testMerge(inputTablets, tableId, start, end, expectationsSetter, removedCompaction -> fail()); + } + + private static void testMerge(List inputTablets, TableId tableId, String start, + String end, Consumer expectationsSetter, + Consumer removedCompactionConsumer) throws Exception { MergeInfo mergeInfo = new MergeInfo(tableId, NamespaceId.of("1"), start == null ? null : start.getBytes(UTF_8), end == null ? null : end.getBytes(UTF_8), MergeInfo.Operation.MERGE); @@ -434,6 +452,25 @@ private static void testMerge(List inputTablets, TableId tableId EasyMock.mock(ConditionalTabletsMutatorImpl.class); ConditionalTabletMutatorImpl tabletMutator = EasyMock.mock(ConditionalTabletMutatorImpl.class); + Ample.RemovedCompactionStore removedCompactionStore = new Ample.RemovedCompactionStore() { + @Override + public Stream list() { + throw new UnsupportedOperationException(); + } + + @Override + public void add(Collection removedCompactions) { + System.out.println("removedCompactions : " + removedCompactions); + removedCompactions.forEach(removedCompactionConsumer); + } + + @Override + public void delete(Collection removedCompactions) { + throw new UnsupportedOperationException(); + } + }; + EasyMock.expect(ample.removedCompactions()).andReturn(removedCompactionStore); + ServiceLock managerLock = EasyMock.mock(ServiceLock.class); EasyMock.expect(context.getServiceLock()).andReturn(managerLock).anyTimes(); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java index 6ceac3d6810..044cf08da07 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java @@ -24,7 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; +import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.OptionalLong; @@ -37,6 +39,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.accumulo.core.client.admin.TabletAvailability; import org.apache.accumulo.core.client.admin.TabletMergeability; @@ -250,6 +253,24 @@ public void testManyColumns() throws Exception { EasyMock.expect(manager.getFileRangeCache()).andReturn(fileRangeCache).atLeastOnce(); EasyMock.expect(manager.getSteadyTime()).andReturn(SteadyTime.from(100_000, TimeUnit.SECONDS)) .atLeastOnce(); + Set removedCompactionSet = new HashSet<>(); + Ample.RemovedCompactionStore rcs = new Ample.RemovedCompactionStore() { + @Override + public Stream list() { + throw new UnsupportedOperationException(); + } + + @Override + public void add(Collection removedCompactions) { + removedCompactionSet.addAll(removedCompactions); + } + + @Override + public void delete(Collection removedCompactions) { + throw new UnsupportedOperationException(); + } + }; + EasyMock.expect(ample.removedCompactions()).andReturn(rcs).atLeastOnce(); ServiceLock managerLock = EasyMock.mock(ServiceLock.class); EasyMock.expect(context.getServiceLock()).andReturn(managerLock).anyTimes(); @@ -289,6 +310,7 @@ public void testManyColumns() throws Exception { UnSplittableMetadata.toUnSplittable(origExtent, 1000, 1001, 1002, tabletFiles.keySet()); EasyMock.expect(tabletMeta.getUnSplittable()).andReturn(usm).atLeastOnce(); EasyMock.expect(tabletMeta.getMigration()).andReturn(migration).atLeastOnce(); + EasyMock.expect(tabletMeta.getDirName()).andReturn("td1").atLeastOnce(); EasyMock.expect(ample.readTablet(origExtent)).andReturn(tabletMeta); @@ -408,6 +430,10 @@ public void testManyColumns() throws Exception { EasyMock.verify(manager, context, ample, tabletMeta, fileRangeCache, tabletsMutator, tablet1Mutator, tablet2Mutator, tablet3Mutator, cr, compactions); + + assertEquals(Set.of(new Ample.RemovedCompaction(cid1, tableId, "td1"), + new Ample.RemovedCompaction(cid2, tableId, "td1"), + new Ample.RemovedCompaction(cid3, tableId, "td1")), removedCompactionSet); } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java index 0101865b3fa..1ed8e81fecd 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java @@ -150,14 +150,16 @@ public void testMergeCancelsExternalCompaction() throws Exception { .collect(Collectors.toSet()); } } - // We need to cancel the compaction or delete the table here because we initiate a user - // compaction above in the test. Even though the external compaction was cancelled - // because we split the table, FaTE will continue to queue up a compaction - client.tableOperations().delete(table1); // Verify that the tmp file are cleaned up Wait.waitFor(() -> FindCompactionTmpFiles .findTempFiles(getCluster().getServerContext(), tid.canonical()).size() == 0); + + // We need to cancel the compaction or delete the table here because we initiate a user + // compaction above in the test. Even though the external compaction was cancelled + // because we split the table, FaTE would continue to queue up a compaction + client.tableOperations().cancelCompaction(table1); + } finally { getCluster().getClusterControl().stop(ServerType.COMPACTOR); getCluster().getClusterControl().start(ServerType.COMPACTOR); From 6ad2090f91238bfe96373a92ec0cd452bb7c3562 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 31 Mar 2026 23:08:36 +0000 Subject: [PATCH 32/65] consolidate code --- .../server/metadata/RemovedCompactionStoreImpl.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java index ceb405e13ea..5dafd5a2746 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java @@ -64,6 +64,10 @@ public Stream list() { private void write(Collection removedCompactions, Function converter) { + if (removedCompactions.isEmpty()) { + return; + } + Map> byLevel = removedCompactions.stream() .collect(Collectors.groupingBy(rc -> Ample.DataLevel.of(rc.table()))); // Do not expect the root to split or merge so it should never have this data @@ -81,10 +85,6 @@ private void write(Collection removedCompactions, @Override public void add(Collection removedCompactions) { - if (removedCompactions.isEmpty()) { - return; - } - write(removedCompactions, rc -> { Mutation m = new Mutation(RemovedCompactionSection.encodeRow(rc)); m.put("", "", ""); @@ -95,10 +95,6 @@ public void add(Collection removedCompactions) { @Override public void delete(Collection removedCompactions) { - if (removedCompactions.isEmpty()) { - return; - } - write(removedCompactions, rc -> { Mutation m = new Mutation(RemovedCompactionSection.encodeRow(rc)); m.putDelete("", ""); From 0cac83ffd8b12f9eb545c986f1c69e6470f82426 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 1 Apr 2026 15:43:58 +0000 Subject: [PATCH 33/65] WIP --- .../manager/compaction/coordinator/CompactionCoordinator.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index d6b1b56f3f5..3ccfc37a5e6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -191,8 +191,6 @@ static FailureCounts incrementSuccess(Object key, FailureCounts counts) { private final LoadingCache compactorCounts; - private volatile long coordinatorStartTime; - private final Map reservationPools; private final Set activeCompactorReservationRequest = ConcurrentHashMap.newKeySet(); @@ -297,8 +295,6 @@ private void checkForConfigChanges() { @Override public void run() { - - this.coordinatorStartTime = System.currentTimeMillis(); startConfigMonitor(ctx.getScheduledExecutor()); startFailureSummaryLogging(); From fb401daa3ca7c1d2b7d5ba7cecb9cec690918eff Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 1 Apr 2026 16:49:20 +0000 Subject: [PATCH 34/65] fix build --- .../manager/compaction/coordinator/CoordinatorManager.java | 4 ++-- .../manager/compaction/queue/CompactionJobQueues.java | 4 ++++ .../apache/accumulo/manager/multi/ManagerAssignmentTest.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index 309ba1b88b3..9149d0c95c6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -19,6 +19,7 @@ package org.apache.accumulo.manager.compaction.coordinator; import static java.util.stream.Collectors.toSet; +import static org.apache.accumulo.core.util.LazySingletons.RANDOM; import static org.apache.accumulo.manager.multi.ManagerAssignment.computeAssignments; import static org.apache.accumulo.server.compaction.CompactionPluginUtils.getConfiguredCompactionResourceGroups; @@ -42,7 +43,6 @@ import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; import org.apache.accumulo.core.trace.TraceUtil; -import org.apache.accumulo.core.util.LazySingletons; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; @@ -82,7 +82,7 @@ private void managerCoordinators() { while (true) { try { - long updateId = LazySingletons.RANDOM.get().nextLong(); + long updateId = RANDOM.get().nextLong(); var current = getCurrentAssignments(updateId); log.trace("Current assignments {}", current); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java index 70b3bc20b71..2c11db5e041 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java @@ -37,6 +37,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + public class CompactionJobQueues { private static final Logger log = LoggerFactory.getLogger(CompactionJobQueues.class); @@ -168,6 +170,8 @@ public void resetMaxSize(long size) { priorityQueues.values().forEach(cjpq -> cjpq.resetMaxSize(this.queueSize)); } + @SuppressFBWarnings(value = "UG_SYNC_SET_UNSYNC_GET", + justification = "reads from atomic ref in getter, setter only wants one thread cleaning queues so it syncs") public Set getAllowedGroups() { return allowedGroups.get(); } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java index 154d9b474c9..9276eafc5a1 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/multi/ManagerAssignmentTest.java @@ -26,9 +26,9 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.util.Sets; import org.junit.jupiter.api.Test; +import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; public class ManagerAssignmentTest { From e6310dc90ff6173fe2848aa0b8477668076e5ad4 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 1 Apr 2026 17:18:21 +0000 Subject: [PATCH 35/65] do some TODOs --- .../org/apache/accumulo/compactor/Compactor.java | 3 +-- .../coordinator/CompactionCoordinator.java | 13 ------------- .../compaction/CompactionCoordinatorTest.java | 3 ++- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 8bb2b4a3aef..e8186b0eb55 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -569,8 +569,7 @@ protected CompactionCoordinatorService.Client getCoordinatorClient() throws TTra if (coordinatorHost == null) { throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper"); } - // TODO change back to trace - LOG.debug("CompactionCoordinator address is: {}", coordinatorHost); + LOG.trace("CompactionCoordinator address is: {}", coordinatorHost); return ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost, getContext()); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 3ccfc37a5e6..751d119fd20 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -32,8 +32,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; @@ -95,7 +93,6 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.securityImpl.thrift.TCredentials; -import org.apache.accumulo.core.spi.compaction.CompactionJob; import org.apache.accumulo.core.spi.compaction.CompactionKind; import org.apache.accumulo.core.tabletserver.thrift.InputFile; import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig; @@ -637,16 +634,6 @@ public void setResourceGroups(TInfo tinfo, TCredentials credentials, long update } } - // TODO remove - public void addJobs(TabletMetadata tabletMetadata, Collection jobs) { - ArrayList resolvedJobs = new ArrayList<>(jobs.size()); - for (var job : jobs) { - resolvedJobs.add(new ResolvedCompactionJob(job, tabletMetadata)); - } - - jobQueues.add(tabletMetadata.getExtent(), resolvedJobs); - } - public CompactionCoordinatorService.Iface getThriftService() { return this; } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index 7812ee97dda..3e24d12f6b7 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -237,7 +237,8 @@ public void testGetCompactionJob() throws Exception { coordinator.getJobQueues().setAllowedGroups(Set.of(GROUP_ID)); CompactionJob job = new CompactionJobImpl((short) 1, GROUP_ID, Collections.emptyList(), CompactionKind.SYSTEM); - coordinator.addJobs(tm, Collections.singleton(job)); + coordinator.addJobs(new TInfo(), rpcCreds, + List.of(new ResolvedCompactionJob(job, tm).toThrift())); CompactionJobPriorityQueue queue = coordinator.getJobQueues().getQueue(GROUP_ID); assertEquals(1, queue.getQueuedJobs()); From 8b26c0003e41b2a4a3f1b0a684caf70767d011b9 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 1 Apr 2026 20:28:22 +0000 Subject: [PATCH 36/65] fix bug with adding jobs to queue found by CompactionIT --- .../coordinator/CompactionCoordinator.java | 17 ++++++++++++----- .../coordinator/CoordinatorManager.java | 9 ++++++--- .../manager/multi/ManagerAssignment.java | 5 ----- .../compaction/CompactionCoordinatorTest.java | 14 +++++++++----- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 751d119fd20..3e3dfd29a19 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -32,6 +32,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; @@ -93,6 +94,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.securityImpl.thrift.TCredentials; +import org.apache.accumulo.core.spi.compaction.CompactionJob; import org.apache.accumulo.core.spi.compaction.CompactionKind; import org.apache.accumulo.core.tabletserver.thrift.InputFile; import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig; @@ -560,7 +562,7 @@ public void beginFullJobScan(TInfo tinfo, TCredentials credentials, String dataL } @Override - public void addJobs(TInfo tinfo, TCredentials credentials, List jobs) + public void addJobs(TInfo tinfo, TCredentials credentials, List tjobs) throws TException { if (!security.canPerformSystemActions(credentials)) { LOG.warn("Thrift call attempted to add job and did not have proper access. {}", @@ -568,12 +570,17 @@ public void addJobs(TInfo tinfo, TCredentials credentials, List> jobs = new HashMap<>(); + for (var tjob : tjobs) { var job = ResolvedCompactionJob.fromThrift(tjob); - LOG.trace("Adding compaction job {} {} {} {}", job.getGroup(), job.getPriority(), - job.getExtent(), job.getJobFiles().size()); - jobQueues.add(job.getExtent(), List.of(job)); + LOG.trace("Adding compaction job {} {} {} {} {}", job.getGroup(), job.getPriority(), + job.getKind(), job.getExtent(), job.getJobFiles().size()); + jobs.computeIfAbsent(job.getExtent(), e -> new ArrayList<>()).add(job); } + + // its important to add all jobs for an extent at once instead of one by one because the job + // queue deletes all existing jobs for an extent when adding an extent + jobs.forEach(jobQueues::add); } @Override diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index 9149d0c95c6..6d4a16bcf4d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -79,8 +79,9 @@ public CoordinatorManager(ServerContext context, private final DeadCompactionDetector deadCompactionDetector; private void managerCoordinators() { + final long initialSleepTime = 10; + long sleepTime = initialSleepTime; while (true) { - try { long updateId = RANDOM.get().nextLong(); var current = getCurrentAssignments(updateId); @@ -96,7 +97,10 @@ private void managerCoordinators() { log.trace("Desired assignments {}", desired); if (!current.equals(desired)) { + sleepTime = initialSleepTime; setAssignments(updateId, desired); + } else { + sleepTime = Math.min(sleepTime + 10, 5000); } updateZookeeper(desired); @@ -105,8 +109,7 @@ private void managerCoordinators() { throw new RuntimeException(e); } - // TODO not good for tests - UtilWaitThread.sleep(5000); + UtilWaitThread.sleep(sleepTime); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java b/server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java index 419878b15a8..7fe86e88e3b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/multi/ManagerAssignment.java @@ -42,9 +42,6 @@ public static Map> computeAssignments(Map> assignments = new HashMap<>(); Set assigned = new HashSet<>(); @@ -85,8 +82,6 @@ public static Map> computeAssignments(Map Date: Wed, 1 Apr 2026 20:49:47 +0000 Subject: [PATCH 37/65] fixes manager failover and BackupManagerIT --- .../coordinator/CoordinatorManager.java | 18 +++++++++--------- .../test/functional/BackupManagerIT.java | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index 6d4a16bcf4d..767b5129dc2 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -80,6 +80,7 @@ public CoordinatorManager(ServerContext context, private void managerCoordinators() { final long initialSleepTime = 10; + final long maxSleepTime = 5000; long sleepTime = initialSleepTime; while (true) { try { @@ -100,13 +101,13 @@ private void managerCoordinators() { sleepTime = initialSleepTime; setAssignments(updateId, desired); } else { - sleepTime = Math.min(sleepTime + 10, 5000); + sleepTime = Math.min(sleepTime + 10, maxSleepTime); } updateZookeeper(desired); - } catch (ReflectiveOperationException | InterruptedException | KeeperException e) { - // TODO - throw new RuntimeException(e); + } catch (Exception e) { + log.warn("Failed to set compaction coordinator assignments", e); + sleepTime = maxSleepTime; } UtilWaitThread.sleep(sleepTime); @@ -139,14 +140,14 @@ private Map> getCurrentAssignments(long updateI for (var assistant : assistants) { CompactionCoordinatorService.Client client = null; + var hp = HostAndPort.fromString(assistant.getServer()); try { - var hp = HostAndPort.fromString(assistant.getServer()); client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, hp, context); var groups = client.getResourceGroups(TraceUtil.traceInfo(), context.rpcCreds(), updateId); assignments.put(hp, groups.stream().map(ResourceGroupId::of).collect(toSet())); } catch (TException e) { - // TODO - throw new RuntimeException(e); + log.warn("Failed to get current assignments from {}", hp, e); + return Map.of(); } finally { ThriftUtil.returnClient(client, context); } @@ -168,8 +169,7 @@ private void setResourceGroups(HostAndPort hp, Set groups, long client.setResourceGroups(TraceUtil.traceInfo(), context.rpcCreds(), updateId, groups.stream().map(AbstractId::canonical).collect(toSet())); } catch (TException e) { - // TODO - throw new RuntimeException(e); + log.warn("Failed to set resource groups for {}", hp, e); } finally { ThriftUtil.returnClient(client, context); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java index 5f788cd18fe..cffe4e622d8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BackupManagerIT.java @@ -69,5 +69,4 @@ public void test() throws Exception { backup.destroy(); } } - } From 88d48bdc63b64dd87d5135703c3f967a5b2c9298 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 1 Apr 2026 20:59:23 +0000 Subject: [PATCH 38/65] remove logging config changes --- test/src/main/resources/log4j2-test.properties | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/src/main/resources/log4j2-test.properties b/test/src/main/resources/log4j2-test.properties index 8eb1ce5f422..f75e6ec8f57 100644 --- a/test/src/main/resources/log4j2-test.properties +++ b/test/src/main/resources/log4j2-test.properties @@ -163,8 +163,5 @@ logger.45.level = trace logger.46.name = org.apache.accumulo.core.spi.scan logger.46.level = info -logger.47.name = org.apache.accumulo.manager.tableOps.compact -logger.47.level = trace - rootLogger.level = debug rootLogger.appenderRef.console.ref = STDOUT From a4050d9bd922f82442ee41d3ceb95d6fe5f277f0 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 1 Apr 2026 21:42:27 +0000 Subject: [PATCH 39/65] fixes flaky fate cancel command IT --- .../apache/accumulo/test/fate/FateOpsCommandsITBase.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java b/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java index 280efe6312b..15b4798bb04 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsITBase.java @@ -584,9 +584,12 @@ protected void testFateCancelCommand(FateStore store, ServerContex assertTrue(result .contains("transaction " + fateId1.canonical() + " was cancelled or already completed")); + var expected = Map.of(fateId1.canonical(), "FAILED", fateId2.canonical(), "NEW"); + // The fate operation may temporarily be in the FAILED_IN_PROGRESS state, wait for this to + // pass + Wait.waitFor(() -> expected.equals(getFateIdsFromSummary())); fateIdsFromSummary = getFateIdsFromSummary(); - assertEquals(Map.of(fateId1.canonical(), "FAILED", fateId2.canonical(), "NEW"), - fateIdsFromSummary); + assertEquals(expected, fateIdsFromSummary); } finally { fate.shutdown(1, TimeUnit.MINUTES); } From 2c5fb3bcc5f984de8bfd3488d6e5af3c06cb0133 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 1 Apr 2026 23:38:51 +0000 Subject: [PATCH 40/65] fixes ExitCodesIT --- .../java/org/apache/accumulo/compactor/Compactor.java | 3 ++- .../org/apache/accumulo/test/functional/ExitCodesIT.java | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index e8186b0eb55..f4ce8f06f11 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -540,8 +540,9 @@ protected TNextCompactionJob getNextJob(Supplier uuid) throws RetriesExcee RetryableThriftCall nextJobThriftCall = new RetryableThriftCall<>(startingWaitTime, maxWaitTime, 0, () -> { - Client coordinatorClient = getCoordinatorClient(); + Client coordinatorClient = null; try { + coordinatorClient = getCoordinatorClient(); ExternalCompactionId eci = ExternalCompactionId.generate(uuid.get()); LOG.trace("Attempting to get next job, eci = {}", eci); currentCompactionId.set(eci); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ExitCodesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ExitCodesIT.java index 5adbcef89c5..3b7c976569e 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ExitCodesIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ExitCodesIT.java @@ -35,6 +35,7 @@ import org.apache.accumulo.core.cli.ServerOpts; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.spi.compaction.RatioBasedCompactionPlanner; import org.apache.accumulo.gc.SimpleGarbageCollector; import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.manager.Manager; @@ -262,6 +263,14 @@ static Stream generateWorkerProcessArguments() { @ParameterizedTest @MethodSource("generateWorkerProcessArguments") public void testWorkerProcesses(ServerType server, TerminalBehavior behavior) throws Exception { + // Need to configure a planner that uses the TEST compaction resource group so that it will be + // present in zookeeper + getCluster().getServerContext().instanceOperations().setProperty( + Property.COMPACTION_SERVICE_PREFIX.getKey() + "test.planner.opts.groups", + "[{'group':'TEST'}]"); + getCluster().getServerContext().instanceOperations().setProperty( + Property.COMPACTION_SERVICE_PREFIX.getKey() + "test.planner", + RatioBasedCompactionPlanner.class.getName()); Map properties = new HashMap<>(); properties.put(PROXY_METHOD_BEHAVIOR, behavior.name()); getCluster().getConfig().setSystemProperties(properties); From 69c76856febd9abde035d790004d782f3aeef1c6 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Thu, 2 Apr 2026 17:24:15 +0000 Subject: [PATCH 41/65] fixes bug found by CompactionConfigChangeIT There was a bug where a compactor running a compaction on a deconfigured group could never commit the compaction. --- .../apache/accumulo/server/ServerContext.java | 10 ++-- ....java => CoordinatorLocationsFactory.java} | 17 ++++-- .../server/init/ZooKeeperInitializer.java | 4 +- .../apache/accumulo/compactor/Compactor.java | 53 +++++++++++++++---- .../compaction/CompactionJobClient.java | 2 +- .../coordinator/CoordinatorManager.java | 8 +-- .../test/MultipleManagerCompactionIT.java | 22 ++++---- .../compaction/CompactionConfigChangeIT.java | 2 +- 8 files changed, 82 insertions(+), 36 deletions(-) rename server/base/src/main/java/org/apache/accumulo/server/compaction/{CoordinatorLocations.java => CoordinatorLocationsFactory.java} (83%) diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java index c5b66175cab..5b54e86d8ce 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java @@ -66,7 +66,8 @@ import org.apache.accumulo.core.util.AddressUtil; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; -import org.apache.accumulo.server.compaction.CoordinatorLocations; +import org.apache.accumulo.server.compaction.CoordinatorLocationsFactory; +import org.apache.accumulo.server.compaction.CoordinatorLocationsFactory.CoordinatorLocations; import org.apache.accumulo.server.conf.NamespaceConfiguration; import org.apache.accumulo.server.conf.ServerConfigurationFactory; import org.apache.accumulo.server.conf.TableConfiguration; @@ -90,7 +91,6 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; -import com.google.common.net.HostAndPort; /** * Provides a server context for Accumulo server components that operate with the system credentials @@ -121,7 +121,7 @@ public class ServerContext extends ClientContext { private final AtomicBoolean sharedSchedExecutorCreated = new AtomicBoolean(false); private final AtomicBoolean sharedMetadataWriterCreated = new AtomicBoolean(false); private final AtomicBoolean sharedUserWriterCreated = new AtomicBoolean(false); - private final AtomicReference coordinatorLocationsRef = + private final AtomicReference coordinatorLocationsRef = new AtomicReference<>(); public ServerContext(SiteConfiguration siteConfig) { @@ -532,10 +532,10 @@ public Supplier getSharedUserWriter() { return sharedUserWriter; } - public Map getCoordinatorLocations(boolean useCache) { + public CoordinatorLocations getCoordinatorLocations(boolean useCache) { var cl = coordinatorLocationsRef.get(); if (cl == null) { - coordinatorLocationsRef.compareAndSet(null, new CoordinatorLocations(this)); + coordinatorLocationsRef.compareAndSet(null, new CoordinatorLocationsFactory(this)); cl = coordinatorLocationsRef.get(); } return cl.getLocations(useCache); diff --git a/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocations.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocationsFactory.java similarity index 83% rename from server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocations.java rename to server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocationsFactory.java index 373c8e74660..268034e7490 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocations.java +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocationsFactory.java @@ -21,7 +21,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.accumulo.core.util.LazySingletons.GSON; +import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.accumulo.core.Constants; @@ -38,17 +40,21 @@ * Reads and writes a map of coordinators to zookeeper. The map contains compactor resource group to * manager address mappings. */ -public class CoordinatorLocations { +public class CoordinatorLocationsFactory { private final ServerContext context; private long lastUpdateCount; - private Map lastLocations; + private CoordinatorLocations lastLocations; - public CoordinatorLocations(ServerContext context) { + public CoordinatorLocationsFactory(ServerContext context) { this.context = context; } - public synchronized Map getLocations(boolean useCache) { + public record CoordinatorLocations(Map locations, + List sortedUniqueHost) { + } + + public synchronized CoordinatorLocations getLocations(boolean useCache) { var zooCache = context.getZooCache(); // TODO how frequently does updateCount change? TODO could have an update counter per path if (lastLocations == null || lastUpdateCount != zooCache.getUpdateCount() || !useCache) { @@ -62,7 +68,8 @@ public synchronized Map getLocations(boolean useCac Map locations = new HashMap<>(); stringMap .forEach((rg, hp) -> locations.put(ResourceGroupId.of(rg), HostAndPort.fromString(hp))); - lastLocations = Map.copyOf(locations); + lastLocations = new CoordinatorLocations(Map.copyOf(locations), locations.values().stream() + .distinct().sorted(Comparator.comparing(HostAndPort::toString)).toList()); } return lastLocations; } diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java index fcdc884fa50..d602d453ba9 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java @@ -45,7 +45,7 @@ import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.util.tables.TableMapping; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.compaction.CoordinatorLocations; +import org.apache.accumulo.server.compaction.CoordinatorLocationsFactory; import org.apache.accumulo.server.conf.codec.VersionedPropCodec; import org.apache.accumulo.server.conf.codec.VersionedProperties; import org.apache.accumulo.server.conf.store.ResourceGroupPropKey; @@ -185,7 +185,7 @@ void initialize(final ServerContext context, final String rootTabletDirName, zrwChroot.putPersistentData(Constants.ZMANAGER_ASSIGNMENTS, EMPTY_BYTE_ARRAY, ZooUtil.NodeExistsPolicy.FAIL); FateLocations.storeLocations(zrwChroot, Map.of(), ZooUtil.NodeExistsPolicy.FAIL); - CoordinatorLocations.setLocations(zrwChroot, Map.of(), ZooUtil.NodeExistsPolicy.FAIL); + CoordinatorLocationsFactory.setLocations(zrwChroot, Map.of(), ZooUtil.NodeExistsPolicy.FAIL); } /** diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index f4ce8f06f11..ec1cc30961a 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -19,6 +19,9 @@ package org.apache.accumulo.compactor; import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.accumulo.compactor.Compactor.CoordinatorLocationType.ANY; +import static org.apache.accumulo.compactor.Compactor.CoordinatorLocationType.GROUP; import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_READ; import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_ENTRIES_WRITTEN; import static org.apache.accumulo.core.metrics.Metric.COMPACTOR_MAJC_CANCELLED; @@ -139,6 +142,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.hash.Hashing; import com.google.common.net.HostAndPort; import io.micrometer.core.instrument.FunctionCounter; @@ -488,7 +492,7 @@ protected void updateCompactionFailed(TExternalCompactionJob job, TCompactionSta String message) throws RetriesExceededException { RetryableThriftCall thriftCall = new RetryableThriftCall<>(1000, RetryableThriftCall.MAX_WAIT_TIME, 25, () -> { - Client coordinatorClient = getCoordinatorClient(); + Client coordinatorClient = getCoordinatorClient(ANY); try { coordinatorClient.compactionFailed(TraceUtil.traceInfo(), getContext().rpcCreds(), job.getExternalCompactionId(), job.extent, message, why, @@ -512,7 +516,11 @@ protected void updateCompactionCompleted(TExternalCompactionJob job, TCompaction throws RetriesExceededException { RetryableThriftCall thriftCall = new RetryableThriftCall<>(1000, RetryableThriftCall.MAX_WAIT_TIME, 25, () -> { - Client coordinatorClient = getCoordinatorClient(); + // Any coordinator can process a completed compactions request, so spread the load there + // is no need to limit to the coordinator queuing compactions for this compactors group. + // Also its possible the compaction group has been deconfigured and is no longer in + // zookeeper, however we can still record the completion by going to any coordinator. + Client coordinatorClient = getCoordinatorClient(ANY); try { coordinatorClient.compactionCompleted(TraceUtil.traceInfo(), getContext().rpcCreds(), job.getExternalCompactionId(), job.extent, stats, getResourceGroup().canonical(), @@ -542,7 +550,7 @@ protected TNextCompactionJob getNextJob(Supplier uuid) throws RetriesExcee new RetryableThriftCall<>(startingWaitTime, maxWaitTime, 0, () -> { Client coordinatorClient = null; try { - coordinatorClient = getCoordinatorClient(); + coordinatorClient = getCoordinatorClient(GROUP); ExternalCompactionId eci = ExternalCompactionId.generate(uuid.get()); LOG.trace("Attempting to get next job, eci = {}", eci); currentCompactionId.set(eci); @@ -559,18 +567,45 @@ protected TNextCompactionJob getNextJob(Supplier uuid) throws RetriesExcee return nextJobThriftCall.run(); } + public enum CoordinatorLocationType { + GROUP, ANY + }; + /** * Get the client to the CompactionCoordinator * * @return compaction coordinator client * @throws TTransportException when unable to get client */ - protected CompactionCoordinatorService.Client getCoordinatorClient() throws TTransportException { - var coordinatorHost = getContext().getCoordinatorLocations(true).get(getResourceGroup()); - if (coordinatorHost == null) { - throw new TTransportException("Unable to get CompactionCoordinator address from ZooKeeper"); - } - LOG.trace("CompactionCoordinator address is: {}", coordinatorHost); + protected CompactionCoordinatorService.Client + getCoordinatorClient(CoordinatorLocationType locationType) throws TTransportException { + + var coordinatorHost = switch (locationType) { + case GROUP -> { + var allCoordinatorLocations = getContext().getCoordinatorLocations(true).locations(); + var host = allCoordinatorLocations.get(getResourceGroup()); + if (host == null) { + throw new TTransportException("Did not find group " + getResourceGroup() + + " in coordinators " + allCoordinatorLocations); + } + yield host; + } + case ANY -> { + var hosts = getContext().getCoordinatorLocations(true).sortedUniqueHost(); + if (hosts.isEmpty()) { + throw new TTransportException("There are no coordinators in zookeeper."); + } + int hashedClientAddr = Math + .abs(Hashing.murmur3_128().hashString(getAdvertiseAddress().toString(), UTF_8).asInt()); + // Hashing compactors to coordinators will spread them evenly across coordinators, however + // each coordinator will only have connections for #compactors/#coordinators. If compactors + // randomly choose a coordinator, then each coordinator would have a lot more connections. + // Hashing also send the same compactor to the same coordinator. + yield hosts.get(hashedClientAddr % hosts.size()); + } + }; + + LOG.trace("CompactionCoordinator address is: {} {}", locationType, coordinatorHost); return ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost, getContext()); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java index e2eda1fdd1d..22189a57a4f 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java @@ -68,7 +68,7 @@ public CompactionJobClient(ServerContext context, Ample.DataLevel dataLevel, boo this.dataLevel = dataLevel; this.fullScan = fullScan; - this.coordinatorLocations = context.getCoordinatorLocations(true); + this.coordinatorLocations = context.getCoordinatorLocations(true).locations(); log.trace("Using coordinator locations {}", coordinatorLocations); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index 767b5129dc2..f190d595ee9 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -48,7 +48,7 @@ import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.compaction.CoordinatorLocations; +import org.apache.accumulo.server.compaction.CoordinatorLocationsFactory; import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -123,10 +123,10 @@ private void updateZookeeper(Map> desired) rgs.forEach(rg -> transformed.put(rg, hp)); }); - var assignmentsInZk = context.getCoordinatorLocations(true); + var assignmentsInZk = context.getCoordinatorLocations(true).locations(); if (!transformed.equals(assignmentsInZk)) { - CoordinatorLocations.setLocations(context.getZooSession().asReaderWriter(), transformed, - NodeExistsPolicy.OVERWRITE); + CoordinatorLocationsFactory.setLocations(context.getZooSession().asReaderWriter(), + transformed, NodeExistsPolicy.OVERWRITE); log.debug("Set new coordinator locations {}", desired); } diff --git a/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java index 005a554236e..244a90a34cb 100644 --- a/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MultipleManagerCompactionIT.java @@ -65,9 +65,10 @@ public void test() throws Exception { Accumulo.newClient().from(getCluster().getClientProperties()).build()) { // wait for three coordinator locations to show up in zookeeper - Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() - .distinct().count() == 3); - assertEquals(getExpectedGroups(), getServerContext().getCoordinatorLocations(true).keySet()); + Wait.waitFor( + () -> getServerContext().getCoordinatorLocations(true).sortedUniqueHost().size() == 3); + assertEquals(getExpectedGroups(), + getServerContext().getCoordinatorLocations(true).locations().keySet()); String table1 = names[0]; createTable(client, table1, "cs1"); @@ -92,9 +93,10 @@ public void test() throws Exception { getCluster().getClusterControl().start(ServerType.MANAGER); // wait for five coordinator locations to show up in zookeeper - Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() - .distinct().count() == 5); - assertEquals(getExpectedGroups(), getServerContext().getCoordinatorLocations(true).keySet()); + Wait.waitFor( + () -> getServerContext().getCoordinatorLocations(true).sortedUniqueHost().size() == 5); + assertEquals(getExpectedGroups(), + getServerContext().getCoordinatorLocations(true).locations().keySet()); compact(client, table1, 3, GROUP1, true); verify(client, table1, 6); @@ -109,9 +111,11 @@ public void test() throws Exception { } // wait for three coordinator locations to show up in zookeeper - Wait.waitFor(() -> getServerContext().getCoordinatorLocations(true).values().stream() - .distinct().count() == 2, 120_000); - assertEquals(getExpectedGroups(), getServerContext().getCoordinatorLocations(true).keySet()); + Wait.waitFor( + () -> getServerContext().getCoordinatorLocations(true).sortedUniqueHost().size() == 2, + 120_000); + assertEquals(getExpectedGroups(), + getServerContext().getCoordinatorLocations(true).locations().keySet()); compact(client, table1, 5, GROUP1, true); verify(client, table1, 30); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java index fe1783b8412..2df58c89902 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java @@ -45,7 +45,7 @@ public class CompactionConfigChangeIT extends AccumuloClusterHarness { @Override public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { - cfg.getClusterServerConfiguration().addCompactorResourceGroup("little", 1); + cfg.getClusterServerConfiguration().addCompactorResourceGroup("little", 2); cfg.getClusterServerConfiguration().addCompactorResourceGroup("big", 1); cfg.setProperty(Property.COMPACTION_SERVICE_PREFIX.getKey() + "cs1.planner", From e7dfa1cf4f01dc1b24cb594fc21e73f532049299 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Thu, 2 Apr 2026 19:42:46 +0000 Subject: [PATCH 42/65] fix bug found by ExitCodesIT Manager process could die if FateManager had a connection error --- .../accumulo/manager/fate/FateManager.java | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java index f990fc0247f..7833c9ca9ee 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateManager.java @@ -78,86 +78,89 @@ public FateManager(ServerContext context) { private final AtomicBoolean stop = new AtomicBoolean(false); - private void manageAssistants() throws TException, InterruptedException { + private void manageAssistants() { log.debug("Started Fate Manager"); long stableCount = 0; long unstableCount = 0; outer: while (!stop.get()) { - - long sleepTime = Math.min(stableCount * 100, 5_000); - Thread.sleep(sleepTime); - - // This map will contain all current workers even if their partitions are empty - Map currentPartitions; try { - currentPartitions = getCurrentAssignments(context); - } catch (TException e) { - log.warn("Failed to get current partitions ", e); - continue; - } - Map> currentAssignments = new HashMap<>(); - currentPartitions.forEach((k, v) -> currentAssignments.put(k, v.partitions())); - Map> desiredParititions = - getDesiredPartitions(currentAssignments.size()); + long sleepTime = Math.min(stableCount * 100, 5_000); + Thread.sleep(sleepTime); - Map> desired = - computeDesiredAssignments(currentAssignments, desiredParititions); - - if (desired.equals(currentAssignments)) { - if (stableCount == 0) { - FateLocations.storeLocations(context, currentAssignments); + // This map will contain all current workers even if their partitions are empty + Map currentPartitions; + try { + currentPartitions = getCurrentAssignments(context); + } catch (TException e) { + log.warn("Failed to get current partitions ", e); + continue; } - stableCount++; - unstableCount = 0; - continue; - } else { - if (unstableCount == 0) { - FateLocations.storeLocations(context, Map.of()); + Map> currentAssignments = new HashMap<>(); + currentPartitions.forEach((k, v) -> currentAssignments.put(k, v.partitions())); + Map> desiredParititions = + getDesiredPartitions(currentAssignments.size()); + + Map> desired = + computeDesiredAssignments(currentAssignments, desiredParititions); + + if (desired.equals(currentAssignments)) { + if (stableCount == 0) { + FateLocations.storeLocations(context, currentAssignments); + } + stableCount++; + unstableCount = 0; + continue; + } else { + if (unstableCount == 0) { + FateLocations.storeLocations(context, Map.of()); + } + stableCount = 0; + unstableCount++; } - stableCount = 0; - unstableCount++; - } - // are there any workers with extra partitions? If so need to unload those first. - int unloads = 0; - for (Map.Entry> entry : desired.entrySet()) { - HostAndPort worker = entry.getKey(); - Set partitions = entry.getValue(); - var curr = currentAssignments.getOrDefault(worker, Set.of()); - if (!Sets.difference(curr, partitions).isEmpty()) { - // This worker has extra partitions that are not desired - var intersection = Sets.intersection(curr, partitions); - if (!setPartitions(worker, currentPartitions.get(worker).updateId(), intersection)) { - log.debug("Failed to set partitions for {} to {}", worker, intersection); - // could not set, so start completely over - continue outer; - } else { - log.debug("Set partitions for {} to {} from {}", worker, intersection, curr); - unloads++; + // are there any workers with extra partitions? If so need to unload those first. + int unloads = 0; + for (Map.Entry> entry : desired.entrySet()) { + HostAndPort worker = entry.getKey(); + Set partitions = entry.getValue(); + var curr = currentAssignments.getOrDefault(worker, Set.of()); + if (!Sets.difference(curr, partitions).isEmpty()) { + // This worker has extra partitions that are not desired + var intersection = Sets.intersection(curr, partitions); + if (!setPartitions(worker, currentPartitions.get(worker).updateId(), intersection)) { + log.debug("Failed to set partitions for {} to {}", worker, intersection); + // could not set, so start completely over + continue outer; + } else { + log.debug("Set partitions for {} to {} from {}", worker, intersection, curr); + unloads++; + } } } - } - if (unloads > 0) { - // some tablets were unloaded, so start over and get new update ids and the current - // partitions - continue outer; - } + if (unloads > 0) { + // some tablets were unloaded, so start over and get new update ids and the current + // partitions + continue outer; + } - // Load all partitions on all workers.. - for (Map.Entry> entry : desired.entrySet()) { - HostAndPort worker = entry.getKey(); - Set partitions = entry.getValue(); - var curr = currentAssignments.getOrDefault(worker, Set.of()); - if (!curr.equals(partitions)) { - if (!setPartitions(worker, currentPartitions.get(worker).updateId(), partitions)) { - log.debug("Failed to set partitions for {} to {}", worker, partitions); - // could not set, so start completely over - continue outer; - } else { - log.debug("Set partitions for {} to {} from {}", worker, partitions, curr); + // Load all partitions on all workers.. + for (Map.Entry> entry : desired.entrySet()) { + HostAndPort worker = entry.getKey(); + Set partitions = entry.getValue(); + var curr = currentAssignments.getOrDefault(worker, Set.of()); + if (!curr.equals(partitions)) { + if (!setPartitions(worker, currentPartitions.get(worker).updateId(), partitions)) { + log.debug("Failed to set partitions for {} to {}", worker, partitions); + // could not set, so start completely over + continue outer; + } else { + log.debug("Set partitions for {} to {} from {}", worker, partitions, curr); + } } } + } catch (Exception e) { + log.warn("Failed to assign fate partitions to other managers, will try again later", e); } } } @@ -168,13 +171,7 @@ public synchronized void start() { Preconditions.checkState(assignmentThread == null); Preconditions.checkState(!stop.get()); - assignmentThread = Threads.createCriticalThread("Fate Manager", () -> { - try { - manageAssistants(); - } catch (Exception e) { - throw new IllegalStateException(e); - } - }); + assignmentThread = Threads.createCriticalThread("Fate Manager", this::manageAssistants); assignmentThread.start(); } @@ -265,6 +262,9 @@ private boolean setPartitions(HostAndPort address, long updateId, Set Date: Fri, 3 Apr 2026 17:28:46 +0000 Subject: [PATCH 43/65] remove wrong TODOs --- .../manager/compaction/coordinator/CompactionCoordinator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 3e3dfd29a19..6d2445286da 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -604,7 +604,6 @@ private static class UpdateId { public Set getResourceGroups(TInfo tinfo, TCredentials credentials, long updateId) throws ThriftSecurityException, TException { if (!security.canPerformSystemActions(credentials)) { - // TODO does not seem like the correct exception, also this code snippet was copied. throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } @@ -621,7 +620,6 @@ public Set getResourceGroups(TInfo tinfo, TCredentials credentials, long public void setResourceGroups(TInfo tinfo, TCredentials credentials, long updateId, Set groups) throws ThriftSecurityException, TException { if (!security.canPerformSystemActions(credentials)) { - // TODO does not seem like the correct exception, also this code snippet was copied. throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } From 363252abac15e58e80b6fda757508354258ceec2 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 3 Apr 2026 17:53:16 +0000 Subject: [PATCH 44/65] do TODOs --- .../compaction/CoordinatorLocationsFactory.java | 1 - .../org/apache/accumulo/manager/Manager.java | 2 ++ .../manager/compaction/CompactionJobClient.java | 13 ++++++++----- .../coordinator/CoordinatorManager.java | 17 ++++++++++++++--- .../test/ComprehensiveMultiManagerIT.java | 2 -- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocationsFactory.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocationsFactory.java index 268034e7490..2638cadd8e8 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocationsFactory.java +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/CoordinatorLocationsFactory.java @@ -56,7 +56,6 @@ public record CoordinatorLocations(Map locations, public synchronized CoordinatorLocations getLocations(boolean useCache) { var zooCache = context.getZooCache(); - // TODO how frequently does updateCount change? TODO could have an update counter per path if (lastLocations == null || lastUpdateCount != zooCache.getUpdateCount() || !useCache) { lastUpdateCount = zooCache.getUpdateCount(); if (!useCache) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 711616cbc13..6486451dc29 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -1236,6 +1236,8 @@ boolean canSuspendTablets() { UtilWaitThread.sleep(100); } + coordinatorManager.stop(); + log.debug("Shutting down fate."); fateManager.stop(FateInstanceType.USER, Duration.ZERO); fateManager.stop(FateInstanceType.META, Duration.ZERO); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java index 22189a57a4f..4451bc68c9b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java @@ -74,18 +74,21 @@ public CompactionJobClient(ServerContext context, Ample.DataLevel dataLevel, boo var uniqueHosts = new HashSet<>(coordinatorLocations.values()); for (var hostPort : uniqueHosts) { + CompactionCoordinatorService.Client client = null; try { - var client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, hostPort, context); + client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, hostPort, context); if (fullScan) { client.beginFullJobScan(TraceUtil.traceInfo(), context.rpcCreds(), dataLevel.name()); } + } catch (TException | RuntimeException e) { + log.warn("Failed to connect to coordinator {}", hostPort, e); + ThriftUtil.returnClient(client, context); + client = null; + } + if (client != null) { coordinatorConnections.put(hostPort, new CoordinatorConnection(client, new ArrayList<>(BUFFER_SIZE), hostPort)); - } catch (TException e) { - // TODO only log - // TODO need to return client or add it - throw new RuntimeException(e); } } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index f190d595ee9..eb6117b842d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -29,6 +29,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import org.apache.accumulo.core.Constants; @@ -67,6 +68,8 @@ public class CoordinatorManager { private final ServerContext context; + private final AtomicBoolean stopped = new AtomicBoolean(false); + public CoordinatorManager(ServerContext context, Function> fateClients) { this.context = context; @@ -82,7 +85,7 @@ private void managerCoordinators() { final long initialSleepTime = 10; final long maxSleepTime = 5000; long sleepTime = initialSleepTime; - while (true) { + while (!stopped.get()) { try { long updateId = RANDOM.get().nextLong(); var current = getCurrentAssignments(updateId); @@ -177,6 +180,7 @@ private void setResourceGroups(HostAndPort hp, Set groups, long public synchronized void start() { Preconditions.checkState(assignmentThread == null); + Preconditions.checkState(!stopped.get()); startCompactorZKCleaner(context.getScheduledExecutor()); @@ -192,6 +196,15 @@ public synchronized void start() { assignmentThread.start(); } + public synchronized void stop() { + stopped.set(true); + try { + assignmentThread.join(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + protected void startCompactorZKCleaner(ScheduledThreadPoolExecutor schedExecutor) { ScheduledFuture future = schedExecutor .scheduleWithFixedDelay(this::cleanUpEmptyCompactorPathInZK, 0, 5, TimeUnit.MINUTES); @@ -240,6 +253,4 @@ private void cleanUpEmptyCompactorPathInZK() { throw new IllegalStateException(e); } } - - // TODO stop() } diff --git a/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java b/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java index 79d362f9cd7..fc0d87369ef 100644 --- a/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ComprehensiveMultiManagerIT.java @@ -56,8 +56,6 @@ public static void setup() throws Exception { client.securityOperations().changeUserAuthorizations("root", AUTHORIZATIONS); } - // TODO could add a compaction RG to comprehensive IT - // TODO could wait for coordinators? // Wait for 3 managers to have a fate partition assigned to them var srvCtx = getCluster().getServerContext(); Wait.waitFor(() -> { From 5d4f0f5c2e0fd2c1f4fa8224f0bdb56f25ad1460 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 3 Apr 2026 18:02:57 +0000 Subject: [PATCH 45/65] do TODOs --- core/src/main/thrift/tabletserver.thrift | 4 ++-- .../coordinator/CompactionCoordinator.java | 10 +--------- .../queue/ResolvedCompactionJob.java | 20 ++++++++++--------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/core/src/main/thrift/tabletserver.thrift b/core/src/main/thrift/tabletserver.thrift index f66ffefee33..5ccc092aebf 100644 --- a/core/src/main/thrift/tabletserver.thrift +++ b/core/src/main/thrift/tabletserver.thrift @@ -103,7 +103,7 @@ struct InputFile { 3:i64 entries 4:i64 timestamp } -//TODO this should move outside of the tablet server file, tabletserver no longer has anything to with compactions + struct TExternalCompactionJob { 1:string externalCompactionId 2:data.TKeyExtent extent @@ -232,7 +232,7 @@ service TabletServerClientService { ) throws ( 1:NoSuchScanIDException nssi ) - + list refreshTablets( 1:client.TInfo tinfo 2:security.TCredentials credentials diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 6d2445286da..ade372e23a6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -19,7 +19,6 @@ package org.apache.accumulo.manager.compaction.coordinator; import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP; @@ -83,12 +82,10 @@ import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.CompactableFileImpl; import org.apache.accumulo.core.metadata.ReferencedTabletFile; -import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.Ample.RejectionHandler; import org.apache.accumulo.core.metadata.schema.CompactionMetadata; -import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; @@ -96,7 +93,6 @@ import org.apache.accumulo.core.securityImpl.thrift.TCredentials; import org.apache.accumulo.core.spi.compaction.CompactionJob; import org.apache.accumulo.core.spi.compaction.CompactionKind; -import org.apache.accumulo.core.tabletserver.thrift.InputFile; import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig; import org.apache.accumulo.core.tabletserver.thrift.TCompactionKind; import org.apache.accumulo.core.tabletserver.thrift.TCompactionStats; @@ -525,11 +521,7 @@ protected TExternalCompactionJob createThriftJob(String externalCompactionId, IteratorConfig iteratorSettings = SystemIteratorUtil .toIteratorConfig(compactionConfig.map(CompactionConfig::getIterators).orElse(List.of())); - var files = rcJob.getJobFilesMap().entrySet().stream().map(e -> { - StoredTabletFile file = e.getKey(); - DataFileValue dfv = e.getValue(); - return new InputFile(file.getMetadata(), dfv.getSize(), dfv.getNumEntries(), dfv.getTime()); - }).collect(toList()); + var files = rcJob.getThriftFiles(); // The fateId here corresponds to the Fate transaction that is driving a user initiated // compaction. A system initiated compaction has no Fate transaction driving it so its ok to set diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java index 85b37f34dfb..85148bf32d0 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -181,6 +182,14 @@ public Map getJobFilesMap() { return jobFiles; } + public List getThriftFiles() { + return jobFiles.entrySet().stream().map(e -> { + StoredTabletFile file = e.getKey(); + DataFileValue dfv = e.getValue(); + return new InputFile(file.getMetadata(), dfv.getSize(), dfv.getNumEntries(), dfv.getTime()); + }).toList(); + } + public Set getCompactionFiles() { return jobFiles.entrySet().stream().map(e -> new CompactableFileImpl(e.getKey(), e.getValue())) .collect(Collectors.toSet()); @@ -233,16 +242,9 @@ public int hashCode() { } public TResolvedCompactionJob toThrift() { - // TODO consolidate code to do this conversion - var files = jobFiles.entrySet().stream().map(e -> { - StoredTabletFile file = e.getKey(); - DataFileValue dfv = e.getValue(); - return new InputFile(file.getMetadata(), dfv.getSize(), dfv.getNumEntries(), dfv.getTime()); - }).toList(); - return new TResolvedCompactionJob(selectedFateId == null ? null : selectedFateId.canonical(), - files, kind.name(), compactingAll, extent.toThrift(), priority, group.canonical(), - tabletDir, overlapsSelectedFiles); + getThriftFiles(), kind.name(), compactingAll, extent.toThrift(), priority, + group.canonical(), tabletDir, overlapsSelectedFiles); } public static ResolvedCompactionJob fromThrift(TResolvedCompactionJob trj) { From 7c45850d994c99ac2b964646b8640616e1db6f31 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 3 Apr 2026 19:04:08 +0000 Subject: [PATCH 46/65] Adds missig opentelem library to tarball --- assemble/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/assemble/pom.xml b/assemble/pom.xml index 4db8052fd14..b006e352d6b 100644 --- a/assemble/pom.xml +++ b/assemble/pom.xml @@ -157,6 +157,11 @@ opentelemetry-api true + + io.opentelemetry + opentelemetry-common + true + io.opentelemetry opentelemetry-context From b84f304698fb5529485baa97db72e85c1e89214e Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Mon, 6 Apr 2026 21:24:23 +0000 Subject: [PATCH 47/65] fixes hang in SplitIT due to dead compaction A test got stuck waiting for a table to go offline because of a dead compaction. Made the dead compaction detector run more frequently to handle this. Left some logging that was added to track this down. --- .../accumulo/core/clientImpl/TableOperationsImpl.java | 6 +++++- .../java/org/apache/accumulo/test/functional/SplitIT.java | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java index 5680fa4f21d..b19a82c07f9 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java @@ -1398,13 +1398,17 @@ private void waitForTableStateTransition(TableId tableId, TableState expectedSta if ((expectedState == TableState.ONLINE && (availability == TabletAvailability.HOSTED - || (availability == TabletAvailability.ONDEMAND) && tablet.getHostingRequested()) + || availability == TabletAvailability.ONDEMAND && tablet.getHostingRequested()) && (loc == null || loc.getType() == LocationType.FUTURE)) || (expectedState == TableState.OFFLINE && (loc != null || opid != null || !externalCompactions.isEmpty()))) { if (continueRow == null) { continueRow = tablet.getExtent().toMetaRow(); } + log.trace( + "Waiting on tablet {} exectedState:{} availability:{} hostingRequested:{} loc:{} opid:{} compactions:{}", + tablet.getExtent(), expectedState, availability, tablet.getHostingRequested(), loc, + opid, externalCompactions.keySet()); waitFor++; lastRow = tablet.getExtent().toMetaRow(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java index b22d10ed29b..2acdc478cc5 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java @@ -105,6 +105,9 @@ protected Duration defaultTimeout() { public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { cfg.setProperty(Property.TSERV_MAXMEM, "5K"); cfg.setMemory(ServerType.TABLET_SERVER, 384, MemoryUnit.MEGABYTE); + // Splitting a tablet w/ a compaction can result in a dead compaction. Run the detector more + // frequently to clean them up as they could cause tests to hang. + cfg.setProperty(Property.COMPACTION_COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL, "5s"); } private String tservMaxMem; From f4b88148836efc33b068bd5ed1f33c40923caaa3 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Mon, 6 Apr 2026 22:56:26 +0000 Subject: [PATCH 48/65] fixes Fate burning CPU when no ranges assigned --- .../main/java/org/apache/accumulo/core/fate/FateExecutor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/org/apache/accumulo/core/fate/FateExecutor.java b/core/src/main/java/org/apache/accumulo/core/fate/FateExecutor.java index 2457095209f..de4bf9c9953 100644 --- a/core/src/main/java/org/apache/accumulo/core/fate/FateExecutor.java +++ b/core/src/main/java/org/apache/accumulo/core/fate/FateExecutor.java @@ -318,6 +318,10 @@ public void run() { while (fate.getKeepRunning().get() && !isShutdown()) { try { var localPartitions = partitions.get(); + if (localPartitions.isEmpty()) { + Thread.sleep(250); + continue; + } // if the set of partitions changes, we should stop looking for work w/ the old set of // partitions BooleanSupplier keepRunning = From d4040f6cdbdf3c02263140eff4e1822274a75c19 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 7 Apr 2026 00:28:00 +0000 Subject: [PATCH 49/65] spread reservation work across all coordinators --- .../thrift/CompactionCoordinatorService.java | 1663 +++++++++++++++-- .../thrift/TDequeuedCompactionJob.java | 511 +++++ .../main/thrift/compaction-coordinator.thrift | 22 +- .../apache/accumulo/compactor/Compactor.java | 29 +- .../coordinator/CompactionCoordinator.java | 72 +- .../compaction/CompactionCoordinatorTest.java | 21 +- 6 files changed, 2074 insertions(+), 244 deletions(-) create mode 100644 core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TDequeuedCompactionJob.java diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index 6aa7dc826ff..5c9054c168a 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -31,7 +31,9 @@ public interface Iface { public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactor) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; - public TNextCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; + public TDequeuedCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; + + public TNextCompactionJob reserveCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, TResolvedCompactionJob job, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactor) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException; @@ -51,7 +53,9 @@ public interface AsyncIface { public void compactionCompleted(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, org.apache.accumulo.core.tabletserver.thrift.TCompactionStats stats, java.lang.String groupName, java.lang.String compactor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void reserveCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, TResolvedCompactionJob job, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; @@ -120,24 +124,22 @@ public void recv_compactionCompleted() throws org.apache.accumulo.core.clientImp } @Override - public TNextCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + public TDequeuedCompactionJob getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { - send_getCompactionJob(tinfo, credentials, groupName, compactor, externalCompactionId); + send_getCompactionJob(tinfo, credentials, groupName); return recv_getCompactionJob(); } - public void send_getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.thrift.TException + public void send_getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName) throws org.apache.thrift.TException { getCompactionJob_args args = new getCompactionJob_args(); args.setTinfo(tinfo); args.setCredentials(credentials); args.setGroupName(groupName); - args.setCompactor(compactor); - args.setExternalCompactionId(externalCompactionId); sendBase("getCompactionJob", args); } - public TNextCompactionJob recv_getCompactionJob() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + public TDequeuedCompactionJob recv_getCompactionJob() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { getCompactionJob_result result = new getCompactionJob_result(); receiveBase(result, "getCompactionJob"); @@ -150,6 +152,37 @@ public TNextCompactionJob recv_getCompactionJob() throws org.apache.accumulo.cor throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getCompactionJob failed: unknown result"); } + @Override + public TNextCompactionJob reserveCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, TResolvedCompactionJob job, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + { + send_reserveCompactionJob(tinfo, credentials, job, compactor, externalCompactionId); + return recv_reserveCompactionJob(); + } + + public void send_reserveCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, TResolvedCompactionJob job, java.lang.String compactor, java.lang.String externalCompactionId) throws org.apache.thrift.TException + { + reserveCompactionJob_args args = new reserveCompactionJob_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setJob(job); + args.setCompactor(compactor); + args.setExternalCompactionId(externalCompactionId); + sendBase("reserveCompactionJob", args); + } + + public TNextCompactionJob recv_reserveCompactionJob() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException + { + reserveCompactionJob_result result = new reserveCompactionJob_result(); + receiveBase(result, "reserveCompactionJob"); + if (result.isSetSuccess()) { + return result.success; + } + if (result.sec != null) { + throw result.sec; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "reserveCompactionJob failed: unknown result"); + } + @Override public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactor) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { @@ -378,26 +411,22 @@ public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.Thrift } @Override - public void getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + public void getCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); - getCompactionJob_call method_call = new getCompactionJob_call(tinfo, credentials, groupName, compactor, externalCompactionId, resultHandler, this, ___protocolFactory, ___transport); + getCompactionJob_call method_call = new getCompactionJob_call(tinfo, credentials, groupName, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; ___manager.call(method_call); } - public static class getCompactionJob_call extends org.apache.thrift.async.TAsyncMethodCall { + public static class getCompactionJob_call extends org.apache.thrift.async.TAsyncMethodCall { private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; private java.lang.String groupName; - private java.lang.String compactor; - private java.lang.String externalCompactionId; - public getCompactionJob_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + public getCompactionJob_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String groupName, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { super(client, protocolFactory, transport, resultHandler, false); this.tinfo = tinfo; this.credentials = credentials; this.groupName = groupName; - this.compactor = compactor; - this.externalCompactionId = externalCompactionId; } @Override @@ -407,6 +436,51 @@ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apa args.setTinfo(tinfo); args.setCredentials(credentials); args.setGroupName(groupName); + args.write(prot); + prot.writeMessageEnd(); + } + + @Override + public TDequeuedCompactionJob getResult() throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new java.lang.IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_getCompactionJob(); + } + } + + @Override + public void reserveCompactionJob(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, TResolvedCompactionJob job, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + reserveCompactionJob_call method_call = new reserveCompactionJob_call(tinfo, credentials, job, compactor, externalCompactionId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class reserveCompactionJob_call extends org.apache.thrift.async.TAsyncMethodCall { + private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; + private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; + private TResolvedCompactionJob job; + private java.lang.String compactor; + private java.lang.String externalCompactionId; + public reserveCompactionJob_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, TResolvedCompactionJob job, java.lang.String compactor, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.tinfo = tinfo; + this.credentials = credentials; + this.job = job; + this.compactor = compactor; + this.externalCompactionId = externalCompactionId; + } + + @Override + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("reserveCompactionJob", org.apache.thrift.protocol.TMessageType.CALL, 0)); + reserveCompactionJob_args args = new reserveCompactionJob_args(); + args.setTinfo(tinfo); + args.setCredentials(credentials); + args.setJob(job); args.setCompactor(compactor); args.setExternalCompactionId(externalCompactionId); args.write(prot); @@ -420,7 +494,7 @@ public TNextCompactionJob getResult() throws org.apache.accumulo.core.clientImpl } org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); - return (new Client(prot)).recv_getCompactionJob(); + return (new Client(prot)).recv_reserveCompactionJob(); } } @@ -707,6 +781,7 @@ protected Processor(I iface, java.util.Map java.util.Map> getProcessMap(java.util.Map> processMap) { processMap.put("compactionCompleted", new compactionCompleted()); processMap.put("getCompactionJob", new getCompactionJob()); + processMap.put("reserveCompactionJob", new reserveCompactionJob()); processMap.put("compactionFailed", new compactionFailed()); processMap.put("beginFullJobScan", new beginFullJobScan()); processMap.put("addJobs", new addJobs()); @@ -772,7 +847,39 @@ protected boolean rethrowUnhandledExceptions() { public getCompactionJob_result getResult(I iface, getCompactionJob_args args) throws org.apache.thrift.TException { getCompactionJob_result result = new getCompactionJob_result(); try { - result.success = iface.getCompactionJob(args.tinfo, args.credentials, args.groupName, args.compactor, args.externalCompactionId); + result.success = iface.getCompactionJob(args.tinfo, args.credentials, args.groupName); + } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + result.sec = sec; + } + return result; + } + } + + public static class reserveCompactionJob extends org.apache.thrift.ProcessFunction { + public reserveCompactionJob() { + super("reserveCompactionJob"); + } + + @Override + public reserveCompactionJob_args getEmptyArgsInstance() { + return new reserveCompactionJob_args(); + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + protected boolean rethrowUnhandledExceptions() { + return false; + } + + @Override + public reserveCompactionJob_result getResult(I iface, reserveCompactionJob_args args) throws org.apache.thrift.TException { + reserveCompactionJob_result result = new reserveCompactionJob_result(); + try { + result.success = iface.reserveCompactionJob(args.tinfo, args.credentials, args.job, args.compactor, args.externalCompactionId); } catch (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { result.sec = sec; } @@ -982,6 +1089,7 @@ protected AsyncProcessor(I iface, java.util.Map java.util.Map> getProcessMap(java.util.Map> processMap) { processMap.put("compactionCompleted", new compactionCompleted()); processMap.put("getCompactionJob", new getCompactionJob()); + processMap.put("reserveCompactionJob", new reserveCompactionJob()); processMap.put("compactionFailed", new compactionFailed()); processMap.put("beginFullJobScan", new beginFullJobScan()); processMap.put("addJobs", new addJobs()); @@ -1061,7 +1169,7 @@ public void start(I iface, compactionCompleted_args args, org.apache.thrift.asyn } } - public static class getCompactionJob extends org.apache.thrift.AsyncProcessFunction { + public static class getCompactionJob extends org.apache.thrift.AsyncProcessFunction { public getCompactionJob() { super("getCompactionJob"); } @@ -1071,13 +1179,84 @@ public getCompactionJob_args getEmptyArgsInstance() { return new getCompactionJob_args(); } + @Override + public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new org.apache.thrift.async.AsyncMethodCallback() { + @Override + public void onComplete(TDequeuedCompactionJob o) { + getCompactionJob_result result = new getCompactionJob_result(); + result.success = o; + try { + fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + } catch (org.apache.thrift.transport.TTransportException e) { + _LOGGER.error("TTransportException writing to internal frame buffer", e); + fb.close(); + } catch (java.lang.Exception e) { + _LOGGER.error("Exception writing to internal frame buffer", e); + onError(e); + } + } + @Override + public void onError(java.lang.Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TSerializable msg; + getCompactionJob_result result = new getCompactionJob_result(); + if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { + result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; + result.setSecIsSet(true); + msg = result; + } else if (e instanceof org.apache.thrift.transport.TTransportException) { + _LOGGER.error("TTransportException inside handler", e); + fb.close(); + return; + } else if (e instanceof org.apache.thrift.TApplicationException) { + _LOGGER.error("TApplicationException inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TApplicationException)e; + } else { + _LOGGER.error("Exception inside handler", e); + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + } catch (java.lang.Exception ex) { + _LOGGER.error("Exception writing to internal frame buffer", ex); + fb.close(); + } + } + }; + } + + @Override + protected boolean isOneway() { + return false; + } + + @Override + public void start(I iface, getCompactionJob_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + iface.getCompactionJob(args.tinfo, args.credentials, args.groupName,resultHandler); + } + } + + public static class reserveCompactionJob extends org.apache.thrift.AsyncProcessFunction { + public reserveCompactionJob() { + super("reserveCompactionJob"); + } + + @Override + public reserveCompactionJob_args getEmptyArgsInstance() { + return new reserveCompactionJob_args(); + } + @Override public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { final org.apache.thrift.AsyncProcessFunction fcall = this; return new org.apache.thrift.async.AsyncMethodCallback() { @Override public void onComplete(TNextCompactionJob o) { - getCompactionJob_result result = new getCompactionJob_result(); + reserveCompactionJob_result result = new reserveCompactionJob_result(); result.success = o; try { fcall.sendResponse(fb, result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); @@ -1093,7 +1272,7 @@ public void onComplete(TNextCompactionJob o) { public void onError(java.lang.Exception e) { byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; org.apache.thrift.TSerializable msg; - getCompactionJob_result result = new getCompactionJob_result(); + reserveCompactionJob_result result = new reserveCompactionJob_result(); if (e instanceof org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) { result.sec = (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) e; result.setSecIsSet(true); @@ -1127,8 +1306,8 @@ protected boolean isOneway() { } @Override - public void start(I iface, getCompactionJob_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.getCompactionJob(args.tinfo, args.credentials, args.groupName, args.compactor, args.externalCompactionId,resultHandler); + public void start(I iface, reserveCompactionJob_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + iface.reserveCompactionJob(args.tinfo, args.credentials, args.job, args.compactor, args.externalCompactionId,resultHandler); } } @@ -2941,8 +3120,6 @@ public static class getCompactionJob_args implements org.apache.thrift.TBase byName = new java.util.HashMap(); @@ -2981,10 +3154,6 @@ public static _Fields findByThriftId(int fieldId) { return CREDENTIALS; case 3: // GROUP_NAME return GROUP_NAME; - case 4: // COMPACTOR - return COMPACTOR; - case 5: // EXTERNAL_COMPACTION_ID - return EXTERNAL_COMPACTION_ID; default: return null; } @@ -3037,10 +3206,6 @@ public java.lang.String getFieldName() { new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); tmpMap.put(_Fields.GROUP_NAME, new org.apache.thrift.meta_data.FieldMetaData("groupName", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.COMPACTOR, new org.apache.thrift.meta_data.FieldMetaData("compactor", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("externalCompactionId", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getCompactionJob_args.class, metaDataMap); } @@ -3051,16 +3216,12 @@ public getCompactionJob_args() { public getCompactionJob_args( org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, - java.lang.String groupName, - java.lang.String compactor, - java.lang.String externalCompactionId) + java.lang.String groupName) { this(); this.tinfo = tinfo; this.credentials = credentials; this.groupName = groupName; - this.compactor = compactor; - this.externalCompactionId = externalCompactionId; } /** @@ -3076,12 +3237,6 @@ public getCompactionJob_args(getCompactionJob_args other) { if (other.isSetGroupName()) { this.groupName = other.groupName; } - if (other.isSetCompactor()) { - this.compactor = other.compactor; - } - if (other.isSetExternalCompactionId()) { - this.externalCompactionId = other.externalCompactionId; - } } @Override @@ -3094,8 +3249,6 @@ public void clear() { this.tinfo = null; this.credentials = null; this.groupName = null; - this.compactor = null; - this.externalCompactionId = null; } @org.apache.thrift.annotation.Nullable @@ -3173,56 +3326,6 @@ public void setGroupNameIsSet(boolean value) { } } - @org.apache.thrift.annotation.Nullable - public java.lang.String getCompactor() { - return this.compactor; - } - - public getCompactionJob_args setCompactor(@org.apache.thrift.annotation.Nullable java.lang.String compactor) { - this.compactor = compactor; - return this; - } - - public void unsetCompactor() { - this.compactor = null; - } - - /** Returns true if field compactor is set (has been assigned a value) and false otherwise */ - public boolean isSetCompactor() { - return this.compactor != null; - } - - public void setCompactorIsSet(boolean value) { - if (!value) { - this.compactor = null; - } - } - - @org.apache.thrift.annotation.Nullable - public java.lang.String getExternalCompactionId() { - return this.externalCompactionId; - } - - public getCompactionJob_args setExternalCompactionId(@org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId) { - this.externalCompactionId = externalCompactionId; - return this; - } - - public void unsetExternalCompactionId() { - this.externalCompactionId = null; - } - - /** Returns true if field externalCompactionId is set (has been assigned a value) and false otherwise */ - public boolean isSetExternalCompactionId() { - return this.externalCompactionId != null; - } - - public void setExternalCompactionIdIsSet(boolean value) { - if (!value) { - this.externalCompactionId = null; - } - } - @Override public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { @@ -3250,22 +3353,6 @@ public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable } break; - case COMPACTOR: - if (value == null) { - unsetCompactor(); - } else { - setCompactor((java.lang.String)value); - } - break; - - case EXTERNAL_COMPACTION_ID: - if (value == null) { - unsetExternalCompactionId(); - } else { - setExternalCompactionId((java.lang.String)value); - } - break; - } } @@ -3282,12 +3369,6 @@ public java.lang.Object getFieldValue(_Fields field) { case GROUP_NAME: return getGroupName(); - case COMPACTOR: - return getCompactor(); - - case EXTERNAL_COMPACTION_ID: - return getExternalCompactionId(); - } throw new java.lang.IllegalStateException(); } @@ -3306,10 +3387,6 @@ public boolean isSet(_Fields field) { return isSetCredentials(); case GROUP_NAME: return isSetGroupName(); - case COMPACTOR: - return isSetCompactor(); - case EXTERNAL_COMPACTION_ID: - return isSetExternalCompactionId(); } throw new java.lang.IllegalStateException(); } @@ -3354,10 +3431,1201 @@ public boolean equals(getCompactionJob_args that) { return false; } - boolean this_present_compactor = true && this.isSetCompactor(); - boolean that_present_compactor = true && that.isSetCompactor(); - if (this_present_compactor || that_present_compactor) { - if (!(this_present_compactor && that_present_compactor)) + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); + if (isSetTinfo()) + hashCode = hashCode * 8191 + tinfo.hashCode(); + + hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); + if (isSetCredentials()) + hashCode = hashCode * 8191 + credentials.hashCode(); + + hashCode = hashCode * 8191 + ((isSetGroupName()) ? 131071 : 524287); + if (isSetGroupName()) + hashCode = hashCode * 8191 + groupName.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(getCompactionJob_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTinfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCredentials()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetGroupName(), other.isSetGroupName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetGroupName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groupName, other.groupName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("getCompactionJob_args("); + boolean first = true; + + sb.append("tinfo:"); + if (this.tinfo == null) { + sb.append("null"); + } else { + sb.append(this.tinfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("credentials:"); + if (this.credentials == null) { + sb.append("null"); + } else { + sb.append(this.credentials); + } + first = false; + if (!first) sb.append(", "); + sb.append("groupName:"); + if (this.groupName == null) { + sb.append("null"); + } else { + sb.append(this.groupName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (tinfo != null) { + tinfo.validate(); + } + if (credentials != null) { + credentials.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getCompactionJob_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public getCompactionJob_argsStandardScheme getScheme() { + return new getCompactionJob_argsStandardScheme(); + } + } + + private static class getCompactionJob_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // TINFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // CREDENTIALS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // GROUP_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.groupName = iprot.readString(); + struct.setGroupNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.tinfo != null) { + oprot.writeFieldBegin(TINFO_FIELD_DESC); + struct.tinfo.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.credentials != null) { + oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); + struct.credentials.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.groupName != null) { + oprot.writeFieldBegin(GROUP_NAME_FIELD_DESC); + oprot.writeString(struct.groupName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getCompactionJob_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public getCompactionJob_argsTupleScheme getScheme() { + return new getCompactionJob_argsTupleScheme(); + } + } + + private static class getCompactionJob_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetTinfo()) { + optionals.set(0); + } + if (struct.isSetCredentials()) { + optionals.set(1); + } + if (struct.isSetGroupName()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetTinfo()) { + struct.tinfo.write(oprot); + } + if (struct.isSetCredentials()) { + struct.credentials.write(oprot); + } + if (struct.isSetGroupName()) { + oprot.writeString(struct.groupName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); + struct.tinfo.read(iprot); + struct.setTinfoIsSet(true); + } + if (incoming.get(1)) { + struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); + struct.credentials.read(iprot); + struct.setCredentialsIsSet(true); + } + if (incoming.get(2)) { + struct.groupName = iprot.readString(); + struct.setGroupNameIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class getCompactionJob_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getCompactionJob_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getCompactionJob_resultStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getCompactionJob_resultTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable TDequeuedCompactionJob success; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"), + SEC((short)1, "sec"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + case 1: // SEC + return SEC; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TDequeuedCompactionJob.class))); + tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getCompactionJob_result.class, metaDataMap); + } + + public getCompactionJob_result() { + } + + public getCompactionJob_result( + TDequeuedCompactionJob success, + org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) + { + this(); + this.success = success; + this.sec = sec; + } + + /** + * Performs a deep copy on other. + */ + public getCompactionJob_result(getCompactionJob_result other) { + if (other.isSetSuccess()) { + this.success = new TDequeuedCompactionJob(other.success); + } + if (other.isSetSec()) { + this.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(other.sec); + } + } + + @Override + public getCompactionJob_result deepCopy() { + return new getCompactionJob_result(this); + } + + @Override + public void clear() { + this.success = null; + this.sec = null; + } + + @org.apache.thrift.annotation.Nullable + public TDequeuedCompactionJob getSuccess() { + return this.success; + } + + public getCompactionJob_result setSuccess(@org.apache.thrift.annotation.Nullable TDequeuedCompactionJob success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec() { + return this.sec; + } + + public getCompactionJob_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + this.sec = sec; + return this; + } + + public void unsetSec() { + this.sec = null; + } + + /** Returns true if field sec is set (has been assigned a value) and false otherwise */ + public boolean isSetSec() { + return this.sec != null; + } + + public void setSecIsSet(boolean value) { + if (!value) { + this.sec = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((TDequeuedCompactionJob)value); + } + break; + + case SEC: + if (value == null) { + unsetSec(); + } else { + setSec((org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + case SEC: + return getSec(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case SEC: + return isSetSec(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof getCompactionJob_result) + return this.equals((getCompactionJob_result)that); + return false; + } + + public boolean equals(getCompactionJob_result that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + boolean this_present_sec = true && this.isSetSec(); + boolean that_present_sec = true && that.isSetSec(); + if (this_present_sec || that_present_sec) { + if (!(this_present_sec && that_present_sec)) + return false; + if (!this.sec.equals(that.sec)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetSuccess()) ? 131071 : 524287); + if (isSetSuccess()) + hashCode = hashCode * 8191 + success.hashCode(); + + hashCode = hashCode * 8191 + ((isSetSec()) ? 131071 : 524287); + if (isSetSec()) + hashCode = hashCode * 8191 + sec.hashCode(); + + return hashCode; + } + + @Override + public int compareTo(getCompactionJob_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetSuccess(), other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetSec(), other.isSetSec()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSec()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, other.sec); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("getCompactionJob_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + if (!first) sb.append(", "); + sb.append("sec:"); + if (this.sec == null) { + sb.append("null"); + } else { + sb.append(this.sec); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (success != null) { + success.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class getCompactionJob_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public getCompactionJob_resultStandardScheme getScheme() { + return new getCompactionJob_resultStandardScheme(); + } + } + + private static class getCompactionJob_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new TDequeuedCompactionJob(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 1: // SEC + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + if (struct.sec != null) { + oprot.writeFieldBegin(SEC_FIELD_DESC); + struct.sec.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class getCompactionJob_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public getCompactionJob_resultTupleScheme getScheme() { + return new getCompactionJob_resultTupleScheme(); + } + } + + private static class getCompactionJob_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetSec()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + if (struct.isSetSec()) { + struct.sec.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = new TDequeuedCompactionJob(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.sec = new org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException(); + struct.sec.read(iprot); + struct.setSecIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + } + + @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) + public static class reserveCompactionJob_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("reserveCompactionJob_args"); + + private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); + private static final org.apache.thrift.protocol.TField JOB_FIELD_DESC = new org.apache.thrift.protocol.TField("job", org.apache.thrift.protocol.TType.STRUCT, (short)3); + private static final org.apache.thrift.protocol.TField COMPACTOR_FIELD_DESC = new org.apache.thrift.protocol.TField("compactor", org.apache.thrift.protocol.TType.STRING, (short)4); + private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)5); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new reserveCompactionJob_argsStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new reserveCompactionJob_argsTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required + public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required + public @org.apache.thrift.annotation.Nullable TResolvedCompactionJob job; // required + public @org.apache.thrift.annotation.Nullable java.lang.String compactor; // required + public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + TINFO((short)1, "tinfo"), + CREDENTIALS((short)2, "credentials"), + JOB((short)3, "job"), + COMPACTOR((short)4, "compactor"), + EXTERNAL_COMPACTION_ID((short)5, "externalCompactionId"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // TINFO + return TINFO; + case 2: // CREDENTIALS + return CREDENTIALS; + case 3: // JOB + return JOB; + case 4: // COMPACTOR + return COMPACTOR; + case 5: // EXTERNAL_COMPACTION_ID + return EXTERNAL_COMPACTION_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); + tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); + tmpMap.put(_Fields.JOB, new org.apache.thrift.meta_data.FieldMetaData("job", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResolvedCompactionJob.class))); + tmpMap.put(_Fields.COMPACTOR, new org.apache.thrift.meta_data.FieldMetaData("compactor", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("externalCompactionId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(reserveCompactionJob_args.class, metaDataMap); + } + + public reserveCompactionJob_args() { + } + + public reserveCompactionJob_args( + org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, + org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, + TResolvedCompactionJob job, + java.lang.String compactor, + java.lang.String externalCompactionId) + { + this(); + this.tinfo = tinfo; + this.credentials = credentials; + this.job = job; + this.compactor = compactor; + this.externalCompactionId = externalCompactionId; + } + + /** + * Performs a deep copy on other. + */ + public reserveCompactionJob_args(reserveCompactionJob_args other) { + if (other.isSetTinfo()) { + this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); + } + if (other.isSetCredentials()) { + this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); + } + if (other.isSetJob()) { + this.job = new TResolvedCompactionJob(other.job); + } + if (other.isSetCompactor()) { + this.compactor = other.compactor; + } + if (other.isSetExternalCompactionId()) { + this.externalCompactionId = other.externalCompactionId; + } + } + + @Override + public reserveCompactionJob_args deepCopy() { + return new reserveCompactionJob_args(this); + } + + @Override + public void clear() { + this.tinfo = null; + this.credentials = null; + this.job = null; + this.compactor = null; + this.externalCompactionId = null; + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { + return this.tinfo; + } + + public reserveCompactionJob_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { + this.tinfo = tinfo; + return this; + } + + public void unsetTinfo() { + this.tinfo = null; + } + + /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ + public boolean isSetTinfo() { + return this.tinfo != null; + } + + public void setTinfoIsSet(boolean value) { + if (!value) { + this.tinfo = null; + } + } + + @org.apache.thrift.annotation.Nullable + public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { + return this.credentials; + } + + public reserveCompactionJob_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { + this.credentials = credentials; + return this; + } + + public void unsetCredentials() { + this.credentials = null; + } + + /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ + public boolean isSetCredentials() { + return this.credentials != null; + } + + public void setCredentialsIsSet(boolean value) { + if (!value) { + this.credentials = null; + } + } + + @org.apache.thrift.annotation.Nullable + public TResolvedCompactionJob getJob() { + return this.job; + } + + public reserveCompactionJob_args setJob(@org.apache.thrift.annotation.Nullable TResolvedCompactionJob job) { + this.job = job; + return this; + } + + public void unsetJob() { + this.job = null; + } + + /** Returns true if field job is set (has been assigned a value) and false otherwise */ + public boolean isSetJob() { + return this.job != null; + } + + public void setJobIsSet(boolean value) { + if (!value) { + this.job = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getCompactor() { + return this.compactor; + } + + public reserveCompactionJob_args setCompactor(@org.apache.thrift.annotation.Nullable java.lang.String compactor) { + this.compactor = compactor; + return this; + } + + public void unsetCompactor() { + this.compactor = null; + } + + /** Returns true if field compactor is set (has been assigned a value) and false otherwise */ + public boolean isSetCompactor() { + return this.compactor != null; + } + + public void setCompactorIsSet(boolean value) { + if (!value) { + this.compactor = null; + } + } + + @org.apache.thrift.annotation.Nullable + public java.lang.String getExternalCompactionId() { + return this.externalCompactionId; + } + + public reserveCompactionJob_args setExternalCompactionId(@org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId) { + this.externalCompactionId = externalCompactionId; + return this; + } + + public void unsetExternalCompactionId() { + this.externalCompactionId = null; + } + + /** Returns true if field externalCompactionId is set (has been assigned a value) and false otherwise */ + public boolean isSetExternalCompactionId() { + return this.externalCompactionId != null; + } + + public void setExternalCompactionIdIsSet(boolean value) { + if (!value) { + this.externalCompactionId = null; + } + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case TINFO: + if (value == null) { + unsetTinfo(); + } else { + setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); + } + break; + + case CREDENTIALS: + if (value == null) { + unsetCredentials(); + } else { + setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); + } + break; + + case JOB: + if (value == null) { + unsetJob(); + } else { + setJob((TResolvedCompactionJob)value); + } + break; + + case COMPACTOR: + if (value == null) { + unsetCompactor(); + } else { + setCompactor((java.lang.String)value); + } + break; + + case EXTERNAL_COMPACTION_ID: + if (value == null) { + unsetExternalCompactionId(); + } else { + setExternalCompactionId((java.lang.String)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case TINFO: + return getTinfo(); + + case CREDENTIALS: + return getCredentials(); + + case JOB: + return getJob(); + + case COMPACTOR: + return getCompactor(); + + case EXTERNAL_COMPACTION_ID: + return getExternalCompactionId(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case TINFO: + return isSetTinfo(); + case CREDENTIALS: + return isSetCredentials(); + case JOB: + return isSetJob(); + case COMPACTOR: + return isSetCompactor(); + case EXTERNAL_COMPACTION_ID: + return isSetExternalCompactionId(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof reserveCompactionJob_args) + return this.equals((reserveCompactionJob_args)that); + return false; + } + + public boolean equals(reserveCompactionJob_args that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_tinfo = true && this.isSetTinfo(); + boolean that_present_tinfo = true && that.isSetTinfo(); + if (this_present_tinfo || that_present_tinfo) { + if (!(this_present_tinfo && that_present_tinfo)) + return false; + if (!this.tinfo.equals(that.tinfo)) + return false; + } + + boolean this_present_credentials = true && this.isSetCredentials(); + boolean that_present_credentials = true && that.isSetCredentials(); + if (this_present_credentials || that_present_credentials) { + if (!(this_present_credentials && that_present_credentials)) + return false; + if (!this.credentials.equals(that.credentials)) + return false; + } + + boolean this_present_job = true && this.isSetJob(); + boolean that_present_job = true && that.isSetJob(); + if (this_present_job || that_present_job) { + if (!(this_present_job && that_present_job)) + return false; + if (!this.job.equals(that.job)) + return false; + } + + boolean this_present_compactor = true && this.isSetCompactor(); + boolean that_present_compactor = true && that.isSetCompactor(); + if (this_present_compactor || that_present_compactor) { + if (!(this_present_compactor && that_present_compactor)) return false; if (!this.compactor.equals(that.compactor)) return false; @@ -3387,9 +4655,9 @@ public int hashCode() { if (isSetCredentials()) hashCode = hashCode * 8191 + credentials.hashCode(); - hashCode = hashCode * 8191 + ((isSetGroupName()) ? 131071 : 524287); - if (isSetGroupName()) - hashCode = hashCode * 8191 + groupName.hashCode(); + hashCode = hashCode * 8191 + ((isSetJob()) ? 131071 : 524287); + if (isSetJob()) + hashCode = hashCode * 8191 + job.hashCode(); hashCode = hashCode * 8191 + ((isSetCompactor()) ? 131071 : 524287); if (isSetCompactor()) @@ -3403,7 +4671,7 @@ public int hashCode() { } @Override - public int compareTo(getCompactionJob_args other) { + public int compareTo(reserveCompactionJob_args other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } @@ -3430,12 +4698,12 @@ public int compareTo(getCompactionJob_args other) { return lastComparison; } } - lastComparison = java.lang.Boolean.compare(isSetGroupName(), other.isSetGroupName()); + lastComparison = java.lang.Boolean.compare(isSetJob(), other.isSetJob()); if (lastComparison != 0) { return lastComparison; } - if (isSetGroupName()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.groupName, other.groupName); + if (isSetJob()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.job, other.job); if (lastComparison != 0) { return lastComparison; } @@ -3481,7 +4749,7 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("getCompactionJob_args("); + java.lang.StringBuilder sb = new java.lang.StringBuilder("reserveCompactionJob_args("); boolean first = true; sb.append("tinfo:"); @@ -3500,11 +4768,11 @@ public java.lang.String toString() { } first = false; if (!first) sb.append(", "); - sb.append("groupName:"); - if (this.groupName == null) { + sb.append("job:"); + if (this.job == null) { sb.append("null"); } else { - sb.append(this.groupName); + sb.append(this.job); } first = false; if (!first) sb.append(", "); @@ -3536,6 +4804,9 @@ public void validate() throws org.apache.thrift.TException { if (credentials != null) { credentials.validate(); } + if (job != null) { + job.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -3554,17 +4825,17 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class getCompactionJob_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + private static class reserveCompactionJob_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { @Override - public getCompactionJob_argsStandardScheme getScheme() { - return new getCompactionJob_argsStandardScheme(); + public reserveCompactionJob_argsStandardScheme getScheme() { + return new reserveCompactionJob_argsStandardScheme(); } } - private static class getCompactionJob_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { + private static class reserveCompactionJob_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, reserveCompactionJob_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -3592,10 +4863,11 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_ar org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 3: // GROUP_NAME - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.groupName = iprot.readString(); - struct.setGroupNameIsSet(true); + case 3: // JOB + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.job = new TResolvedCompactionJob(); + struct.job.read(iprot); + struct.setJobIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -3628,7 +4900,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_ar } @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, reserveCompactionJob_args struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); @@ -3642,9 +4914,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_a struct.credentials.write(oprot); oprot.writeFieldEnd(); } - if (struct.groupName != null) { - oprot.writeFieldBegin(GROUP_NAME_FIELD_DESC); - oprot.writeString(struct.groupName); + if (struct.job != null) { + oprot.writeFieldBegin(JOB_FIELD_DESC); + struct.job.write(oprot); oprot.writeFieldEnd(); } if (struct.compactor != null) { @@ -3663,17 +4935,17 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_a } - private static class getCompactionJob_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + private static class reserveCompactionJob_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { @Override - public getCompactionJob_argsTupleScheme getScheme() { - return new getCompactionJob_argsTupleScheme(); + public reserveCompactionJob_argsTupleScheme getScheme() { + return new reserveCompactionJob_argsTupleScheme(); } } - private static class getCompactionJob_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { + private static class reserveCompactionJob_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, reserveCompactionJob_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; java.util.BitSet optionals = new java.util.BitSet(); if (struct.isSetTinfo()) { @@ -3682,7 +4954,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_ar if (struct.isSetCredentials()) { optionals.set(1); } - if (struct.isSetGroupName()) { + if (struct.isSetJob()) { optionals.set(2); } if (struct.isSetCompactor()) { @@ -3698,8 +4970,8 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_ar if (struct.isSetCredentials()) { struct.credentials.write(oprot); } - if (struct.isSetGroupName()) { - oprot.writeString(struct.groupName); + if (struct.isSetJob()) { + struct.job.write(oprot); } if (struct.isSetCompactor()) { oprot.writeString(struct.compactor); @@ -3710,7 +4982,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_ar } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, reserveCompactionJob_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; java.util.BitSet incoming = iprot.readBitSet(5); if (incoming.get(0)) { @@ -3724,8 +4996,9 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_arg struct.setCredentialsIsSet(true); } if (incoming.get(2)) { - struct.groupName = iprot.readString(); - struct.setGroupNameIsSet(true); + struct.job = new TResolvedCompactionJob(); + struct.job.read(iprot); + struct.setJobIsSet(true); } if (incoming.get(3)) { struct.compactor = iprot.readString(); @@ -3744,14 +5017,14 @@ private static S scheme(org.apache. } @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class getCompactionJob_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getCompactionJob_result"); + public static class reserveCompactionJob_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("reserveCompactionJob_result"); private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new getCompactionJob_resultStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new getCompactionJob_resultTupleSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new reserveCompactionJob_resultStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new reserveCompactionJob_resultTupleSchemeFactory(); public @org.apache.thrift.annotation.Nullable TNextCompactionJob success; // required public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec; // required @@ -3830,13 +5103,13 @@ public java.lang.String getFieldName() { tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getCompactionJob_result.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(reserveCompactionJob_result.class, metaDataMap); } - public getCompactionJob_result() { + public reserveCompactionJob_result() { } - public getCompactionJob_result( + public reserveCompactionJob_result( TNextCompactionJob success, org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { @@ -3848,7 +5121,7 @@ public getCompactionJob_result( /** * Performs a deep copy on other. */ - public getCompactionJob_result(getCompactionJob_result other) { + public reserveCompactionJob_result(reserveCompactionJob_result other) { if (other.isSetSuccess()) { this.success = new TNextCompactionJob(other.success); } @@ -3858,8 +5131,8 @@ public getCompactionJob_result(getCompactionJob_result other) { } @Override - public getCompactionJob_result deepCopy() { - return new getCompactionJob_result(this); + public reserveCompactionJob_result deepCopy() { + return new reserveCompactionJob_result(this); } @Override @@ -3873,7 +5146,7 @@ public TNextCompactionJob getSuccess() { return this.success; } - public getCompactionJob_result setSuccess(@org.apache.thrift.annotation.Nullable TNextCompactionJob success) { + public reserveCompactionJob_result setSuccess(@org.apache.thrift.annotation.Nullable TNextCompactionJob success) { this.success = success; return this; } @@ -3898,7 +5171,7 @@ public org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException getSec return this.sec; } - public getCompactionJob_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { + public reserveCompactionJob_result setSec(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException sec) { this.sec = sec; return this; } @@ -3972,12 +5245,12 @@ public boolean isSet(_Fields field) { @Override public boolean equals(java.lang.Object that) { - if (that instanceof getCompactionJob_result) - return this.equals((getCompactionJob_result)that); + if (that instanceof reserveCompactionJob_result) + return this.equals((reserveCompactionJob_result)that); return false; } - public boolean equals(getCompactionJob_result that) { + public boolean equals(reserveCompactionJob_result that) { if (that == null) return false; if (this == that) @@ -4020,7 +5293,7 @@ public int hashCode() { } @Override - public int compareTo(getCompactionJob_result other) { + public int compareTo(reserveCompactionJob_result other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } @@ -4067,7 +5340,7 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("getCompactionJob_result("); + java.lang.StringBuilder sb = new java.lang.StringBuilder("reserveCompactionJob_result("); boolean first = true; sb.append("success:"); @@ -4113,17 +5386,17 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class getCompactionJob_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + private static class reserveCompactionJob_resultStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { @Override - public getCompactionJob_resultStandardScheme getScheme() { - return new getCompactionJob_resultStandardScheme(); + public reserveCompactionJob_resultStandardScheme getScheme() { + return new reserveCompactionJob_resultStandardScheme(); } } - private static class getCompactionJob_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { + private static class reserveCompactionJob_resultStandardScheme extends org.apache.thrift.scheme.StandardScheme { @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, reserveCompactionJob_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -4163,7 +5436,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getCompactionJob_re } @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, reserveCompactionJob_result struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); @@ -4183,17 +5456,17 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getCompactionJob_r } - private static class getCompactionJob_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + private static class reserveCompactionJob_resultTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { @Override - public getCompactionJob_resultTupleScheme getScheme() { - return new getCompactionJob_resultTupleScheme(); + public reserveCompactionJob_resultTupleScheme getScheme() { + return new reserveCompactionJob_resultTupleScheme(); } } - private static class getCompactionJob_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { + private static class reserveCompactionJob_resultTupleScheme extends org.apache.thrift.scheme.TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, reserveCompactionJob_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; java.util.BitSet optionals = new java.util.BitSet(); if (struct.isSetSuccess()) { @@ -4212,7 +5485,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_re } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, getCompactionJob_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, reserveCompactionJob_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TDequeuedCompactionJob.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TDequeuedCompactionJob.java new file mode 100644 index 00000000000..11630d92803 --- /dev/null +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TDequeuedCompactionJob.java @@ -0,0 +1,511 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Autogenerated by Thrift Compiler (0.17.0) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.accumulo.core.compaction.thrift; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) +public class TDequeuedCompactionJob implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TDequeuedCompactionJob"); + + private static final org.apache.thrift.protocol.TField JOB_FIELD_DESC = new org.apache.thrift.protocol.TField("job", org.apache.thrift.protocol.TType.STRUCT, (short)1); + private static final org.apache.thrift.protocol.TField COMPACTOR_COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("compactorCount", org.apache.thrift.protocol.TType.I32, (short)2); + + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new TDequeuedCompactionJobStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new TDequeuedCompactionJobTupleSchemeFactory(); + + public @org.apache.thrift.annotation.Nullable TResolvedCompactionJob job; // required + public int compactorCount; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + JOB((short)1, "job"), + COMPACTOR_COUNT((short)2, "compactorCount"); + + private static final java.util.Map byName = new java.util.HashMap(); + + static { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // JOB + return JOB; + case 2: // COMPACTOR_COUNT + return COMPACTOR_COUNT; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + @org.apache.thrift.annotation.Nullable + public static _Fields findByName(java.lang.String name) { + return byName.get(name); + } + + private final short _thriftId; + private final java.lang.String _fieldName; + + _Fields(short thriftId, java.lang.String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + @Override + public short getThriftFieldId() { + return _thriftId; + } + + @Override + public java.lang.String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __COMPACTORCOUNT_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.JOB, new org.apache.thrift.meta_data.FieldMetaData("job", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TResolvedCompactionJob.class))); + tmpMap.put(_Fields.COMPACTOR_COUNT, new org.apache.thrift.meta_data.FieldMetaData("compactorCount", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TDequeuedCompactionJob.class, metaDataMap); + } + + public TDequeuedCompactionJob() { + } + + public TDequeuedCompactionJob( + TResolvedCompactionJob job, + int compactorCount) + { + this(); + this.job = job; + this.compactorCount = compactorCount; + setCompactorCountIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public TDequeuedCompactionJob(TDequeuedCompactionJob other) { + __isset_bitfield = other.__isset_bitfield; + if (other.isSetJob()) { + this.job = new TResolvedCompactionJob(other.job); + } + this.compactorCount = other.compactorCount; + } + + @Override + public TDequeuedCompactionJob deepCopy() { + return new TDequeuedCompactionJob(this); + } + + @Override + public void clear() { + this.job = null; + setCompactorCountIsSet(false); + this.compactorCount = 0; + } + + @org.apache.thrift.annotation.Nullable + public TResolvedCompactionJob getJob() { + return this.job; + } + + public TDequeuedCompactionJob setJob(@org.apache.thrift.annotation.Nullable TResolvedCompactionJob job) { + this.job = job; + return this; + } + + public void unsetJob() { + this.job = null; + } + + /** Returns true if field job is set (has been assigned a value) and false otherwise */ + public boolean isSetJob() { + return this.job != null; + } + + public void setJobIsSet(boolean value) { + if (!value) { + this.job = null; + } + } + + public int getCompactorCount() { + return this.compactorCount; + } + + public TDequeuedCompactionJob setCompactorCount(int compactorCount) { + this.compactorCount = compactorCount; + setCompactorCountIsSet(true); + return this; + } + + public void unsetCompactorCount() { + __isset_bitfield = org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield, __COMPACTORCOUNT_ISSET_ID); + } + + /** Returns true if field compactorCount is set (has been assigned a value) and false otherwise */ + public boolean isSetCompactorCount() { + return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield, __COMPACTORCOUNT_ISSET_ID); + } + + public void setCompactorCountIsSet(boolean value) { + __isset_bitfield = org.apache.thrift.EncodingUtils.setBit(__isset_bitfield, __COMPACTORCOUNT_ISSET_ID, value); + } + + @Override + public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { + switch (field) { + case JOB: + if (value == null) { + unsetJob(); + } else { + setJob((TResolvedCompactionJob)value); + } + break; + + case COMPACTOR_COUNT: + if (value == null) { + unsetCompactorCount(); + } else { + setCompactorCount((java.lang.Integer)value); + } + break; + + } + } + + @org.apache.thrift.annotation.Nullable + @Override + public java.lang.Object getFieldValue(_Fields field) { + switch (field) { + case JOB: + return getJob(); + + case COMPACTOR_COUNT: + return getCompactorCount(); + + } + throw new java.lang.IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + @Override + public boolean isSet(_Fields field) { + if (field == null) { + throw new java.lang.IllegalArgumentException(); + } + + switch (field) { + case JOB: + return isSetJob(); + case COMPACTOR_COUNT: + return isSetCompactorCount(); + } + throw new java.lang.IllegalStateException(); + } + + @Override + public boolean equals(java.lang.Object that) { + if (that instanceof TDequeuedCompactionJob) + return this.equals((TDequeuedCompactionJob)that); + return false; + } + + public boolean equals(TDequeuedCompactionJob that) { + if (that == null) + return false; + if (this == that) + return true; + + boolean this_present_job = true && this.isSetJob(); + boolean that_present_job = true && that.isSetJob(); + if (this_present_job || that_present_job) { + if (!(this_present_job && that_present_job)) + return false; + if (!this.job.equals(that.job)) + return false; + } + + boolean this_present_compactorCount = true; + boolean that_present_compactorCount = true; + if (this_present_compactorCount || that_present_compactorCount) { + if (!(this_present_compactorCount && that_present_compactorCount)) + return false; + if (this.compactorCount != that.compactorCount) + return false; + } + + return true; + } + + @Override + public int hashCode() { + int hashCode = 1; + + hashCode = hashCode * 8191 + ((isSetJob()) ? 131071 : 524287); + if (isSetJob()) + hashCode = hashCode * 8191 + job.hashCode(); + + hashCode = hashCode * 8191 + compactorCount; + + return hashCode; + } + + @Override + public int compareTo(TDequeuedCompactionJob other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = java.lang.Boolean.compare(isSetJob(), other.isSetJob()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetJob()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.job, other.job); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = java.lang.Boolean.compare(isSetCompactorCount(), other.isSetCompactorCount()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetCompactorCount()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compactorCount, other.compactorCount); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + @org.apache.thrift.annotation.Nullable + @Override + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + scheme(iprot).read(iprot, this); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + scheme(oprot).write(oprot, this); + } + + @Override + public java.lang.String toString() { + java.lang.StringBuilder sb = new java.lang.StringBuilder("TDequeuedCompactionJob("); + boolean first = true; + + sb.append("job:"); + if (this.job == null) { + sb.append("null"); + } else { + sb.append(this.job); + } + first = false; + if (!first) sb.append(", "); + sb.append("compactorCount:"); + sb.append(this.compactorCount); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (job != null) { + job.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class TDequeuedCompactionJobStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public TDequeuedCompactionJobStandardScheme getScheme() { + return new TDequeuedCompactionJobStandardScheme(); + } + } + + private static class TDequeuedCompactionJobStandardScheme extends org.apache.thrift.scheme.StandardScheme { + + @Override + public void read(org.apache.thrift.protocol.TProtocol iprot, TDequeuedCompactionJob struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // JOB + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.job = new TResolvedCompactionJob(); + struct.job.read(iprot); + struct.setJobIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // COMPACTOR_COUNT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.compactorCount = iprot.readI32(); + struct.setCompactorCountIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + @Override + public void write(org.apache.thrift.protocol.TProtocol oprot, TDequeuedCompactionJob struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.job != null) { + oprot.writeFieldBegin(JOB_FIELD_DESC); + struct.job.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(COMPACTOR_COUNT_FIELD_DESC); + oprot.writeI32(struct.compactorCount); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class TDequeuedCompactionJobTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { + @Override + public TDequeuedCompactionJobTupleScheme getScheme() { + return new TDequeuedCompactionJobTupleScheme(); + } + } + + private static class TDequeuedCompactionJobTupleScheme extends org.apache.thrift.scheme.TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, TDequeuedCompactionJob struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetJob()) { + optionals.set(0); + } + if (struct.isSetCompactorCount()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetJob()) { + struct.job.write(oprot); + } + if (struct.isSetCompactorCount()) { + oprot.writeI32(struct.compactorCount); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, TDequeuedCompactionJob struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.job = new TResolvedCompactionJob(); + struct.job.read(iprot); + struct.setJobIsSet(true); + } + if (incoming.get(1)) { + struct.compactorCount = iprot.readI32(); + struct.setCompactorCountIsSet(true); + } + } + } + + private static S scheme(org.apache.thrift.protocol.TProtocol proto) { + return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); + } + private static void unusedMethod() {} +} + diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index 973255c0513..1da0b3a02ea 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -78,6 +78,13 @@ struct TResolvedCompactionJob { 9:bool overlapsSelectedFiles } +struct TDequeuedCompactionJob { + 1:TResolvedCompactionJob job + // The total number of compactors servicing the queue this job was requested for + 2:i32 compactorCount + +} + exception UnknownCompactionIdException {} service CompactionCoordinatorService { @@ -98,12 +105,23 @@ service CompactionCoordinatorService { ) /* - * Called by Compactor to get the next compaction job + * Called by Compactor to get the next compaction job off the queue */ - TNextCompactionJob getCompactionJob( + TDequeuedCompactionJob getCompactionJob( 1:client.TInfo tinfo 2:security.TCredentials credentials 3:string groupName + )throws( + 1:client.ThriftSecurityException sec + ) + + /* + * Called by a compactor to reserve a job returned by getCompactionJob() + */ + TNextCompactionJob reserveCompactionJob( + 1:client.TInfo tinfo + 2:security.TCredentials credentials + 3:TResolvedCompactionJob job 4:string compactor 5:string externalCompactionId )throws( diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index ec1cc30961a..456ae4ca6b5 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -70,6 +70,7 @@ import org.apache.accumulo.core.compaction.thrift.CompactorService; import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate; +import org.apache.accumulo.core.compaction.thrift.TDequeuedCompactionJob; import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; import org.apache.accumulo.core.compaction.thrift.UnknownCompactionIdException; @@ -548,15 +549,35 @@ protected TNextCompactionJob getNextJob(Supplier uuid) throws RetriesExcee RetryableThriftCall nextJobThriftCall = new RetryableThriftCall<>(startingWaitTime, maxWaitTime, 0, () -> { + Client coordinatorClient = null; + TDequeuedCompactionJob unreservedJob; try { + // Must go to the coordinator that is hosting the queue for the compactor group. All + // compactors in the group will go to this single coordinator for this, but the RPC to + // pull from the queue is quick in memory operation. coordinatorClient = getCoordinatorClient(GROUP); + unreservedJob = coordinatorClient.getCompactionJob(TraceUtil.traceInfo(), + getContext().rpcCreds(), this.getResourceGroup().canonical()); + } finally { + ThriftUtil.returnClient(coordinatorClient, getContext()); + } + + if (unreservedJob.getJob() == null) { + return new TNextCompactionJob(new TExternalCompactionJob(), + unreservedJob.getCompactorCount()); + } + + try { + // Go to any coordinator to reserve the job, this spreads the metadata operations to + // reserve a compaction across all coordinators regardless of compactor group. + coordinatorClient = getCoordinatorClient(ANY); ExternalCompactionId eci = ExternalCompactionId.generate(uuid.get()); - LOG.trace("Attempting to get next job, eci = {}", eci); + LOG.trace("Attempting to reserve next job, eci = {}", eci); currentCompactionId.set(eci); - return coordinatorClient.getCompactionJob(TraceUtil.traceInfo(), - getContext().rpcCreds(), this.getResourceGroup().canonical(), - getAdvertiseAddress().toString(), eci.toString()); + return coordinatorClient.reserveCompactionJob(TraceUtil.traceInfo(), + getContext().rpcCreds(), unreservedJob.getJob(), getAdvertiseAddress().toString(), + eci.toString()); } catch (Exception e) { currentCompactionId.set(null); throw e; diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index e9c90962633..f6a4a7da097 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -63,6 +63,7 @@ import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.compaction.thrift.TCompactionState; +import org.apache.accumulo.core.compaction.thrift.TDequeuedCompactionJob; import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; import org.apache.accumulo.core.compaction.thrift.TResolvedCompactionJob; import org.apache.accumulo.core.conf.Property; @@ -307,64 +308,65 @@ public void run() { * Return the next compaction job from the queue to a Compactor * * @param groupName group - * @param compactorAddress compactor address * @throws ThriftSecurityException when permission error * @return compaction job */ @Override - public TNextCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials, - String groupName, String compactorAddress, String externalCompactionId) - throws ThriftSecurityException { - + public TDequeuedCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials, + String groupName) throws ThriftSecurityException, TException { // do not expect users to call this directly, expect compactors to call this method if (!security.canPerformSystemActions(credentials)) { throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } ResourceGroupId groupId = ResourceGroupId.of(groupName); - LOG.trace("getCompactionJob called for group {} by compactor {}", groupId, compactorAddress); + ResolvedCompactionJob rcJob = (ResolvedCompactionJob) jobQueues.poll(groupId); + return new TDequeuedCompactionJob(rcJob == null ? null : rcJob.toThrift(), + compactorCounts.get(groupId)); + } - TExternalCompactionJob result = null; + @Override + public TNextCompactionJob reserveCompactionJob(TInfo tinfo, TCredentials credentials, + TResolvedCompactionJob job, String compactorAddress, String externalCompactionId) + throws ThriftSecurityException, TException { - ResolvedCompactionJob rcJob = (ResolvedCompactionJob) jobQueues.poll(groupId); + // do not expect users to call this directly, expect compactors to call this method + if (!security.canPerformSystemActions(credentials)) { + throw new AccumuloSecurityException(credentials.getPrincipal(), + SecurityErrorCode.PERMISSION_DENIED).asThriftException(); + } + ResourceGroupId groupId = ResourceGroupId.of(job.group); + LOG.trace("reserveCompactionJob called for group {} by compactor {}", groupId, + compactorAddress); - while (rcJob != null) { + TExternalCompactionJob result = null; - Optional compactionConfig = getCompactionConfig(rcJob); + ResolvedCompactionJob rcJob = ResolvedCompactionJob.fromThrift(job); - // this method may reread the metadata, do not use the metadata in rcJob for anything after - // this method - CompactionMetadata ecm = null; + Optional compactionConfig = getCompactionConfig(rcJob); - var kind = rcJob.getKind(); + // this method may reread the metadata, do not use the metadata in rcJob for anything after + // this method + CompactionMetadata ecm = null; - // Only reserve user compactions when the config is present. When compactions are canceled the - // config is deleted. - var cid = ExternalCompactionId.from(externalCompactionId); - if (kind == CompactionKind.SYSTEM - || (kind == CompactionKind.USER && compactionConfig.isPresent())) { - ecm = reserveCompaction(rcJob, compactorAddress, cid); - } + var kind = rcJob.getKind(); - if (ecm != null) { - result = createThriftJob(externalCompactionId, ecm, rcJob, compactionConfig); - TabletLogger.compacting(rcJob.getExtent(), rcJob.getSelectedFateId(), cid, compactorAddress, - rcJob, ecm.getCompactTmpName()); - break; - } else { - LOG.debug( - "Unable to reserve compaction job for {}, pulling another off the queue for group {}", - rcJob.getExtent(), groupName); - rcJob = (ResolvedCompactionJob) jobQueues.poll(ResourceGroupId.of(groupName)); - } + // Only reserve user compactions when the config is present. When compactions are canceled the + // config is deleted. + var cid = ExternalCompactionId.from(externalCompactionId); + if (kind == CompactionKind.SYSTEM + || (kind == CompactionKind.USER && compactionConfig.isPresent())) { + ecm = reserveCompaction(rcJob, compactorAddress, cid); } - if (rcJob == null) { - LOG.trace("No jobs found in group {} ", groupName); + if (ecm != null) { + result = createThriftJob(externalCompactionId, ecm, rcJob, compactionConfig); + TabletLogger.compacting(rcJob.getExtent(), rcJob.getSelectedFateId(), cid, compactorAddress, + rcJob, ecm.getCompactTmpName()); } if (result == null) { - LOG.trace("No jobs found for group {}, returning empty job to compactor {}", groupName, + LOG.trace("No jobs found for group {}, returning empty job to compactor {}", groupId, compactorAddress); result = new TExternalCompactionJob(); } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index 67c864346a1..c2b765761d4 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -25,6 +25,7 @@ import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import java.util.ArrayList; @@ -41,6 +42,7 @@ import org.apache.accumulo.core.clientImpl.thrift.TInfo; import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.compaction.thrift.TCompactionState; +import org.apache.accumulo.core.compaction.thrift.TDequeuedCompactionJob; import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; import org.apache.accumulo.core.conf.DefaultConfiguration; @@ -248,10 +250,13 @@ public void testGetCompactionJob() throws Exception { // Get the next job ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID()); - TNextCompactionJob nextJob = coordinator.getCompactionJob(new TInfo(), rpcCreds, - GROUP_ID.toString(), "localhost:10241", eci.toString()); - assertEquals(3, nextJob.getCompactorCount()); - TExternalCompactionJob createdJob = nextJob.getJob(); + TDequeuedCompactionJob unreservedJob = + coordinator.getCompactionJob(new TInfo(), rpcCreds, GROUP_ID.toString()); + assertEquals(3, unreservedJob.getCompactorCount()); + assertNotNull(unreservedJob.getJob()); + TNextCompactionJob reservedJob = coordinator.reserveCompactionJob(new TInfo(), rpcCreds, + unreservedJob.getJob(), "localhost:10241", eci.toString()); + TExternalCompactionJob createdJob = reservedJob.getJob(); assertEquals(eci.toString(), createdJob.getExternalCompactionId()); assertEquals(ke, KeyExtent.fromThrift(createdJob.getExtent())); @@ -263,9 +268,9 @@ public void testGetCompactionJob() throws Exception { @Test public void testGetCompactionJobNoJobs() throws Exception { var coordinator = new TestCoordinator(manager, new ArrayList<>()); - TNextCompactionJob nextJob = coordinator.getCompactionJob(TraceUtil.traceInfo(), rpcCreds, - GROUP_ID.toString(), "localhost:10240", UUID.randomUUID().toString()); - assertEquals(3, nextJob.getCompactorCount()); - assertNull(nextJob.getJob().getExternalCompactionId()); + TDequeuedCompactionJob unreservedJob = + coordinator.getCompactionJob(TraceUtil.traceInfo(), rpcCreds, GROUP_ID.toString()); + assertEquals(3, unreservedJob.getCompactorCount()); + assertNull(unreservedJob.getJob()); } } From 4fcd7211526f22d0a1aa2d5abbcbbdc6d623aada Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 7 Apr 2026 23:30:22 +0000 Subject: [PATCH 50/65] handles same host+port in different resource groups in LiveTServerSet The same host+port in different resource groups in ZK would cause LiveTServerSet to continually add and remove the host port. Changed an internal map that keyed on host+port to instead key on the ZK path to fix this. fixes #6298 --- .../server/manager/LiveTServerSet.java | 57 ++++---- .../server/manager/LiveTServerSetTest.java | 125 +++++++++++++++++- 2 files changed, 152 insertions(+), 30 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java index 8d7f5ee6539..3e86c0441d4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java @@ -78,7 +78,7 @@ public interface Listener { private static final Logger log = LoggerFactory.getLogger(LiveTServerSet.class); - private final AtomicReference cback; + protected final AtomicReference cback; private final ServerContext context; public class TServerConnection { @@ -204,7 +204,7 @@ static class TServerInfo { // The set of active tservers with locks, indexed by their name in zookeeper. When the contents of // this map are modified, tServersSnapshot should be set to null. - private final Map current = new HashMap<>(); + private final Map current = new HashMap<>(); private LiveTServersSnapshot tServersSnapshot = null; @@ -233,12 +233,17 @@ public synchronized void startListeningForTabletServerChanges(Listener cback) { .scheduleWithFixedDelay(this::scanServers, 5000, 5000, TimeUnit.MILLISECONDS)); } + @VisibleForTesting + protected Set getTserverPaths() { + return context.getServerPaths().getTabletServer(ResourceGroupPredicate.ANY, + AddressSelector.all(), false); + } + public synchronized void scanServers() { try { final Set updates = new HashSet<>(); final Set doomed = new HashSet<>(); - final Set tservers = context.getServerPaths() - .getTabletServer(ResourceGroupPredicate.ANY, AddressSelector.all(), false); + final Set tservers = getTserverPaths(); locklessServers.keySet().retainAll(tservers); @@ -262,6 +267,11 @@ private void deleteServerNode(String serverNode) throws InterruptedException, Ke } } + @VisibleForTesting + protected Optional getLockData(ServiceLockPath tserverPath, ZcStat stat) { + return ServiceLock.getLockData(context.getZooCache(), tserverPath, stat); + } + private synchronized void checkServer(final Set updates, final Set doomed, final ServiceLockPath tserverPath) throws InterruptedException, KeeperException { @@ -269,33 +279,32 @@ private synchronized void checkServer(final Set updates, // invalidate the snapshot forcing it to be recomputed the next time its requested tServersSnapshot = null; - final TServerInfo info = current.get(tserverPath.getServer()); + final TServerInfo info = current.get(tserverPath); ZcStat stat = new ZcStat(); - Optional sld = - ServiceLock.getLockData(context.getZooCache(), tserverPath, stat); + Optional sld = getLockData(tserverPath, stat); if (sld.isEmpty()) { - log.trace("lock does not exist for server: {}", tserverPath.getServer()); + log.trace("lock does not exist for server: {}", tserverPath); if (info != null) { doomed.add(info.instance); - current.remove(tserverPath.getServer()); - log.trace("removed {} from current set and adding to doomed list", tserverPath.getServer()); + current.remove(tserverPath); + log.trace("removed {} from current set and adding to doomed list", tserverPath); } Long firstSeen = locklessServers.get(tserverPath); if (firstSeen == null) { locklessServers.put(tserverPath, System.currentTimeMillis()); - log.trace("first seen, added {} to list of lockless servers", tserverPath.getServer()); + log.trace("first seen, added {} to list of lockless servers", tserverPath); } else if (System.currentTimeMillis() - firstSeen > MINUTES.toMillis(10)) { deleteServerNode(tserverPath.toString()); locklessServers.remove(tserverPath); log.trace( "deleted zookeeper node for server: {}, has been without lock for over 10 minutes", - tserverPath.getServer()); + tserverPath); } } else { - log.trace("Lock exists for server: {}, adding to current set", tserverPath.getServer()); + log.trace("Lock exists for server: {}, adding to current set", tserverPath); locklessServers.remove(tserverPath); HostAndPort address = sld.orElseThrow().getAddress(ServiceLockData.ThriftService.TSERV); ResourceGroupId resourceGroup = @@ -306,13 +315,13 @@ private synchronized void checkServer(final Set updates, updates.add(instance); TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(address), resourceGroup); - current.put(tserverPath.getServer(), tServerInfo); + current.put(tserverPath, tServerInfo); } else if (!info.instance.equals(instance)) { doomed.add(info.instance); updates.add(instance); TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(address), resourceGroup); - current.put(tserverPath.getServer(), tServerInfo); + current.put(tserverPath, tServerInfo); } } } @@ -459,7 +468,7 @@ public synchronized TServerInstance find(String tabletServer) { return find(current, tabletServer); } - static TServerInstance find(Map servers, String tabletServer) { + static TServerInstance find(Map servers, String tabletServer) { HostAndPort addr; String sessionId = null; if (tabletServer.charAt(tabletServer.length() - 1) == ']') { @@ -473,11 +482,11 @@ static TServerInstance find(Map servers, String tabletServer } else { addr = AddressUtil.parseAddress(tabletServer); } - for (Entry entry : servers.entrySet()) { - if (entry.getValue().instance.getHostAndPort().equals(addr)) { + for (TServerInfo tServerInfo : servers.values()) { + if (tServerInfo.instance.getHostAndPort().equals(addr)) { // Return the instance if we have no desired session ID, or we match the desired session ID - if (sessionId == null || sessionId.equals(entry.getValue().instance.getSession())) { - return entry.getValue().instance; + if (sessionId == null || sessionId.equals(tServerInfo.instance.getSession())) { + return tServerInfo.instance; } } } @@ -491,17 +500,17 @@ public synchronized void remove(TServerInstance server) { Optional resourceGroup = Optional.empty(); Optional address = Optional.empty(); - for (Entry entry : current.entrySet()) { + for (Entry entry : current.entrySet()) { if (entry.getValue().instance.equals(server)) { - address = Optional.of(HostAndPort.fromString(entry.getKey())); + address = Optional.of(HostAndPort.fromString(entry.getKey().getServer())); resourceGroup = Optional.of(entry.getValue().resourceGroup); + current.remove(entry.getKey()); break; } } if (resourceGroup.isEmpty() || address.isEmpty()) { return; } - current.remove(address.orElseThrow().toString()); ResourceGroupPredicate rgPredicate = resourceGroup.map(rg -> { ResourceGroupPredicate rgp = rg2 -> rg.equals(rg2); @@ -511,7 +520,7 @@ public synchronized void remove(TServerInstance server) { address.map(AddressSelector::exact).orElse(AddressSelector.all()); Set paths = context.getServerPaths().getTabletServer(rgPredicate, addrPredicate, false); - if (paths.isEmpty() || paths.size() > 1) { + if (paths.size() != 1) { log.error("Zero or many zookeeper entries match input arguments."); } else { ServiceLockPath slp = paths.iterator().next(); diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java index 15e9d5e3ac5..837b5f7facb 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java @@ -18,32 +18,49 @@ */ package org.apache.accumulo.server.manager; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createStrictMock; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import org.apache.accumulo.core.data.ResourceGroupId; +import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; +import org.apache.accumulo.core.lock.ServiceLockPaths; +import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.zookeeper.ZcStat; +import org.apache.accumulo.core.zookeeper.ZooCache; +import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection; import org.apache.accumulo.server.manager.LiveTServerSet.TServerInfo; -import org.easymock.EasyMock; import org.junit.jupiter.api.Test; +import com.google.common.annotations.VisibleForTesting; import com.google.common.net.HostAndPort; public class LiveTServerSetTest { @Test public void testSessionIds() { - Map servers = new HashMap<>(); - TServerConnection mockConn = EasyMock.createMock(TServerConnection.class); + Map servers = new HashMap<>(); + TServerConnection mockConn = createMock(TServerConnection.class); + + var hostPort = HostAndPort.fromParts("localhost", 1234); TServerInfo server1 = - new TServerInfo(new TServerInstance(HostAndPort.fromParts("localhost", 1234), "5555"), - mockConn, ResourceGroupId.DEFAULT); - servers.put("server1", server1); + new TServerInfo(new TServerInstance(hostPort, "5555"), mockConn, ResourceGroupId.DEFAULT); + + ServiceLockPaths lockPaths = new ServiceLockPaths(createMock(ZooCache.class)); + + servers.put(lockPaths.createTabletServerPath(ResourceGroupId.DEFAULT, hostPort), server1); assertEquals(server1.instance, LiveTServerSet.find(servers, "localhost:1234")); assertNull(LiveTServerSet.find(servers, "localhost:4321")); @@ -51,4 +68,100 @@ public void testSessionIds() { assertNull(LiveTServerSet.find(servers, "localhost:1234[55755]")); } + record LockInfo(ServiceLockData sld, long sessionId) { + } + + static class TestLiveTserverSet extends LiveTServerSet { + private final Set paths; + private final Map locks; + + public TestLiveTserverSet(Set paths, Map locks, + Listener listener) { + super(createStrictMock(ServerContext.class)); + this.paths = paths; + this.locks = locks; + this.cback.set(listener); + } + + @VisibleForTesting + protected Set getTserverPaths() { + return paths; + } + + @VisibleForTesting + protected Optional getLockData(ServiceLockPath tserverPath, ZcStat stat) { + var lockInfo = locks.get(tserverPath); + if (lockInfo == null) { + return Optional.empty(); + } + + stat.setEphemeralOwner(lockInfo.sessionId); + + return Optional.of(lockInfo.sld); + } + } + + @Test + public void testSameHostPortInDiffResourceGroups() { + // This tests two tservers with the same host port in different resource groups where only one + // has a lock. There was a bug where LiveTserverSet would add and remove that host port from its + // set and pass both add/remove to the callback listener. + + ServiceLockPaths lockPaths = new ServiceLockPaths(createMock(ZooCache.class)); + + var hp1 = HostAndPort.fromParts("host1", 9800); + var g1 = ResourceGroupId.of("tg1"); + var path1 = lockPaths.createTabletServerPath(ResourceGroupId.DEFAULT, hp1); + + var path2 = lockPaths.createTabletServerPath(g1, hp1); + + var paths = new HashSet(); + paths.add(path1); + paths.add(path2); + + var locks = new HashMap(); + locks.put(path2, new LockInfo( + new ServiceLockData(UUID.randomUUID(), hp1.toString(), ThriftService.TSERV, g1), 123456)); + + var deletedSeen = new HashSet(); + var addedSeen = new HashSet(); + LiveTServerSet tservers = new TestLiveTserverSet(paths, locks, ((current, deleted, added) -> { + deletedSeen.addAll(deleted); + addedSeen.addAll(added); + })); + tservers.scanServers(); + + var expected1 = Set.of(new TServerInstance(hp1, 123456)); + assertEquals(expected1, addedSeen); + assertEquals(Set.of(), deletedSeen); + assertEquals(expected1, tservers.getSnapshot().getTservers()); + assertEquals(Map.of(g1, expected1), tservers.getSnapshot().getTserverGroups()); + + // change which tserver has the lock + locks.clear(); + locks.put(path1, new LockInfo(new ServiceLockData(UUID.randomUUID(), hp1.toString(), + ThriftService.TSERV, ResourceGroupId.DEFAULT), 654321)); + + addedSeen.clear(); + tservers.scanServers(); + + var expected2 = Set.of(new TServerInstance(hp1, 654321)); + assertEquals(expected2, addedSeen); + assertEquals(expected1, deletedSeen); + assertEquals(expected2, tservers.getSnapshot().getTservers()); + assertEquals(Map.of(ResourceGroupId.DEFAULT, expected2), + tservers.getSnapshot().getTserverGroups()); + + // test when nothing has changed since last scan + addedSeen.clear(); + deletedSeen.clear(); + tservers.scanServers(); + // Should not see any add/removes + assertEquals(Set.of(), addedSeen); + assertEquals(Set.of(), deletedSeen); + assertEquals(expected2, tservers.getSnapshot().getTservers()); + assertEquals(Map.of(ResourceGroupId.DEFAULT, expected2), + tservers.getSnapshot().getTserverGroups()); + + } } From 1c60481633e843360efed24dc05376dfeeed5125 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 7 Apr 2026 10:43:40 -0700 Subject: [PATCH 51/65] Update server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java Co-authored-by: Daniel Roberts --- .../accumulo/server/metadata/RemovedCompactionStoreImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java index 5dafd5a2746..96b997ea2ec 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java @@ -58,8 +58,8 @@ private Stream createStream(String tableName) { @Override public Stream list() { - return Stream.concat(createStream(SystemTables.ROOT.tableName()), - createStream(SystemTables.METADATA.tableName())); + return Stream.concat(createStream(Ample.DataLevel.METADATA.metaTable()), + createStream(Ample.DataLevel.USER.metaTable())); } private void write(Collection removedCompactions, From 94f076b059dc64a135b94061724a2321f39a8ab6 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 7 Apr 2026 10:43:50 -0700 Subject: [PATCH 52/65] Update server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java Co-authored-by: Daniel Roberts --- .../manager/compaction/coordinator/DeadCompactionDetector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java index ae11a88500c..0161713ff47 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java @@ -151,7 +151,7 @@ private void detectDeadCompactions() { }); } - // Get the list of compaction entires that were removed from the metadata table by a split or + // Get the list of compaction entries that were removed from the metadata table by a split or // merge operation. Must get this data before getting the running set of compactions. List removedCompactions; try (Stream listing = context.getAmple().removedCompactions().list()) { From d40c39f6eb4eeeea375bd82fffa8875d414b6bb5 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 7 Apr 2026 19:06:22 +0000 Subject: [PATCH 53/65] code review update --- .../server/util/FindCompactionTmpFiles.java | 134 ++++++++++-------- .../coordinator/CompactionCoordinator.java | 6 +- .../coordinator/DeadCompactionDetector.java | 9 +- .../compaction/ExternalCompaction2ITBase.java | 4 +- .../compaction/ExternalCompaction_3_IT.java | 2 +- 5 files changed, 86 insertions(+), 69 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java index c27a65639a4..7064b12de37 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java @@ -33,6 +33,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.cli.ServerOpts; @@ -58,6 +59,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.google.auto.service.AutoService; +import com.google.common.util.concurrent.MoreExecutors; import io.opentelemetry.api.trace.Span; import io.opentelemetry.context.Scope; @@ -143,63 +145,8 @@ public static class DeleteStats { public int error = 0; } - public static DeleteStats deleteTempFiles(ServerContext context, Set filesToDelete) - throws InterruptedException { - - final ExecutorService delSvc = Executors.newFixedThreadPool(8); - // use a linked list to make removal from the middle of the list quick - final List> futures = new LinkedList<>(); - final DeleteStats stats = new DeleteStats(); - - filesToDelete.forEach(p -> { - futures.add(delSvc.submit(() -> { - if (context.getVolumeManager().exists(p)) { - boolean result = context.getVolumeManager().delete(p); - if (result) { - LOG.debug("Removed old temp file {}", p); - } else { - LOG.error( - "Unable to remove old temp file {}, operation returned false with no exception", p); - } - return result; - } - return true; - })); - }); - delSvc.shutdown(); - - int expectedResponses = filesToDelete.size(); - while (expectedResponses > 0) { - Iterator> iter = futures.iterator(); - while (iter.hasNext()) { - Future future = iter.next(); - if (future.isDone()) { - expectedResponses--; - iter.remove(); - try { - if (future.get()) { - stats.success++; - } else { - stats.failure++; - } - } catch (ExecutionException e) { - stats.error++; - LOG.error("Error deleting a compaction tmp file", e); - } - } - } - LOG.debug("Waiting on {} background delete operations", expectedResponses); - if (expectedResponses > 0) { - UtilWaitThread.sleep(3_000); - } - } - delSvc.awaitTermination(10, TimeUnit.MINUTES); - return stats; - } - - // Finds any tmp files matching the given compaction ids in table dir and deletes them. - public static void deleteTmpFiles(ServerContext ctx, TableId tableId, String dirName, - Set ecidsForTablet) { + public static void findTmpFiles(ServerContext ctx, TableId tableId, String dirName, + Set ecidsForTablet, Consumer findConsumer) { final Collection vols = ctx.getVolumeManager().getVolumes(); for (Volume vol : vols) { try { @@ -216,11 +163,7 @@ public static void deleteTmpFiles(ServerContext ctx, TableId tableId, String dir } if (files != null) { for (FileStatus file : files) { - if (!fs.delete(file.getPath(), false)) { - LOG.warn("Unable to delete ecid tmp file: {}: ", file.getPath()); - } else { - LOG.debug("Deleted ecid tmp file: {}", file.getPath()); - } + findConsumer.accept(file.getPath()); } } } @@ -230,6 +173,73 @@ public static void deleteTmpFiles(ServerContext ctx, TableId tableId, String dir } } + private static boolean deleteTmpFile(ServerContext context, Path p) throws IOException { + if (context.getVolumeManager().exists(p)) { + boolean result = context.getVolumeManager().delete(p); + if (result) { + LOG.debug("Removed old temp file {}", p); + } else { + LOG.error("Unable to remove old temp file {}, operation returned false with no exception", + p); + } + return result; + } + return true; + } + + public static DeleteStats deleteTempFiles(ServerContext context, Set filesToDelete) { + + final ExecutorService delSvc; + if (filesToDelete.size() < 4) { + // Do not bother creating a thread pool and threads for a few files. + delSvc = MoreExecutors.newDirectExecutorService(); + } else { + delSvc = Executors.newFixedThreadPool(8); + } + + final DeleteStats stats = new DeleteStats(); + + // use a linked list to make removal from the middle of the list quick + final List> futures = new LinkedList<>(); + + filesToDelete.forEach(p -> { + futures.add(delSvc.submit(() -> deleteTmpFile(context, p))); + }); + delSvc.shutdown(); + + try { + int expectedResponses = filesToDelete.size(); + while (expectedResponses > 0) { + Iterator> iter = futures.iterator(); + while (iter.hasNext()) { + Future future = iter.next(); + if (future.isDone()) { + expectedResponses--; + iter.remove(); + try { + if (future.get()) { + stats.success++; + } else { + stats.failure++; + } + } catch (ExecutionException e) { + stats.error++; + LOG.error("Error deleting a compaction tmp file", e); + } + } + } + if (expectedResponses > 0) { + LOG.debug("Waiting on {} background delete operations", expectedResponses); + UtilWaitThread.sleep(1_000); + } + } + delSvc.awaitTermination(10, TimeUnit.MINUTES); + return stats; + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + public FindCompactionTmpFiles() { super(new FindOpts()); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index f6a4a7da097..5ed83fe7d87 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -828,12 +828,14 @@ static void compactionFailedForLevel(ServerContext ctx, // death that the dead compaction will be detected in the future and the files removed then. try (var tablets = ctx.getAmple().readTablets() .forTablets(compactions.keySet(), Optional.empty()).fetch(ColumnType.DIR).build()) { + Set tmpFilesToDelete = new HashSet<>(); for (TabletMetadata tm : tablets) { var extent = tm.getExtent(); var ecidsForTablet = compactions.get(extent); - FindCompactionTmpFiles.deleteTmpFiles(ctx, extent.tableId(), tm.getDirName(), - ecidsForTablet); + FindCompactionTmpFiles.findTmpFiles(ctx, extent.tableId(), tm.getDirName(), ecidsForTablet, + tmpFilesToDelete::add); } + FindCompactionTmpFiles.deleteTempFiles(ctx, tmpFilesToDelete); } try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java index 0161713ff47..e83828a792d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -50,6 +51,7 @@ import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.util.FindCompactionTmpFiles; +import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -169,10 +171,13 @@ private void detectDeadCompactions() { if (!removedCompactions.isEmpty()) { var runningSet = Objects.requireNonNull(running); removedCompactions.removeIf(rc -> runningSet.contains(rc.id())); + Set tmpFilesToDelete = new HashSet<>(); removedCompactions.forEach(rc -> { - log.trace("attempting to delete tmp files for removed compaction {}", rc); - FindCompactionTmpFiles.deleteTmpFiles(context, rc.table(), rc.dir(), Set.of(rc.id())); + log.trace("attempting to find tmp files for removed compaction {}", rc); + FindCompactionTmpFiles.findTmpFiles(context, rc.table(), rc.dir(), Set.of(rc.id()), + tmpFilesToDelete::add); }); + FindCompactionTmpFiles.deleteTempFiles(context, tmpFilesToDelete); context.getAmple().removedCompactions().delete(removedCompactions); } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java index ad7b19325bb..fd4d228f6ce 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2ITBase.java @@ -138,7 +138,7 @@ public void testSplitCancelsExternalCompaction() throws Exception { // Verify that the tmp file are cleaned up Wait.waitFor(() -> FindCompactionTmpFiles - .findTempFiles(getCluster().getServerContext(), tid.canonical()).size() == 0, 60_000); + .findTempFiles(getCluster().getServerContext(), tid.canonical()).isEmpty(), 60_000); } } @@ -200,7 +200,7 @@ public void testUserCompactionCancellation() throws Exception { // Verify that the tmp file are cleaned up Wait.waitFor(() -> FindCompactionTmpFiles - .findTempFiles(getCluster().getServerContext(), tid.canonical()).size() == 0); + .findTempFiles(getCluster().getServerContext(), tid.canonical()).isEmpty()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java index 1ed8e81fecd..b011f3fb9f1 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_3_IT.java @@ -153,7 +153,7 @@ public void testMergeCancelsExternalCompaction() throws Exception { // Verify that the tmp file are cleaned up Wait.waitFor(() -> FindCompactionTmpFiles - .findTempFiles(getCluster().getServerContext(), tid.canonical()).size() == 0); + .findTempFiles(getCluster().getServerContext(), tid.canonical()).isEmpty()); // We need to cancel the compaction or delete the table here because we initiate a user // compaction above in the test. Even though the external compaction was cancelled From b49b1a3959d2535728e488cb0c9bc4d66adbd15a Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 8 Apr 2026 15:42:27 +0000 Subject: [PATCH 54/65] renamed RemovedCompaction to OrphanedCompaction --- .../accumulo/core/metadata/schema/Ample.java | 14 ++++---- .../core/metadata/schema/MetadataSchema.java | 12 +++---- ....java => OrphanedCompactionStoreImpl.java} | 34 +++++++++---------- .../server/metadata/ServerAmpleImpl.java | 4 +-- .../coordinator/DeadCompactionDetector.java | 17 +++++----- .../manager/tableOps/merge/MergeTablets.java | 18 +++++----- .../manager/tableOps/split/UpdateTablets.java | 4 +-- .../tableOps/merge/MergeTabletsTest.java | 25 +++++++------- .../tableOps/split/UpdateTabletsTest.java | 20 +++++------ 9 files changed, 74 insertions(+), 74 deletions(-) rename server/base/src/main/java/org/apache/accumulo/server/metadata/{RemovedCompactionStoreImpl.java => OrphanedCompactionStoreImpl.java} (70%) diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java index 1d874dccf1b..7e66f370682 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java @@ -671,23 +671,23 @@ default ScanServerRefStore scanServerRefs() { throw new UnsupportedOperationException(); } - record RemovedCompaction(ExternalCompactionId id, TableId table, String dir) { - }; + record OrphanedCompaction(ExternalCompactionId id, TableId table, String dir) { + } /** * Tracks compactions that were removed from the metadata table but may still be running on * compactors. The tmp files associated with these compactions can eventually be removed when the * compaction is no longer running. */ - interface RemovedCompactionStore { - Stream list(); + interface OrphanedCompactionStore { + Stream list(); - void add(Collection removedCompactions); + void add(Collection orphanedCompactions); - void delete(Collection removedCompactions); + void delete(Collection orphanedCompactions); } - default RemovedCompactionStore removedCompactions() { + default OrphanedCompactionStore orphanedCompactions() { throw new UnsupportedOperationException(); } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java index bb1309583e0..fa7a98ce38f 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java @@ -549,11 +549,11 @@ public static String getRowPrefix() { /** * Holds information about compactions that were deleted from tablets metadata by split or merge - * operations. These may have tmp files that need to be cleaned up. + * operations. These may still be running and may have tmp files that need to be cleaned up. */ - public static class RemovedCompactionSection { + public static class OrphanedCompactionSection { private static final Section section = - new Section(RESERVED_PREFIX + "rcomp", true, RESERVED_PREFIX + "rcomq", false); + new Section(RESERVED_PREFIX + "ocomp", true, RESERVED_PREFIX + "ocomq", false); public static Range getRange() { return section.getRange(); @@ -563,15 +563,15 @@ public static String getRowPrefix() { return section.getRowPrefix(); } - public static Ample.RemovedCompaction decodeRow(String row) { + public static Ample.OrphanedCompaction decodeRow(String row) { String[] fields = row.split("#"); Preconditions.checkArgument(fields.length == 4); Preconditions.checkArgument(getRowPrefix().equals(fields[0])); - return new Ample.RemovedCompaction(ExternalCompactionId.from(fields[1]), + return new Ample.OrphanedCompaction(ExternalCompactionId.from(fields[1]), TableId.of(fields[2]), fields[3]); } - public static String encodeRow(Ample.RemovedCompaction rc) { + public static String encodeRow(Ample.OrphanedCompaction rc) { // put the compaction id first in the row because its uuid will spread out nicely and avoid // hot spotting return getRowPrefix() + "#" + rc.id().canonical() + "#" + rc.table().canonical() + "#" diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/OrphanedCompactionStoreImpl.java similarity index 70% rename from server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java rename to server/base/src/main/java/org/apache/accumulo/server/metadata/OrphanedCompactionStoreImpl.java index 96b997ea2ec..0226c000f4f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/RemovedCompactionStoreImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/OrphanedCompactionStoreImpl.java @@ -31,44 +31,44 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.schema.Ample; -import org.apache.accumulo.core.metadata.schema.MetadataSchema.RemovedCompactionSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.OrphanedCompactionSection; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.server.ServerContext; import com.google.common.base.Preconditions; -public class RemovedCompactionStoreImpl implements Ample.RemovedCompactionStore { +public class OrphanedCompactionStoreImpl implements Ample.OrphanedCompactionStore { private final ServerContext context; - public RemovedCompactionStoreImpl(ServerContext context) { + public OrphanedCompactionStoreImpl(ServerContext context) { this.context = context; } - private Stream createStream(String tableName) { + private Stream createStream(String tableName) { Scanner scanner = null; try { scanner = context.createScanner(tableName, Authorizations.EMPTY); } catch (TableNotFoundException e) { throw new IllegalStateException(e); } - scanner.setRange(RemovedCompactionSection.getRange()); + scanner.setRange(OrphanedCompactionSection.getRange()); return scanner.stream().map(e -> e.getKey().getRowData().toString()) - .map(RemovedCompactionSection::decodeRow).onClose(scanner::close); + .map(OrphanedCompactionSection::decodeRow).onClose(scanner::close); } @Override - public Stream list() { + public Stream list() { return Stream.concat(createStream(Ample.DataLevel.METADATA.metaTable()), createStream(Ample.DataLevel.USER.metaTable())); } - private void write(Collection removedCompactions, - Function converter) { - if (removedCompactions.isEmpty()) { + private void write(Collection orphanedCompactions, + Function converter) { + if (orphanedCompactions.isEmpty()) { return; } - Map> byLevel = removedCompactions.stream() + Map> byLevel = orphanedCompactions.stream() .collect(Collectors.groupingBy(rc -> Ample.DataLevel.of(rc.table()))); // Do not expect the root to split or merge so it should never have this data Preconditions.checkArgument(!byLevel.containsKey(Ample.DataLevel.ROOT)); @@ -84,9 +84,9 @@ private void write(Collection removedCompactions, } @Override - public void add(Collection removedCompactions) { - write(removedCompactions, rc -> { - Mutation m = new Mutation(RemovedCompactionSection.encodeRow(rc)); + public void add(Collection orphanedCompactions) { + write(orphanedCompactions, oc -> { + Mutation m = new Mutation(OrphanedCompactionSection.encodeRow(oc)); m.put("", "", ""); return m; }); @@ -94,9 +94,9 @@ public void add(Collection removedCompactions) { } @Override - public void delete(Collection removedCompactions) { - write(removedCompactions, rc -> { - Mutation m = new Mutation(RemovedCompactionSection.encodeRow(rc)); + public void delete(Collection orphanedCompactions) { + write(orphanedCompactions, oc -> { + Mutation m = new Mutation(OrphanedCompactionSection.encodeRow(oc)); m.putDelete("", ""); return m; }); diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java index 35fc63fe3ff..ee17d962dee 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java @@ -282,8 +282,8 @@ public ScanServerRefStore scanServerRefs() { } @Override - public RemovedCompactionStore removedCompactions() { - return new RemovedCompactionStoreImpl(getContext()); + public OrphanedCompactionStore orphanedCompactions() { + return new OrphanedCompactionStoreImpl(getContext()); } @VisibleForTesting diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java index e83828a792d..411a4a76993 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java @@ -155,30 +155,31 @@ private void detectDeadCompactions() { // Get the list of compaction entries that were removed from the metadata table by a split or // merge operation. Must get this data before getting the running set of compactions. - List removedCompactions; - try (Stream listing = context.getAmple().removedCompactions().list()) { - removedCompactions = listing.collect(Collectors.toCollection(ArrayList::new)); + List orphanedCompactions; + try (Stream listing = + context.getAmple().orphanedCompactions().list()) { + orphanedCompactions = listing.collect(Collectors.toCollection(ArrayList::new)); } // Must get the set of running compactions after reading compaction ids from the metadata table Set running = null; - if (!removedCompactions.isEmpty() || !tabletCompactions.isEmpty()) { + if (!orphanedCompactions.isEmpty() || !tabletCompactions.isEmpty()) { running = ExternalCompactionUtil.getCompactionIdsRunningOnCompactors(context); } // Delete any tmp files related to compaction metadata entries that were removed by split or // merge and are no longer running. - if (!removedCompactions.isEmpty()) { + if (!orphanedCompactions.isEmpty()) { var runningSet = Objects.requireNonNull(running); - removedCompactions.removeIf(rc -> runningSet.contains(rc.id())); + orphanedCompactions.removeIf(rc -> runningSet.contains(rc.id())); Set tmpFilesToDelete = new HashSet<>(); - removedCompactions.forEach(rc -> { + orphanedCompactions.forEach(rc -> { log.trace("attempting to find tmp files for removed compaction {}", rc); FindCompactionTmpFiles.findTmpFiles(context, rc.table(), rc.dir(), Set.of(rc.id()), tmpFilesToDelete::add); }); FindCompactionTmpFiles.deleteTempFiles(context, tmpFilesToDelete); - context.getAmple().removedCompactions().delete(removedCompactions); + context.getAmple().orphanedCompactions().delete(orphanedCompactions); } if (tabletCompactions.isEmpty()) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java index 5ec0954e146..d184b39cb4e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java @@ -75,7 +75,7 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { Map newFiles = new HashMap<>(); TabletMetadata firstTabletMeta = null; TabletMetadata lastTabletMeta = null; - List removedCompactions = new ArrayList<>(); + List orphanedCompactions = new ArrayList<>(); try (var tabletsMetadata = env.getContext().getAmple().readTablets().forTable(range.tableId()) .overlapping(range.prevEndRow(), range.endRow()).build()) { @@ -146,14 +146,14 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { // These compaction metadata entries will be deleted, queue up removal of the tmp file once // the compaction is no longer running tabletMeta.getExternalCompactions().keySet().stream() - .map(ecid -> new Ample.RemovedCompaction(ecid, tabletMeta.getExtent().tableId(), + .map(ecid -> new Ample.OrphanedCompaction(ecid, tabletMeta.getExtent().tableId(), tabletMeta.getDirName())) - .forEach(removedCompactions::add); - if (removedCompactions.size() > 1000 && tabletsSeen > 1) { - removedCompactions + .forEach(orphanedCompactions::add); + if (orphanedCompactions.size() > 1000 && tabletsSeen > 1) { + orphanedCompactions .forEach(rc -> log.trace("{} adding removed compaction {}", fateId, rc)); - env.getContext().getAmple().removedCompactions().add(removedCompactions); - removedCompactions.clear(); + env.getContext().getAmple().orphanedCompactions().add(orphanedCompactions); + orphanedCompactions.clear(); } } @@ -168,8 +168,8 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { lastTabletMeta); } - removedCompactions.forEach(rc -> log.trace("{} adding removed compaction {}", fateId, rc)); - env.getContext().getAmple().removedCompactions().add(removedCompactions); + orphanedCompactions.forEach(rc -> log.trace("{} adding removed compaction {}", fateId, rc)); + env.getContext().getAmple().orphanedCompactions().add(orphanedCompactions); log.info("{} merge low tablet {}", fateId, firstTabletMeta.getExtent()); log.info("{} merge high tablet {}", fateId, lastTabletMeta.getExtent()); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java index 7f387f64de8..f4d2d6df00f 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java @@ -270,11 +270,11 @@ private void updateExistingTablet(FateId fateId, ServerContext ctx, TabletMetada // queue up the tmp files related to these compaction metadata entries to be eventually deleted // once the compaction is no longer running var removedCompactions = tabletMetadata.getExternalCompactions().keySet().stream() - .map(ecid -> new Ample.RemovedCompaction(ecid, tabletMetadata.getExtent().tableId(), + .map(ecid -> new Ample.OrphanedCompaction(ecid, tabletMetadata.getExtent().tableId(), tabletMetadata.getDirName())) .toList(); removedCompactions.forEach(rc -> log.trace("{} adding removed compaction {}", fateId, rc)); - ctx.getAmple().removedCompactions().add(removedCompactions); + ctx.getAmple().orphanedCompactions().add(removedCompactions); try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) { var newExtent = newTablets.navigableKeySet().last(); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java index c61faa2b9c0..002ffbeea80 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java @@ -211,7 +211,7 @@ public void testManyColumns() throws Exception { EasyMock.replay(lastTabletMeta, compactions); - Set removedCompactions = new HashSet<>(); + Set orphanedCompactions = new HashSet<>(); testMerge(List.of(tablet1, tablet2, lastTabletMeta), tableId, null, null, tabletMutator -> { EasyMock.expect(tabletMutator.putTime(MetadataTime.parse("L30"))).andReturn(tabletMutator) @@ -251,13 +251,13 @@ public void testManyColumns() throws Exception { .andReturn(tabletMutator).once(); EasyMock.expect(tabletMutator.deleteMigration()).andReturn(tabletMutator); - }, removedCompactions::add); + }, orphanedCompactions::add); EasyMock.verify(lastTabletMeta, compactions); - assertEquals(Set.of(new Ample.RemovedCompaction(cid1, tableId, "td3"), - new Ample.RemovedCompaction(cid2, tableId, "td3"), - new Ample.RemovedCompaction(cid3, tableId, "td3")), removedCompactions); + assertEquals(Set.of(new Ample.OrphanedCompaction(cid1, tableId, "td3"), + new Ample.OrphanedCompaction(cid2, tableId, "td3"), + new Ample.OrphanedCompaction(cid3, tableId, "td3")), orphanedCompactions); } @Test @@ -437,7 +437,7 @@ private static void testMerge(List inputTablets, TableId tableId private static void testMerge(List inputTablets, TableId tableId, String start, String end, Consumer expectationsSetter, - Consumer removedCompactionConsumer) throws Exception { + Consumer orphanedCompactionConsumer) throws Exception { MergeInfo mergeInfo = new MergeInfo(tableId, NamespaceId.of("1"), start == null ? null : start.getBytes(UTF_8), end == null ? null : end.getBytes(UTF_8), MergeInfo.Operation.MERGE); @@ -452,24 +452,23 @@ private static void testMerge(List inputTablets, TableId tableId EasyMock.mock(ConditionalTabletsMutatorImpl.class); ConditionalTabletMutatorImpl tabletMutator = EasyMock.mock(ConditionalTabletMutatorImpl.class); - Ample.RemovedCompactionStore removedCompactionStore = new Ample.RemovedCompactionStore() { + Ample.OrphanedCompactionStore orphanedCompactionStore = new Ample.OrphanedCompactionStore() { @Override - public Stream list() { + public Stream list() { throw new UnsupportedOperationException(); } @Override - public void add(Collection removedCompactions) { - System.out.println("removedCompactions : " + removedCompactions); - removedCompactions.forEach(removedCompactionConsumer); + public void add(Collection removedCompactions) { + removedCompactions.forEach(orphanedCompactionConsumer); } @Override - public void delete(Collection removedCompactions) { + public void delete(Collection removedCompactions) { throw new UnsupportedOperationException(); } }; - EasyMock.expect(ample.removedCompactions()).andReturn(removedCompactionStore); + EasyMock.expect(ample.orphanedCompactions()).andReturn(orphanedCompactionStore); ServiceLock managerLock = EasyMock.mock(ServiceLock.class); EasyMock.expect(context.getServiceLock()).andReturn(managerLock).anyTimes(); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java index ad0c77b03f0..13ad3bde70e 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java @@ -253,24 +253,24 @@ public void testManyColumns() throws Exception { EasyMock.expect(fateEnv.getFileRangeCache()).andReturn(fileRangeCache).atLeastOnce(); EasyMock.expect(fateEnv.getSteadyTime()).andReturn(SteadyTime.from(100_000, TimeUnit.SECONDS)) .atLeastOnce(); - Set removedCompactionSet = new HashSet<>(); - Ample.RemovedCompactionStore rcs = new Ample.RemovedCompactionStore() { + Set orphanedCompactionSet = new HashSet<>(); + Ample.OrphanedCompactionStore ocs = new Ample.OrphanedCompactionStore() { @Override - public Stream list() { + public Stream list() { throw new UnsupportedOperationException(); } @Override - public void add(Collection removedCompactions) { - removedCompactionSet.addAll(removedCompactions); + public void add(Collection removedCompactions) { + orphanedCompactionSet.addAll(removedCompactions); } @Override - public void delete(Collection removedCompactions) { + public void delete(Collection removedCompactions) { throw new UnsupportedOperationException(); } }; - EasyMock.expect(ample.removedCompactions()).andReturn(rcs).atLeastOnce(); + EasyMock.expect(ample.orphanedCompactions()).andReturn(ocs).atLeastOnce(); ServiceLock managerLock = EasyMock.mock(ServiceLock.class); EasyMock.expect(context.getServiceLock()).andReturn(managerLock).anyTimes(); @@ -431,9 +431,9 @@ public void delete(Collection removedCompactions) { EasyMock.verify(fateEnv, context, ample, tabletMeta, fileRangeCache, tabletsMutator, tablet1Mutator, tablet2Mutator, tablet3Mutator, cr, compactions); - assertEquals(Set.of(new Ample.RemovedCompaction(cid1, tableId, "td1"), - new Ample.RemovedCompaction(cid2, tableId, "td1"), - new Ample.RemovedCompaction(cid3, tableId, "td1")), removedCompactionSet); + assertEquals(Set.of(new Ample.OrphanedCompaction(cid1, tableId, "td1"), + new Ample.OrphanedCompaction(cid2, tableId, "td1"), + new Ample.OrphanedCompaction(cid3, tableId, "td1")), orphanedCompactionSet); } @Test From 55f50d87cf444d34365247ee9789c56739a535a6 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 8 Apr 2026 17:49:44 +0000 Subject: [PATCH 55/65] remove import --- .../accumulo/server/metadata/OrphanedCompactionStoreImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/OrphanedCompactionStoreImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/OrphanedCompactionStoreImpl.java index 0226c000f4f..4fe09966b90 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/OrphanedCompactionStoreImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/OrphanedCompactionStoreImpl.java @@ -29,7 +29,6 @@ import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.MetadataSchema.OrphanedCompactionSection; import org.apache.accumulo.core.security.Authorizations; From a7503c9d2e136617904d016531faf4d45b4bb874 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 8 Apr 2026 19:29:35 +0000 Subject: [PATCH 56/65] improve bulknewIT --- .../accumulo/test/functional/BulkNewIT.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java index d308404c919..1ba19ca08fa 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java @@ -123,6 +123,8 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -154,7 +156,7 @@ private static class Callback implements MiniClusterConfigurationCallback { @Override public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration conf) { cfg.setMemory(ServerType.TABLET_SERVER, 512, MemoryUnit.MEGABYTE); - + cfg.getClusterServerConfiguration().setNumDefaultCompactors(4); // use raw local file system conf.set("fs.file.impl", RawLocalFileSystem.class.getName()); } @@ -993,15 +995,9 @@ public void testManyFiles() throws Exception { } } - @Test - public void testConcurrentCompactions() throws Exception { - // run test with bulk imports happening in parallel - testConcurrentCompactions(true); - // run the test with bulk imports happening serially - testConcurrentCompactions(false); - } - - private void testConcurrentCompactions(boolean parallelBulkImports) throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testConcurrentCompactions(boolean parallelBulkImports) throws Exception { // Tests compactions running concurrently with bulk import to ensure that data is not bulk // imported twice. Doing a large number of bulk imports should naturally cause compactions to // happen. This test ensures that compactions running concurrently with bulk import does not @@ -1087,6 +1083,10 @@ private void testConcurrentCompactions(boolean parallelBulkImports) throws Excep // expect to see 10 tablets assertEquals(10, rowCounts.size()); } + + // since this is a parameterized test it will use the same tableName, so delete for the next + // parameterized iteration + c.tableOperations().delete(tableName); } } From d9f90460604f4e05bfee69457177799c94587f6c Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 8 Apr 2026 22:03:47 +0000 Subject: [PATCH 57/65] maybe fix metrics --- .../src/main/java/org/apache/accumulo/manager/Manager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 6486451dc29..5a9438f9874 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -904,6 +904,8 @@ private void checkForHeldServer(SortedMap ts private void setupAssistantMetrics(MetricsProducer... producers) { MetricsInfo metricsInfo = getContext().getMetricsInfo(); metricsInfo.addMetricsProducers(producers); + // TODO should tests compaction metrics from multiple managers + metricsInfo.addMetricsProducers(requireNonNull(compactionCoordinator)); metricsInfo.init(MetricsInfo.serviceTags(getContext().getInstanceName(), getApplicationName(), getAdvertiseAddress(), getResourceGroup())); } @@ -915,7 +917,6 @@ private void setupPrimaryMetrics() { // ensure all tablet group watchers are setup Preconditions.checkState(watchers.size() == DataLevel.values().length); watchers.forEach(watcher -> metricsInfo.addMetricsProducers(watcher.getMetrics())); - metricsInfo.addMetricsProducers(requireNonNull(compactionCoordinator)); // ensure fate is completely setup metricsInfo.addMetricsProducers(new MetaFateMetrics(getContext(), getConfiguration().getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL))); From 73f87d04299262f2c3355aeed1862b534599e560 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Thu, 9 Apr 2026 15:35:11 +0000 Subject: [PATCH 58/65] add TODO --- .../org/apache/accumulo/monitor/next/InformationFetcher.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java index 1f5f9151147..bebb85a0c8c 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java @@ -280,6 +280,7 @@ public void run() { final List> futures = new ArrayList<>(); final SystemInformation summary = new SystemInformation(allMetrics, this.ctx); Set managers = this.ctx.instanceOperations().getServers(ServerId.Type.MANAGER); + // TODO this assumes a single coordinator HostAndPort coordinatorHost = null; if (!managers.isEmpty()) { ServerId manager = managers.iterator().next(); From 3211d11122836f6f2f2b5a9a4b533f51073c7238 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 10 Apr 2026 15:35:51 +0000 Subject: [PATCH 59/65] remove done TODO --- .../apache/accumulo/core/client/admin/InstanceOperations.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java index b679edc9efe..72d602704c0 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/InstanceOperations.java @@ -247,7 +247,6 @@ Map modifyProperties(Consumer> mapMutator) * @return set of servers of the supplied type * @since 4.0.0 */ - // TODO this needs to change for multiple managers Set getServers(ServerId.Type type); /** From d9b6a698250c06e7b0ace861d3b6b7a68e993f1b Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 10 Apr 2026 16:34:29 +0000 Subject: [PATCH 60/65] Adds monitor endpoint to show manager responsibilites Needed for #6190 --- .../accumulo/monitor/next/Endpoints.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index 9a809e9d3ca..4d428573631 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -23,7 +23,9 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -46,6 +48,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.metrics.flatbuffers.FMetric; import org.apache.accumulo.core.process.thrift.MetricResponse; +import org.apache.accumulo.core.rpc.clients.ManagerClient; import org.apache.accumulo.core.util.compaction.RunningCompactionInfo; import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.monitor.next.InformationFetcher.InstanceSummary; @@ -55,6 +58,7 @@ import org.apache.accumulo.monitor.next.ec.CompactorsSummary; import org.apache.accumulo.monitor.next.ec.CoordinatorSummary; import org.apache.accumulo.monitor.next.sservers.ScanServerView; +import org.apache.accumulo.server.manager.FateLocations; import io.micrometer.core.instrument.Meter.Id; import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; @@ -168,6 +172,30 @@ public List getManagerMetrics() { return List.of(); } + @GET + @Path("manager/responsibilities") + @Produces(MediaType.APPLICATION_JSON) + @Description("Returns each managers responsibilities") + public Map> getManagerResponsibilities() { + var fateLocs = new FateLocations(monitor.getContext()); + Map> responsibilities = new HashMap<>(); + fateLocs.getLocations().forEach((addr, partitions) -> { + partitions.forEach(partition -> { + responsibilities.computeIfAbsent(addr.toString(), a -> new ArrayList<>()) + .add(partition.toString()); + }); + }); + + var primary = ManagerClient.getPrimaryManagerLocation(monitor.getContext()); + if (primary != null) { + responsibilities.computeIfAbsent(primary, a -> new ArrayList<>()) + .addAll(List.of("TABLET_MANAGEMENT", "BALANCING", "CLIENT_RPC", "TSERVER_MONITORING", + "CLUSTER_MAINTENANCE", "COMPACTION_COORDINATION")); + } + + return responsibilities; + } + @GET @Path("gc") @Produces(MediaType.APPLICATION_JSON) From ac5a37b57d0cfb4474f9d793a74be0e111658cb5 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 10 Apr 2026 17:13:22 +0000 Subject: [PATCH 61/65] adds compactions to manager endpoint --- .../accumulo/monitor/next/Endpoints.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index 4d428573631..cecb6f9d33c 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -63,6 +64,10 @@ import io.micrometer.core.instrument.Meter.Id; import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; + @Path("/") public class Endpoints { /** @@ -167,7 +172,7 @@ public MetricResponse getManager() { public List getManagerMetrics() { var managerMetrics = getManager().getMetrics(); if (managerMetrics != null) { - return managerMetrics.stream().map(FMetric::getRootAsFMetric).collect(Collectors.toList()); + return managerMetrics.stream().map(FMetric::getRootAsFMetric).collect(toList()); } return List.of(); } @@ -190,9 +195,15 @@ public Map> getManagerResponsibilities() { if (primary != null) { responsibilities.computeIfAbsent(primary, a -> new ArrayList<>()) .addAll(List.of("TABLET_MANAGEMENT", "BALANCING", "CLIENT_RPC", "TSERVER_MONITORING", - "CLUSTER_MAINTENANCE", "COMPACTION_COORDINATION")); + "CLUSTER_MAINTENANCE")); } + monitor.getContext().getCoordinatorLocations(true).locations().entrySet().stream() + .collect(groupingBy(Entry::getValue, mapping(Entry::getKey, toList()))).forEach((addr, groups) -> { + responsibilities.computeIfAbsent(addr.toString(), a -> new ArrayList<>()) + .add("COMPACTOR_GROUPS:" + groups); + }); + return responsibilities; } @@ -361,7 +372,7 @@ public List getCompactions() { monitor.getInformationFetcher().getSummaryForEndpoint().getTopRunningCompactions(); return longRunning.values().stream().flatMap(TimeOrderedRunningCompactionSet::stream).distinct() .sorted(TimeOrderedRunningCompactionSet.OLDEST_FIRST_COMPARATOR) - .collect(Collectors.toList()); + .collect(toList()); } @GET @@ -376,7 +387,7 @@ public List getCompactions() { if (longRunning == null) { return List.of(); } - return longRunning.stream().collect(Collectors.toList()); + return longRunning.stream().collect(toList()); } @GET From f505d0403dedba7eb440823a44106ce4a2925725 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 10 Apr 2026 17:26:18 +0000 Subject: [PATCH 62/65] fixes accumulo-cluster start --managers --- assemble/bin/accumulo-cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster index 085ad3185e7..0b3126ce01e 100755 --- a/assemble/bin/accumulo-cluster +++ b/assemble/bin/accumulo-cluster @@ -125,7 +125,7 @@ function parse_args() { exit 2 fi - if ! PARSE_OUTPUT=$(getopt -o "" --long "dry-run,local,manager,gc,monitor,tservers::,sservers::,compactors::" -n 'accumulo-cluster' -- "$@"); then + if ! PARSE_OUTPUT=$(getopt -o "" --long "dry-run,local,managers,gc,monitor,tservers::,sservers::,compactors::" -n 'accumulo-cluster' -- "$@"); then print_usage exit 1 fi From b8af2df649e31e5c8369546020d3da1daf400821 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Tue, 14 Apr 2026 22:11:11 +0000 Subject: [PATCH 63/65] server/monitor/src/ --- .../apache/accumulo/monitor/next/Endpoints.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index cecb6f9d33c..98625161da4 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -18,6 +18,10 @@ */ package org.apache.accumulo.monitor.next; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -64,10 +68,6 @@ import io.micrometer.core.instrument.Meter.Id; import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.mapping; -import static java.util.stream.Collectors.toList; - @Path("/") public class Endpoints { /** @@ -199,7 +199,8 @@ public Map> getManagerResponsibilities() { } monitor.getContext().getCoordinatorLocations(true).locations().entrySet().stream() - .collect(groupingBy(Entry::getValue, mapping(Entry::getKey, toList()))).forEach((addr, groups) -> { + .collect(groupingBy(Entry::getValue, mapping(Entry::getKey, toList()))) + .forEach((addr, groups) -> { responsibilities.computeIfAbsent(addr.toString(), a -> new ArrayList<>()) .add("COMPACTOR_GROUPS:" + groups); }); @@ -371,8 +372,7 @@ public List getCompactions() { Map longRunning = monitor.getInformationFetcher().getSummaryForEndpoint().getTopRunningCompactions(); return longRunning.values().stream().flatMap(TimeOrderedRunningCompactionSet::stream).distinct() - .sorted(TimeOrderedRunningCompactionSet.OLDEST_FIRST_COMPARATOR) - .collect(toList()); + .sorted(TimeOrderedRunningCompactionSet.OLDEST_FIRST_COMPARATOR).collect(toList()); } @GET From 2961d0fa8929c8e4faf3bf54064688f7e37bd402 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Wed, 15 Apr 2026 17:25:46 +0000 Subject: [PATCH 64/65] revert changes merged in from #6309 --- .../accumulo/monitor/next/Endpoints.java | 47 ++----------------- 1 file changed, 4 insertions(+), 43 deletions(-) diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java index 8256047ae64..e42e230183f 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java @@ -18,21 +18,14 @@ */ package org.apache.accumulo.monitor.next; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.mapping; -import static java.util.stream.Collectors.toList; - import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.reflect.Method; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -54,7 +47,6 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.metrics.flatbuffers.FMetric; import org.apache.accumulo.core.process.thrift.MetricResponse; -import org.apache.accumulo.core.rpc.clients.ManagerClient; import org.apache.accumulo.core.util.compaction.RunningCompactionInfo; import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.monitor.next.InformationFetcher.InstanceSummary; @@ -64,7 +56,6 @@ import org.apache.accumulo.monitor.next.ec.CompactorsSummary; import org.apache.accumulo.monitor.next.ec.CoordinatorSummary; import org.apache.accumulo.monitor.next.views.ServersView; -import org.apache.accumulo.server.manager.FateLocations; import io.micrometer.core.instrument.Meter.Id; import io.micrometer.core.instrument.cumulative.CumulativeDistributionSummary; @@ -178,42 +169,11 @@ public MetricResponse getManager() { public List getManagerMetrics() { var managerMetrics = getManager().getMetrics(); if (managerMetrics != null) { - return managerMetrics.stream().map(FMetric::getRootAsFMetric).collect(toList()); + return managerMetrics.stream().map(FMetric::getRootAsFMetric).collect(Collectors.toList()); } return List.of(); } - @GET - @Path("manager/responsibilities") - @Produces(MediaType.APPLICATION_JSON) - @Description("Returns each managers responsibilities") - public Map> getManagerResponsibilities() { - var fateLocs = new FateLocations(monitor.getContext()); - Map> responsibilities = new HashMap<>(); - fateLocs.getLocations().forEach((addr, partitions) -> { - partitions.forEach(partition -> { - responsibilities.computeIfAbsent(addr.toString(), a -> new ArrayList<>()) - .add(partition.toString()); - }); - }); - - var primary = ManagerClient.getPrimaryManagerLocation(monitor.getContext()); - if (primary != null) { - responsibilities.computeIfAbsent(primary, a -> new ArrayList<>()) - .addAll(List.of("TABLET_MANAGEMENT", "BALANCING", "CLIENT_RPC", "TSERVER_MONITORING", - "CLUSTER_MAINTENANCE")); - } - - monitor.getContext().getCoordinatorLocations(true).locations().entrySet().stream() - .collect(groupingBy(Entry::getValue, mapping(Entry::getKey, toList()))) - .forEach((addr, groups) -> { - responsibilities.computeIfAbsent(addr.toString(), a -> new ArrayList<>()) - .add("COMPACTOR_GROUPS:" + groups); - }); - - return responsibilities; - } - @GET @Path("gc") @Produces(MediaType.APPLICATION_JSON) @@ -384,7 +344,8 @@ public List getCompactions() { Map longRunning = monitor.getInformationFetcher().getSummaryForEndpoint().getTopRunningCompactions(); return longRunning.values().stream().flatMap(TimeOrderedRunningCompactionSet::stream).distinct() - .sorted(TimeOrderedRunningCompactionSet.OLDEST_FIRST_COMPARATOR).collect(toList()); + .sorted(TimeOrderedRunningCompactionSet.OLDEST_FIRST_COMPARATOR) + .collect(Collectors.toList()); } @GET @@ -399,7 +360,7 @@ public List getCompactions() { if (longRunning == null) { return List.of(); } - return longRunning.stream().collect(toList()); + return longRunning.stream().collect(Collectors.toList()); } @GET From c9cfcd6a87ddc2af6876e5c1884728c7248b9c72 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Fri, 17 Apr 2026 20:14:47 +0000 Subject: [PATCH 65/65] Adds overview documentation --- .../compaction/CompactionJobClient.java | 3 ++ .../coordinator/CompactionCoordinator.java | 42 +++++++++++++++++++ .../coordinator/CoordinatorManager.java | 7 +++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java index 4451bc68c9b..3b9ee60b1d6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionJobClient.java @@ -44,6 +44,9 @@ /** * Sends compaction jobs to remote compaction coordinators. + * + * @see org.apache.accumulo.manager.compaction.coordinator.CompactionCoordinator for a general + * overview */ public class CompactionJobClient implements AutoCloseable { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 5cb132c7383..a7a6aee3634 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -132,6 +132,48 @@ import io.micrometer.core.instrument.MeterRegistry; +/** + * This class is the focal point in a distributed system for coordinating compactions. It has the + * following functionality. + *

    + *
  • Maintains priority queues of compaction jos for some subset of the compactor groups
    • + *
  • Accepts assignments of compactor groups over RPC
    • + *
  • Handles RPC request to add compaction jobs to a queue. Only accepts the request if its + * assigned that compactor group.
    • + *
  • Handles RPC request to pull a compaction job of a queue.
    • + *
  • Handles RPC request to reserve a compaction job in the metadata table.
    • + *
  • Handles RPC request to complete a compaction job. This initiates a fate operation to commit + * the job.
    • + *
  • Handles RPC request to fail a compaction job which updates the metadata table.
    • + *
+ * + *

+ * This class is part of a larger distributed system and overview of that larger system is given + * here. Other parts of the code point to this overview. The following are the pieces of the system + * and how they work together. + *

+ * + *
    + *
  • An instance of this class runs in every manager process and accepts RPC request.
    • + *
  • {@link CoordinatorManager} runs in the primary manager. It examines Accumulo configuration + * and finds the set of compactor resource groups. It then evenly assigns those compactor resource + * groups to instances of this class running in different manager processes. After it has made + * assignments it uses {@link org.apache.accumulo.server.compaction.CoordinatorLocationsFactory} to + * store those in zookeeper.
    • + *
  • {@link org.apache.accumulo.manager.TabletGroupWatcher}
    • analyzes tablets and generate + * compaction jobs for tablet that need to compact. When it finds a job is uses + * {@link org.apache.accumulo.manager.compaction.CompactionJobClient} to send them to an instance of + * this class. The {@link org.apache.accumulo.manager.compaction.CompactionJobClient} uses the + * {@link org.apache.accumulo.server.compaction.CoordinatorLocationsFactory} to find remote + * coordinators to send jobs to. + *
  • Compactors processes use the + * {@link org.apache.accumulo.server.compaction.CoordinatorLocationsFactory} to find the coordinator + * that has their queue. They make RPC request to that specific coordinator to pull jobs from their + * queue. Compactors use any coordinator to reserve, complete, and fail jobs in order to spread + * metadata and fate table load across all managers.
    • + *
+ * + */ public class CompactionCoordinator implements CompactionCoordinatorService.Iface, Runnable, MetricsProducer { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java index eb6117b842d..7677df01c22 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CoordinatorManager.java @@ -59,8 +59,11 @@ import com.google.common.net.HostAndPort; /** - * Handles assigning compactor resource groups to different manager processes. This is run by the - * primary manager to distribute compaction coordination. + * Handles assigning compactor resource groups to different manager processes. Also runs periodic + * maintenance task like detecting dead compactions and cleaning empty compactor entries out of + * zookeeper. This is run by the primary manager to distribute compaction coordination. + * + * @see CompactionCoordinator for a general overview */ public class CoordinatorManager {