diff --git a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java index b11285c0e34..149c12e12be 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java @@ -340,5 +340,4 @@ private static Optional findCompactorRunningCompaction(ClientContex throw new IllegalStateException("Exception calling cancel compaction for " + ecid, e); } } - } diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java index cf40536be89..4f446373ac9 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactionCoordinatorService.java @@ -37,8 +37,6 @@ public interface Iface { public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactor) throws org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException, org.apache.accumulo.core.clientImpl.thrift.ThriftNotActiveServiceException, org.apache.thrift.TException; - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException; - } public interface AsyncIface { @@ -51,8 +49,6 @@ public interface AsyncIface { public void compactionFailed(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.accumulo.core.dataImpl.thrift.TKeyExtent extent, java.lang.String exceptionClassName, TCompactionState failureState, java.lang.String groupName, java.lang.String compactor, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -209,21 +205,6 @@ public void recv_compactionFailed() throws org.apache.accumulo.core.clientImpl.t return; } - @Override - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException - { - send_recordCompletion(tinfo, credentials, externalCompactionId); - } - - public void send_recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId) throws org.apache.thrift.TException - { - recordCompletion_args args = new recordCompletion_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.setExternalCompactionId(externalCompactionId); - sendBaseOneway("recordCompletion", args); - } - } public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -449,47 +430,6 @@ public Void getResult() throws org.apache.accumulo.core.clientImpl.thrift.Thrift } } - @Override - public void recordCompletion(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - checkReady(); - recordCompletion_call method_call = new recordCompletion_call(tinfo, credentials, externalCompactionId, resultHandler, this, ___protocolFactory, ___transport); - this.___currentMethod = method_call; - ___manager.call(method_call); - } - - public static class recordCompletion_call extends org.apache.thrift.async.TAsyncMethodCall { - private org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; - private org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; - private java.lang.String externalCompactionId; - public recordCompletion_call(org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, java.lang.String externalCompactionId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { - super(client, protocolFactory, transport, resultHandler, true); - this.tinfo = tinfo; - this.credentials = credentials; - this.externalCompactionId = externalCompactionId; - } - - @Override - public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { - prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("recordCompletion", org.apache.thrift.protocol.TMessageType.ONEWAY, 0)); - recordCompletion_args args = new recordCompletion_args(); - args.setTinfo(tinfo); - args.setCredentials(credentials); - args.setExternalCompactionId(externalCompactionId); - args.write(prot); - prot.writeMessageEnd(); - } - - @Override - public Void getResult() throws org.apache.thrift.TException { - if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { - throw new java.lang.IllegalStateException("Method call not finished!"); - } - org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); - org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); - return null; - } - } - } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -507,7 +447,6 @@ protected Processor(I iface, java.util.Map extends org.apache.thrift.ProcessFunction { - public recordCompletion() { - super("recordCompletion"); - } - - @Override - public recordCompletion_args getEmptyArgsInstance() { - return new recordCompletion_args(); - } - - @Override - protected boolean isOneway() { - return true; - } - - @Override - protected boolean rethrowUnhandledExceptions() { - return false; - } - - @Override - public org.apache.thrift.TBase getResult(I iface, recordCompletion_args args) throws org.apache.thrift.TException { - iface.recordCompletion(args.tinfo, args.credentials, args.externalCompactionId); - return null; - } - } - } public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { @@ -691,7 +603,6 @@ protected AsyncProcessor(I iface, java.util.Map extends org.apache.thrift.AsyncProcessFunction { - public recordCompletion() { - super("recordCompletion"); - } - - @Override - public recordCompletion_args getEmptyArgsInstance() { - return new recordCompletion_args(); - } - - @Override - public org.apache.thrift.async.AsyncMethodCallback getResultHandler(final org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer fb, final int seqid) { - final org.apache.thrift.AsyncProcessFunction fcall = this; - return new org.apache.thrift.async.AsyncMethodCallback() { - @Override - public void onComplete(Void o) { - } - @Override - public void onError(java.lang.Exception e) { - if (e instanceof org.apache.thrift.transport.TTransportException) { - _LOGGER.error("TTransportException inside handler", e); - fb.close(); - } else { - _LOGGER.error("Exception inside oneway handler", e); - } - } - }; - } - - @Override - protected boolean isOneway() { - return true; - } - - @Override - public void start(I iface, recordCompletion_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.recordCompletion(args.tinfo, args.credentials, args.externalCompactionId,resultHandler); - } - } - } @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) @@ -6884,604 +6755,5 @@ private static S scheme(org.apache. } } - @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) - public static class recordCompletion_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("recordCompletion_args"); - - private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1); - private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2); - private static final org.apache.thrift.protocol.TField EXTERNAL_COMPACTION_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("externalCompactionId", org.apache.thrift.protocol.TType.STRING, (short)3); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new recordCompletion_argsStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new recordCompletion_argsTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo; // required - public @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials; // required - public @org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - TINFO((short)1, "tinfo"), - CREDENTIALS((short)2, "credentials"), - EXTERNAL_COMPACTION_ID((short)3, "externalCompactionId"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // TINFO - return TINFO; - case 2: // CREDENTIALS - return CREDENTIALS; - case 3: // EXTERNAL_COMPACTION_ID - return EXTERNAL_COMPACTION_ID; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.clientImpl.thrift.TInfo.class))); - tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.securityImpl.thrift.TCredentials.class))); - tmpMap.put(_Fields.EXTERNAL_COMPACTION_ID, new org.apache.thrift.meta_data.FieldMetaData("externalCompactionId", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(recordCompletion_args.class, metaDataMap); - } - - public recordCompletion_args() { - } - - public recordCompletion_args( - org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo, - org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials, - java.lang.String externalCompactionId) - { - this(); - this.tinfo = tinfo; - this.credentials = credentials; - this.externalCompactionId = externalCompactionId; - } - - /** - * Performs a deep copy on other. - */ - public recordCompletion_args(recordCompletion_args other) { - if (other.isSetTinfo()) { - this.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(other.tinfo); - } - if (other.isSetCredentials()) { - this.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(other.credentials); - } - if (other.isSetExternalCompactionId()) { - this.externalCompactionId = other.externalCompactionId; - } - } - - @Override - public recordCompletion_args deepCopy() { - return new recordCompletion_args(this); - } - - @Override - public void clear() { - this.tinfo = null; - this.credentials = null; - this.externalCompactionId = null; - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.clientImpl.thrift.TInfo getTinfo() { - return this.tinfo; - } - - public recordCompletion_args setTinfo(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.clientImpl.thrift.TInfo tinfo) { - this.tinfo = tinfo; - return this; - } - - public void unsetTinfo() { - this.tinfo = null; - } - - /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */ - public boolean isSetTinfo() { - return this.tinfo != null; - } - - public void setTinfoIsSet(boolean value) { - if (!value) { - this.tinfo = null; - } - } - - @org.apache.thrift.annotation.Nullable - public org.apache.accumulo.core.securityImpl.thrift.TCredentials getCredentials() { - return this.credentials; - } - - public recordCompletion_args setCredentials(@org.apache.thrift.annotation.Nullable org.apache.accumulo.core.securityImpl.thrift.TCredentials credentials) { - this.credentials = credentials; - return this; - } - - public void unsetCredentials() { - this.credentials = null; - } - - /** Returns true if field credentials is set (has been assigned a value) and false otherwise */ - public boolean isSetCredentials() { - return this.credentials != null; - } - - public void setCredentialsIsSet(boolean value) { - if (!value) { - this.credentials = null; - } - } - - @org.apache.thrift.annotation.Nullable - public java.lang.String getExternalCompactionId() { - return this.externalCompactionId; - } - - public recordCompletion_args setExternalCompactionId(@org.apache.thrift.annotation.Nullable java.lang.String externalCompactionId) { - this.externalCompactionId = externalCompactionId; - return this; - } - - public void unsetExternalCompactionId() { - this.externalCompactionId = null; - } - - /** Returns true if field externalCompactionId is set (has been assigned a value) and false otherwise */ - public boolean isSetExternalCompactionId() { - return this.externalCompactionId != null; - } - - public void setExternalCompactionIdIsSet(boolean value) { - if (!value) { - this.externalCompactionId = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case TINFO: - if (value == null) { - unsetTinfo(); - } else { - setTinfo((org.apache.accumulo.core.clientImpl.thrift.TInfo)value); - } - break; - - case CREDENTIALS: - if (value == null) { - unsetCredentials(); - } else { - setCredentials((org.apache.accumulo.core.securityImpl.thrift.TCredentials)value); - } - break; - - case EXTERNAL_COMPACTION_ID: - if (value == null) { - unsetExternalCompactionId(); - } else { - setExternalCompactionId((java.lang.String)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case TINFO: - return getTinfo(); - - case CREDENTIALS: - return getCredentials(); - - case EXTERNAL_COMPACTION_ID: - return getExternalCompactionId(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case TINFO: - return isSetTinfo(); - case CREDENTIALS: - return isSetCredentials(); - case EXTERNAL_COMPACTION_ID: - return isSetExternalCompactionId(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof recordCompletion_args) - return this.equals((recordCompletion_args)that); - return false; - } - - public boolean equals(recordCompletion_args that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_tinfo = true && this.isSetTinfo(); - boolean that_present_tinfo = true && that.isSetTinfo(); - if (this_present_tinfo || that_present_tinfo) { - if (!(this_present_tinfo && that_present_tinfo)) - return false; - if (!this.tinfo.equals(that.tinfo)) - return false; - } - - boolean this_present_credentials = true && this.isSetCredentials(); - boolean that_present_credentials = true && that.isSetCredentials(); - if (this_present_credentials || that_present_credentials) { - if (!(this_present_credentials && that_present_credentials)) - return false; - if (!this.credentials.equals(that.credentials)) - return false; - } - - boolean this_present_externalCompactionId = true && this.isSetExternalCompactionId(); - boolean that_present_externalCompactionId = true && that.isSetExternalCompactionId(); - if (this_present_externalCompactionId || that_present_externalCompactionId) { - if (!(this_present_externalCompactionId && that_present_externalCompactionId)) - return false; - if (!this.externalCompactionId.equals(that.externalCompactionId)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetTinfo()) ? 131071 : 524287); - if (isSetTinfo()) - hashCode = hashCode * 8191 + tinfo.hashCode(); - - hashCode = hashCode * 8191 + ((isSetCredentials()) ? 131071 : 524287); - if (isSetCredentials()) - hashCode = hashCode * 8191 + credentials.hashCode(); - - hashCode = hashCode * 8191 + ((isSetExternalCompactionId()) ? 131071 : 524287); - if (isSetExternalCompactionId()) - hashCode = hashCode * 8191 + externalCompactionId.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(recordCompletion_args other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetTinfo(), other.isSetTinfo()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetTinfo()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, other.tinfo); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetCredentials(), other.isSetCredentials()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCredentials()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, other.credentials); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = java.lang.Boolean.compare(isSetExternalCompactionId(), other.isSetExternalCompactionId()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetExternalCompactionId()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.externalCompactionId, other.externalCompactionId); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("recordCompletion_args("); - boolean first = true; - - sb.append("tinfo:"); - if (this.tinfo == null) { - sb.append("null"); - } else { - sb.append(this.tinfo); - } - first = false; - if (!first) sb.append(", "); - sb.append("credentials:"); - if (this.credentials == null) { - sb.append("null"); - } else { - sb.append(this.credentials); - } - first = false; - if (!first) sb.append(", "); - sb.append("externalCompactionId:"); - if (this.externalCompactionId == null) { - sb.append("null"); - } else { - sb.append(this.externalCompactionId); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - if (tinfo != null) { - tinfo.validate(); - } - if (credentials != null) { - credentials.validate(); - } - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class recordCompletion_argsStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public recordCompletion_argsStandardScheme getScheme() { - return new recordCompletion_argsStandardScheme(); - } - } - - private static class recordCompletion_argsStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, recordCompletion_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // TINFO - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // CREDENTIALS - if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 3: // EXTERNAL_COMPACTION_ID - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.externalCompactionId = iprot.readString(); - struct.setExternalCompactionIdIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, recordCompletion_args struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.tinfo != null) { - oprot.writeFieldBegin(TINFO_FIELD_DESC); - struct.tinfo.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.credentials != null) { - oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC); - struct.credentials.write(oprot); - oprot.writeFieldEnd(); - } - if (struct.externalCompactionId != null) { - oprot.writeFieldBegin(EXTERNAL_COMPACTION_ID_FIELD_DESC); - oprot.writeString(struct.externalCompactionId); - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class recordCompletion_argsTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public recordCompletion_argsTupleScheme getScheme() { - return new recordCompletion_argsTupleScheme(); - } - } - - private static class recordCompletion_argsTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, recordCompletion_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetTinfo()) { - optionals.set(0); - } - if (struct.isSetCredentials()) { - optionals.set(1); - } - if (struct.isSetExternalCompactionId()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); - if (struct.isSetTinfo()) { - struct.tinfo.write(oprot); - } - if (struct.isSetCredentials()) { - struct.credentials.write(oprot); - } - if (struct.isSetExternalCompactionId()) { - oprot.writeString(struct.externalCompactionId); - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, recordCompletion_args struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(3); - if (incoming.get(0)) { - struct.tinfo = new org.apache.accumulo.core.clientImpl.thrift.TInfo(); - struct.tinfo.read(iprot); - struct.setTinfoIsSet(true); - } - if (incoming.get(1)) { - struct.credentials = new org.apache.accumulo.core.securityImpl.thrift.TCredentials(); - struct.credentials.read(iprot); - struct.setCredentialsIsSet(true); - } - if (incoming.get(2)) { - struct.externalCompactionId = iprot.readString(); - struct.setExternalCompactionIdIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - } - private static void unusedMethod() {} } diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java index d9d867d8d2d..c59d19d98f6 100644 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java +++ b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/CompactorService.java @@ -3668,14 +3668,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, getActiveCompaction case 0: // SUCCESS if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { { - org.apache.thrift.protocol.TList _list28 = iprot.readListBegin(); - struct.success = new java.util.ArrayList(_list28.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem29; - for (int _i30 = 0; _i30 < _list28.size; ++_i30) + org.apache.thrift.protocol.TList _list18 = iprot.readListBegin(); + struct.success = new java.util.ArrayList(_list18.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem19; + for (int _i20 = 0; _i20 < _list18.size; ++_i20) { - _elem29 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem29.read(iprot); - struct.success.add(_elem29); + _elem19 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem19.read(iprot); + struct.success.add(_elem19); } iprot.readListEnd(); } @@ -3713,9 +3713,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, getActiveCompactio oprot.writeFieldBegin(SUCCESS_FIELD_DESC); { oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.success.size())); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter31 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter21 : struct.success) { - _iter31.write(oprot); + _iter21.write(oprot); } oprot.writeListEnd(); } @@ -3755,9 +3755,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, getActiveCompaction if (struct.isSetSuccess()) { { oprot.writeI32(struct.success.size()); - for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter32 : struct.success) + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _iter22 : struct.success) { - _iter32.write(oprot); + _iter22.write(oprot); } } } @@ -3772,14 +3772,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, getActiveCompactions java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { { - org.apache.thrift.protocol.TList _list33 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); - struct.success = new java.util.ArrayList(_list33.size); - @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem34; - for (int _i35 = 0; _i35 < _list33.size; ++_i35) + org.apache.thrift.protocol.TList _list23 = iprot.readListBegin(org.apache.thrift.protocol.TType.STRUCT); + struct.success = new java.util.ArrayList(_list23.size); + @org.apache.thrift.annotation.Nullable org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction _elem24; + for (int _i25 = 0; _i25 < _list23.size; ++_i25) { - _elem34 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); - _elem34.read(iprot); - struct.success.add(_elem34); + _elem24 = new org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction(); + _elem24.read(iprot); + struct.success.add(_elem24); } } struct.setSuccessIsSet(true); diff --git a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java b/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java deleted file mode 100644 index a5b3d74a2b9..00000000000 --- a/core/src/main/thrift-gen-java/org/apache/accumulo/core/compaction/thrift/TExternalCompactionMap.java +++ /dev/null @@ -1,471 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Autogenerated by Thrift Compiler (0.17.0) - * - * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - * @generated - */ -package org.apache.accumulo.core.compaction.thrift; - -@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"}) -public class TExternalCompactionMap implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TExternalCompactionMap"); - - private static final org.apache.thrift.protocol.TField COMPACTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compactions", org.apache.thrift.protocol.TType.MAP, (short)1); - - private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new TExternalCompactionMapStandardSchemeFactory(); - private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new TExternalCompactionMapTupleSchemeFactory(); - - public @org.apache.thrift.annotation.Nullable java.util.Map compactions; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - COMPACTIONS((short)1, "compactions"); - - private static final java.util.Map byName = new java.util.HashMap(); - - static { - for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } - } - - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // COMPACTIONS - return COMPACTIONS; - default: - return null; - } - } - - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new java.lang.IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; - } - - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - @org.apache.thrift.annotation.Nullable - public static _Fields findByName(java.lang.String name) { - return byName.get(name); - } - - private final short _thriftId; - private final java.lang.String _fieldName; - - _Fields(short thriftId, java.lang.String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; - } - - @Override - public short getThriftFieldId() { - return _thriftId; - } - - @Override - public java.lang.String getFieldName() { - return _fieldName; - } - } - - // isset id assignments - public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.COMPACTIONS, new org.apache.thrift.meta_data.FieldMetaData("compactions", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), - new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TExternalCompaction.class)))); - metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TExternalCompactionMap.class, metaDataMap); - } - - public TExternalCompactionMap() { - } - - public TExternalCompactionMap( - java.util.Map compactions) - { - this(); - this.compactions = compactions; - } - - /** - * Performs a deep copy on other. - */ - public TExternalCompactionMap(TExternalCompactionMap other) { - if (other.isSetCompactions()) { - java.util.Map __this__compactions = new java.util.HashMap(other.compactions.size()); - for (java.util.Map.Entry other_element : other.compactions.entrySet()) { - - java.lang.String other_element_key = other_element.getKey(); - TExternalCompaction other_element_value = other_element.getValue(); - - java.lang.String __this__compactions_copy_key = other_element_key; - - TExternalCompaction __this__compactions_copy_value = new TExternalCompaction(other_element_value); - - __this__compactions.put(__this__compactions_copy_key, __this__compactions_copy_value); - } - this.compactions = __this__compactions; - } - } - - @Override - public TExternalCompactionMap deepCopy() { - return new TExternalCompactionMap(this); - } - - @Override - public void clear() { - this.compactions = null; - } - - public int getCompactionsSize() { - return (this.compactions == null) ? 0 : this.compactions.size(); - } - - public void putToCompactions(java.lang.String key, TExternalCompaction val) { - if (this.compactions == null) { - this.compactions = new java.util.HashMap(); - } - this.compactions.put(key, val); - } - - @org.apache.thrift.annotation.Nullable - public java.util.Map getCompactions() { - return this.compactions; - } - - public TExternalCompactionMap setCompactions(@org.apache.thrift.annotation.Nullable java.util.Map compactions) { - this.compactions = compactions; - return this; - } - - public void unsetCompactions() { - this.compactions = null; - } - - /** Returns true if field compactions is set (has been assigned a value) and false otherwise */ - public boolean isSetCompactions() { - return this.compactions != null; - } - - public void setCompactionsIsSet(boolean value) { - if (!value) { - this.compactions = null; - } - } - - @Override - public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { - switch (field) { - case COMPACTIONS: - if (value == null) { - unsetCompactions(); - } else { - setCompactions((java.util.Map)value); - } - break; - - } - } - - @org.apache.thrift.annotation.Nullable - @Override - public java.lang.Object getFieldValue(_Fields field) { - switch (field) { - case COMPACTIONS: - return getCompactions(); - - } - throw new java.lang.IllegalStateException(); - } - - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - @Override - public boolean isSet(_Fields field) { - if (field == null) { - throw new java.lang.IllegalArgumentException(); - } - - switch (field) { - case COMPACTIONS: - return isSetCompactions(); - } - throw new java.lang.IllegalStateException(); - } - - @Override - public boolean equals(java.lang.Object that) { - if (that instanceof TExternalCompactionMap) - return this.equals((TExternalCompactionMap)that); - return false; - } - - public boolean equals(TExternalCompactionMap that) { - if (that == null) - return false; - if (this == that) - return true; - - boolean this_present_compactions = true && this.isSetCompactions(); - boolean that_present_compactions = true && that.isSetCompactions(); - if (this_present_compactions || that_present_compactions) { - if (!(this_present_compactions && that_present_compactions)) - return false; - if (!this.compactions.equals(that.compactions)) - return false; - } - - return true; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode * 8191 + ((isSetCompactions()) ? 131071 : 524287); - if (isSetCompactions()) - hashCode = hashCode * 8191 + compactions.hashCode(); - - return hashCode; - } - - @Override - public int compareTo(TExternalCompactionMap other) { - if (!getClass().equals(other.getClass())) { - return getClass().getName().compareTo(other.getClass().getName()); - } - - int lastComparison = 0; - - lastComparison = java.lang.Boolean.compare(isSetCompactions(), other.isSetCompactions()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetCompactions()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.compactions, other.compactions); - if (lastComparison != 0) { - return lastComparison; - } - } - return 0; - } - - @org.apache.thrift.annotation.Nullable - @Override - public _Fields fieldForId(int fieldId) { - return _Fields.findByThriftId(fieldId); - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { - scheme(iprot).read(iprot, this); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { - scheme(oprot).write(oprot, this); - } - - @Override - public java.lang.String toString() { - java.lang.StringBuilder sb = new java.lang.StringBuilder("TExternalCompactionMap("); - boolean first = true; - - sb.append("compactions:"); - if (this.compactions == null) { - sb.append("null"); - } else { - sb.append(this.compactions); - } - first = false; - sb.append(")"); - return sb.toString(); - } - - public void validate() throws org.apache.thrift.TException { - // check for required fields - // check for sub-struct validity - } - - private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { - try { - write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, java.lang.ClassNotFoundException { - try { - read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); - } catch (org.apache.thrift.TException te) { - throw new java.io.IOException(te); - } - } - - private static class TExternalCompactionMapStandardSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public TExternalCompactionMapStandardScheme getScheme() { - return new TExternalCompactionMapStandardScheme(); - } - } - - private static class TExternalCompactionMapStandardScheme extends org.apache.thrift.scheme.StandardScheme { - - @Override - public void read(org.apache.thrift.protocol.TProtocol iprot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TField schemeField; - iprot.readStructBegin(); - while (true) - { - schemeField = iprot.readFieldBegin(); - if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { - break; - } - switch (schemeField.id) { - case 1: // COMPACTIONS - if (schemeField.type == org.apache.thrift.protocol.TType.MAP) { - { - org.apache.thrift.protocol.TMap _map18 = iprot.readMapBegin(); - struct.compactions = new java.util.HashMap(2*_map18.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key19; - @org.apache.thrift.annotation.Nullable TExternalCompaction _val20; - for (int _i21 = 0; _i21 < _map18.size; ++_i21) - { - _key19 = iprot.readString(); - _val20 = new TExternalCompaction(); - _val20.read(iprot); - struct.compactions.put(_key19, _val20); - } - iprot.readMapEnd(); - } - struct.setCompactionsIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - default: - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - iprot.readFieldEnd(); - } - iprot.readStructEnd(); - - // check for required fields of primitive type, which can't be checked in the validate method - struct.validate(); - } - - @Override - public void write(org.apache.thrift.protocol.TProtocol oprot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - struct.validate(); - - oprot.writeStructBegin(STRUCT_DESC); - if (struct.compactions != null) { - oprot.writeFieldBegin(COMPACTIONS_FIELD_DESC); - { - oprot.writeMapBegin(new org.apache.thrift.protocol.TMap(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT, struct.compactions.size())); - for (java.util.Map.Entry _iter22 : struct.compactions.entrySet()) - { - oprot.writeString(_iter22.getKey()); - _iter22.getValue().write(oprot); - } - oprot.writeMapEnd(); - } - oprot.writeFieldEnd(); - } - oprot.writeFieldStop(); - oprot.writeStructEnd(); - } - - } - - private static class TExternalCompactionMapTupleSchemeFactory implements org.apache.thrift.scheme.SchemeFactory { - @Override - public TExternalCompactionMapTupleScheme getScheme() { - return new TExternalCompactionMapTupleScheme(); - } - } - - private static class TExternalCompactionMapTupleScheme extends org.apache.thrift.scheme.TupleScheme { - - @Override - public void write(org.apache.thrift.protocol.TProtocol prot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet optionals = new java.util.BitSet(); - if (struct.isSetCompactions()) { - optionals.set(0); - } - oprot.writeBitSet(optionals, 1); - if (struct.isSetCompactions()) { - { - oprot.writeI32(struct.compactions.size()); - for (java.util.Map.Entry _iter23 : struct.compactions.entrySet()) - { - oprot.writeString(_iter23.getKey()); - _iter23.getValue().write(oprot); - } - } - } - } - - @Override - public void read(org.apache.thrift.protocol.TProtocol prot, TExternalCompactionMap struct) throws org.apache.thrift.TException { - org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; - java.util.BitSet incoming = iprot.readBitSet(1); - if (incoming.get(0)) { - { - org.apache.thrift.protocol.TMap _map24 = iprot.readMapBegin(org.apache.thrift.protocol.TType.STRING, org.apache.thrift.protocol.TType.STRUCT); - struct.compactions = new java.util.HashMap(2*_map24.size); - @org.apache.thrift.annotation.Nullable java.lang.String _key25; - @org.apache.thrift.annotation.Nullable TExternalCompaction _val26; - for (int _i27 = 0; _i27 < _map24.size; ++_i27) - { - _key25 = iprot.readString(); - _val26 = new TExternalCompaction(); - _val26.read(iprot); - struct.compactions.put(_key25, _val26); - } - } - struct.setCompactionsIsSet(true); - } - } - } - - private static S scheme(org.apache.thrift.protocol.TProtocol proto) { - return (org.apache.thrift.scheme.StandardScheme.class.equals(proto.getScheme()) ? STANDARD_SCHEME_FACTORY : TUPLE_SCHEME_FACTORY).getScheme(); - } - private static void unusedMethod() {} -} - diff --git a/core/src/main/thrift/compaction-coordinator.thrift b/core/src/main/thrift/compaction-coordinator.thrift index 29e4dde868d..8effd88aff5 100644 --- a/core/src/main/thrift/compaction-coordinator.thrift +++ b/core/src/main/thrift/compaction-coordinator.thrift @@ -60,10 +60,6 @@ struct TExternalCompactionList { 1:list compactions } -struct TExternalCompactionMap { - 1:map compactions -} - struct TNextCompactionJob { 1:tabletserver.TExternalCompactionJob job // The total number of compactors servicing the queue this job was requested for @@ -135,13 +131,6 @@ service CompactionCoordinatorService { 1:client.ThriftSecurityException sec 2:client.ThriftNotActiveServiceException tnase ) - - - oneway void recordCompletion( - 1:client.TInfo tinfo - 2:security.TCredentials credentials - 3:string externalCompactionId - ) } service CompactorService { diff --git a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java index 854f096ebd7..514a110f2af 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java +++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionPluginUtils.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -46,6 +47,7 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.AbstractId; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.RowRange; import org.apache.accumulo.core.data.TableId; @@ -62,9 +64,13 @@ import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.accumulo.core.spi.compaction.CompactionDispatcher; +import org.apache.accumulo.core.spi.compaction.CompactionPlanner; +import org.apache.accumulo.core.spi.compaction.CompactionServiceId; import org.apache.accumulo.core.summary.SummarizerFactory; import org.apache.accumulo.core.summary.SummaryCollection; import org.apache.accumulo.core.summary.SummaryReader; +import org.apache.accumulo.core.util.compaction.CompactionPlannerInitParams; +import org.apache.accumulo.core.util.compaction.CompactionServicesConfig; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.ServiceEnvironmentImpl; import org.apache.accumulo.server.tablets.TabletNameGenerator; @@ -345,4 +351,33 @@ public ServiceEnvironment getServiceEnv() { return dispatcher; } + + /** + * Inspect configuration and determines what resource groups are configured for compaction. + */ + public static Set getConfiguredCompactionResourceGroups(ServerContext ctx) + throws ReflectiveOperationException { + + Set groups = new HashSet<>(); + AccumuloConfiguration config = ctx.getConfiguration(); + CompactionServicesConfig servicesConfig = new CompactionServicesConfig(config); + + for (var entry : servicesConfig.getPlanners().entrySet()) { + String serviceId = entry.getKey(); + String plannerClassName = entry.getValue(); + + Class plannerClass = + Class.forName(plannerClassName).asSubclass(CompactionPlanner.class); + CompactionPlanner planner = plannerClass.getDeclaredConstructor().newInstance(); + + var initParams = new CompactionPlannerInitParams(CompactionServiceId.of(serviceId), + servicesConfig.getPlannerPrefix(serviceId), servicesConfig.getOptions().get(serviceId), + new ServiceEnvironmentImpl(ctx)); + + planner.init(initParams); + + initParams.getRequestedGroups().stream().map(AbstractId::canonical).forEach(groups::add); + } + return groups; + } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 5ff232765c4..60e09f3dd89 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -107,7 +107,6 @@ import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metrics.MetricsInfo; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.trace.TraceUtil; @@ -609,11 +608,6 @@ public CompactionCoordinator getCompactionCoordinator() { return compactionCoordinator; } - @Override - public void recordCompactionCompletion(ExternalCompactionId ecid) { - getCompactionCoordinator().recordCompletion(ecid); - } - public void hostOndemand(List extents) { extents.forEach(e -> Preconditions.checkArgument(DataLevel.of(e.tableId()) == DataLevel.USER)); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 0ff67d48501..4da7b76fa51 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -27,7 +27,6 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED; -import static org.apache.accumulo.core.util.threads.ThreadPoolNames.COMPACTOR_RUNNING_COMPACTIONS_POOL; import java.io.FileNotFoundException; import java.io.IOException; @@ -62,16 +61,13 @@ import org.apache.accumulo.core.client.TableDeletedException; import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.compaction.CompactableFile; -import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode; import org.apache.accumulo.core.clientImpl.thrift.TInfo; import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.compaction.thrift.TCompactionState; import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate; -import org.apache.accumulo.core.compaction.thrift.TExternalCompaction; import org.apache.accumulo.core.compaction.thrift.TNextCompactionJob; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ResourceGroupId; import org.apache.accumulo.core.data.TableId; @@ -104,16 +100,12 @@ import org.apache.accumulo.core.securityImpl.thrift.TCredentials; import org.apache.accumulo.core.spi.compaction.CompactionJob; import org.apache.accumulo.core.spi.compaction.CompactionKind; -import org.apache.accumulo.core.spi.compaction.CompactionPlanner; -import org.apache.accumulo.core.spi.compaction.CompactionServiceId; import org.apache.accumulo.core.tabletserver.thrift.InputFile; import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig; import org.apache.accumulo.core.tabletserver.thrift.TCompactionKind; import org.apache.accumulo.core.tabletserver.thrift.TCompactionStats; import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob; import org.apache.accumulo.core.util.cache.Caches.CacheName; -import org.apache.accumulo.core.util.compaction.CompactionPlannerInitParams; -import org.apache.accumulo.core.util.compaction.CompactionServicesConfig; import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; @@ -127,7 +119,6 @@ import org.apache.accumulo.manager.compaction.queue.ResolvedCompactionJob; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.ServiceEnvironmentImpl; import org.apache.accumulo.server.compaction.CompactionConfigStorage; import org.apache.accumulo.server.compaction.CompactionPluginUtils; import org.apache.accumulo.server.security.AuditedSecurityOperation; @@ -135,7 +126,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,7 +136,6 @@ import com.github.benmanes.caffeine.cache.Weigher; import com.google.common.base.Preconditions; import com.google.common.base.Suppliers; -import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import io.micrometer.core.instrument.MeterRegistry; @@ -191,16 +180,6 @@ static FailureCounts incrementSuccess(Object key, FailureCounts counts) { public static final String RESTART_UPDATE_MSG = "Coordinator restarted, compaction found in progress"; - /* - * Map of compactionId to TExternalCompaction. This is an informational cache of what external - * compactions may be running. Its possible it may contain external compactions that are not - * actually running. It may not contain compactions that are actually running. The metadata table - * is the most authoritative source of what external compactions are currently running, but it - * does not have the stats that this map has. - */ - protected final Map RUNNING_CACHE = - new ConcurrentHashMap<>(); - private final ServerContext ctx; private final AuditedSecurityOperation security; private final CompactionJobQueues jobQueues; @@ -308,7 +287,7 @@ protected void startCompactorZKCleaner(ScheduledThreadPoolExecutor schedExecutor protected void startInternalStateCleaner(ScheduledThreadPoolExecutor schedExecutor) { ScheduledFuture future = - schedExecutor.scheduleWithFixedDelay(this::cleanUpInternalState, 0, 5, TimeUnit.MINUTES); + schedExecutor.scheduleWithFixedDelay(this::resizeThreadPools, 0, 5, TimeUnit.MINUTES); ThreadPools.watchNonCriticalScheduledTask(future); } @@ -331,30 +310,6 @@ public void run() { startConfigMonitor(ctx.getScheduledExecutor()); startCompactorZKCleaner(ctx.getScheduledExecutor()); - // On a re-start of the coordinator it's possible that external compactions are in-progress. - // Attempt to get the running compactions on the compactors and then resolve which tserver - // the external compaction came from to re-populate the RUNNING collection. - LOG.info("Checking for running external compactions"); - // On re-start contact the running Compactors to try and seed the list of running compactions - try { - List running = getCompactionsRunningOnCompactors(); - if (running.isEmpty()) { - LOG.info("No running external compactions found"); - } else { - LOG.info("Found {} running external compactions", running.size()); - running.forEach(tec -> { - TCompactionStatusUpdate update = new TCompactionStatusUpdate(); - update.setState(TCompactionState.IN_PROGRESS); - update.setMessage(RESTART_UPDATE_MSG); - tec.putToUpdates(coordinatorStartTime, update); - RUNNING_CACHE.put(ExternalCompactionId.of(tec.getJob().getExternalCompactionId()), tec); - }); - } - } catch (InterruptedException e) { - throw new IllegalStateException( - "Thread interrupted while retrieving running compactions from compactors", e); - } - startDeadCompactionDetector(); startFailureSummaryLogging(); startInternalStateCleaner(ctx.getScheduledExecutor()); @@ -372,14 +327,6 @@ protected void startDeadCompactionDetector() { deadCompactionDetector.start(); } - protected long getMissingCompactorWarningTime() { - return this.ctx.getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME) * 3; - } - - public long getNumRunningCompactions() { - return RUNNING_CACHE.size(); - } - /** * Return the next compaction job from the queue to a Compactor * @@ -425,13 +372,6 @@ public TNextCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials if (ecm != null) { result = createThriftJob(externalCompactionId, ecm, rcJob, compactionConfig); - // It is possible that by the time this added that the the compactor that made this request - // is dead. In this cases the compaction is not actually running. - TExternalCompaction tec = new TExternalCompaction(); - tec.setCompactor(compactorAddress); - tec.setGroupName(groupName); - tec.setJob(result); - RUNNING_CACHE.put(ExternalCompactionId.of(result.getExternalCompactionId()), tec); TabletLogger.compacting(rcJob.getExtent(), rcJob.getSelectedFateId(), cid, compactorAddress, rcJob, ecm.getCompactTmpName()); break; @@ -913,8 +853,6 @@ public boolean test(TabletMetadata tabletMetadata) { } }); } - - compactions.values().forEach(ecids -> ecids.forEach(this::recordCompletion)); } /** @@ -938,26 +876,6 @@ public void updateCompactionStatus(TInfo tinfo, TCredentials credentials, } LOG.debug("Compaction status update, id: {}, timestamp: {}, update: {}", externalCompactionId, timestamp, update); - final TExternalCompaction tec = - RUNNING_CACHE.get(ExternalCompactionId.of(externalCompactionId)); - if (null != tec) { - if (update.getState() == TCompactionState.STARTED) { - tec.setStartTime(timestamp); - } - tec.putToUpdates(timestamp, update); - } - } - - @Override - public void recordCompletion(TInfo tinfo, TCredentials credentials, String externalCompactionId) - throws TException { - if (security.canPerformSystemActions(credentials)) { - recordCompletion(ExternalCompactionId.of(externalCompactionId)); - } - } - - public void recordCompletion(ExternalCompactionId ecid) { - var tec = RUNNING_CACHE.remove(ecid); } protected Set readExternalCompactionIds() { @@ -974,35 +892,6 @@ public CompactionJobQueues getJobQueues() { return jobQueues; } - /* Method exists to be overridden in test to hide static method */ - protected List getCompactionsRunningOnCompactors() - throws InterruptedException { - int numCompactors = this.ctx.instanceOperations().getServers(ServerId.Type.COMPACTOR).size(); - final ExecutorService executor = - ThreadPools.getServerThreadPools().getPoolBuilder(COMPACTOR_RUNNING_COMPACTIONS_POOL) - .numCoreThreads(numCompactors / 10).build(); - try { - List running = new ArrayList<>(); - @SuppressWarnings("unused") - List failures = ExternalCompactionUtil.getCompactionsRunningOnCompactors(this.ctx, - executor, (t) -> running.add(t)); - return running; - } finally { - executor.shutdownNow(); - } - } - - /* Method exists to be overridden in test to hide static method */ - protected Set getRunningCompactors() { - return ctx.instanceOperations().getServers(ServerId.Type.COMPACTOR); - } - - /* Method exists to be overridden in test to hide static method */ - protected void cancelCompactionOnCompactor(String address, String externalCompactionId) { - HostAndPort hostPort = HostAndPort.fromString(address); - ExternalCompactionUtil.cancelCompaction(this.ctx, hostPort, externalCompactionId); - } - private void deleteEmpty(ZooReaderWriter zoorw, String path) throws KeeperException, InterruptedException { try { @@ -1052,44 +941,7 @@ private void cleanUpEmptyCompactorPathInZK() { } } - private Set getCompactionServicesConfigurationGroups() - throws ReflectiveOperationException, IllegalArgumentException, SecurityException { - - Set groups = new HashSet<>(); - AccumuloConfiguration config = ctx.getConfiguration(); - CompactionServicesConfig servicesConfig = new CompactionServicesConfig(config); - - for (var entry : servicesConfig.getPlanners().entrySet()) { - String serviceId = entry.getKey(); - String plannerClassName = entry.getValue(); - - Class plannerClass = - Class.forName(plannerClassName).asSubclass(CompactionPlanner.class); - CompactionPlanner planner = plannerClass.getDeclaredConstructor().newInstance(); - - var initParams = new CompactionPlannerInitParams(CompactionServiceId.of(serviceId), - servicesConfig.getPlannerPrefix(serviceId), servicesConfig.getOptions().get(serviceId), - new ServiceEnvironmentImpl(ctx)); - - planner.init(initParams); - - groups.addAll(initParams.getRequestedGroups()); - } - return groups; - } - - public void cleanUpInternalState() { - - // This method does the following: - // - // 1. Removes entries from RUNNING_CACHE and LONG_RUNNING_COMPACTIONS_BY_RG that are not really - // running - // 2. Cancels running compactions for groups that are not in the current configuration - // 3. Remove groups not in configuration from TIME_COMPACTOR_LAST_CHECKED - // 4. Log groups with no compactors - // 5. Log compactors with no groups - // 6. Log groups with compactors and queued jos that have not checked in - + public void resizeThreadPools() { var config = ctx.getConfiguration(); ThreadPools.resizePool(reservationPools.get(DataLevel.ROOT), config, Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_ROOT); @@ -1097,81 +949,5 @@ public void cleanUpInternalState() { Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_META); ThreadPools.resizePool(reservationPools.get(DataLevel.USER), config, Property.COMPACTION_COORDINATOR_RESERVATION_THREADS_USER); - - // grab a snapshot of the ids in the set before reading the metadata table. This is done to - // avoid removing things that are added while reading the metadata. - final Set idsSnapshot = Set.copyOf(RUNNING_CACHE.keySet()); - - // grab the ids that are listed as running in the metadata table. It important that this is done - // after getting the snapshot. - final Set idsInMetadata = readExternalCompactionIds(); - LOG.trace("Current ECIDs in metadata: {}", idsInMetadata.size()); - LOG.trace("Current ECIDs in running cache: {}", idsSnapshot.size()); - - final Set idsToRemove = Sets.difference(idsSnapshot, idsInMetadata); - - // remove ids that are in the running set but not in the metadata table - idsToRemove.forEach(this::recordCompletion); - if (idsToRemove.size() > 0) { - LOG.debug("Removed stale entries from RUNNING_CACHE : {}", idsToRemove); - } - - // Get the set of groups being referenced in the current configuration - Set groupsInConfiguration = null; - try { - groupsInConfiguration = getCompactionServicesConfigurationGroups(); - } catch (RuntimeException | ReflectiveOperationException e) { - LOG.error( - "Error getting groups from the compaction services configuration. Unable to clean up internal state.", - e); - return; - } - - // Compaction jobs are created in the TabletGroupWatcher and added to the Coordinator - // via the addJobs method which adds the job to the CompactionJobQueues object. - final Set groupsWithJobs = jobQueues.getQueueIds(); - - final Set jobGroupsNotInConfiguration = - Sets.difference(groupsWithJobs, groupsInConfiguration); - - if (jobGroupsNotInConfiguration != null && !jobGroupsNotInConfiguration.isEmpty()) { - RUNNING_CACHE.values().forEach(tec -> { - if (jobGroupsNotInConfiguration.contains(ResourceGroupId.of(tec.getGroupName()))) { - LOG.warn( - "External compaction {} running in group {} on compactor {}," - + " but group not found in current configuration. Failing compaction...", - tec.getJob().getExternalCompactionId(), tec.getGroupName(), tec.getCompactor()); - cancelCompactionOnCompactor(tec.getCompactor(), tec.getJob().getExternalCompactionId()); - } - }); - - } - - final Set runningCompactors = getRunningCompactors(); - - final Set runningCompactorGroups = new HashSet<>(); - runningCompactors.forEach( - c -> runningCompactorGroups.add(ResourceGroupId.of(c.getResourceGroup().canonical()))); - - final Set groupsWithNoCompactors = - Sets.difference(groupsInConfiguration, runningCompactorGroups); - if (groupsWithNoCompactors != null && !groupsWithNoCompactors.isEmpty()) { - for (ResourceGroupId group : groupsWithNoCompactors) { - long queuedJobCount = jobQueues.getQueuedJobs(group); - if (queuedJobCount > 0) { - LOG.warn("Compactor group {} has {} queued compactions but no running compactors", group, - queuedJobCount); - } - } - } - - final Set compactorsWithNoGroups = - Sets.difference(runningCompactorGroups, groupsInConfiguration); - if (compactorsWithNoGroups != null && !compactorsWithNoGroups.isEmpty()) { - LOG.warn( - "The following groups have running compactors, but are not in the current configuration: {}", - compactorsWithNoGroups); - } - } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java index 3933a625048..89cdedab8ec 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/PutGcCandidates.java @@ -20,7 +20,6 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; @@ -41,7 +40,6 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { env.getContext().getAmple().putGcCandidates(commitData.getTableId(), commitData.getJobFiles()); if (refreshLocation == null) { - env.recordCompactionCompletion(ExternalCompactionId.of(commitData.ecid)); return null; } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java index d3be75c0128..cdcf649e02b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/RefreshTablet.java @@ -27,7 +27,6 @@ import org.apache.accumulo.core.fate.FateId; import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.manager.tableOps.AbstractFateOperation; import org.apache.accumulo.manager.tableOps.FateEnv; @@ -63,8 +62,6 @@ public Repo call(FateId fateId, FateEnv env) throws Exception { executorService.shutdownNow(); } - env.recordCompactionCompletion(ExternalCompactionId.of(compactionId)); - return null; } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java index e7909dd0a71..01cf2baea20 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/fate/FateWorkerEnv.java @@ -28,18 +28,15 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; import org.apache.accumulo.core.trace.TraceUtil; -import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.core.util.time.SteadyTime; @@ -176,26 +173,6 @@ public EventPublisher getEventPublisher() { return eventHandler; } - @Override - public void recordCompactionCompletion(ExternalCompactionId ecid) { - var coordinatorHost = ExternalCompactionUtil.findCompactionCoordinator(getContext()); - if (coordinatorHost.isPresent()) { - CompactionCoordinatorService.Client client = null; - try { - client = ThriftUtil.getClient(ThriftClientTypes.COORDINATOR, coordinatorHost.orElseThrow(), - getContext()); - client.recordCompletion(TraceUtil.traceInfo(), getContext().rpcCreds(), ecid.canonical()); - log.trace("Sent compaction completion {} {}", coordinatorHost, ecid); - } catch (TException te) { - log.trace("Failed to send compaction completion {} {}", coordinatorHost, ecid, te); - } finally { - ThriftUtil.returnClient(client, getContext()); - } - } else { - log.trace("No coordinator found, dropping compaction completion for {}", ecid); - } - } - @Override public Set onlineTabletServers() { return liveTServerSet.getSnapshot().getTservers(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java index f64b3851ae6..9eece1208fd 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/FateEnv.java @@ -23,7 +23,6 @@ import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.manager.EventPublisher; import org.apache.accumulo.manager.split.FileRangeCache; @@ -36,8 +35,6 @@ public interface FateEnv { EventPublisher getEventPublisher(); - void recordCompactionCompletion(ExternalCompactionId ecid); - Set onlineTabletServers(); TableManager getTableManager(); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index 3aa3f848d69..fcb8d9afb13 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -18,7 +18,6 @@ */ package org.apache.accumulo.manager.compaction; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; @@ -32,7 +31,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -40,7 +38,6 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.admin.CompactionConfig; -import org.apache.accumulo.core.client.admin.servers.ServerId; import org.apache.accumulo.core.clientImpl.thrift.TInfo; import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.compaction.thrift.TCompactionState; @@ -84,14 +81,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; -import com.google.common.net.HostAndPort; - public class CompactionCoordinatorTest { private static final ResourceGroupId GROUP_ID = ResourceGroupId.of("R2DQ"); - private final HostAndPort tserverAddr = HostAndPort.fromParts("192.168.1.1", 9090); - public MetricsInfo getMockMetrics() { MetricsInfo metricsInfo = createMock(MetricsInfo.class); metricsInfo.addMetricsProducers(anyObject()); @@ -150,34 +143,10 @@ void setMetadataCompactionIds(Set mci) { metadataCompactionIds = mci; } - @Override - protected Set readExternalCompactionIds() { - if (metadataCompactionIds == null) { - return RUNNING_CACHE.keySet(); - } else { - return metadataCompactionIds; - } - } - - public Map getRunning() { - return RUNNING_CACHE; - } - public void resetInternals() { - getRunning().clear(); metadataCompactionIds = null; } - @Override - protected List getCompactionsRunningOnCompactors() { - return runningCompactions; - } - - @Override - protected Set getRunningCompactors() { - return Set.of(); - } - @Override protected CompactionMetadata reserveCompaction(ResolvedCompactionJob rcJob, String compactorAddress, ExternalCompactionId externalCompactionId) { @@ -206,10 +175,6 @@ protected TExternalCompactionJob createThriftJob(String externalCompactionId, .toThrift(), Map.of()); } - - @Override - protected void cancelCompactionOnCompactor(String address, String externalCompactionId) {} - } private TableId tableId; @@ -259,50 +224,10 @@ public void verifyMocks() { public void testCoordinatorColdStart() throws Exception { var coordinator = new TestCoordinator(manager, new ArrayList<>()); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); coordinator.run(); coordinator.shutdown(); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); - } - - @Test - public void testCoordinatorRestartOneRunningCompaction() throws Exception { - List runningCompactions = new ArrayList<>(); - ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID()); - - TExternalCompactionJob job = createMock(TExternalCompactionJob.class); - expect(job.getExternalCompactionId()).andReturn(eci.toString()).atLeastOnce(); - TKeyExtent extent = new TKeyExtent(); - extent.setTable("1".getBytes(UTF_8)); - - TExternalCompaction current = new TExternalCompaction(); - current.setCompactor(tserverAddr.toString()); - current.setGroupName(GROUP_ID.canonical()); - current.setJob(job); - - runningCompactions.add(current); - - replay(job); - - var coordinator = new TestCoordinator(manager, runningCompactions); - coordinator.resetInternals(); - assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); - coordinator.run(); - coordinator.shutdown(); - assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(1, coordinator.getRunning().size()); - - Map running = coordinator.getRunning(); - Entry ecomp = running.entrySet().iterator().next(); - assertEquals(eci, ecomp.getKey()); - TExternalCompaction tec = ecomp.getValue(); - assertEquals(GROUP_ID, ResourceGroupId.of(tec.getGroupName())); - assertEquals(tserverAddr.toString(), tec.getCompactor()); - - verify(job); } @Test @@ -318,14 +243,12 @@ public void testGetCompactionJob() throws Exception { var coordinator = new TestCoordinator(manager, new ArrayList<>()); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); // Use coordinator.run() to populate the internal data structures. This is tested in a different // test. coordinator.run(); coordinator.shutdown(); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(0, coordinator.getRunning().size()); // Add a job to the job queue CompactionJob job = @@ -344,12 +267,6 @@ public void testGetCompactionJob() throws Exception { assertEquals(ke, KeyExtent.fromThrift(createdJob.getExtent())); assertEquals(0, coordinator.getJobQueues().getQueuedJobCount()); - assertEquals(1, coordinator.getRunning().size()); - Entry entry = - coordinator.getRunning().entrySet().iterator().next(); - assertEquals(eci.toString(), entry.getKey().toString()); - assertEquals("localhost:10241", entry.getValue().getCompactor()); - assertEquals(eci.toString(), entry.getValue().getJob().getExternalCompactionId()); verify(tm); } @@ -362,32 +279,4 @@ public void testGetCompactionJobNoJobs() throws Exception { assertEquals(3, nextJob.getCompactorCount()); assertNull(nextJob.getJob().getExternalCompactionId()); } - - @Test - public void testCleanUpRunning() throws Exception { - TExternalCompaction ext1 = createMock(TExternalCompaction.class); - TExternalCompaction ext2 = createMock(TExternalCompaction.class); - TExternalCompaction ext3 = createMock(TExternalCompaction.class); - replay(ext1, ext2, ext3); - - TestCoordinator coordinator = new TestCoordinator(manager, new ArrayList<>()); - - var ecid1 = ExternalCompactionId.generate(UUID.randomUUID()); - var ecid2 = ExternalCompactionId.generate(UUID.randomUUID()); - var ecid3 = ExternalCompactionId.generate(UUID.randomUUID()); - - coordinator.getRunning().put(ecid1, ext1); - coordinator.getRunning().put(ecid2, ext2); - coordinator.getRunning().put(ecid3, ext3); - coordinator.cleanUpInternalState(); - - assertEquals(Set.of(ecid1, ecid2, ecid3), coordinator.getRunning().keySet()); - - coordinator.setMetadataCompactionIds(Set.of(ecid1, ecid2)); - coordinator.cleanUpInternalState(); - - assertEquals(Set.of(ecid1, ecid2), coordinator.getRunning().keySet()); - - verify(ext1, ext2, ext3); - } } diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java index cb67c61fc4f..9b27141f0d8 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.compaction.CompactionPluginUtils; import org.checkerframework.checker.nullness.qual.Nullable; import org.eclipse.jetty.util.NanoTime; import org.slf4j.Logger; @@ -305,6 +306,15 @@ public void run() { futures.add(this.pool.submit(new TableInformationFetcher(this.ctx, tableId, summary))); } + futures.add(this.pool.submit(() -> { + try { + var groups = CompactionPluginUtils.getConfiguredCompactionResourceGroups(ctx); + summary.addConfiguredCompactionGroups(groups); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + })); + long monitorFetchTimeout = ctx.getConfiguration().getTimeInMillis(Property.MONITOR_FETCH_TIMEOUT); long allFuturesAdded = NanoTime.now(); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java index 2a5148010fb..3de740a1b8f 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java @@ -415,6 +415,8 @@ public Stream stream() { private final Set suggestions = new ConcurrentSkipListSet<>(); + private final Set configuredCompactionResourceGroups = ConcurrentHashMap.newKeySet(); + private long timestamp = 0; private ScanServerView scanServerView; private final int rgLongRunningCompactionSize; @@ -448,6 +450,7 @@ public void clear() { suggestions.clear(); runningCompactionsPerGroup.clear(); runningCompactionsPerTable.clear(); + configuredCompactionResourceGroups.clear(); scanServerView = null; } @@ -580,6 +583,10 @@ public void processError(ServerId server) { problemHosts.add(server); } + public void addConfiguredCompactionGroups(Set groups) { + configuredCompactionResourceGroups.addAll(groups); + } + public void finish() { // Update the deployment not-responded numbers based // on the problem hosts. @@ -627,6 +634,14 @@ public void finish() { } } } + + for (var compactorGroup : compactors.keySet()) { + if (!configuredCompactionResourceGroups.contains(compactorGroup)) { + suggestions.add("Compactor group " + compactorGroup + + " has running compactors, but no configuration uses them."); + } + } + Set scanServers = new HashSet<>(); sservers.values().forEach(scanServers::addAll); int problemScanServerCount = (int) problemHosts.stream()