From e87984a99e69b6f60106a429fd0c5cc4ac7afa14 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 7 Feb 2026 15:23:23 +0800 Subject: [PATCH 1/9] HDDS-14561. SCMRatisRequest/ResponseProto should use the shaded protobuf from Ratis --- .../hadoop/hdds/scm/ha/SCMRatisRequest.java | 10 ++++++---- .../hadoop/hdds/scm/ha/SCMRatisResponse.java | 2 +- .../hdds/scm/ha/TestSCMRatisRequest.java | 20 +++++++++---------- .../hdds/scm/ha/TestSCMRatisResponse.java | 8 ++++---- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java index 28fca97e422..ac1daff1f56 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.Method; @@ -111,9 +112,10 @@ public Message encode() throws InvalidProtocolBufferException { } methodBuilder.addAllArgs(args); requestProtoBuilder.setMethod(methodBuilder.build()); + final SCMRatisRequestProto requestProto = requestProtoBuilder.build(); return Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - requestProtoBuilder.build().toByteArray())); + UnsafeByteOperations.unsafeWrap( + requestProto.toByteString().asReadOnlyByteBuffer())); } /** @@ -122,7 +124,7 @@ public Message encode() throws InvalidProtocolBufferException { public static SCMRatisRequest decode(Message message) throws InvalidProtocolBufferException { final SCMRatisRequestProto requestProto = - SCMRatisRequestProto.parseFrom(message.getContent().toByteArray()); + SCMRatisRequestProto.parseFrom(message.getContent().asReadOnlyByteBuffer()); // proto2 required-equivalent checks if (!requestProto.hasType()) { @@ -173,7 +175,7 @@ public static String smProtoToString(StateMachineLogEntryProto proto) { StringBuilder builder = new StringBuilder(); try { builder.append(TextFormat.shortDebugString( - SCMRatisRequestProto.parseFrom(proto.getLogData().toByteArray()))); + SCMRatisRequestProto.parseFrom(proto.getLogData().asReadOnlyByteBuffer()))); } catch (Throwable ex) { LOG.error("smProtoToString failed", ex); builder.append("smProtoToString failed with"); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java index 34e5bea3401..eba1f5b7ec7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java @@ -92,7 +92,7 @@ public static SCMRatisResponse decode(RaftClientReply reply) return new SCMRatisResponse(); } - final SCMRatisResponseProto responseProto = SCMRatisResponseProto.parseFrom(response.toByteArray()); + final SCMRatisResponseProto responseProto = SCMRatisResponseProto.parseFrom(response.asReadOnlyByteBuffer()); // proto2 required-equivalent checks if (!responseProto.hasType()) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java index baf5d379ebf..e35eb162462 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java @@ -105,8 +105,8 @@ public void testDecodeMissingRequestTypeShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - proto.toByteArray())); + org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -124,8 +124,8 @@ public void testDecodeMissingMethodShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - proto.toByteArray())); + org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -145,8 +145,8 @@ public void testDecodeMissingMethodNameShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - proto.toByteArray())); + org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -173,8 +173,8 @@ public void testDecodeMissingArgumentTypeShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - proto.toByteArray())); + org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -201,8 +201,8 @@ public void testDecodeMissingArgumentValueShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - proto.toByteArray())); + org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java index b0c67a1460e..e669effeb81 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java @@ -108,8 +108,8 @@ public void testResponseDecodeMissingTypeShouldFail() throws Exception { RaftClientReply reply = mock(RaftClientReply.class); when(reply.isSuccess()).thenReturn(true); when(reply.getMessage()).thenReturn(Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - proto.toByteArray()))); + org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer()))); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -130,8 +130,8 @@ public void testResponseDecodeMissingValueShouldFail() throws Exception { RaftClientReply reply = mock(RaftClientReply.class); when(reply.isSuccess()).thenReturn(true); when(reply.getMessage()).thenReturn(Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom( - proto.toByteArray()))); + org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer()))); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, From c0b314537bc27b10f2826074e283f8222860336b Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 7 Feb 2026 15:44:08 +0800 Subject: [PATCH 2/9] Fix checkstyle error --- .../java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java index ac1daff1f56..5086636dace 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java @@ -20,7 +20,6 @@ import com.google.common.base.Preconditions; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; -import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.Method; @@ -30,6 +29,7 @@ import org.apache.hadoop.hdds.scm.ha.io.CodecFactory; import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto; import org.apache.ratis.protocol.Message; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 4aa8930dc41b8de2f094392ec14e95b10dada49a Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Wed, 11 Feb 2026 00:28:29 +0800 Subject: [PATCH 3/9] Apply review feedback on protobuf handling --- hadoop-hdds/interface-server/pom.xml | 2 + .../hadoop/hdds/scm/ha/SCMRatisRequest.java | 4 +- .../hadoop/hdds/scm/ha/SCMRatisResponse.java | 2 +- .../hdds/scm/ha/io/BigIntegerCodec.java | 6 +-- .../hadoop/hdds/scm/ha/io/BooleanCodec.java | 2 +- .../hdds/scm/ha/io/ByteStringCodec.java | 4 +- .../apache/hadoop/hdds/scm/ha/io/Codec.java | 4 +- .../hadoop/hdds/scm/ha/io/CodecFactory.java | 4 +- .../hadoop/hdds/scm/ha/io/EnumCodec.java | 9 ++-- .../hdds/scm/ha/io/GeneratedMessageCodec.java | 46 +++++++++++++------ .../hadoop/hdds/scm/ha/io/IntegerCodec.java | 8 ++-- .../hadoop/hdds/scm/ha/io/ListCodec.java | 4 +- .../hadoop/hdds/scm/ha/io/LongCodec.java | 8 ++-- .../hdds/scm/ha/io/ManagedSecretKeyCodec.java | 20 +++++--- .../hadoop/hdds/scm/ha/io/StringCodec.java | 6 +-- .../hdds/scm/ha/io/X509CertificateCodec.java | 8 ++-- .../ha/TestSCMRatisProtocolCompatibility.java | 4 +- .../hdds/scm/ha/TestSCMRatisRequest.java | 4 +- .../hdds/scm/ha/TestSCMRatisResponse.java | 4 +- .../hdds/scm/ha/io/TestBigIntegerCodec.java | 2 +- .../scm/ha/io/TestX509CertificateCodec.java | 8 ++-- 21 files changed, 93 insertions(+), 66 deletions(-) diff --git a/hadoop-hdds/interface-server/pom.xml b/hadoop-hdds/interface-server/pom.xml index c38927df500..b914a3d4ab4 100644 --- a/hadoop-hdds/interface-server/pom.xml +++ b/hadoop-hdds/interface-server/pom.xml @@ -83,6 +83,7 @@ InterSCMProtocol.proto SCMUpdateProtocol.proto + SCMRatisProtocol.proto target/generated-sources/proto-java-for-ratis false @@ -101,6 +102,7 @@ InterSCMProtocol.proto SCMUpdateProtocol.proto + SCMRatisProtocol.proto target/generated-sources/proto-java-for-protobuf-${protobuf.version} false diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java index 5086636dace..7270f850680 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hdds.scm.ha; import com.google.common.base.Preconditions; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.TextFormat; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.Method; @@ -29,6 +27,8 @@ import org.apache.hadoop.hdds.scm.ha.io.CodecFactory; import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto; import org.apache.ratis.protocol.Message; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat; import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java index eba1f5b7ec7..55f55cab4f5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisResponse.java @@ -17,12 +17,12 @@ package org.apache.hadoop.hdds.scm.ha; -import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.SCMRatisResponseProto; import org.apache.hadoop.hdds.scm.ha.io.CodecFactory; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BigIntegerCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BigIntegerCodec.java index bee5c59f008..afc7da03fa1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BigIntegerCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BigIntegerCodec.java @@ -17,9 +17,9 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; -import com.google.protobuf.ProtoUtils; import java.math.BigInteger; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * Codec for type BigInteger. @@ -29,7 +29,7 @@ public class BigIntegerCodec implements Codec { @Override public ByteString serialize(Object object) { // BigInteger returns a new byte[]. - return ProtoUtils.unsafeByteString(((BigInteger)object).toByteArray()); + return UnsafeByteOperations.unsafeWrap(((BigInteger) object).toByteArray()); } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BooleanCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BooleanCodec.java index 899d10814a4..2f79e8e9cfe 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BooleanCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/BooleanCodec.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; /** * {@link Codec} for {@code Boolean} objects. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java index d2599a4c1ab..e6762935a95 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java @@ -17,8 +17,8 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; /** * A dummy codec that serializes a ByteString object to ByteString. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/Codec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/Codec.java index 49520ab6dd8..1a552283658 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/Codec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/Codec.java @@ -17,8 +17,8 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; /** * Codec interface to marshall/unmarshall data to/from {@link ByteString}. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java index 1cd3ad1bf92..8928136156a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java @@ -17,8 +17,6 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import com.google.protobuf.ProtocolMessageEnum; import java.math.BigInteger; @@ -29,6 +27,8 @@ import java.util.Map; import org.apache.commons.lang3.ClassUtils; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; /** * Maps types to the corresponding {@link Codec} implementation. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/EnumCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/EnumCodec.java index 32108b2da8b..24ea6af1424 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/EnumCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/EnumCodec.java @@ -18,12 +18,12 @@ package org.apache.hadoop.hdds.scm.ha.io; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.ProtoUtils; import com.google.protobuf.ProtocolMessageEnum; import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * {@link Codec} for {@link ProtocolMessageEnum} objects. @@ -34,8 +34,7 @@ public class EnumCodec implements Codec { public ByteString serialize(Object object) throws InvalidProtocolBufferException { // toByteArray returns a new array - return ProtoUtils.unsafeByteString(Ints.toByteArray( - ((ProtocolMessageEnum) object).getNumber())); + return UnsafeByteOperations.unsafeWrap(Ints.toByteArray(((ProtocolMessageEnum) object).getNumber())); } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java index b4adf805276..d9ff56f8355 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java @@ -17,11 +17,11 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.Message; /** * {@link Codec} for {@link Message} objects. @@ -29,22 +29,40 @@ public class GeneratedMessageCodec implements Codec { @Override - public ByteString serialize(Object object) { - return ((Message)object).toByteString(); + public ByteString serialize(Object object) throws InvalidProtocolBufferException { + try { + Object bs = ReflectionUtil + .getMethod(object.getClass(), "toByteString") + .invoke(object); + if (bs instanceof org.apache.ratis.thirdparty.com.google.protobuf.ByteString) { + return (org.apache.ratis.thirdparty.com.google.protobuf.ByteString) bs; + } + return org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(((com.google.protobuf.ByteString) bs).asReadOnlyByteBuffer()); + + } catch (Exception e) { + throw new InvalidProtocolBufferException( + "Message cannot be encoded: " + e.getMessage()); + } } @Override - public Message deserialize(Class type, ByteString value) + public Object deserialize(Class type, ByteString value) throws InvalidProtocolBufferException { try { - return (Message) ReflectionUtil.getMethod(type, - "parseFrom", byte[].class) - .invoke(null, (Object) value.toByteArray()); - } catch (NoSuchMethodException | IllegalAccessException - | InvocationTargetException ex) { - ex.printStackTrace(); + try { + return ReflectionUtil + .getMethod(type, "parseFrom", ByteBuffer.class) + .invoke(null, value.asReadOnlyByteBuffer()); + } catch (NoSuchMethodException ignored) { + // fallbackļ¼šparseFrom(byte[]) + return ReflectionUtil + .getMethod(type, "parseFrom", byte[].class) + .invoke(null, value.toByteArray()); + } + } catch (Exception e) { throw new InvalidProtocolBufferException( - "Message cannot be decoded: " + ex.getMessage()); + "Message cannot be decoded: " + e.getMessage()); } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/IntegerCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/IntegerCodec.java index a7d00e535e0..81ccae87052 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/IntegerCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/IntegerCodec.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hdds.scm.ha.io; import com.google.common.primitives.Ints; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.ProtoUtils; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * Encodes/decodes an integer to a byte string. @@ -30,7 +30,7 @@ public class IntegerCodec implements Codec { public ByteString serialize(Object object) throws InvalidProtocolBufferException { // toByteArray returns a new array - return ProtoUtils.unsafeByteString(Ints.toByteArray((Integer) object)); + return UnsafeByteOperations.unsafeWrap(Ints.toByteArray((Integer) object)); } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java index 591fb17e321..798076b6a97 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ListCodec.java @@ -17,13 +17,13 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.ListArgument; import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; /** * {@link Codec} for {@link List} objects. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/LongCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/LongCodec.java index b8c35eae478..ad5792cb5a1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/LongCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/LongCodec.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hdds.scm.ha.io; import com.google.common.primitives.Longs; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.ProtoUtils; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * {@link Codec} for {@code Long} objects. @@ -31,7 +31,7 @@ public class LongCodec implements Codec { public ByteString serialize(Object object) throws InvalidProtocolBufferException { // toByteArray returns a new array - return ProtoUtils.unsafeByteString(Longs.toByteArray((Long) object)); + return UnsafeByteOperations.unsafeWrap(Longs.toByteArray((Long) object)); } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java index ec4a7c261e4..0a6ad2e1fcd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java @@ -17,10 +17,11 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hdds.protocol.proto.SCMSecretKeyProtocolProtos; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * A codec for {@link ManagedSecretKey} objects. @@ -30,14 +31,21 @@ public class ManagedSecretKeyCodec implements Codec { public ByteString serialize(Object object) throws InvalidProtocolBufferException { ManagedSecretKey secretKey = (ManagedSecretKey) object; - return secretKey.toProtobuf().toByteString(); + return UnsafeByteOperations.unsafeWrap( + secretKey.toProtobuf().toByteString().asReadOnlyByteBuffer()); } @Override public Object deserialize(Class type, ByteString value) throws InvalidProtocolBufferException { - SCMSecretKeyProtocolProtos.ManagedSecretKey message = - SCMSecretKeyProtocolProtos.ManagedSecretKey.parseFrom(value); - return ManagedSecretKey.fromProtobuf(message); + try { + SCMSecretKeyProtocolProtos.ManagedSecretKey message = + SCMSecretKeyProtocolProtos.ManagedSecretKey.parseFrom( + value.asReadOnlyByteBuffer()); + return ManagedSecretKey.fromProtobuf(message); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw new org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()); + } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/StringCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/StringCodec.java index 47d8917872d..c53e4e82f94 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/StringCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/StringCodec.java @@ -19,8 +19,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.protobuf.ByteString; -import com.google.protobuf.ProtoUtils; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * {@link Codec} for {@code String} objects. @@ -29,7 +29,7 @@ public class StringCodec implements Codec { @Override public ByteString serialize(Object object) { // getBytes returns a new array - return ProtoUtils.unsafeByteString(((String) object).getBytes(UTF_8)); + return UnsafeByteOperations.unsafeWrap(((String) object).getBytes(UTF_8)); } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/X509CertificateCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/X509CertificateCodec.java index e68eaf481b2..408e69e9a29 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/X509CertificateCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/X509CertificateCodec.java @@ -19,11 +19,11 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.ProtoUtils; import java.security.cert.X509Certificate; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * Codec for type X509Certificate. @@ -36,7 +36,7 @@ public ByteString serialize(Object object) String certString = CertificateCodec.getPEMEncodedString((X509Certificate) object); // getBytes returns a new array - return ProtoUtils.unsafeByteString(certString.getBytes(UTF_8)); + return UnsafeByteOperations.unsafeWrap(certString.getBytes(UTF_8)); } catch (Exception ex) { throw new InvalidProtocolBufferException( "X509Certificate cannot be decoded: " + ex.getMessage()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java index ba6d7b2005c..9af06fa3c4b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java @@ -68,7 +68,7 @@ public void testProto2RequestCanBeParsedByProto3() throws Exception { assertEquals("testOp", proto3.getMethod().getName()); assertEquals(1, proto3.getMethod().getArgsCount()); assertEquals("java.lang.String", proto3.getMethod().getArgs(0).getType()); - assertEquals(ByteString.copyFromUtf8("v"), proto3.getMethod().getArgs(0).getValue()); + assertEquals("v", proto3.getMethod().getArgs(0).getValue().toStringUtf8()); } /** @@ -93,6 +93,6 @@ public void testProto2ResponseCanBeParsedByProto3() throws Exception { assertTrue(proto3.hasType(), "proto3 should see type presence from proto2 bytes"); assertTrue(proto3.hasValue(), "proto3 should see value presence from proto2 bytes"); assertEquals("java.lang.String", proto3.getType()); - assertEquals(ByteString.copyFromUtf8("ok"), proto3.getValue()); + assertEquals("ok", proto3.getValue().toStringUtf8()); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java index e35eb162462..0e5d8f37d48 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java @@ -22,8 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -31,6 +29,8 @@ import org.apache.hadoop.hdds.scm.ha.io.ListCodec; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.ratis.protocol.Message; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.junit.jupiter.api.Test; /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java index e669effeb81..9072ebb8fcc 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java @@ -26,8 +26,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; @@ -37,6 +35,8 @@ import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.protocol.exceptions.LeaderNotReadyException; import org.apache.ratis.protocol.exceptions.RaftException; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java index 096b1e7f506..4142a69eb57 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java @@ -19,8 +19,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import com.google.protobuf.ByteString; import java.math.BigInteger; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.junit.jupiter.api.Test; /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestX509CertificateCodec.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestX509CertificateCodec.java index 2ae1aac94ee..8b5ad7d8fa1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestX509CertificateCodec.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestX509CertificateCodec.java @@ -21,14 +21,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.ProtoUtils; import java.security.KeyPair; import java.security.cert.X509Certificate; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -63,7 +63,7 @@ public void codec() throws Exception { public void testCodecError() { X509CertificateCodec x509CertificateCodec = new X509CertificateCodec(); - final ByteString byteString = ProtoUtils.unsafeByteString("dummy".getBytes(UTF_8)); + final ByteString byteString = UnsafeByteOperations.unsafeWrap("dummy".getBytes(UTF_8)); assertThrows(InvalidProtocolBufferException.class, () -> x509CertificateCodec.deserialize(X509Certificate.class, byteString)); From 8eeaa6e862212571e813b63f10e499cf4dab5cf8 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Wed, 11 Feb 2026 12:36:46 +0800 Subject: [PATCH 4/9] Fix shaded protobuf mismatch in TestSCMRatisProtocolCompatibility test --- .../ha/TestSCMRatisProtocolCompatibility.java | 76 ++++++++++++++----- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java index 3d58cecb37c..a791a206bb8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java @@ -17,11 +17,10 @@ package org.apache.hadoop.hdds.scm.ha; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.google.protobuf.ByteString; -import com.google.protobuf.TextFormat; import java.nio.ByteBuffer; import java.util.Random; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol; @@ -35,7 +34,7 @@ public class TestSCMRatisProtocolCompatibility { static final Random RANDOM = new Random(); static final Class[] TYPES = {String.class, Integer.class, byte[].class}; - static ByteString randomValue(Class clazz) { + static com.google.protobuf.ByteString randomValueProto2(Class clazz) { if (clazz == String.class) { final int length = RANDOM.nextInt(3); final StringBuilder builder = new StringBuilder(length); @@ -44,15 +43,37 @@ static ByteString randomValue(Class clazz) { } final String string = builder.toString(); assertEquals(length, string.length()); - return ByteString.copyFromUtf8(string); + return com.google.protobuf.ByteString.copyFromUtf8(string); } else if (clazz == Integer.class) { final ByteBuffer buffer = ByteBuffer.allocate(4); buffer.putInt(RANDOM.nextInt()); - return ByteString.copyFrom(buffer.array()); + return com.google.protobuf.ByteString.copyFrom(buffer.array()); } else if (clazz == byte[].class) { final byte[] bytes = new byte[RANDOM.nextInt(3)]; RANDOM.nextBytes(bytes); - return ByteString.copyFrom(bytes); + return com.google.protobuf.ByteString.copyFrom(bytes); + } + throw new IllegalArgumentException("Unrecognized class " + clazz); + } + + static org.apache.ratis.thirdparty.com.google.protobuf.ByteString randomValueProto3(Class clazz) { + if (clazz == String.class) { + final int length = RANDOM.nextInt(3); + final StringBuilder builder = new StringBuilder(length); + for (int i = 0; i < length; i++) { + builder.append(RANDOM.nextInt(10)); + } + final String string = builder.toString(); + assertEquals(length, string.length()); + return org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFromUtf8(string); + } else if (clazz == Integer.class) { + final ByteBuffer buffer = ByteBuffer.allocate(4); + buffer.putInt(RANDOM.nextInt()); + return org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom(buffer.array()); + } else if (clazz == byte[].class) { + final byte[] bytes = new byte[RANDOM.nextInt(3)]; + RANDOM.nextBytes(bytes); + return org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom(bytes); } throw new IllegalArgumentException("Unrecognized class " + clazz); } @@ -61,7 +82,7 @@ static Proto2SCMRatisProtocolForTesting.MethodArgument randomProto2MethodArgumen final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; return Proto2SCMRatisProtocolForTesting.MethodArgument.newBuilder() .setType(type.getName()) - .setValue(randomValue(type)) + .setValue(randomValueProto2(type)) .build(); } @@ -69,7 +90,7 @@ static SCMRatisProtocol.MethodArgument randomProto3MethodArgument() { final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; return SCMRatisProtocol.MethodArgument.newBuilder() .setType(type.getName()) - .setValue(randomValue(type)) + .setValue(randomValueProto3(type)) .build(); } @@ -77,7 +98,7 @@ static Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto randomProto2SCMRat final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; return Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto.newBuilder() .setType(type.getName()) - .setValue(randomValue(type)) + .setValue(randomValueProto2(type)) .build(); } @@ -85,7 +106,7 @@ static SCMRatisProtocol.SCMRatisResponseProto randomProto3SCMRatisResponseProto( final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; return SCMRatisProtocol.SCMRatisResponseProto.newBuilder() .setType(type.getName()) - .setValue(randomValue(type)) + .setValue(randomValueProto3(type)) .build(); } @@ -155,20 +176,28 @@ static void runTestProto2RequestCanBeParsedByProto3( } assertEquals(proto2.toString(), proto3.toString()); - assertEquals(TextFormat.shortDebugString(proto2), TextFormat.shortDebugString(proto3)); + assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); assertEquals(proto2, Proto2SCMRatisProtocolForTesting.SCMRatisRequestProto.parseFrom(proto3.toByteArray())); } + private static byte[] bytesOf(com.google.protobuf.ByteString b) { + return b.toByteArray(); + } + + private static byte[] bytesOf(org.apache.ratis.thirdparty.com.google.protobuf.ByteString b) { + return b.toByteArray(); + } + static void assertMethodArgument(Proto2SCMRatisProtocolForTesting.MethodArgument proto2, SCMRatisProtocol.MethodArgument proto3) { - assertEquals(proto2.getValue(), proto3.getValue()); assertEquals(proto2.getType(), proto3.getType()); + assertArrayEquals(bytesOf(proto2.getValue()), bytesOf(proto3.getValue())); } static void assertSCMRatisResponseProto(Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto proto2, SCMRatisProtocol.SCMRatisResponseProto proto3) { - assertEquals(proto2.getValue(), proto3.getValue()); assertEquals(proto2.getType(), proto3.getType()); + assertArrayEquals(bytesOf(proto2.getValue()), bytesOf(proto3.getValue())); } /** @@ -195,7 +224,7 @@ static void runTestProto2ResponseCanBeParsedByProto3() throws Exception { assertSCMRatisResponseProto(proto2, proto3); assertEquals(proto2.toString(), proto3.toString()); - assertEquals(TextFormat.shortDebugString(proto2), TextFormat.shortDebugString(proto3)); + assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); assertEquals(proto2, Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto.parseFrom(proto3.toByteArray())); } @@ -246,11 +275,12 @@ static void runTestProto3RequestCanBeParsedByProto2( for (int i = 0; i < numArgs; i++) { assertEquals(proto3.getMethod().getArgs(i).getType(), proto2.getMethod().getArgs(i).getType()); - assertEquals(proto3.getMethod().getArgs(i).getValue(), proto2.getMethod().getArgs(i).getValue()); + assertArrayEquals(bytesOf(proto3.getMethod().getArgs(i).getValue()), + bytesOf(proto2.getMethod().getArgs(i).getValue())); } assertEquals(proto2.toString(), proto3.toString()); - assertEquals(TextFormat.shortDebugString(proto2), TextFormat.shortDebugString(proto3)); + assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); assertEquals(proto3, SCMRatisProtocol.SCMRatisRequestProto.parseFrom(proto2.toByteArray())); } @@ -268,10 +298,20 @@ static void runTestProto3ResponseCanBeParsedByProto2() throws Exception { Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto.parseFrom(proto3.toByteArray()); assertEquals(proto3.getType(), proto2.getType()); - assertEquals(proto3.getValue(), proto2.getValue()); + assertArrayEquals(bytesOf(proto3.getValue()), bytesOf(proto2.getValue())); assertEquals(proto2.toString(), proto3.toString()); - assertEquals(TextFormat.shortDebugString(proto2), TextFormat.shortDebugString(proto3)); + assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); assertEquals(proto3, SCMRatisProtocol.SCMRatisResponseProto.parseFrom(proto2.toByteArray())); } + + private static String shortDebugStringProto2( + com.google.protobuf.MessageOrBuilder msg) { + return com.google.protobuf.TextFormat.shortDebugString(msg); + } + + private static String shortDebugStringProto3( + org.apache.ratis.thirdparty.com.google.protobuf.MessageOrBuilder msg) { + return org.apache.ratis.thirdparty.com.google.protobuf.TextFormat.shortDebugString(msg); + } } From 298f5de50d40f15a91cbe07eb14d816b9e7be979 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Wed, 11 Feb 2026 23:49:42 +0800 Subject: [PATCH 5/9] Fix unit test error --- .../hadoop/hdds/scm/ha/io/ByteStringCodec.java | 16 ++++++++++++++++ .../hadoop/hdds/scm/ha/io/CodecFactory.java | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java index e6762935a95..6cf14f390a4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java @@ -28,12 +28,28 @@ public class ByteStringCodec implements Codec { @Override public ByteString serialize(Object object) throws InvalidProtocolBufferException { + if (object instanceof ByteString) { + return (ByteString) object; + } + if (object instanceof com.google.protobuf.ByteString) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) object; + return org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations + .unsafeWrap(bs.asReadOnlyByteBuffer()); + } return (ByteString) object; } @Override public Object deserialize(Class type, ByteString value) throws InvalidProtocolBufferException { + if (type == ByteString.class || ByteString.class.isAssignableFrom(type)) { + return value; + } + if (type == com.google.protobuf.ByteString.class + || com.google.protobuf.ByteString.class.isAssignableFrom(type)) { + return com.google.protobuf.UnsafeByteOperations + .unsafeWrap(value.asReadOnlyByteBuffer()); + } return value; } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java index 8928136156a..d8ab0ea718a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java @@ -27,7 +27,6 @@ import java.util.Map; import org.apache.commons.lang3.ClassUtils; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; -import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; /** @@ -47,7 +46,9 @@ public final class CodecFactory { codecs.put(Boolean.class, new BooleanCodec()); codecs.put(BigInteger.class, new BigIntegerCodec()); codecs.put(X509Certificate.class, new X509CertificateCodec()); - codecs.put(ByteString.class, new ByteStringCodec()); + codecs.put(com.google.protobuf.ByteString.class, new ByteStringCodec()); + codecs.put(org.apache.ratis.thirdparty.com.google.protobuf.ByteString.class, + new ByteStringCodec()); codecs.put(ManagedSecretKey.class, new ManagedSecretKeyCodec()); } From ac43a62d7c4c0319438e305230812027446ea74f Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 14 Feb 2026 00:40:51 +0800 Subject: [PATCH 6/9] Refactor implementation and update CodecFactory --- .../hadoop/hdds/scm/ha/SCMRatisRequest.java | 4 +- .../hdds/scm/ha/io/ByteStringCodec.java | 16 ---- .../hadoop/hdds/scm/ha/io/CodecFactory.java | 10 +- .../hdds/scm/ha/io/GeneratedMessageCodec.java | 64 +++---------- .../hdds/scm/ha/io/ManagedSecretKeyCodec.java | 3 +- .../scm/ha/io/ScmGeneratedMessageCodec.java | 50 ++++++++++ ...SCMRatisProtocolCompatibilityTestUtil.java | 86 +++++++++++++++++ .../ha/TestSCMRatisProtocolCompatibility.java | 96 ++++--------------- .../hdds/scm/ha/TestSCMRatisResponse.java | 7 +- 9 files changed, 178 insertions(+), 158 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmGeneratedMessageCodec.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java index 7270f850680..0305b4f82e8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java @@ -113,9 +113,7 @@ public Message encode() throws InvalidProtocolBufferException { methodBuilder.addAllArgs(args); requestProtoBuilder.setMethod(methodBuilder.build()); final SCMRatisRequestProto requestProto = requestProtoBuilder.build(); - return Message.valueOf( - UnsafeByteOperations.unsafeWrap( - requestProto.toByteString().asReadOnlyByteBuffer())); + return Message.valueOf(requestProto.toByteString()); } /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java index 6cf14f390a4..e6762935a95 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java @@ -28,28 +28,12 @@ public class ByteStringCodec implements Codec { @Override public ByteString serialize(Object object) throws InvalidProtocolBufferException { - if (object instanceof ByteString) { - return (ByteString) object; - } - if (object instanceof com.google.protobuf.ByteString) { - com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) object; - return org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(bs.asReadOnlyByteBuffer()); - } return (ByteString) object; } @Override public Object deserialize(Class type, ByteString value) throws InvalidProtocolBufferException { - if (type == ByteString.class || ByteString.class.isAssignableFrom(type)) { - return value; - } - if (type == com.google.protobuf.ByteString.class - || com.google.protobuf.ByteString.class.isAssignableFrom(type)) { - return com.google.protobuf.UnsafeByteOperations - .unsafeWrap(value.asReadOnlyByteBuffer()); - } return value; } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java index d8ab0ea718a..8786acff614 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.ha.io; -import com.google.protobuf.Message; import com.google.protobuf.ProtocolMessageEnum; import java.math.BigInteger; import java.security.cert.X509Certificate; @@ -27,7 +26,9 @@ import java.util.Map; import org.apache.commons.lang3.ClassUtils; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.Message; /** * Maps types to the corresponding {@link Codec} implementation. @@ -37,7 +38,8 @@ public final class CodecFactory { private static Map, Codec> codecs = new HashMap<>(); static { - codecs.put(Message.class, new GeneratedMessageCodec()); + codecs.put(com.google.protobuf.Message.class, new GeneratedMessageCodec()); + codecs.put(Message.class, new ScmGeneratedMessageCodec()); codecs.put(ProtocolMessageEnum.class, new EnumCodec()); codecs.put(List.class, new ListCodec()); codecs.put(Integer.class, new IntegerCodec()); @@ -46,9 +48,7 @@ public final class CodecFactory { codecs.put(Boolean.class, new BooleanCodec()); codecs.put(BigInteger.class, new BigIntegerCodec()); codecs.put(X509Certificate.class, new X509CertificateCodec()); - codecs.put(com.google.protobuf.ByteString.class, new ByteStringCodec()); - codecs.put(org.apache.ratis.thirdparty.com.google.protobuf.ByteString.class, - new ByteStringCodec()); + codecs.put(ByteString.class, new ByteStringCodec()); codecs.put(ManagedSecretKey.class, new ManagedSecretKeyCodec()); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java index d9ff56f8355..ddfe70916ab 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java @@ -1,68 +1,30 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.apache.hadoop.hdds.scm.ha.io; -import java.nio.ByteBuffer; +import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; +import com.google.protobuf.Message; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; -import org.apache.ratis.thirdparty.com.google.protobuf.Message; -/** - * {@link Codec} for {@link Message} objects. - */ public class GeneratedMessageCodec implements Codec { @Override - public ByteString serialize(Object object) throws InvalidProtocolBufferException { - try { - Object bs = ReflectionUtil - .getMethod(object.getClass(), "toByteString") - .invoke(object); - if (bs instanceof org.apache.ratis.thirdparty.com.google.protobuf.ByteString) { - return (org.apache.ratis.thirdparty.com.google.protobuf.ByteString) bs; - } - return org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(((com.google.protobuf.ByteString) bs).asReadOnlyByteBuffer()); - - } catch (Exception e) { - throw new InvalidProtocolBufferException( - "Message cannot be encoded: " + e.getMessage()); - } + public ByteString serialize(Object object) + throws org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException { + final byte[] bytes = ((Message) object).toByteString().toByteArray(); + return ByteString.copyFrom(bytes); } @Override public Object deserialize(Class type, ByteString value) - throws InvalidProtocolBufferException { + throws org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException { try { - try { - return ReflectionUtil - .getMethod(type, "parseFrom", ByteBuffer.class) - .invoke(null, value.asReadOnlyByteBuffer()); - } catch (NoSuchMethodException ignored) { - // fallbackļ¼šparseFrom(byte[]) - return ReflectionUtil - .getMethod(type, "parseFrom", byte[].class) - .invoke(null, value.toByteArray()); - } - } catch (Exception e) { - throw new InvalidProtocolBufferException( - "Message cannot be decoded: " + e.getMessage()); + return ReflectionUtil.getMethod(type, "parseFrom", byte[].class) + .invoke(null, (Object) value.toByteArray()); + } catch (NoSuchMethodException | IllegalAccessException + | InvocationTargetException ex) { + ex.printStackTrace(); + throw new InvalidProtocolBufferException("Message cannot be decoded: " + ex.getMessage()); } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java index 0a6ad2e1fcd..a82943f730e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ManagedSecretKeyCodec.java @@ -44,8 +44,7 @@ public Object deserialize(Class type, ByteString value) value.asReadOnlyByteBuffer()); return ManagedSecretKey.fromProtobuf(message); } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw new org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()); + throw new InvalidProtocolBufferException("Failed to deserialize value for " + type, e); } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmGeneratedMessageCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmGeneratedMessageCodec.java new file mode 100644 index 00000000000..5a8be986fa8 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmGeneratedMessageCodec.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.ha.io; + +import java.lang.reflect.InvocationTargetException; +import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.Message; + +/** + * {@link Codec} for {@link Message} objects. + */ +public class ScmGeneratedMessageCodec implements Codec { + + @Override + public ByteString serialize(Object object) throws InvalidProtocolBufferException { + return ((Message)object).toByteString(); + } + + @Override + public Message deserialize(Class type, ByteString value) + throws InvalidProtocolBufferException { + try { + return (Message) ReflectionUtil.getMethod(type, + "parseFrom", byte[].class) + .invoke(null, (Object) value.toByteArray()); + } catch (NoSuchMethodException | IllegalAccessException + | InvocationTargetException ex) { + ex.printStackTrace(); + throw new InvalidProtocolBufferException( + "Message cannot be decoded: " + ex.getMessage()); + } + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java new file mode 100644 index 00000000000..9a6bdc9d984 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.ha; + +import com.google.protobuf.ByteString; +import org.apache.hadoop.hdds.protocol.proto.testing.Proto2SCMRatisProtocolForTesting; + +import java.nio.ByteBuffer; + +import static org.apache.hadoop.hdds.scm.ha.TestSCMRatisProtocolCompatibility.RANDOM; +import static org.apache.hadoop.hdds.scm.ha.TestSCMRatisProtocolCompatibility.TYPES; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests proto2 to proto3 compatibility for SCMRatisProtocol. + */ +public class SCMRatisProtocolCompatibilityTestUtil { + static ByteString randomValueProto2(Class clazz) { + if (clazz == String.class) { + final int length = RANDOM.nextInt(3); + final StringBuilder builder = new StringBuilder(length); + for (int i = 0; i < length; i++) { + builder.append(RANDOM.nextInt(10)); + } + final String string = builder.toString(); + assertEquals(length, string.length()); + return ByteString.copyFromUtf8(string); + } else if (clazz == Integer.class) { + final ByteBuffer buffer = ByteBuffer.allocate(4); + buffer.putInt(RANDOM.nextInt()); + return ByteString.copyFrom(buffer.array()); + } else if (clazz == byte[].class) { + final byte[] bytes = new byte[RANDOM.nextInt(3)]; + RANDOM.nextBytes(bytes); + return ByteString.copyFrom(bytes); + } + throw new IllegalArgumentException("Unrecognized class " + clazz); + } + + static Proto2SCMRatisProtocolForTesting.MethodArgument randomProto2MethodArgument() { + final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; + return Proto2SCMRatisProtocolForTesting.MethodArgument.newBuilder() + .setType(type.getName()) + .setValue(randomValueProto2(type)) + .build(); + } + + static Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto randomProto2SCMRatisResponseProto() { + final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; + return Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto.newBuilder() + .setType(type.getName()) + .setValue(randomValueProto2(type)) + .build(); + } + + static Proto2SCMRatisProtocolForTesting.SCMRatisRequestProto proto2Request( + String name, Proto2SCMRatisProtocolForTesting.RequestType type, int numArgs) { + // Build request using proto2 (test-only schema) + final Proto2SCMRatisProtocolForTesting.Method.Builder b = + Proto2SCMRatisProtocolForTesting.Method.newBuilder() + .setName(name); + for (int i = 0; i < numArgs; i++) { + b.addArgs(randomProto2MethodArgument()); + } + + return Proto2SCMRatisProtocolForTesting.SCMRatisRequestProto.newBuilder() + .setType(type) + .setMethod(b) + .build(); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java index a791a206bb8..40ff9baaf2d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java @@ -17,7 +17,8 @@ package org.apache.hadoop.hdds.scm.ha; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.apache.hadoop.hdds.scm.ha.SCMRatisProtocolCompatibilityTestUtil.proto2Request; +import static org.apache.hadoop.hdds.scm.ha.SCMRatisProtocolCompatibilityTestUtil.randomProto2SCMRatisResponseProto; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -25,6 +26,8 @@ import java.util.Random; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol; import org.apache.hadoop.hdds.protocol.proto.testing.Proto2SCMRatisProtocolForTesting; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.junit.jupiter.api.Test; /** @@ -34,28 +37,6 @@ public class TestSCMRatisProtocolCompatibility { static final Random RANDOM = new Random(); static final Class[] TYPES = {String.class, Integer.class, byte[].class}; - static com.google.protobuf.ByteString randomValueProto2(Class clazz) { - if (clazz == String.class) { - final int length = RANDOM.nextInt(3); - final StringBuilder builder = new StringBuilder(length); - for (int i = 0; i < length; i++) { - builder.append(RANDOM.nextInt(10)); - } - final String string = builder.toString(); - assertEquals(length, string.length()); - return com.google.protobuf.ByteString.copyFromUtf8(string); - } else if (clazz == Integer.class) { - final ByteBuffer buffer = ByteBuffer.allocate(4); - buffer.putInt(RANDOM.nextInt()); - return com.google.protobuf.ByteString.copyFrom(buffer.array()); - } else if (clazz == byte[].class) { - final byte[] bytes = new byte[RANDOM.nextInt(3)]; - RANDOM.nextBytes(bytes); - return com.google.protobuf.ByteString.copyFrom(bytes); - } - throw new IllegalArgumentException("Unrecognized class " + clazz); - } - static org.apache.ratis.thirdparty.com.google.protobuf.ByteString randomValueProto3(Class clazz) { if (clazz == String.class) { final int length = RANDOM.nextInt(3); @@ -78,14 +59,6 @@ static org.apache.ratis.thirdparty.com.google.protobuf.ByteString randomValu throw new IllegalArgumentException("Unrecognized class " + clazz); } - static Proto2SCMRatisProtocolForTesting.MethodArgument randomProto2MethodArgument() { - final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; - return Proto2SCMRatisProtocolForTesting.MethodArgument.newBuilder() - .setType(type.getName()) - .setValue(randomValueProto2(type)) - .build(); - } - static SCMRatisProtocol.MethodArgument randomProto3MethodArgument() { final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; return SCMRatisProtocol.MethodArgument.newBuilder() @@ -94,14 +67,6 @@ static SCMRatisProtocol.MethodArgument randomProto3MethodArgument() { .build(); } - static Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto randomProto2SCMRatisResponseProto() { - final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; - return Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto.newBuilder() - .setType(type.getName()) - .setValue(randomValueProto2(type)) - .build(); - } - static SCMRatisProtocol.SCMRatisResponseProto randomProto3SCMRatisResponseProto() { final Class type = TYPES[RANDOM.nextInt(TYPES.length)]; return SCMRatisProtocol.SCMRatisResponseProto.newBuilder() @@ -110,21 +75,6 @@ static SCMRatisProtocol.SCMRatisResponseProto randomProto3SCMRatisResponseProto( .build(); } - static Proto2SCMRatisProtocolForTesting.SCMRatisRequestProto proto2Request( - String name, Proto2SCMRatisProtocolForTesting.RequestType type, int numArgs) { - // Build request using proto2 (test-only schema) - final Proto2SCMRatisProtocolForTesting.Method.Builder b = Proto2SCMRatisProtocolForTesting.Method.newBuilder() - .setName(name); - for (int i = 0; i < numArgs; i++) { - b.addArgs(randomProto2MethodArgument()); - } - - return Proto2SCMRatisProtocolForTesting.SCMRatisRequestProto.newBuilder() - .setType(type) - .setMethod(b) - .build(); - } - static SCMRatisProtocol.SCMRatisRequestProto proto3Request( String name, SCMRatisProtocol.RequestType type, int numArgs) { final SCMRatisProtocol.Method.Builder b = SCMRatisProtocol.Method.newBuilder() @@ -176,28 +126,24 @@ static void runTestProto2RequestCanBeParsedByProto3( } assertEquals(proto2.toString(), proto3.toString()); - assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); + assertShortDebugString(proto2, proto3); assertEquals(proto2, Proto2SCMRatisProtocolForTesting.SCMRatisRequestProto.parseFrom(proto3.toByteArray())); } - private static byte[] bytesOf(com.google.protobuf.ByteString b) { - return b.toByteArray(); - } - - private static byte[] bytesOf(org.apache.ratis.thirdparty.com.google.protobuf.ByteString b) { - return b.toByteArray(); + private static void assertByteStringEquals(com.google.protobuf.ByteString proto2, ByteString proto3) { + assertEquals(UnsafeByteOperations.unsafeWrap(proto2.asReadOnlyByteBuffer()), proto3); } static void assertMethodArgument(Proto2SCMRatisProtocolForTesting.MethodArgument proto2, SCMRatisProtocol.MethodArgument proto3) { assertEquals(proto2.getType(), proto3.getType()); - assertArrayEquals(bytesOf(proto2.getValue()), bytesOf(proto3.getValue())); + assertByteStringEquals(proto2.getValue(), proto3.getValue()); } static void assertSCMRatisResponseProto(Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto proto2, SCMRatisProtocol.SCMRatisResponseProto proto3) { assertEquals(proto2.getType(), proto3.getType()); - assertArrayEquals(bytesOf(proto2.getValue()), bytesOf(proto3.getValue())); + assertByteStringEquals(proto2.getValue(), proto3.getValue()); } /** @@ -224,7 +170,7 @@ static void runTestProto2ResponseCanBeParsedByProto3() throws Exception { assertSCMRatisResponseProto(proto2, proto3); assertEquals(proto2.toString(), proto3.toString()); - assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); + assertShortDebugString(proto2, proto3); assertEquals(proto2, Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto.parseFrom(proto3.toByteArray())); } @@ -275,12 +221,12 @@ static void runTestProto3RequestCanBeParsedByProto2( for (int i = 0; i < numArgs; i++) { assertEquals(proto3.getMethod().getArgs(i).getType(), proto2.getMethod().getArgs(i).getType()); - assertArrayEquals(bytesOf(proto3.getMethod().getArgs(i).getValue()), - bytesOf(proto2.getMethod().getArgs(i).getValue())); + assertByteStringEquals(proto2.getMethod().getArgs(i).getValue(), + proto3.getMethod().getArgs(i).getValue()); } assertEquals(proto2.toString(), proto3.toString()); - assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); + assertShortDebugString(proto2, proto3); assertEquals(proto3, SCMRatisProtocol.SCMRatisRequestProto.parseFrom(proto2.toByteArray())); } @@ -298,20 +244,16 @@ static void runTestProto3ResponseCanBeParsedByProto2() throws Exception { Proto2SCMRatisProtocolForTesting.SCMRatisResponseProto.parseFrom(proto3.toByteArray()); assertEquals(proto3.getType(), proto2.getType()); - assertArrayEquals(bytesOf(proto3.getValue()), bytesOf(proto2.getValue())); + assertByteStringEquals(proto2.getValue(), proto3.getValue()); assertEquals(proto2.toString(), proto3.toString()); - assertEquals(shortDebugStringProto2(proto2), shortDebugStringProto3(proto3)); + assertShortDebugString(proto2, proto3); assertEquals(proto3, SCMRatisProtocol.SCMRatisResponseProto.parseFrom(proto2.toByteArray())); } - private static String shortDebugStringProto2( - com.google.protobuf.MessageOrBuilder msg) { - return com.google.protobuf.TextFormat.shortDebugString(msg); - } - - private static String shortDebugStringProto3( - org.apache.ratis.thirdparty.com.google.protobuf.MessageOrBuilder msg) { - return org.apache.ratis.thirdparty.com.google.protobuf.TextFormat.shortDebugString(msg); + private static void assertShortDebugString(com.google.protobuf.MessageOrBuilder proto2, + org.apache.ratis.thirdparty.com.google.protobuf.MessageOrBuilder proto3) { + assertEquals(com.google.protobuf.TextFormat.shortDebugString(proto2), + org.apache.ratis.thirdparty.com.google.protobuf.TextFormat.shortDebugString(proto3)); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java index 9072ebb8fcc..b65aa3ac885 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisResponse.java @@ -37,6 +37,7 @@ import org.apache.ratis.protocol.exceptions.RaftException; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -108,8 +109,7 @@ public void testResponseDecodeMissingTypeShouldFail() throws Exception { RaftClientReply reply = mock(RaftClientReply.class); when(reply.isSuccess()).thenReturn(true); when(reply.getMessage()).thenReturn(Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer()))); + UnsafeByteOperations.unsafeWrap(proto.toByteString().asReadOnlyByteBuffer()))); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -130,8 +130,7 @@ public void testResponseDecodeMissingValueShouldFail() throws Exception { RaftClientReply reply = mock(RaftClientReply.class); when(reply.isSuccess()).thenReturn(true); when(reply.getMessage()).thenReturn(Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer()))); + UnsafeByteOperations.unsafeWrap(proto.toByteString().asReadOnlyByteBuffer()))); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, From 2351cd1ab73aa1bb34254c54b9b488f250703eea Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 14 Feb 2026 01:35:57 +0800 Subject: [PATCH 7/9] Fix checkstyle error --- .../hadoop/hdds/scm/ha/SCMRatisRequest.java | 1 - .../hdds/scm/ha/io/GeneratedMessageCodec.java | 23 ++++++++++++++++++- ...SCMRatisProtocolCompatibilityTestUtil.java | 15 +++++++----- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java index 0305b4f82e8..0eeec239c9a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisRequest.java @@ -29,7 +29,6 @@ import org.apache.ratis.protocol.Message; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat; -import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java index ddfe70916ab..6b387d69486 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java @@ -1,11 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hdds.scm.ha.io; +import com.google.protobuf.Message; import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; -import com.google.protobuf.Message; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +/** + * {@link Codec} implementation for non-shaded + * {@link com.google.protobuf.Message} objects. + */ public class GeneratedMessageCodec implements Codec { @Override diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java index 9a6bdc9d984..aaf7ba6e1f4 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/SCMRatisProtocolCompatibilityTestUtil.java @@ -17,19 +17,22 @@ package org.apache.hadoop.hdds.scm.ha; -import com.google.protobuf.ByteString; -import org.apache.hadoop.hdds.protocol.proto.testing.Proto2SCMRatisProtocolForTesting; - -import java.nio.ByteBuffer; - import static org.apache.hadoop.hdds.scm.ha.TestSCMRatisProtocolCompatibility.RANDOM; import static org.apache.hadoop.hdds.scm.ha.TestSCMRatisProtocolCompatibility.TYPES; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.google.protobuf.ByteString; +import java.nio.ByteBuffer; +import org.apache.hadoop.hdds.protocol.proto.testing.Proto2SCMRatisProtocolForTesting; + /** * Tests proto2 to proto3 compatibility for SCMRatisProtocol. */ -public class SCMRatisProtocolCompatibilityTestUtil { +public final class SCMRatisProtocolCompatibilityTestUtil { + + private SCMRatisProtocolCompatibilityTestUtil() { + } + static ByteString randomValueProto2(Class clazz) { if (clazz == String.class) { final int length = RANDOM.nextInt(3); From df9720b1b760c725a6bfee89d798b81c5f354f6e Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 14 Feb 2026 08:10:14 +0800 Subject: [PATCH 8/9] Add ByteString handling support, adopt UnsafeByteOperations, and clean up imports --- .../hdds/scm/ha/io/ByteStringCodec.java | 9 +++-- .../hadoop/hdds/scm/ha/io/CodecFactory.java | 3 +- .../hdds/scm/ha/io/GeneratedMessageCodec.java | 9 +++-- .../hdds/scm/ha/io/ScmByteStringCodec.java | 39 +++++++++++++++++++ .../ha/TestSCMRatisProtocolCompatibility.java | 8 ++-- .../hdds/scm/ha/TestSCMRatisRequest.java | 16 +++----- 6 files changed, 62 insertions(+), 22 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmByteStringCodec.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java index e6762935a95..d5175a53ea7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java @@ -19,21 +19,24 @@ import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** - * A dummy codec that serializes a ByteString object to ByteString. + * {@link Codec} implementation for non-shaded + * {@link com.google.protobuf.ByteString} objects. */ public class ByteStringCodec implements Codec { @Override public ByteString serialize(Object object) throws InvalidProtocolBufferException { - return (ByteString) object; + return UnsafeByteOperations.unsafeWrap( + ((com.google.protobuf.ByteString) object).asReadOnlyByteBuffer()); } @Override public Object deserialize(Class type, ByteString value) throws InvalidProtocolBufferException { - return value; + return com.google.protobuf.ByteString.copyFrom(value.toByteArray()); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java index 8786acff614..3141754a53c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/CodecFactory.java @@ -48,7 +48,8 @@ public final class CodecFactory { codecs.put(Boolean.class, new BooleanCodec()); codecs.put(BigInteger.class, new BigIntegerCodec()); codecs.put(X509Certificate.class, new X509CertificateCodec()); - codecs.put(ByteString.class, new ByteStringCodec()); + codecs.put(com.google.protobuf.ByteString.class, new ByteStringCodec()); + codecs.put(ByteString.class, new ScmByteStringCodec()); codecs.put(ManagedSecretKey.class, new ManagedSecretKeyCodec()); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java index 6b387d69486..14a30c874af 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/GeneratedMessageCodec.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; /** * {@link Codec} implementation for non-shaded @@ -31,14 +32,14 @@ public class GeneratedMessageCodec implements Codec { @Override public ByteString serialize(Object object) - throws org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException { - final byte[] bytes = ((Message) object).toByteString().toByteArray(); - return ByteString.copyFrom(bytes); + throws InvalidProtocolBufferException { + return UnsafeByteOperations.unsafeWrap( + ((Message) object).toByteString().asReadOnlyByteBuffer()); } @Override public Object deserialize(Class type, ByteString value) - throws org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException { + throws InvalidProtocolBufferException { try { return ReflectionUtil.getMethod(type, "parseFrom", byte[].class) .invoke(null, (Object) value.toByteArray()); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmByteStringCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmByteStringCodec.java new file mode 100644 index 00000000000..2b479ff5559 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ScmByteStringCodec.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.ha.io; + +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; + +/** + * A dummy codec that serializes a ByteString object to ByteString. + */ +public class ScmByteStringCodec implements Codec { + + @Override + public ByteString serialize(Object object) + throws InvalidProtocolBufferException { + return (ByteString) object; + } + + @Override + public Object deserialize(Class type, ByteString value) + throws InvalidProtocolBufferException { + return value; + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java index 40ff9baaf2d..043f9f4e563 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisProtocolCompatibility.java @@ -37,7 +37,7 @@ public class TestSCMRatisProtocolCompatibility { static final Random RANDOM = new Random(); static final Class[] TYPES = {String.class, Integer.class, byte[].class}; - static org.apache.ratis.thirdparty.com.google.protobuf.ByteString randomValueProto3(Class clazz) { + static ByteString randomValueProto3(Class clazz) { if (clazz == String.class) { final int length = RANDOM.nextInt(3); final StringBuilder builder = new StringBuilder(length); @@ -46,15 +46,15 @@ static org.apache.ratis.thirdparty.com.google.protobuf.ByteString randomValu } final String string = builder.toString(); assertEquals(length, string.length()); - return org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFromUtf8(string); + return ByteString.copyFromUtf8(string); } else if (clazz == Integer.class) { final ByteBuffer buffer = ByteBuffer.allocate(4); buffer.putInt(RANDOM.nextInt()); - return org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom(buffer.array()); + return ByteString.copyFrom(buffer.array()); } else if (clazz == byte[].class) { final byte[] bytes = new byte[RANDOM.nextInt(3)]; RANDOM.nextBytes(bytes); - return org.apache.ratis.thirdparty.com.google.protobuf.ByteString.copyFrom(bytes); + return ByteString.copyFrom(bytes); } throw new IllegalArgumentException("Unrecognized class " + clazz); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java index 0e5d8f37d48..2f4ebf94369 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMRatisRequest.java @@ -31,6 +31,7 @@ import org.apache.ratis.protocol.Message; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.junit.jupiter.api.Test; /** @@ -105,8 +106,7 @@ public void testDecodeMissingRequestTypeShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); + UnsafeByteOperations.unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -124,8 +124,7 @@ public void testDecodeMissingMethodShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); + UnsafeByteOperations.unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -145,8 +144,7 @@ public void testDecodeMissingMethodNameShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); + UnsafeByteOperations.unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -173,8 +171,7 @@ public void testDecodeMissingArgumentTypeShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); + UnsafeByteOperations.unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, @@ -201,8 +198,7 @@ public void testDecodeMissingArgumentValueShouldFail() throws Exception { .build(); Message msg = Message.valueOf( - org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations - .unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); + UnsafeByteOperations.unsafeWrap(proto.toByteString().asReadOnlyByteBuffer())); InvalidProtocolBufferException ex = assertThrows( InvalidProtocolBufferException.class, From b1955b134bd2211a92f25ab6f10dd06aefb158f6 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 14 Feb 2026 11:31:40 +0800 Subject: [PATCH 9/9] Use UnsafeByteOperations to reduce unnecessary ByteString allocations --- .../java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java index d5175a53ea7..1ffabcf328a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/io/ByteStringCodec.java @@ -37,6 +37,7 @@ public ByteString serialize(Object object) @Override public Object deserialize(Class type, ByteString value) throws InvalidProtocolBufferException { - return com.google.protobuf.ByteString.copyFrom(value.toByteArray()); + return com.google.protobuf.UnsafeByteOperations. + unsafeWrap(value.asReadOnlyByteBuffer()); } }