diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml index 6240d7be1144..260f92265374 100644 --- a/hadoop-hdds/client/pom.xml +++ b/hadoop-hdds/client/pom.xml @@ -86,6 +86,12 @@ + + + commons-io + commons-io + test + org.apache.ozone hdds-common diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index 9a3ad947807a..46ade2a57c40 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -63,10 +63,6 @@ commons-codec commons-codec - - commons-io - commons-io - io.opentelemetry opentelemetry-api @@ -160,10 +156,6 @@ org.slf4j slf4j-api - - org.yaml - snakeyaml - org.apache.ratis ratis-metrics-dropwizard3 @@ -180,14 +172,19 @@ slf4j-reload4j runtime + + + + commons-io + commons-io + test + org.apache.ozone hdds-config test-jar test - - org.apache.ozone hdds-test-utils diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNode.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNode.java index 2fca31c38a53..ac9ec514a80b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNode.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNode.java @@ -99,9 +99,4 @@ Node getLeaf(int leafIndex, List excludedScopes, @Override int hashCode(); - - static InnerNode fromProtobuf( - HddsProtos.InnerNode innerNode) { - return InnerNodeImpl.fromProtobuf(innerNode); - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/Node.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/Node.java index 54d720e3c6f6..849111161bce 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/Node.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/Node.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.net; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; /** @@ -134,16 +133,4 @@ default HddsProtos.NetworkNode toProtobuf( int clientVersion) { return null; } - - static Node fromProtobuf( - HddsProtos.NetworkNode networkNode) { - if (networkNode.hasDatanodeDetails()) { - return DatanodeDetails.getFromProtoBuf( - networkNode.getDatanodeDetails()); - } else if (networkNode.hasInnerNode()) { - return InnerNode.fromProtobuf(networkNode.getInnerNode()); - } else { - return null; - } - } } diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index a703a7be5924..9e455b757277 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -124,6 +124,10 @@ javax.servlet javax.servlet-api + + org.apache.commons + commons-collections4 + org.apache.commons commons-compress diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java similarity index 99% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java index c55646ad1ab9..c38a8800b5d6 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java @@ -109,7 +109,7 @@ public Builder setChildrenMap( String networkName = childrenMapProto.hasNetworkName() ? childrenMapProto.getNetworkName() : null; Node node = childrenMapProto.hasNetworkNode() ? - Node.fromProtobuf(childrenMapProto.getNetworkNode()) : null; + NodeSchemaManager.fromProtobuf(childrenMapProto.getNetworkNode()) : null; newChildrenMap.put(networkName, node); } this.childrenMap = newChildrenMap; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaLoader.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaLoader.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaLoader.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaLoader.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java similarity index 91% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java index 326b16a7b6f9..0d6ad032969e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java @@ -23,6 +23,8 @@ import java.util.Arrays; import java.util.List; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.net.NodeSchemaLoader.NodeSchemaLoadResult; import org.slf4j.Logger; @@ -138,4 +140,14 @@ public String complete(String path) { } return null; } + + static Node fromProtobuf(HddsProtos.NetworkNode networkNode) { + if (networkNode.hasDatanodeDetails()) { + return DatanodeDetails.getFromProtoBuf(networkNode.getDatanodeDetails()); + } else if (networkNode.hasInnerNode()) { + return InnerNodeImpl.fromProtobuf(networkNode.getInnerNode()); + } else { + return null; + } + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/package-info.java new file mode 100644 index 000000000000..d363c3c5cf75 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/net/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The network topology supported by Ozone. + */ +package org.apache.hadoop.hdds.scm.net; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/YamlUtils.java diff --git a/hadoop-hdds/common/src/main/resources/network-topology-default.xml b/hadoop-hdds/framework/src/main/resources/network-topology-default.xml similarity index 100% rename from hadoop-hdds/common/src/main/resources/network-topology-default.xml rename to hadoop-hdds/framework/src/main/resources/network-topology-default.xml diff --git a/hadoop-hdds/common/src/main/resources/network-topology-default.yaml b/hadoop-hdds/framework/src/main/resources/network-topology-default.yaml similarity index 100% rename from hadoop-hdds/common/src/main/resources/network-topology-default.yaml rename to hadoop-hdds/framework/src/main/resources/network-topology-default.yaml diff --git a/hadoop-hdds/common/src/main/resources/network-topology-nodegroup.xml b/hadoop-hdds/framework/src/main/resources/network-topology-nodegroup.xml similarity index 100% rename from hadoop-hdds/common/src/main/resources/network-topology-nodegroup.xml rename to hadoop-hdds/framework/src/main/resources/network-topology-nodegroup.xml diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java similarity index 100% rename from hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java rename to hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestNetworkTopologyImpl.java diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaLoader.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaLoader.java similarity index 100% rename from hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaLoader.java rename to hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaLoader.java diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java similarity index 100% rename from hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java rename to hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestNodeSchemaManager.java diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestYamlSchemaLoader.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestYamlSchemaLoader.java similarity index 100% rename from hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/net/TestYamlSchemaLoader.java rename to hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/net/TestYamlSchemaLoader.java diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/enforce-error.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/enforce-error.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/enforce-error.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/enforce-error.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/external-entity.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/external-entity.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/external-entity.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/external-entity.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/good.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/good.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/good.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/good.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/good.yaml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/good.yaml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/good.yaml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/good.yaml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/invalid-cost.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/invalid-cost.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/invalid-cost.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/invalid-cost.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/invalid-version.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/invalid-version.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/invalid-version.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/invalid-version.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/middle-leaf.yaml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/middle-leaf.yaml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/middle-leaf.yaml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/middle-leaf.yaml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-leaf.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-leaf.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-leaf.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-leaf.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-root.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-root.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-root.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-root.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-root.yaml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-root.yaml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-root.yaml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-root.yaml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-topology.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-topology.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/multiple-topology.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/multiple-topology.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/no-leaf.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/no-leaf.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/no-leaf.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/no-leaf.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/no-root.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/no-root.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/no-root.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/no-root.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/no-topology.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/no-topology.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/no-topology.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/no-topology.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/path-layers-size-mismatch.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/path-layers-size-mismatch.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/path-layers-size-mismatch.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/path-layers-size-mismatch.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/path-with-id-reference-failure.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/path-with-id-reference-failure.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/path-with-id-reference-failure.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/path-with-id-reference-failure.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/unknown-layer-type.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/unknown-layer-type.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/unknown-layer-type.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/unknown-layer-type.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/wrong-path-order-1.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/wrong-path-order-1.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/wrong-path-order-1.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/wrong-path-order-1.xml diff --git a/hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/wrong-path-order-2.xml b/hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/wrong-path-order-2.xml similarity index 100% rename from hadoop-hdds/common/src/test/resources/networkTopologyTestFiles/wrong-path-order-2.xml rename to hadoop-hdds/framework/src/test/resources/networkTopologyTestFiles/wrong-path-order-2.xml diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index 5ac1bf86d43b..6772d4f85c9b 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -151,6 +151,12 @@ runtime + + + commons-io + commons-io + test + io.grpc grpc-inprocess diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 79fb802e38f9..17723b208cf6 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -90,8 +90,8 @@ run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/scm-audit-log4j2.properties" "e run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/s3g-audit-log4j2.properties" "etc/hadoop" run cp "${ROOT}/hadoop-ozone/dist/src/shell/conf/ozone-site.xml" "etc/hadoop" run cp -f "${ROOT}/hadoop-ozone/dist/src/shell/conf/log4j.properties" "etc/hadoop" -run cp "${ROOT}/hadoop-hdds/common/src/main/resources/network-topology-default.xml" "etc/hadoop" -run cp "${ROOT}/hadoop-hdds/common/src/main/resources/network-topology-nodegroup.xml" "etc/hadoop" +run cp "${ROOT}/hadoop-hdds/framework/src/main/resources/network-topology-default.xml" "etc/hadoop" +run cp "${ROOT}/hadoop-hdds/framework/src/main/resources/network-topology-nodegroup.xml" "etc/hadoop" run cp -r "${ROOT}/hadoop-ozone/dist/src/main/dockerlibexec/." "libexec/"