Hdds 11784

.github/workflows/ci.yml

@@ -30,6 +30,7 @@ env:
   FAIL_FAST: ${{ github.event_name == 'pull_request' }}
   # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
   TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
+  MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml --show-version
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
   HADOOP_IMAGE: ghcr.io/apache/hadoop
   OZONE_IMAGE: ghcr.io/apache/ozone

.mvn/extensions.xml

@@ -24,7 +24,7 @@
   <extension>
     <groupId>com.gradle</groupId>
     <artifactId>develocity-maven-extension</artifactId>
-    <version>1.22.1</version>
+    <version>1.23</version>
   </extension>
   <extension>
     <groupId>com.gradle</groupId>

dev-support/ci/maven-settings.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <mirrors>
+    <mirror>
+      <id>block-snapshots1</id>
+      <mirrorOf>apache.snapshots</mirrorOf>
+      <name>Block access to Apache Snapshots</name>
+      <url>https://repository.apache.org/snapshots</url>
+      <blocked>true</blocked>
+    </mirror>
+    <mirror>
+      <id>block-snapshots2</id>
+      <mirrorOf>apache.snapshots.https</mirrorOf>
+      <name>Block access to Apache Snapshots</name>
+      <url>https://repository.apache.org/content/repositories/snapshots</url>
+      <blocked>true</blocked>
+    </mirror>
+  </mirrors>
+</settings>

dev-support/pom.xml

@@ -12,27 +12,25 @@
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
   <parent>
-    <artifactId>ozone-main</artifactId>
     <groupId>org.apache.ozone</groupId>
+    <artifactId>ozone-main</artifactId>
     <version>2.0.0-SNAPSHOT</version>
   </parent>
-  <modelVersion>4.0.0</modelVersion>
   <artifactId>ozone-dev-support</artifactId>
-  <description>Helper module for sharing resources among projects</description>
   <name>Apache Ozone Dev Support</name>
+  <description>Helper module for sharing resources among projects</description>
 
   <properties>
     <failIfNoTests>false</failIfNoTests>
   </properties>
   <build>
     <resources>
       <resource>
-        <directory>${project.build.directory}/extra-resources</directory>
         <targetPath>META-INF</targetPath>
+        <directory>${project.build.directory}/extra-resources</directory>
         <includes>
           <include>LICENSE.txt</include>
           <include>NOTICE.txt</include>
@@ -54,10 +52,10 @@
         <executions>
           <execution>
             <id>copy-resources</id>
-            <phase>validate</phase>
             <goals>
               <goal>copy-resources</goal>
             </goals>
+            <phase>validate</phase>
             <configuration>
               <outputDirectory>${project.build.directory}/extra-resources</outputDirectory>
               <resources>
@@ -77,21 +75,21 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-remote-resources-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>process-resources</phase>
-            <goals>
-              <goal>bundle</goal>
-            </goals>
-          </execution>
-        </executions>
         <configuration>
           <resourcesDirectory>${project.build.outputDirectory}</resourcesDirectory>
           <includes>
             <include>META-INF/LICENSE.txt</include>
             <include>META-INF/NOTICE.txt</include>
           </includes>
         </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>bundle</goal>
+            </goals>
+            <phase>process-resources</phase>
+          </execution>
+        </executions>
       </plugin>
     </plugins>
   </build>

hadoop-hdds/annotations/pom.xml

@@ -12,10 +12,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.ozone</groupId>
@@ -25,14 +22,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
   <artifactId>hdds-annotation-processing</artifactId>
   <version>2.0.0-SNAPSHOT</version>
-  <description>Apache Ozone annotation processing tools for validating custom
-    annotations at compile time.
-  </description>
-  <name>Apache Ozone Annotation Processing</name>
   <packaging>jar</packaging>
+  <name>Apache Ozone Annotation Processing</name>
+  <description>Apache Ozone annotation processing tools for validating custom
+    annotations at compile time.</description>
 
   <properties>
-    <maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
+    <!-- no tests in this module so far -->
+    <maven.test.skip>true</maven.test.skip>
   </properties>
 
   <build>

hadoop-hdds/client/pom.xml

@@ -12,10 +12,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-https://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.ozone</groupId>
@@ -25,14 +22,31 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
   <artifactId>hdds-client</artifactId>
   <version>2.0.0-SNAPSHOT</version>
-  <description>Apache Ozone Distributed Data Store Client Library</description>
-  <name>Apache Ozone HDDS Client</name>
   <packaging>jar</packaging>
-
-  <properties>
-  </properties>
+  <name>Apache Ozone HDDS Client</name>
+  <description>Apache Ozone Distributed Data Store Client Library</description>
 
   <dependencies>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentracing</groupId>
+      <artifactId>opentracing-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentracing</groupId>
+      <artifactId>opentracing-util</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.annotation</groupId>
+      <artifactId>jakarta.annotation-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-common</artifactId>
@@ -49,11 +63,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-interface-client</artifactId>
     </dependency>
-
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-client</artifactId>
@@ -74,31 +83,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-thirdparty-misc</artifactId>
     </dependency>
-
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>io.opentracing</groupId>
-      <artifactId>opentracing-api</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.opentracing</groupId>
-      <artifactId>opentracing-util</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>jakarta.annotation</groupId>
-      <artifactId>jakarta.annotation-api</artifactId>
-    </dependency>
-
     <!-- Test dependencies -->
     <dependency>
       <groupId>org.apache.ozone</groupId>
@@ -148,7 +137,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <artifactId>maven-enforcer-plugin</artifactId>
         <executions>
           <execution>
-            <id>ban-annotations</id> <!-- override default restriction from root POM -->
+            <id>ban-annotations</id>
+            <!-- override default restriction from root POM -->
             <configuration>
               <rules>
                 <restrictImports>

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockDataStreamOutput.java

@@ -253,7 +253,7 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
     }
     while (len > 0) {
       allocateNewBufferIfNeeded();
-      int writeLen = Math.min(len, currentBuffer.length());
+      int writeLen = Math.min(len, currentBuffer.remaining());
       final StreamBuffer buf = new StreamBuffer(b, off, writeLen);
       currentBuffer.put(buf);
       writeChunkIfNeeded();
@@ -265,7 +265,7 @@ public void write(ByteBuffer b, int off, int len) throws IOException {
   }
 
   private void writeChunkIfNeeded() throws IOException {
-    if (currentBuffer.length() == 0) {
+    if (currentBuffer.remaining() == 0) {
       writeChunk(currentBuffer);
       currentBuffer = null;
     }
@@ -672,6 +672,7 @@ private void writeChunkToContainer(ByteBuffer buf)
             out.writeAsync(buf, StandardWriteOption.SYNC) :
             out.writeAsync(buf))
             .whenCompleteAsync((r, e) -> {
+              metrics.decrPendingContainerOpsMetrics(ContainerProtos.Type.WriteChunk);
               if (e != null || !r.isSuccess()) {
                 if (e == null) {
                   e = new IOException("result is not success");

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java

@@ -39,8 +39,8 @@ public ByteBuffer duplicate() {
     return buffer.duplicate();
   }
 
-  public int length() {
-    return buffer.limit() - buffer.position();
+  public int remaining() {
+    return buffer.remaining();
   }
 
   public int position() {