HBASE-29117 Kerby version conflict with Hadoop 3.4 (#6674)

Use hadoop-minikdc instead of depending on kerby directly Signed-off-by: Andrew Purtell <[email protected]> Signed-off-by: Istvan Toth <[email protected]>
apache · Feb 14, 2025 · d59814c · d59814c
1 parent f37e8ac
commit d59814c
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 95 deletions.
diff --git a/hbase-asyncfs/pom.xml b/hbase-asyncfs/pom.xml
@@ -78,16 +78,6 @@
       <artifactId>bcprov-jdk18on</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-client</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-http</artifactId>

diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
@@ -165,11 +165,6 @@
       <artifactId>log4j-slf4j-impl</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcprov-jdk18on</artifactId>
@@ -349,6 +344,11 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-common</artifactId>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minikdc</artifactId>
+          <scope>test</scope>
+        </dependency>
       </dependencies>
       <build>
         <plugins>

diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/SimpleKdcServerUtil.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/SimpleKdcServerUtil.java
@@ -19,7 +19,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.net.BindException;
 import java.net.InetAddress;
 import java.util.function.Supplier;
 import org.apache.hadoop.hbase.net.BoundSocketMaker;
@@ -85,12 +84,9 @@ static SimpleKdcServer getRunningSimpleKdcServer(File testDir,
         if (kdc != null) {
           kdc.stop();
         }
-        if (ke.getCause() != null && ke.getCause() instanceof BindException) {
-          LOG.info("Clashed using port {}; getting a new random port", kdcPort);
-          continue;
-        } else {
-          throw ke;
-        }
+        // new kerby just eats the cause exception so we can not test for BindException any more.
+        // the only way is to always retry...
+        LOG.info("Failed to init/start kdc server, retry = {}", i, ke);
       } finally {
         if (bsm != null) {
           bsm.close();

diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml
@@ -100,21 +100,11 @@
       <optional>true</optional>
     </dependency>
     <!-- Test dependencies -->
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcprov-jdk18on</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-core</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
@@ -425,12 +415,6 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-minikdc</artifactId>
           <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.directory.api</groupId>
-              <artifactId>api-all</artifactId>
-            </exclusion>
-          </exclusions>
         </dependency>
       </dependencies>
       <build>

diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml
@@ -193,16 +193,6 @@
       <groupId>org.apache.tomcat</groupId>
       <artifactId>tomcat-jasper</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-core</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>

diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
@@ -260,16 +260,6 @@
       <artifactId>hamcrest-library</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-client</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>

diff --git a/hbase-testing-util/pom.xml b/hbase-testing-util/pom.xml
@@ -211,22 +211,6 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-minikdc</artifactId>
           <scope>compile</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>bouncycastle</groupId>
-              <artifactId>bcprov-jdk15</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.kerby</groupId>
-          <artifactId>kerb-client</artifactId>
-          <scope>compile</scope>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.kerby</groupId>
-          <artifactId>kerb-simplekdc</artifactId>
-          <scope>compile</scope>
         </dependency>
       </dependencies>
     </profile>

diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml
@@ -108,11 +108,6 @@
       <groupId>org.apache.tomcat</groupId>
       <artifactId>tomcat-jasper</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>jcl-over-slf4j</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -893,7 +893,6 @@
     <spy.version>2.12.3</spy.version>
     <bouncycastle.version>1.78</bouncycastle.version>
     <skyscreamer.version>1.5.1</skyscreamer.version>
-    <kerby.version>1.0.1</kerby.version>
     <commons-crypto.version>1.1.0</commons-crypto.version>
     <curator.version>4.2.0</curator.version>
     <!-- Plugin Dependencies -->
@@ -1690,21 +1689,6 @@
         <version>${bouncycastle.version}</version>
         <scope>test</scope>
       </dependency>
-      <dependency>
-        <groupId>org.apache.kerby</groupId>
-        <artifactId>kerb-core</artifactId>
-        <version>${kerby.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.kerby</groupId>
-        <artifactId>kerb-client</artifactId>
-        <version>${kerby.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.kerby</groupId>
-        <artifactId>kerb-simplekdc</artifactId>
-        <version>${kerby.version}</version>
-      </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-crypto</artifactId>
@@ -3959,6 +3943,14 @@
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-reload4j</artifactId>
               </exclusion>
+              <exclusion>
+                <groupId>jline</groupId>
+                <artifactId>jline</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.jline</groupId>
+                <artifactId>jline</artifactId>
+              </exclusion>
             </exclusions>
           </dependency>
           <dependency>
@@ -4166,6 +4158,14 @@
               </exclusion>
             </exclusions>
           </dependency>
+          <!--
+            We use hadoop-minikdc everywhere instead of depending on kerby is
+            because hadoop upgrade kerby version in hadoop 3.4.x and kerby 1.x
+            and 2.x are not compatible, so if we specify the version of kerby
+            in our pom will cause compile/runtime error while compiling against
+            different hadoop versions, and also cause problem for downstream
+            projects. See HBASE-29117 for more details.
+          -->
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-minikdc</artifactId>
@@ -4188,6 +4188,18 @@
                 <groupId>bouncycastle</groupId>
                 <artifactId>bcprov-jdk15</artifactId>
               </exclusion>
+              <exclusion>
+                <groupId>org.apache.directory.api</groupId>
+                <artifactId>api-all</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>jline</groupId>
+                <artifactId>jline</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.jline</groupId>
+                <artifactId>jline</artifactId>
+              </exclusion>
             </exclusions>
           </dependency>
           <dependency>