ZOOKEEPER-4954: Use FIPS style hostname verification when no custom t…

ZOOKEEPER-4954: Use FIPS style hostname verification when no custom truststore is specified
Reviewers: anmolnar
Author: stoty
Closes #2283 from stoty/ZOOKEEPER-4954

Author: stoty

zookeeper-server/src/main/java/org/apache/zookeeper/common/ClientX509Util.java

@@ -93,7 +93,7 @@ public SslContext createNettySslContextForClient(ZKConfig config)
 
         SslContext sslContext1 = sslContextBuilder.build();
 
-        if (getFipsMode(config) && isServerHostnameVerificationEnabled(config)) {
+        if ((getFipsMode(config) || tm == null) && isServerHostnameVerificationEnabled(config)) {
             return addHostnameVerification(sslContext1, "Server");
         } else {
             return sslContext1;
@@ -138,7 +138,7 @@ public SslContext createNettySslContextForServer(ZKConfig config, KeyManager key
 
         SslContext sslContext1 = sslContextBuilder.build();
 
-        if (getFipsMode(config) && isClientHostnameVerificationEnabled(config)) {
+        if ((getFipsMode(config) || trustManager == null) && isClientHostnameVerificationEnabled(config)) {
             return addHostnameVerification(sslContext1, "Client");
         } else {
             return sslContext1;

zookeeper-server/src/test/java/org/apache/zookeeper/common/X509UtilTest.java

@@ -23,6 +23,8 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.handler.ssl.SslContext;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -45,6 +47,7 @@
 import javax.net.ssl.HandshakeCompletedEvent;
 import javax.net.ssl.HandshakeCompletedListener;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.SSLServerSocket;
 import javax.net.ssl.SSLSocket;
@@ -58,6 +61,7 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 
+
 public class X509UtilTest extends BaseX509ParameterizedTestCase {
 
     private X509Util x509Util;
@@ -754,6 +758,28 @@ public void testCreateSSLContext_ocspWithJreProvider(
         }
     }
 
+    @ParameterizedTest
+    @MethodSource("data")
+    public void testCreateSSLContext_hostnameVerificationNoCustomTrustStore(X509KeyType caKeyType,
+            X509KeyType certKeyType, String keyPassword, Integer paramIndex) throws Exception {
+        init(caKeyType, certKeyType, keyPassword, paramIndex);
+        // No truststore
+        System.clearProperty(x509Util.getSslTruststoreLocationProperty());
+        // Verify client hostname too
+        System.setProperty(x509Util.getSslClientHostnameVerificationEnabledProperty(), "true");
+        ZKConfig zkConfig = new ZKConfig();
+        try (ClientX509Util clientX509Util = new ClientX509Util();) {
+            UnpooledByteBufAllocator byteBufAllocator = new UnpooledByteBufAllocator(false);
+            SslContext clientContext = clientX509Util.createNettySslContextForClient(zkConfig);
+            SSLEngine clientEngine = clientContext.newEngine(byteBufAllocator);
+            assertEquals(clientEngine.getSSLParameters().getEndpointIdentificationAlgorithm(), "HTTPS");
+
+            SslContext serverContext = clientX509Util.createNettySslContextForServer(zkConfig);
+            SSLEngine serverEngine = serverContext.newEngine(byteBufAllocator);
+            assertEquals(serverEngine.getSSLParameters().getEndpointIdentificationAlgorithm(), "HTTPS");
+        }
+    }
+
     private static void forceClose(Socket s) {
         if (s == null || s.isClosed()) {
             return;