Branch 2.7.7

BUILDING.txt

@@ -219,10 +219,10 @@ If the build process fails with an out of memory error, you should be able to fi
 it by increasing the memory used by maven -which can be done via the environment
 variable MAVEN_OPTS.
 
-Here is an example setting to allocate between 256 and 512 MB of heap space to
+Here is an example setting to allocate between 256 MB and 1.5 GB of heap space to
 Maven
 
-export MAVEN_OPTS="-Xms256m -Xmx512m"
+export MAVEN_OPTS="-Xms256m -Xmx1536m"
 
 ----------------------------------------------------------------------------------
 

dev-support/docker/Dockerfile

@@ -46,25 +46,26 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
 RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
 
 #######
-# Oracle Java
+# Java OpenJDK
 #######
 
 RUN apt-get install -y software-properties-common
 RUN add-apt-repository -y ppa:webupd8team/java
 RUN apt-get update
 
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get install -y oracle-java8-installer
+# Install OpenJDK 7
+RUN apt-get install -y openjdk-7-jdk
 
 ######
-# Install findbugs
+# Install spotbugs (successor of findbugs)
 ######
-RUN mkdir -p /opt/findbugs && \
-    curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
-         -o /opt/findbugs.tar.gz && \
-    tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
-ENV FINDBUGS_HOME /opt/findbugs
+RUN mkdir -p /opt/spotbugs && \
+    curl -L -s -S \
+         http://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/3.1.2/spotbugs-3.1.2.tgz \
+         -o /opt/spotbugs.tar.gz && \
+    tar xzf /opt/spotbugs.tar.gz --strip-components 1 -C /opt/spotbugs
+# Hadoop uses FINDBUGS_HOME to run spotbugs
+ENV FINDBUGS_HOME /opt/spotbugs
 
 ####
 # Install shellcheck
@@ -75,7 +76,7 @@ RUN cabal update && cabal install shellcheck --global
 ###
 # Avoid out of memory errors in builds
 ###
-ENV MAVEN_OPTS -Xms256m -Xmx512m
+ENV MAVEN_OPTS -Xms256m -Xmx1536m
 
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.

hadoop-assemblies/pom.xml

@@ -23,12 +23,12 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>2.7.7</version>
     <relativePath>../hadoop-project</relativePath>
   </parent>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-assemblies</artifactId>
-  <version>2.7.4-SNAPSHOT</version>
+  <version>2.7.7</version>
   <name>Apache Hadoop Assemblies</name>
   <description>Apache Hadoop Assemblies</description>
 

hadoop-build-tools/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>hadoop-main</artifactId>
     <groupId>org.apache.hadoop</groupId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>2.7.7</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>hadoop-build-tools</artifactId>

hadoop-client/pom.xml

@@ -18,12 +18,12 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project-dist</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>2.7.7</version>
     <relativePath>../hadoop-project-dist</relativePath>
   </parent>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-client</artifactId>
-  <version>2.7.4-SNAPSHOT</version>
+  <version>2.7.7</version>
   <packaging>jar</packaging>
 
   <description>Apache Hadoop Client</description>

hadoop-common-project/hadoop-annotations/pom.xml

@@ -20,12 +20,12 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>2.7.7</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-annotations</artifactId>
-  <version>2.7.4-SNAPSHOT</version>
+  <version>2.7.7</version>
   <description>Apache Hadoop Annotations</description>
   <name>Apache Hadoop Annotations</name>
   <packaging>jar</packaging>

hadoop-common-project/hadoop-auth-examples/pom.xml

@@ -20,12 +20,12 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>2.7.7</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-auth-examples</artifactId>
-  <version>2.7.4-SNAPSHOT</version>
+  <version>2.7.7</version>
   <packaging>war</packaging>
 
   <name>Apache Hadoop Auth Examples</name>

hadoop-common-project/hadoop-auth/pom.xml

@@ -20,12 +20,12 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>2.7.4-SNAPSHOT</version>
+    <version>2.7.7</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-auth</artifactId>
-  <version>2.7.4-SNAPSHOT</version>
+  <version>2.7.7</version>
   <packaging>jar</packaging>
 
   <name>Apache Hadoop Auth</name>

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java

@@ -323,8 +323,8 @@ String apply(String[] params) throws IOException {
         }
       }
       if (result != null && nonSimplePattern.matcher(result).find()) {
-        throw new NoMatchingRule("Non-simple name " + result +
-                                 " after auth_to_local rule " + this);
+        LOG.info("Non-simple name {} after auth_to_local rule {}",
+            result, this);
       }
       if (toLowerCase && result != null) {
         result = result.toLowerCase(Locale.ENGLISH);
@@ -377,7 +377,7 @@ public static class NoMatchingRule extends IOException {
   /**
    * Get the translation of the principal name into an operating system
    * user name.
-   * @return the short name
+   * @return the user name
    * @throws IOException throws if something is wrong with the rules
    */
   public String getShortName() throws IOException {
@@ -397,7 +397,8 @@ public String getShortName() throws IOException {
         return result;
       }
     }
-    throw new NoMatchingRule("No rules applied to " + toString());
+    LOG.info("No auth_to_local rules applied to {}", this);
+    return toString();
   }
 
   /**

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java

@@ -15,8 +15,9 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
-import java.nio.charset.Charset;
+import java.security.SecureRandom;
 import java.util.Random;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -32,7 +33,7 @@ public class RandomSignerSecretProvider extends RolloverSignerSecretProvider {
 
   public RandomSignerSecretProvider() {
     super();
-    rand = new Random();
+    rand = new SecureRandom();
   }
 
   /**
@@ -48,6 +49,8 @@ public RandomSignerSecretProvider(long seed) {
 
   @Override
   protected byte[] generateNewSecret() {
-    return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8"));
+    byte[] secret = new byte[32]; // 32 bytes = 256 bits
+    rand.nextBytes(secret);
+    return secret;
   }
 }

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java

@@ -38,7 +38,7 @@
 public abstract class RolloverSignerSecretProvider
     extends SignerSecretProvider {
 
-  private static Logger LOG = LoggerFactory.getLogger(
+  static Logger LOG = LoggerFactory.getLogger(
     RolloverSignerSecretProvider.class);
   /**
    * Stores the currently valid secrets.  The current secret is the 0th element

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java

@@ -16,6 +16,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.security.SecureRandom;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -176,7 +177,7 @@ public class ZKSignerSecretProvider extends RolloverSignerSecretProvider {
 
   public ZKSignerSecretProvider() {
     super();
-    rand = new Random();
+    rand = new SecureRandom();
   }
 
   /**
@@ -369,8 +370,11 @@ private synchronized void pullFromZK(boolean isInit) {
     }
   }
 
-  private byte[] generateRandomSecret() {
-    return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8"));
+  @VisibleForTesting
+  protected byte[] generateRandomSecret() {
+    byte[] secret = new byte[32]; // 32 bytes = 256 bits
+    rand.nextBytes(secret);
+    return secret;
   }
 
   /**

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java

@@ -109,12 +109,7 @@ public void testNameRules() throws Exception {
     kn = new KerberosName("bar@BAR");
     Assert.assertEquals("bar", kn.getShortName());
     kn = new KerberosName("bar@FOO");
-    try {
-      kn.getShortName();
-      Assert.fail();
-    }
-    catch (Exception ex) {      
-    }
+    Assert.assertEquals("bar@FOO", kn.getShortName());
   }
 
   @Test(timeout=60000)

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java

@@ -72,23 +72,14 @@ private void checkBadName(String name) {
     }
   }
 
-  private void checkBadTranslation(String from) {
-    System.out.println("Checking bad translation for " + from);
-    KerberosName nm = new KerberosName(from);
-    try {
-      nm.getShortName();
-      Assert.fail("didn't get exception for " + from);
-    } catch (IOException ie) {
-      // PASS
-    }
-  }
-
   @Test
   public void testAntiPatterns() throws Exception {
     checkBadName("owen/owen/[email protected]");
     checkBadName("owen@foo/bar.com");
-    checkBadTranslation("[email protected]");
-    checkBadTranslation("root/[email protected]");
+
+    // no rules applied, these should pass
+    checkTranslation("[email protected]", "[email protected]");
+    checkTranslation("root/[email protected]", "root/[email protected]");
   }
 
   @Test

hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRandomSignerSecretProvider.java

@@ -14,22 +14,37 @@
 package org.apache.hadoop.security.authentication.util;
 
 import java.util.Random;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+
 public class TestRandomSignerSecretProvider {
 
+  // rollover every 50 msec
+  private final int timeout = 100;
+  private final long rolloverFrequency = timeout / 2;
+
+  {
+    LogManager.getLogger(
+        RolloverSignerSecretProvider.LOG.getName()).setLevel(Level.DEBUG);
+  }
+
   @Test
   public void testGetAndRollSecrets() throws Exception {
-    long rolloverFrequency = 15 * 1000; // rollover every 15 sec
-    // use the same seed so we can predict the RNG
+    // Use the same seed and a "plain" Random so we can predict the RNG
     long seed = System.currentTimeMillis();
     Random rand = new Random(seed);
-    byte[] secret1 = Long.toString(rand.nextLong()).getBytes();
-    byte[] secret2 = Long.toString(rand.nextLong()).getBytes();
-    byte[] secret3 = Long.toString(rand.nextLong()).getBytes();
-    RandomSignerSecretProvider secretProvider =
-        new RandomSignerSecretProvider(seed);
+    byte[] secret1 = generateNewSecret(rand);
+    byte[] secret2 = generateNewSecret(rand);
+    byte[] secret3 = generateNewSecret(rand);
+    MockRandomSignerSecretProvider secretProvider =
+        spy(new MockRandomSignerSecretProvider(seed));
     try {
       secretProvider.init(null, null, rolloverFrequency);
 
@@ -39,25 +54,58 @@ public void testGetAndRollSecrets() throws Exception {
       Assert.assertEquals(2, allSecrets.length);
       Assert.assertArrayEquals(secret1, allSecrets[0]);
       Assert.assertNull(allSecrets[1]);
-      Thread.sleep(rolloverFrequency + 2000);
+      verify(secretProvider, timeout(timeout).atLeastOnce()).rollSecret();
+      secretProvider.realRollSecret();
 
       currentSecret = secretProvider.getCurrentSecret();
       allSecrets = secretProvider.getAllSecrets();
       Assert.assertArrayEquals(secret2, currentSecret);
       Assert.assertEquals(2, allSecrets.length);
       Assert.assertArrayEquals(secret2, allSecrets[0]);
       Assert.assertArrayEquals(secret1, allSecrets[1]);
-      Thread.sleep(rolloverFrequency + 2000);
+      verify(secretProvider, timeout(timeout).atLeast(2)).rollSecret();
+      secretProvider.realRollSecret();
 
       currentSecret = secretProvider.getCurrentSecret();
       allSecrets = secretProvider.getAllSecrets();
       Assert.assertArrayEquals(secret3, currentSecret);
       Assert.assertEquals(2, allSecrets.length);
       Assert.assertArrayEquals(secret3, allSecrets[0]);
       Assert.assertArrayEquals(secret2, allSecrets[1]);
-      Thread.sleep(rolloverFrequency + 2000);
+      verify(secretProvider, timeout(timeout).atLeast(3)).rollSecret();
+      secretProvider.realRollSecret();
     } finally {
       secretProvider.destroy();
     }
   }
+
+  /**
+   * A hack to test RandomSignerSecretProvider.
+   * We want to test that RandomSignerSecretProvider.rollSecret() is
+   * periodically called at the expected frequency, but we want to exclude the
+   * race-condition and not take a long time to run the test.
+   */
+  private class MockRandomSignerSecretProvider
+      extends RandomSignerSecretProvider {
+    MockRandomSignerSecretProvider(long seed) {
+      super(seed);
+    }
+    @Override
+    protected synchronized void rollSecret() {
+      // this is a no-op: simply used for Mockito to verify that rollSecret()
+      // is periodically called at the expected frequency
+    }
+
+    public void realRollSecret() {
+      // the test code manually calls RandomSignerSecretProvider.rollSecret()
+      // to update the state
+      super.rollSecret();
+    }
+  }
+
+  private byte[] generateNewSecret(Random rand) {
+    byte[] secret = new byte[32];
+    rand.nextBytes(secret);
+    return secret;
+  }
 }