diff --git a/BUILDING.txt b/BUILDING.txt index e1302505de69f..ba60e4ed051f3 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -219,10 +219,10 @@ If the build process fails with an out of memory error, you should be able to fi it by increasing the memory used by maven -which can be done via the environment variable MAVEN_OPTS. -Here is an example setting to allocate between 256 and 512 MB of heap space to +Here is an example setting to allocate between 256 MB and 1.5 GB of heap space to Maven -export MAVEN_OPTS="-Xms256m -Xmx512m" +export MAVEN_OPTS="-Xms256m -Xmx1536m" ---------------------------------------------------------------------------------- diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index e1fff8a4c09b0..c1ff9888dc70d 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -46,25 +46,26 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar . ####### -# Oracle Java +# Java OpenJDK ####### RUN apt-get install -y software-properties-common RUN add-apt-repository -y ppa:webupd8team/java RUN apt-get update -# Auto-accept the Oracle JDK license -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get install -y oracle-java8-installer +# Install OpenJDK 7 +RUN apt-get install -y openjdk-7-jdk ###### -# Install findbugs +# Install spotbugs (successor of findbugs) ###### -RUN mkdir -p /opt/findbugs && \ - curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ - -o /opt/findbugs.tar.gz && \ - tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs -ENV FINDBUGS_HOME /opt/findbugs +RUN mkdir -p /opt/spotbugs && \ + curl -L -s -S \ + http://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/3.1.2/spotbugs-3.1.2.tgz \ + -o /opt/spotbugs.tar.gz && \ + tar xzf /opt/spotbugs.tar.gz --strip-components 1 -C /opt/spotbugs +# Hadoop uses FINDBUGS_HOME to run spotbugs +ENV FINDBUGS_HOME /opt/spotbugs #### # Install shellcheck @@ -75,7 +76,7 @@ RUN cabal update && cabal install shellcheck --global ### # Avoid out of memory errors in builds ### -ENV MAVEN_OPTS -Xms256m -Xmx512m +ENV MAVEN_OPTS -Xms256m -Xmx1536m ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml index 00db8ab948dfc..db1e6a0240487 100644 --- a/hadoop-assemblies/pom.xml +++ b/hadoop-assemblies/pom.xml @@ -23,12 +23,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-assemblies - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Assemblies Apache Hadoop Assemblies diff --git a/hadoop-build-tools/pom.xml b/hadoop-build-tools/pom.xml index 4c0a1378cf0b5..a36e4a029b7cc 100644 --- a/hadoop-build-tools/pom.xml +++ b/hadoop-build-tools/pom.xml @@ -18,7 +18,7 @@ hadoop-main org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 hadoop-build-tools diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml index a974f27935c3b..b32a5e6653836 100644 --- a/hadoop-client/pom.xml +++ b/hadoop-client/pom.xml @@ -18,12 +18,12 @@ org.apache.hadoop hadoop-project-dist - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project-dist org.apache.hadoop hadoop-client - 2.7.4-SNAPSHOT + 2.7.7 jar Apache Hadoop Client diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml index 705fccbcd7dce..8e75125568adb 100644 --- a/hadoop-common-project/hadoop-annotations/pom.xml +++ b/hadoop-common-project/hadoop-annotations/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-annotations - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Annotations Apache Hadoop Annotations jar diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml index 757a3ed33ec0f..c910f95317231 100644 --- a/hadoop-common-project/hadoop-auth-examples/pom.xml +++ b/hadoop-common-project/hadoop-auth-examples/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-auth-examples - 2.7.4-SNAPSHOT + 2.7.7 war Apache Hadoop Auth Examples diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index b3ce2c7095b0b..4deaf56aff9e3 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-auth - 2.7.4-SNAPSHOT + 2.7.7 jar Apache Hadoop Auth diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java index 0bc1109e512db..645fbc6fd0dcf 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java @@ -323,8 +323,8 @@ String apply(String[] params) throws IOException { } } if (result != null && nonSimplePattern.matcher(result).find()) { - throw new NoMatchingRule("Non-simple name " + result + - " after auth_to_local rule " + this); + LOG.info("Non-simple name {} after auth_to_local rule {}", + result, this); } if (toLowerCase && result != null) { result = result.toLowerCase(Locale.ENGLISH); @@ -377,7 +377,7 @@ public static class NoMatchingRule extends IOException { /** * Get the translation of the principal name into an operating system * user name. - * @return the short name + * @return the user name * @throws IOException throws if something is wrong with the rules */ public String getShortName() throws IOException { @@ -397,7 +397,8 @@ public String getShortName() throws IOException { return result; } } - throw new NoMatchingRule("No rules applied to " + toString()); + LOG.info("No auth_to_local rules applied to {}", this); + return toString(); } /** diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java index 41059a7e00900..9245887832102 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RandomSignerSecretProvider.java @@ -15,8 +15,9 @@ import com.google.common.annotations.VisibleForTesting; -import java.nio.charset.Charset; +import java.security.SecureRandom; import java.util.Random; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -32,7 +33,7 @@ public class RandomSignerSecretProvider extends RolloverSignerSecretProvider { public RandomSignerSecretProvider() { super(); - rand = new Random(); + rand = new SecureRandom(); } /** @@ -48,6 +49,8 @@ public RandomSignerSecretProvider(long seed) { @Override protected byte[] generateNewSecret() { - return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8")); + byte[] secret = new byte[32]; // 32 bytes = 256 bits + rand.nextBytes(secret); + return secret; } } diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java index bdca3e4eb9429..8ce4b2353262c 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/RolloverSignerSecretProvider.java @@ -38,7 +38,7 @@ public abstract class RolloverSignerSecretProvider extends SignerSecretProvider { - private static Logger LOG = LoggerFactory.getLogger( + static Logger LOG = LoggerFactory.getLogger( RolloverSignerSecretProvider.class); /** * Stores the currently valid secrets. The current secret is the 0th element diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java index 11bfccd05c6e9..91a2efd438d54 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java @@ -16,6 +16,7 @@ import com.google.common.annotations.VisibleForTesting; import java.nio.ByteBuffer; import java.nio.charset.Charset; +import java.security.SecureRandom; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -176,7 +177,7 @@ public class ZKSignerSecretProvider extends RolloverSignerSecretProvider { public ZKSignerSecretProvider() { super(); - rand = new Random(); + rand = new SecureRandom(); } /** @@ -369,8 +370,11 @@ private synchronized void pullFromZK(boolean isInit) { } } - private byte[] generateRandomSecret() { - return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8")); + @VisibleForTesting + protected byte[] generateRandomSecret() { + byte[] secret = new byte[32]; // 32 bytes = 256 bits + rand.nextBytes(secret); + return secret; } /** diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java index 408563f29934f..e3444ef4b92ee 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java @@ -109,12 +109,7 @@ public void testNameRules() throws Exception { kn = new KerberosName("bar@BAR"); Assert.assertEquals("bar", kn.getShortName()); kn = new KerberosName("bar@FOO"); - try { - kn.getShortName(); - Assert.fail(); - } - catch (Exception ex) { - } + Assert.assertEquals("bar@FOO", kn.getShortName()); } @Test(timeout=60000) diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java index 354917efe2080..f85b3e11d5a55 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java @@ -72,23 +72,14 @@ private void checkBadName(String name) { } } - private void checkBadTranslation(String from) { - System.out.println("Checking bad translation for " + from); - KerberosName nm = new KerberosName(from); - try { - nm.getShortName(); - Assert.fail("didn't get exception for " + from); - } catch (IOException ie) { - // PASS - } - } - @Test public void testAntiPatterns() throws Exception { checkBadName("owen/owen/owen@FOO.COM"); checkBadName("owen@foo/bar.com"); - checkBadTranslation("foo@ACME.COM"); - checkBadTranslation("root/joe@FOO.COM"); + + // no rules applied, these should pass + checkTranslation("foo@ACME.COM", "foo@ACME.COM"); + checkTranslation("root/joe@FOO.COM", "root/joe@FOO.COM"); } @Test diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRandomSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRandomSignerSecretProvider.java index 41d4967eaceec..45398c286ba36 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRandomSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestRandomSignerSecretProvider.java @@ -14,22 +14,37 @@ package org.apache.hadoop.security.authentication.util; import java.util.Random; + +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; import org.junit.Assert; import org.junit.Test; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; + public class TestRandomSignerSecretProvider { + // rollover every 50 msec + private final int timeout = 100; + private final long rolloverFrequency = timeout / 2; + + { + LogManager.getLogger( + RolloverSignerSecretProvider.LOG.getName()).setLevel(Level.DEBUG); + } + @Test public void testGetAndRollSecrets() throws Exception { - long rolloverFrequency = 15 * 1000; // rollover every 15 sec - // use the same seed so we can predict the RNG + // Use the same seed and a "plain" Random so we can predict the RNG long seed = System.currentTimeMillis(); Random rand = new Random(seed); - byte[] secret1 = Long.toString(rand.nextLong()).getBytes(); - byte[] secret2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secret3 = Long.toString(rand.nextLong()).getBytes(); - RandomSignerSecretProvider secretProvider = - new RandomSignerSecretProvider(seed); + byte[] secret1 = generateNewSecret(rand); + byte[] secret2 = generateNewSecret(rand); + byte[] secret3 = generateNewSecret(rand); + MockRandomSignerSecretProvider secretProvider = + spy(new MockRandomSignerSecretProvider(seed)); try { secretProvider.init(null, null, rolloverFrequency); @@ -39,7 +54,8 @@ public void testGetAndRollSecrets() throws Exception { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret1, allSecrets[0]); Assert.assertNull(allSecrets[1]); - Thread.sleep(rolloverFrequency + 2000); + verify(secretProvider, timeout(timeout).atLeastOnce()).rollSecret(); + secretProvider.realRollSecret(); currentSecret = secretProvider.getCurrentSecret(); allSecrets = secretProvider.getAllSecrets(); @@ -47,7 +63,8 @@ public void testGetAndRollSecrets() throws Exception { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret2, allSecrets[0]); Assert.assertArrayEquals(secret1, allSecrets[1]); - Thread.sleep(rolloverFrequency + 2000); + verify(secretProvider, timeout(timeout).atLeast(2)).rollSecret(); + secretProvider.realRollSecret(); currentSecret = secretProvider.getCurrentSecret(); allSecrets = secretProvider.getAllSecrets(); @@ -55,9 +72,40 @@ public void testGetAndRollSecrets() throws Exception { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret3, allSecrets[0]); Assert.assertArrayEquals(secret2, allSecrets[1]); - Thread.sleep(rolloverFrequency + 2000); + verify(secretProvider, timeout(timeout).atLeast(3)).rollSecret(); + secretProvider.realRollSecret(); } finally { secretProvider.destroy(); } } + + /** + * A hack to test RandomSignerSecretProvider. + * We want to test that RandomSignerSecretProvider.rollSecret() is + * periodically called at the expected frequency, but we want to exclude the + * race-condition and not take a long time to run the test. + */ + private class MockRandomSignerSecretProvider + extends RandomSignerSecretProvider { + MockRandomSignerSecretProvider(long seed) { + super(seed); + } + @Override + protected synchronized void rollSecret() { + // this is a no-op: simply used for Mockito to verify that rollSecret() + // is periodically called at the expected frequency + } + + public void realRollSecret() { + // the test code manually calls RandomSignerSecretProvider.rollSecret() + // to update the state + super.rollSecret(); + } + } + + private byte[] generateNewSecret(Random rand) { + byte[] secret = new byte[32]; + rand.nextBytes(secret); + return secret; + } } diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java index d7b6e17e11734..628342e40dc4a 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java @@ -13,21 +13,37 @@ */ package org.apache.hadoop.security.authentication.util; -import java.util.Arrays; +import java.nio.charset.Charset; import java.util.Properties; import java.util.Random; import javax.servlet.ServletContext; + import org.apache.curator.test.TestingServer; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestZKSignerSecretProvider { private TestingServer zkServer; + // rollover every 50 msec + private final int timeout = 100; + private final long rolloverFrequency = timeout / 2; + + { + LogManager.getLogger( + RolloverSignerSecretProvider.LOG.getName()).setLevel(Level.DEBUG); + } + @Before public void setup() throws Exception { zkServer = new TestingServer(); @@ -45,14 +61,14 @@ public void teardown() throws Exception { // Test just one ZKSignerSecretProvider to verify that it works in the // simplest case public void testOne() throws Exception { - long rolloverFrequency = 15 * 1000; // rollover every 15 sec - // use the same seed so we can predict the RNG + // Use the same seed and a "plain" Random so we can predict the RNG long seed = System.currentTimeMillis(); Random rand = new Random(seed); - byte[] secret2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secret1 = Long.toString(rand.nextLong()).getBytes(); - byte[] secret3 = Long.toString(rand.nextLong()).getBytes(); - ZKSignerSecretProvider secretProvider = new ZKSignerSecretProvider(seed); + byte[] secret2 = generateNewSecret(rand); + byte[] secret1 = generateNewSecret(rand); + byte[] secret3 = generateNewSecret(rand); + MockZKSignerSecretProvider secretProvider = + spy(new MockZKSignerSecretProvider(seed)); Properties config = new Properties(); config.setProperty( ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING, @@ -68,7 +84,8 @@ public void testOne() throws Exception { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret1, allSecrets[0]); Assert.assertNull(allSecrets[1]); - Thread.sleep((rolloverFrequency + 2000)); + verify(secretProvider, timeout(timeout).atLeastOnce()).rollSecret(); + secretProvider.realRollSecret(); currentSecret = secretProvider.getCurrentSecret(); allSecrets = secretProvider.getAllSecrets(); @@ -76,7 +93,8 @@ public void testOne() throws Exception { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret2, allSecrets[0]); Assert.assertArrayEquals(secret1, allSecrets[1]); - Thread.sleep((rolloverFrequency + 2000)); + verify(secretProvider, timeout(timeout).atLeast(2)).rollSecret(); + secretProvider.realRollSecret(); currentSecret = secretProvider.getCurrentSecret(); allSecrets = secretProvider.getAllSecrets(); @@ -84,33 +102,59 @@ public void testOne() throws Exception { Assert.assertEquals(2, allSecrets.length); Assert.assertArrayEquals(secret3, allSecrets[0]); Assert.assertArrayEquals(secret2, allSecrets[1]); - Thread.sleep((rolloverFrequency + 2000)); + verify(secretProvider, timeout(timeout).atLeast(3)).rollSecret(); + secretProvider.realRollSecret(); } finally { secretProvider.destroy(); } } + /** + * A hack to test ZKSignerSecretProvider. + * We want to test that ZKSignerSecretProvider.rollSecret() is periodically + * called at the expected frequency, but we want to exclude the + * race-condition and not take a long time to run the test. + */ + private class MockZKSignerSecretProvider extends ZKSignerSecretProvider { + MockZKSignerSecretProvider(long seed) { + super(seed); + } + @Override + protected synchronized void rollSecret() { + // this is a no-op: simply used for Mockito to verify that rollSecret() + // is periodically called at the expected frequency + } + + public void realRollSecret() { + // the test code manually calls ZKSignerSecretProvider.rollSecret() + // to update the state + super.rollSecret(); + } + } + @Test - public void testMultipleInit() throws Exception { - long rolloverFrequency = 15 * 1000; // rollover every 15 sec - // use the same seed so we can predict the RNG - long seedA = System.currentTimeMillis(); - Random rand = new Random(seedA); - byte[] secretA2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretA1 = Long.toString(rand.nextLong()).getBytes(); - // use the same seed so we can predict the RNG - long seedB = System.currentTimeMillis() + rand.nextLong(); - rand = new Random(seedB); - byte[] secretB2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretB1 = Long.toString(rand.nextLong()).getBytes(); - // use the same seed so we can predict the RNG - long seedC = System.currentTimeMillis() + rand.nextLong(); - rand = new Random(seedC); - byte[] secretC2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretC1 = Long.toString(rand.nextLong()).getBytes(); - ZKSignerSecretProvider secretProviderA = new ZKSignerSecretProvider(seedA); - ZKSignerSecretProvider secretProviderB = new ZKSignerSecretProvider(seedB); - ZKSignerSecretProvider secretProviderC = new ZKSignerSecretProvider(seedC); + // HADOOP-14246 increased the length of the secret from 160 bits to 256 bits. + // This test verifies that the upgrade goes smoothly. + public void testUpgradeChangeSecretLength() throws Exception { + // Use the same seed and a "plain" Random so we can predict the RNG + long seed = System.currentTimeMillis(); + Random rand = new Random(seed); + byte[] secret2 = Long.toString(rand.nextLong()) + .getBytes(Charset.forName("UTF-8")); + byte[] secret1 = Long.toString(rand.nextLong()) + .getBytes(Charset.forName("UTF-8")); + byte[] secret3 = Long.toString(rand.nextLong()) + .getBytes(Charset.forName("UTF-8")); + rand = new Random(seed); + // Secrets 4 and 5 get thrown away by ZK when the new secret provider tries + // to init + byte[] secret4 = generateNewSecret(rand); + byte[] secret5 = generateNewSecret(rand); + byte[] secret6 = generateNewSecret(rand); + byte[] secret7 = generateNewSecret(rand); + // Initialize the znode data with the old secret length + MockZKSignerSecretProvider oldSecretProvider = + spy(new OldMockZKSignerSecretProvider(seed)); Properties config = new Properties(); config.setProperty( ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING, @@ -118,90 +162,120 @@ public void testMultipleInit() throws Exception { config.setProperty(ZKSignerSecretProvider.ZOOKEEPER_PATH, "/secret"); try { - secretProviderA.init(config, getDummyServletContext(), rolloverFrequency); - secretProviderB.init(config, getDummyServletContext(), rolloverFrequency); - secretProviderC.init(config, getDummyServletContext(), rolloverFrequency); + oldSecretProvider.init(config, getDummyServletContext(), + rolloverFrequency); - byte[] currentSecretA = secretProviderA.getCurrentSecret(); - byte[][] allSecretsA = secretProviderA.getAllSecrets(); - byte[] currentSecretB = secretProviderB.getCurrentSecret(); - byte[][] allSecretsB = secretProviderB.getAllSecrets(); - byte[] currentSecretC = secretProviderC.getCurrentSecret(); - byte[][] allSecretsC = secretProviderC.getAllSecrets(); - Assert.assertArrayEquals(currentSecretA, currentSecretB); - Assert.assertArrayEquals(currentSecretB, currentSecretC); - Assert.assertEquals(2, allSecretsA.length); - Assert.assertEquals(2, allSecretsB.length); - Assert.assertEquals(2, allSecretsC.length); - Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]); - Assert.assertArrayEquals(allSecretsB[0], allSecretsC[0]); - Assert.assertNull(allSecretsA[1]); - Assert.assertNull(allSecretsB[1]); - Assert.assertNull(allSecretsC[1]); - char secretChosen = 'z'; - if (Arrays.equals(secretA1, currentSecretA)) { - Assert.assertArrayEquals(secretA1, allSecretsA[0]); - secretChosen = 'A'; - } else if (Arrays.equals(secretB1, currentSecretB)) { - Assert.assertArrayEquals(secretB1, allSecretsA[0]); - secretChosen = 'B'; - }else if (Arrays.equals(secretC1, currentSecretC)) { - Assert.assertArrayEquals(secretC1, allSecretsA[0]); - secretChosen = 'C'; - } else { - Assert.fail("It appears that they all agreed on the same secret, but " - + "not one of the secrets they were supposed to"); - } - Thread.sleep((rolloverFrequency + 2000)); + byte[] currentSecret = oldSecretProvider.getCurrentSecret(); + byte[][] allSecrets = oldSecretProvider.getAllSecrets(); + Assert.assertArrayEquals(secret1, currentSecret); + Assert.assertEquals(2, allSecrets.length); + Assert.assertArrayEquals(secret1, allSecrets[0]); + Assert.assertNull(allSecrets[1]); + oldSecretProvider.realRollSecret(); - currentSecretA = secretProviderA.getCurrentSecret(); - allSecretsA = secretProviderA.getAllSecrets(); - currentSecretB = secretProviderB.getCurrentSecret(); - allSecretsB = secretProviderB.getAllSecrets(); - currentSecretC = secretProviderC.getCurrentSecret(); - allSecretsC = secretProviderC.getAllSecrets(); - Assert.assertArrayEquals(currentSecretA, currentSecretB); - Assert.assertArrayEquals(currentSecretB, currentSecretC); - Assert.assertEquals(2, allSecretsA.length); - Assert.assertEquals(2, allSecretsB.length); - Assert.assertEquals(2, allSecretsC.length); - Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]); - Assert.assertArrayEquals(allSecretsB[0], allSecretsC[0]); - Assert.assertArrayEquals(allSecretsA[1], allSecretsB[1]); - Assert.assertArrayEquals(allSecretsB[1], allSecretsC[1]); - // The second secret used is prechosen by whoever won the init; so it - // should match with whichever we saw before - if (secretChosen == 'A') { - Assert.assertArrayEquals(secretA2, currentSecretA); - } else if (secretChosen == 'B') { - Assert.assertArrayEquals(secretB2, currentSecretA); - } else if (secretChosen == 'C') { - Assert.assertArrayEquals(secretC2, currentSecretA); - } + currentSecret = oldSecretProvider.getCurrentSecret(); + allSecrets = oldSecretProvider.getAllSecrets(); + Assert.assertArrayEquals(secret2, currentSecret); + Assert.assertEquals(2, allSecrets.length); + Assert.assertArrayEquals(secret2, allSecrets[0]); + Assert.assertArrayEquals(secret1, allSecrets[1]); } finally { - secretProviderC.destroy(); - secretProviderB.destroy(); - secretProviderA.destroy(); + oldSecretProvider.destroy(); + } + // Now use a ZKSignerSecretProvider with the newer length + MockZKSignerSecretProvider newSecretProvider = + spy(new MockZKSignerSecretProvider(seed)); + try { + newSecretProvider.init(config, getDummyServletContext(), + rolloverFrequency); + + byte[] currentSecret = newSecretProvider.getCurrentSecret(); + byte[][] allSecrets = newSecretProvider.getAllSecrets(); + Assert.assertArrayEquals(secret2, currentSecret); + Assert.assertEquals(2, allSecrets.length); + Assert.assertArrayEquals(secret2, allSecrets[0]); + Assert.assertArrayEquals(secret1, allSecrets[1]); + newSecretProvider.realRollSecret(); + + currentSecret = newSecretProvider.getCurrentSecret(); + allSecrets = newSecretProvider.getAllSecrets(); + Assert.assertArrayEquals(secret3, currentSecret); + Assert.assertEquals(2, allSecrets.length); + Assert.assertArrayEquals(secret3, allSecrets[0]); + Assert.assertArrayEquals(secret2, allSecrets[1]); + newSecretProvider.realRollSecret(); + + currentSecret = newSecretProvider.getCurrentSecret(); + allSecrets = newSecretProvider.getAllSecrets(); + Assert.assertArrayEquals(secret6, currentSecret); + Assert.assertEquals(2, allSecrets.length); + Assert.assertArrayEquals(secret6, allSecrets[0]); + Assert.assertArrayEquals(secret3, allSecrets[1]); + newSecretProvider.realRollSecret(); + + currentSecret = newSecretProvider.getCurrentSecret(); + allSecrets = newSecretProvider.getAllSecrets(); + Assert.assertArrayEquals(secret7, currentSecret); + Assert.assertEquals(2, allSecrets.length); + Assert.assertArrayEquals(secret7, allSecrets[0]); + Assert.assertArrayEquals(secret6, allSecrets[1]); + } finally { + newSecretProvider.destroy(); } } + /** + * A version of {@link MockZKSignerSecretProvider} that uses the old way of + * generating secrets (160 bit long). + */ + private class OldMockZKSignerSecretProvider + extends MockZKSignerSecretProvider { + private Random rand; + OldMockZKSignerSecretProvider(long seed) { + super(seed); + rand = new Random(seed); + } + + @Override + protected byte[] generateRandomSecret() { + return Long.toString(rand.nextLong()).getBytes(Charset.forName("UTF-8")); + } + } + + @Test + public void testMultiple1() throws Exception { + testMultiple(1); + } + @Test - public void testMultipleUnsychnronized() throws Exception { - long rolloverFrequency = 15 * 1000; // rollover every 15 sec - // use the same seed so we can predict the RNG + public void testMultiple2() throws Exception { + testMultiple(2); + } + + /** + * @param order: + * 1: secretProviderA wins both realRollSecret races + * 2: secretProviderA wins 1st race, B wins 2nd + * @throws Exception + */ + public void testMultiple(int order) throws Exception { + // Use the same seed and a "plain" Random so we can predict the RNG long seedA = System.currentTimeMillis(); Random rand = new Random(seedA); - byte[] secretA2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretA1 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretA3 = Long.toString(rand.nextLong()).getBytes(); - // use the same seed so we can predict the RNG + byte[] secretA2 = generateNewSecret(rand); + byte[] secretA1 = generateNewSecret(rand); + byte[] secretA3 = generateNewSecret(rand); + byte[] secretA4 = generateNewSecret(rand); long seedB = System.currentTimeMillis() + rand.nextLong(); rand = new Random(seedB); - byte[] secretB2 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretB1 = Long.toString(rand.nextLong()).getBytes(); - byte[] secretB3 = Long.toString(rand.nextLong()).getBytes(); - ZKSignerSecretProvider secretProviderA = new ZKSignerSecretProvider(seedA); - ZKSignerSecretProvider secretProviderB = new ZKSignerSecretProvider(seedB); + byte[] secretB2 = generateNewSecret(rand); + byte[] secretB1 = generateNewSecret(rand); + byte[] secretB3 = generateNewSecret(rand); + byte[] secretB4 = generateNewSecret(rand); + MockZKSignerSecretProvider secretProviderA = + spy(new MockZKSignerSecretProvider(seedA)); + MockZKSignerSecretProvider secretProviderB = + spy(new MockZKSignerSecretProvider(seedB)); Properties config = new Properties(); config.setProperty( ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING, @@ -210,14 +284,24 @@ public void testMultipleUnsychnronized() throws Exception { "/secret"); try { secretProviderA.init(config, getDummyServletContext(), rolloverFrequency); + secretProviderB.init(config, getDummyServletContext(), rolloverFrequency); byte[] currentSecretA = secretProviderA.getCurrentSecret(); byte[][] allSecretsA = secretProviderA.getAllSecrets(); + byte[] currentSecretB = secretProviderB.getCurrentSecret(); + byte[][] allSecretsB = secretProviderB.getAllSecrets(); Assert.assertArrayEquals(secretA1, currentSecretA); + Assert.assertArrayEquals(secretA1, currentSecretB); Assert.assertEquals(2, allSecretsA.length); + Assert.assertEquals(2, allSecretsB.length); Assert.assertArrayEquals(secretA1, allSecretsA[0]); + Assert.assertArrayEquals(secretA1, allSecretsB[0]); Assert.assertNull(allSecretsA[1]); - Thread.sleep((rolloverFrequency + 2000)); + Assert.assertNull(allSecretsB[1]); + verify(secretProviderA, timeout(timeout).atLeastOnce()).rollSecret(); + verify(secretProviderB, timeout(timeout).atLeastOnce()).rollSecret(); + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); currentSecretA = secretProviderA.getCurrentSecret(); allSecretsA = secretProviderA.getAllSecrets(); @@ -225,17 +309,32 @@ public void testMultipleUnsychnronized() throws Exception { Assert.assertEquals(2, allSecretsA.length); Assert.assertArrayEquals(secretA2, allSecretsA[0]); Assert.assertArrayEquals(secretA1, allSecretsA[1]); - Thread.sleep((rolloverFrequency / 5)); - - secretProviderB.init(config, getDummyServletContext(), rolloverFrequency); - byte[] currentSecretB = secretProviderB.getCurrentSecret(); - byte[][] allSecretsB = secretProviderB.getAllSecrets(); + currentSecretB = secretProviderB.getCurrentSecret(); + allSecretsB = secretProviderB.getAllSecrets(); Assert.assertArrayEquals(secretA2, currentSecretB); Assert.assertEquals(2, allSecretsA.length); Assert.assertArrayEquals(secretA2, allSecretsB[0]); Assert.assertArrayEquals(secretA1, allSecretsB[1]); - Thread.sleep((rolloverFrequency)); + verify(secretProviderA, timeout(timeout).atLeast(2)).rollSecret(); + verify(secretProviderB, timeout(timeout).atLeastOnce()).rollSecret(); + + switch (order) { + case 1: + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); + break; + case 2: + secretProviderB.realRollSecret(); + secretProviderA.realRollSecret(); + secretProviderB.realRollSecret(); + secretProviderA.realRollSecret(); + break; + default: + throw new Exception("Invalid order selected"); + } currentSecretA = secretProviderA.getCurrentSecret(); allSecretsA = secretProviderA.getAllSecrets(); @@ -246,13 +345,13 @@ public void testMultipleUnsychnronized() throws Exception { Assert.assertEquals(2, allSecretsB.length); Assert.assertArrayEquals(allSecretsA[0], allSecretsB[0]); Assert.assertArrayEquals(allSecretsA[1], allSecretsB[1]); - if (Arrays.equals(secretA3, currentSecretA)) { - Assert.assertArrayEquals(secretA3, allSecretsA[0]); - } else if (Arrays.equals(secretB3, currentSecretB)) { - Assert.assertArrayEquals(secretB3, allSecretsA[0]); - } else { - Assert.fail("It appears that they all agreed on the same secret, but " - + "not one of the secrets they were supposed to"); + switch (order) { + case 1: + Assert.assertArrayEquals(secretA4, allSecretsA[0]); + break; + case 2: + Assert.assertArrayEquals(secretB4, allSecretsA[0]); + break; } } finally { secretProviderB.destroy(); @@ -261,10 +360,16 @@ public void testMultipleUnsychnronized() throws Exception { } private ServletContext getDummyServletContext() { - ServletContext servletContext = Mockito.mock(ServletContext.class); - Mockito.when(servletContext.getAttribute(ZKSignerSecretProvider - .ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE)) - .thenReturn(null); + ServletContext servletContext = mock(ServletContext.class); + when(servletContext.getAttribute(ZKSignerSecretProvider + .ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE)) + .thenReturn(null); return servletContext; } + + private byte[] generateNewSecret(Random rand) { + byte[] secret = new byte[32]; + rand.nextBytes(secret); + return secret; + } } diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5fdfb2cba8d85..8a11aee549b58 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1,6 +1,146 @@ Hadoop Change Log -Release 2.7.4 - UNRELEASED +Release 2.7.7 - 2018-07-18 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + HADOOP-15486. Make NetworkTopology#netLock fair. + Contributed by Nanda kumar. + + OPTIMIZATIONS + + BUG FIXES + + HADOOP-15473. Configure serialFilter in KeyProvider to avoid + UnrecoverableKeyException caused by JDK-8189997. + Contributed by Gabor Bota. + + Additional check when unpacking archives. + Contributed by Jason Lowe and Akira Ajisaka. + + Skip the proxy user check if the ugi has not been initialized. + Contributed by Daryn Sharp + + HADOOP-14970. MiniHadoopClusterManager doesn't respect lack of + format option. + Contributed by Erik Krogen + +Release 2.7.6 - 2018-04-16 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + HADOOP-14246. Authentication Tokens should use SecureRandom instead of + Random and 256 bit secrets (Conttributed by Robert Kanter via + Daniel Templeton) + + HADOOP-15177. Update the release year to 2018. + (Bharat Viswanadham via aajisaka) + + HADOOP-13263. Reload cached groups in background after expiry. + (Stephen O'Donnell via Arpit Agarwal) + + HADOOP-15212. Add independent secret manager method for logging expired + tokens. (Daryn Sharp via kihwal) + + HADOOP-9477. Add posixGroups support for LDAP groups mapping service. + (Dapeng Sun via Yongjun Zhang) + + HADOOP-12568. Update core-default.xml to describe posixGroups support. + (Wei-Chiu Chuang via aajisaka) + + HADOOP-15283. Upgrade from findbugs 3.0.1 to spotbugs 3.1.2 in branch-2 + to fix docker image build. (aajisaka) + + HADOOP-15279. Increase maven heap size recommendations. + (Allen Wittenauer via aajisaka) + + HADOOP-12472. Make GenericTestUtils.assertExceptionContains robust. + (Steve Loughran via jing9) + + HADOOP-13105. Support timeouts in LDAP queries in LdapGroupsMapping. + (Mingliang Liu via shv) + + HADOOP-15345. Backport HADOOP-12185 to branch-2.7: NetworkTopology is not + efficient adding/getting/removing nodes. (He Xiaoqiao via Inigo Goiri) + + OPTIMIZATIONS + + BUG FIXES + + HADOOP-12181. Fix intermittent test failure of TestZKSignerSecretProvider. + (Masatake Iwasaki via wheat9) + + HADOOP-12611. TestZKSignerSecretProvider#testMultipleInit occasionally fail. + (ebadger via rkanter) + + HADOOP-14842. Hadoop 2.8.2 release build process get stuck due to java + issue. Contributed by Junping Du. + + HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with + '@' to be non-simple. (Bolke de Bruin via stevel). + + HADOOP-13375. o.a.h.security.TestGroupsCaching.testBackgroundRefreshCounters + seems flaky. (Weiwei Yang via Mingliang Liu, shv) + + HADOOP-13508. FsPermission string constructor does not recognize + sticky bit. (Atul Sikaria via Wei-Chiu Chuang, shv) + + HADOOP-15143. NPE due to Invalid KerberosTicket in UGI. + (Mukul Kumar Singh via Wei-Chiu Chuang) + + HADOOP-15206. BZip2 drops and duplicates records when input split size + is small. (Aki Tanaka via jlowe) + + HADOOP-12001. Fixed LdapGroupsMapping to include configurable Posix UID and + GID attributes during the search. (Patrick White via vinodkv) + + Revert HADOOP-13119. Add ability to secure log servlet using proxy users. + (Contribute by Yuanbo Liu via Eric Yang) + + HADOOP-12862. LDAP Group Mapping over SSL can not specify trust store. + (Wei-Chiu Chuang and Konstantin Shvachko) + +Release 2.7.5 - 2017-12-14 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + HADOOP-13442. Optimize UGI group lookups. (Daryn Sharp via kihwal, shv) + + HADOOP-14827. Allow StopWatch to accept a Timer parameter for tests. + (Erik Krogen via jlowe) + + HADOOP-14867. Update HDFS Federation setup document, for incorrect + property name for secondary name node http address. + (Bharat Viswanadham via Arpit Agarwal) + + OPTIMIZATIONS + + BUG FIXES + + HADOOP-14702. Fix formatting issue and regression caused by conversion from + APT to Markdown. (Doris Gu via iwasakims) + + HADOOP-14881. LoadGenerator should use Time.monotonicNow() to measure + durations. (Bharat Viswanadham via jlowe) + + HADOOP-14902. LoadGenerator#genFile write close timing is incorrectly + calculated. (Hanisha Koneru via jlowe) + + HADOOP-14919. BZip2 drops records when reading data in splits. (Jason Lowe) + +Release 2.7.4 - 2017-08-04 INCOMPATIBLE CHANGES @@ -42,6 +182,9 @@ Release 2.7.4 - UNRELEASED HADOOP-14440. Add metrics for connections dropped. (Eric Badger via kihwal) + HADOOP-12472. Make GenericTestUtils.assertExceptionContains robust. + (Steve Loughran via jing9) + OPTIMIZATIONS HADOOP-14138. Remove S3A ref from META-INF service discovery, rely on diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index b304bab8f636c..a644907d7d002 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project-dist - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project-dist org.apache.hadoop hadoop-common - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Common Apache Hadoop Common jar diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index 316c48e7575cd..e435c102dc5c4 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -266,3 +266,14 @@ log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log #log4j.appender.nodemanagerrequestlog.RetainDays=3 + +# WebHdfs request log on datanodes +# Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to +# direct the log to a separate file. +#datanode.webhdfs.logger=INFO,console +#log4j.logger.datanode.webhdfs=${datanode.webhdfs.logger} +#log4j.appender.HTTPDRFA=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.HTTPDRFA.File=${hadoop.log.dir}/hadoop-datanode-webhdfs.log +#log4j.appender.HTTPDRFA.layout=org.apache.log4j.PatternLayout +#log4j.appender.HTTPDRFA.layout.ConversionPattern=%d{ISO8601} %m%n +#log4j.appender.HTTPDRFA.DatePattern=.yyyy-MM-dd diff --git a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html index acd0c568c9a5e..a746a884fdba2 100644 --- a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html +++ b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html @@ -1,5 +1,5 @@ -Hadoop 2.6.0 Release Notes +Hadoop 2.7.7 Release Notes -

Hadoop 2.6.0 Release Notes

+

Hadoop 2.7.7 Release Notes

These release notes include new developer and user-facing incompatibilities, features, and major improvements. -

Changes since Hadoop 2.5.1

+

Changes since Hadoop 2.7.6

 - - -Hadoop 2.4.1 Release Notes - - - -

Hadoop 2.4.1 Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.4.0

- - - -Hadoop 2.4.0 Release Notes - - - -

Hadoop 2.4.0 Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.3.0

- - - -Hadoop 2.3.0 Release Notes - - - -

Hadoop 2.3.0 Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.2.0

- - - - -Hadoop 2.2.0 Release Notes - - - -

Hadoop 2.2.0 Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.1.1-beta

- - - -Hadoop 2.1.1-beta Release Notes - - - -

Hadoop 2.1.1-beta Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.1.0-beta

- - - -Hadoop 2.1.0-beta Release Notes - - - -

Hadoop 2.1.0-beta Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.0.5-alpha

- - - -Hadoop 2.0.5-alpha Release Notes - - - -

Hadoop 2.0.5-alpha Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.0.4-alpha

- - - -Hadoop 2.0.4-alpha Release Notes - - - -

Hadoop 2.0.4-alpha Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.0.3-alpha

- - - -Hadoop 2.0.3-alpha Release Notes - - - -

Hadoop 2.0.3-alpha Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.0.2

- - - -Hadoop 2.0.2-alpha Release Notes - - - -

Hadoop 2.0.2-alpha Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.0.1-alpha

- - - -Hadoop 2.0.1-alpha Release Notes - - - -

Hadoop 2.0.1-alpha Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 2.0.0-alpha

- - - -Hadoop 2.0.0-alpha Release Notes - - - -

Hadoop 2.0.0-alpha Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 0.23.2

- - - -Hadoop 0.23.2 Release Notes - - - -

Hadoop 0.23.2 Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 0.23.1

- - - -Hadoop 0.23.1 Release Notes - - - -

Hadoop 0.23.1 Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 0.23.0

- - - -Hadoop 0.23.0 Release Notes - - - -

Hadoop 0.23.0 Release Notes

-These release notes include new developer and user-facing incompatibilities, features, and major improvements. - -

Changes since Hadoop 1.0.0

- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 1cd6d6c9b314f..5ce3e65686533 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -82,6 +82,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry; import org.apache.hadoop.security.alias.CredentialProviderFactory; @@ -185,19 +186,34 @@ public class Configuration implements Iterable>, private static final String DEFAULT_STRING_CHECK = "testingforemptydefaultvalue"; + private static final String XINCLUDE_NS_URI = + "http://www.w3.org/2001/XInclude"; + + private static boolean restrictSystemPropsDefault = false; + private boolean restrictSystemProps = restrictSystemPropsDefault; private boolean allowNullValueProperties = false; - + private static class Resource { private final Object resource; private final String name; + private final boolean restrictParser; public Resource(Object resource) { this(resource, resource.toString()); } - + + public Resource(Object resource, boolean useRestrictedParser) { + this(resource, resource.toString(), useRestrictedParser); + } + public Resource(Object resource, String name) { + this(resource, name, getRestrictParserDefault(resource)); + } + + public Resource(Object resource, String name, boolean restrictParser) { this.resource = resource; this.name = name; + this.restrictParser = restrictParser; } public String getName(){ @@ -207,11 +223,28 @@ public String getName(){ public Object getResource() { return resource; } - + + public boolean isParserRestricted() { + return restrictParser; + } + @Override public String toString() { return name; } + + private static boolean getRestrictParserDefault(Object resource) { + if (resource instanceof String || !UserGroupInformation.isInitialized()) { + return false; + } + UserGroupInformation user; + try { + user = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + throw new RuntimeException("Unable to determine current user", e); + } + return user.getRealUser() != null; + } } /** @@ -233,7 +266,7 @@ public String toString() { new ConcurrentHashMap()); private boolean loadDefaults = true; - + /** * Configuration objects */ @@ -710,6 +743,7 @@ public Configuration(Configuration other) { this.overlay = (Properties)other.overlay.clone(); } + this.restrictSystemProps = other.restrictSystemProps; this.updatingResource = new ConcurrentHashMap( other.updatingResource); this.finalParameters = Collections.newSetFromMap( @@ -741,6 +775,14 @@ public static synchronized void addDefaultResource(String name) { } } + public static void setRestrictSystemPropertiesDefault(boolean val) { + restrictSystemPropsDefault = val; + } + + public void setRestrictSystemProperties(boolean val) { + this.restrictSystemProps = val; + } + /** * Add a configuration resource. * @@ -754,6 +796,10 @@ public void addResource(String name) { addResourceObject(new Resource(name)); } + public void addResource(String name, boolean restrictedParser) { + addResourceObject(new Resource(name, restrictedParser)); + } + /** * Add a configuration resource. * @@ -768,6 +814,10 @@ public void addResource(URL url) { addResourceObject(new Resource(url)); } + public void addResource(URL url, boolean restrictedParser) { + addResourceObject(new Resource(url, restrictedParser)); + } + /** * Add a configuration resource. * @@ -782,6 +832,10 @@ public void addResource(Path file) { addResourceObject(new Resource(file)); } + public void addResource(Path file, boolean restrictedParser) { + addResourceObject(new Resource(file, restrictedParser)); + } + /** * Add a configuration resource. * @@ -799,6 +853,10 @@ public void addResource(InputStream in) { addResourceObject(new Resource(in)); } + public void addResource(InputStream in, boolean restrictedParser) { + addResourceObject(new Resource(in, restrictedParser)); + } + /** * Add a configuration resource. * @@ -812,7 +870,12 @@ public void addResource(InputStream in) { public void addResource(InputStream in, String name) { addResourceObject(new Resource(in, name)); } - + + public void addResource(InputStream in, String name, + boolean restrictedParser) { + addResourceObject(new Resource(in, name, restrictedParser)); + } + /** * Add a configuration resource. * @@ -842,6 +905,7 @@ public synchronized void reloadConfiguration() { private synchronized void addResourceObject(Resource resource) { resources.add(resource); // add to resources + restrictSystemProps |= resource.isParserRestricted(); reloadConfiguration(); } @@ -940,10 +1004,12 @@ private String substituteVars(String expr) { final String var = eval.substring(varBounds[SUB_START_IDX], varBounds[SUB_END_IDX]); String val = null; - try { - val = System.getProperty(var); - } catch(SecurityException se) { - LOG.warn("Unexpected SecurityException in Configuration", se); + if (!restrictSystemProps) { + try { + val = System.getProperty(var); + } catch (SecurityException se) { + LOG.warn("Unexpected SecurityException in Configuration", se); + } } if (val == null) { val = getRaw(var); @@ -994,6 +1060,10 @@ public void setAllowNullValueProperties( boolean val ) { this.allowNullValueProperties = val; } + public void setRestrictSystemProps(boolean val) { + this.restrictSystemProps = val; + } + /** * Return existence of the name property, but only for * names which have no valid value, usually non-existent or commented @@ -1939,7 +2009,7 @@ public char[] getPassword(String name) throws IOException { * @return password or null if not found * @throws IOException */ - protected char[] getPasswordFromCredentialProviders(String name) + public char[] getPasswordFromCredentialProviders(String name) throws IOException { char[] pass = null; try { @@ -2491,12 +2561,12 @@ private void loadResources(Properties properties, boolean quiet) { if(loadDefaults) { for (String resource : defaultResources) { - loadResource(properties, new Resource(resource), quiet); + loadResource(properties, new Resource(resource, false), quiet); } //support the hadoop-site.xml as a deprecated case if(getResource("hadoop-site.xml")!=null) { - loadResource(properties, new Resource("hadoop-site.xml"), quiet); + loadResource(properties, new Resource("hadoop-site.xml", false), quiet); } } @@ -2521,13 +2591,16 @@ private Resource loadResource(Properties properties, Resource wrapper, boolean q //allow includes in the xml file docBuilderFactory.setNamespaceAware(true); + boolean useXInclude = !wrapper.isParserRestricted(); try { - docBuilderFactory.setXIncludeAware(true); + docBuilderFactory.setXIncludeAware(useXInclude); } catch (UnsupportedOperationException e) { - LOG.error("Failed to set setXIncludeAware(true) for parser " - + docBuilderFactory - + ":" + e, - e); + LOG.error("Failed to set setXIncludeAware(" + useXInclude + + ") for parser " + docBuilderFactory, e); + } + if (wrapper.isParserRestricted()) { + docBuilderFactory.setFeature( + "http://apache.org/xml/features/disallow-doctype-decl", true); } DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = null; @@ -2583,11 +2656,19 @@ private Resource loadResource(Properties properties, Resource wrapper, boolean q continue; Element prop = (Element)propNode; if ("configuration".equals(prop.getTagName())) { - loadResource(toAddTo, new Resource(prop, name), quiet); + loadResource(toAddTo, + new Resource(prop, name, wrapper.isParserRestricted()), quiet); continue; } - if (!"property".equals(prop.getTagName())) - LOG.warn("bad conf file: element not "); + if (!"property".equals(prop.getTagName())) { + if (wrapper.isParserRestricted() + && XINCLUDE_NS_URI.equals(prop.getNamespaceURI())) { + throw new RuntimeException("Error parsing resource " + wrapper + + ": XInclude is not supported for restricted resources"); + } + LOG.warn("Unexpected tag in conf file " + wrapper + + ": expected but found <" + prop.getTagName() + ">"); + } NodeList fields = prop.getChildNodes(); String attr = null; String value = null; @@ -2633,7 +2714,7 @@ private Resource loadResource(Properties properties, Resource wrapper, boolean q if (returnCachedProperties) { overlay(properties, toAddTo); - return new Resource(toAddTo, name); + return new Resource(toAddTo, name, wrapper.isParserRestricted()); } return null; } catch (IOException e) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index 033e2fdcc9da4..0af18dc5ba7d1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -39,6 +39,8 @@ import javax.crypto.KeyGenerator; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER; + /** * A provider of secret key material for Hadoop applications. Provides an * abstraction to separate key storage from users of encryption. It @@ -56,6 +58,14 @@ public abstract class KeyProvider { public static final String DEFAULT_BITLENGTH_NAME = "hadoop.security.key.default.bitlength"; public static final int DEFAULT_BITLENGTH = 128; + public static final String JCEKS_KEY_SERIALFILTER_DEFAULT = + "java.lang.Enum;" + + "java.security.KeyRep;" + + "java.security.KeyRep$Type;" + + "javax.crypto.spec.SecretKeySpec;" + + "org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata;" + + "!*"; + public static final String JCEKS_KEY_SERIAL_FILTER = "jceks.key.serialFilter"; private final Configuration conf; @@ -364,6 +374,14 @@ public String toString() { */ public KeyProvider(Configuration conf) { this.conf = new Configuration(conf); + // Added for HADOOP-15473. Configured serialFilter property fixes + // java.security.UnrecoverableKeyException in JDK 8u171. + if(System.getProperty(JCEKS_KEY_SERIAL_FILTER) == null) { + String serialFilter = + conf.get(HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER, + JCEKS_KEY_SERIALFILTER_DEFAULT); + System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter); + } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 0cb6cfd75acf1..d6a5921d94399 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -271,6 +271,19 @@ public class CommonConfigurationKeysPublic { public static final long HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT = 5000; /** See core-default.xml */ + public static final String HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD = + "hadoop.security.groups.cache.background.reload"; + /** See core-default.xml. */ + public static final boolean + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_DEFAULT = false; + /** See core-default.xml. */ + public static final String + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS = + "hadoop.security.groups.cache.background.reload.threads"; + /** See core-default.xml. */ + public static final int + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS_DEFAULT = 3; + /** See core-default.xml.*/ public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; /** See core-default.xml */ @@ -326,6 +339,13 @@ public class CommonConfigurationKeysPublic { /** See core-default.xml */ public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY = "hadoop.security.crypto.jce.provider"; + /** + * @see + * + * core-default.xml + */ + public static final String HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER = + "hadoop.security.crypto.jceks.key.serialfilter"; /** See core-default.xml */ public static final String HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY = "hadoop.security.crypto.buffer.size"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 2e684f57290a0..f0b6ede3691f6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -26,7 +26,6 @@ import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; -import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -2185,12 +2184,11 @@ static void checkAccessPermissions(FileStatus stat, FsAction mode) FsPermission perm = stat.getPermission(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String user = ugi.getShortUserName(); - List groups = Arrays.asList(ugi.getGroupNames()); if (user.equals(stat.getOwner())) { if (perm.getUserAction().implies(mode)) { return; } - } else if (groups.contains(stat.getGroup())) { + } else if (ugi.getGroups().contains(stat.getGroup())) { if (perm.getGroupAction().implies(mode)) { return; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 23fb946244995..00381fee2785d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -587,16 +587,21 @@ public static long getDU(File dir) { public static void unZip(File inFile, File unzipDir) throws IOException { Enumeration entries; ZipFile zipFile = new ZipFile(inFile); + String targetDirPath = unzipDir.getCanonicalPath() + File.separator; try { entries = zipFile.entries(); while (entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); if (!entry.isDirectory()) { + File file = new File(unzipDir, entry.getName()); + if (!file.getCanonicalPath().startsWith(targetDirPath)) { + throw new IOException("expanding " + entry.getName() + + " would create file outside of " + unzipDir); + } InputStream in = zipFile.getInputStream(entry); try { - File file = new File(unzipDir, entry.getName()); - if (!file.getParentFile().mkdirs()) { + if (!file.getParentFile().mkdirs()) { if (!file.getParentFile().isDirectory()) { throw new IOException("Mkdirs failed to create " + file.getParentFile().toString()); @@ -705,6 +710,13 @@ private static void unTarUsingJava(File inFile, File untarDir, private static void unpackEntries(TarArchiveInputStream tis, TarArchiveEntry entry, File outputDir) throws IOException { + String targetDirPath = outputDir.getCanonicalPath() + File.separator; + File outputFile = new File(outputDir, entry.getName()); + if (!outputFile.getCanonicalPath().startsWith(targetDirPath)) { + throw new IOException("expanding " + entry.getName() + + " would create entry outside of " + outputDir); + } + if (entry.isDirectory()) { File subDir = new File(outputDir, entry.getName()); if (!subDir.mkdirs() && !subDir.isDirectory()) { @@ -719,7 +731,6 @@ private static void unpackEntries(TarArchiveInputStream tis, return; } - File outputFile = new File(outputDir, entry.getName()); if (!outputFile.getParentFile().exists()) { if (!outputFile.getParentFile().mkdirs()) { throw new IOException("Mkdirs failed to create tar internal dir " diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java index 0258293823eb2..a816396b9483d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java @@ -103,7 +103,7 @@ public FsPermission(FsPermission other) { * @throws IllegalArgumentException if mode is invalid */ public FsPermission(String mode) { - this(new UmaskParser(mode).getUMask()); + this(new RawParser(mode).getPermission()); } /** Return user {@link FsAction}. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java new file mode 100644 index 0000000000000..3bbe9cb3929b4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.permission; + +import java.util.regex.Pattern; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +class RawParser extends PermissionParser { + private static Pattern rawOctalPattern = + Pattern.compile("^\\s*([01]?)([0-7]{3})\\s*$"); + private static Pattern rawNormalPattern = + Pattern.compile("\\G\\s*([ugoa]*)([+=-]+)([rwxt]*)([,\\s]*)\\s*"); + + private short permission; + + public RawParser(String modeStr) throws IllegalArgumentException { + super(modeStr, rawNormalPattern, rawOctalPattern); + permission = (short)combineModes(0, false); + } + + public short getPermission() { + return permission; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 4eabb66efea22..b612cab9376fd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -829,7 +829,7 @@ public FileChecksum getFileChecksum(final Path f) public FileStatus getFileStatus(Path f) throws IOException { checkPathIsSlash(f); return new FileStatus(0, true, 0, 0, creationTime, creationTime, - PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0], + PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(), new Path(theInternalDir.fullPath).makeQualified( myUri, ROOT_PATH)); @@ -850,7 +850,7 @@ public FileStatus[] listStatus(Path f) throws AccessControlException, result[i++] = new FileStatus(0, false, 0, 0, creationTime, creationTime, PERMISSION_555, - ugi.getUserName(), ugi.getGroupNames()[0], + ugi.getUserName(), ugi.getPrimaryGroupName(), link.getTargetLink(), new Path(inode.fullPath).makeQualified( myUri, null)); @@ -982,7 +982,7 @@ public void setAcl(Path path, List aclSpec) throws IOException { public AclStatus getAclStatus(Path path) throws IOException { checkPathIsSlash(path); return new AclStatus.Builder().owner(ugi.getUserName()) - .group(ugi.getGroupNames()[0]) + .group(ugi.getPrimaryGroupName()) .addEntries(AclUtil.getMinimalAcl(PERMISSION_555)) .stickyBit(false).build(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 7fb06f58c1d39..fd861bb2ec126 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -844,14 +844,14 @@ public FileChecksum getFileChecksum(final Path f) public FileStatus getFileStatus(final Path f) throws IOException { checkPathIsSlash(f); return new FileStatus(0, true, 0, 0, creationTime, creationTime, - PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0], + PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(), new Path(theInternalDir.fullPath).makeQualified( myUri, null)); } @Override public FileStatus getFileLinkStatus(final Path f) - throws FileNotFoundException { + throws IOException { // look up i internalDirs children - ignore first Slash INode inode = theInternalDir.children.get(f.toUri().toString().substring(1)); @@ -864,13 +864,13 @@ public FileStatus getFileLinkStatus(final Path f) INodeLink inodelink = (INodeLink) inode; result = new FileStatus(0, false, 0, 0, creationTime, creationTime, - PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0], + PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(), inodelink.getTargetLink(), new Path(inode.fullPath).makeQualified( myUri, null)); } else { result = new FileStatus(0, true, 0, 0, creationTime, creationTime, - PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0], + PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(), new Path(inode.fullPath).makeQualified( myUri, null)); } @@ -915,7 +915,7 @@ public FileStatus[] listStatus(final Path f) throws AccessControlException, result[i++] = new FileStatus(0, false, 0, 0, creationTime, creationTime, - PERMISSION_555, ugi.getUserName(), ugi.getGroupNames()[0], + PERMISSION_555, ugi.getUserName(), ugi.getPrimaryGroupName(), link.getTargetLink(), new Path(inode.fullPath).makeQualified( myUri, null)); @@ -1049,7 +1049,7 @@ public void setAcl(Path path, List aclSpec) throws IOException { public AclStatus getAclStatus(Path path) throws IOException { checkPathIsSlash(path); return new AclStatus.Builder().owner(ugi.getUserName()) - .group(ugi.getGroupNames()[0]) + .group(ugi.getPrimaryGroupName()) .addEntries(AclUtil.getMinimalAcl(PERMISSION_555)) .stickyBit(false).build(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java index 1b092ec8da16b..ad238e6b9f9e9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java @@ -23,7 +23,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; -import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -276,7 +275,7 @@ private static void checkStat(File f, String owner, String group, UserGroupInformation.createRemoteUser(expectedOwner); final String adminsGroupString = "Administrators"; success = owner.equals(adminsGroupString) - && Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString); + && ugi.getGroups().contains(adminsGroupString); } else { success = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java index cbae646437a12..b41b91f183385 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java @@ -199,43 +199,8 @@ public SplitCompressionInputStream createInputStream(InputStream seekableIn, Seekable.class.getName()); } - //find the position of first BZip2 start up marker - ((Seekable)seekableIn).seek(0); - - // BZip2 start of block markers are of 6 bytes. But the very first block - // also has "BZh9", making it 10 bytes. This is the common case. But at - // time stream might start without a leading BZ. - final long FIRST_BZIP2_BLOCK_MARKER_POSITION = - CBZip2InputStream.numberOfBytesTillNextMarker(seekableIn); - long adjStart = 0L; - if (start != 0) { - // Other than the first of file, the marker size is 6 bytes. - adjStart = Math.max(0L, start - (FIRST_BZIP2_BLOCK_MARKER_POSITION - - (HEADER_LEN + SUB_HEADER_LEN))); - } - - ((Seekable)seekableIn).seek(adjStart); - SplitCompressionInputStream in = - new BZip2CompressionInputStream(seekableIn, adjStart, end, readMode); - - - // The following if clause handles the following case: - // Assume the following scenario in BZip2 compressed stream where - // . represent compressed data. - // .....[48 bit Block].....[48 bit Block].....[48 bit Block]... - // ........................[47 bits][1 bit].....[48 bit Block]... - // ................................^[Assume a Byte alignment here] - // ........................................^^[current position of stream] - // .....................^^[We go back 10 Bytes in stream and find a Block marker] - // ........................................^^[We align at wrong position!] - // ...........................................................^^[While this pos is correct] - - if (in.getPos() < start) { - ((Seekable)seekableIn).seek(start); - in = new BZip2CompressionInputStream(seekableIn, start, end, readMode); - } - - return in; + ((Seekable)seekableIn).seek(start); + return new BZip2CompressionInputStream(seekableIn, start, end, readMode); } /** @@ -392,9 +357,29 @@ public BZip2CompressionInputStream(InputStream in, long start, long end, bufferedIn = new BufferedInputStream(super.in); this.startingPos = super.getPos(); this.readMode = readMode; + long numSkipped = 0; if (this.startingPos == 0) { // We only strip header if it is start of file bufferedIn = readStreamHeader(); + } else if (this.readMode == READ_MODE.BYBLOCK && + this.startingPos <= HEADER_LEN + SUB_HEADER_LEN) { + // When we're in BYBLOCK mode and the start position is >=0 + // and < HEADER_LEN + SUB_HEADER_LEN, we should skip to after + // start of the first bz2 block to avoid duplicated records + numSkipped = HEADER_LEN + SUB_HEADER_LEN + 1 - this.startingPos; + long skipBytes = numSkipped; + while (skipBytes > 0) { + long s = bufferedIn.skip(skipBytes); + if (s > 0) { + skipBytes -= s; + } else { + if (bufferedIn.read() == -1) { + break; // end of the split + } else { + skipBytes--; + } + } + } } input = new CBZip2InputStream(bufferedIn, readMode); if (this.isHeaderStripped) { @@ -405,7 +390,15 @@ public BZip2CompressionInputStream(InputStream in, long start, long end, input.updateReportedByteCount(SUB_HEADER_LEN); } - this.updatePos(false); + if (numSkipped > 0) { + input.updateReportedByteCount((int) numSkipped); + } + + // To avoid dropped records, not advertising a new byte position + // when we are in BYBLOCK mode and the start position is 0 + if (!(this.readMode == READ_MODE.BYBLOCK && this.startingPos == 0)) { + this.updatePos(false); + } } private BufferedInputStream readStreamHeader() throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java index 57fc07b617a7c..edb37fee848bb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java @@ -52,20 +52,20 @@ * This Ant code was enhanced so that it can de-compress blocks of bzip2 data. * Current position in the stream is an important statistic for Hadoop. For * example in LineRecordReader, we solely depend on the current position in the - * stream to know about the progess. The notion of position becomes complicated + * stream to know about the progress. The notion of position becomes complicated * for compressed files. The Hadoop splitting is done in terms of compressed * file. But a compressed file deflates to a large amount of data. So we have * handled this problem in the following way. * * On object creation time, we find the next block start delimiter. Once such a * marker is found, the stream stops there (we discard any read compressed data - * in this process) and the position is updated (i.e. the caller of this class - * will find out the stream location). At this point we are ready for actual - * reading (i.e. decompression) of data. + * in this process) and the position is reported as the beginning of the block + * start delimiter. At this point we are ready for actual reading + * (i.e. decompression) of data. * * The subsequent read calls give out data. The position is updated when the * caller of this class has read off the current block + 1 bytes. In between the - * block reading, position is not updated. (We can only update the postion on + * block reading, position is not updated. (We can only update the position on * block boundaries). *

* @@ -204,11 +204,12 @@ private int readAByte(InputStream inStream) throws IOException { * in the stream. It can find bit patterns of length <= 63 bits. Specifically * this method is used in CBZip2InputStream to find the end of block (EOB) * delimiter in the stream, starting from the current position of the stream. - * If marker is found, the stream position will be right after marker at the - * end of this call. + * If marker is found, the stream position will be at the byte containing + * the starting bit of the marker. * * @param marker The bit pattern to be found in the stream * @param markerBitLength No of bits in the marker + * @return true if the marker was found otherwise false * * @throws IOException * @throws IllegalArgumentException if marketBitLength is greater than 63 @@ -224,23 +225,33 @@ public boolean skipToNextMarker(long marker, int markerBitLength) long bytes = 0; bytes = this.bsR(markerBitLength); if (bytes == -1) { + this.reportedBytesReadFromCompressedStream = + this.bytesReadFromCompressedStream; return false; } while (true) { if (bytes == marker) { + // Report the byte position where the marker starts + long markerBytesRead = (markerBitLength + this.bsLive + 7) / 8; + this.reportedBytesReadFromCompressedStream = + this.bytesReadFromCompressedStream - markerBytesRead; return true; - } else { bytes = bytes << 1; bytes = bytes & ((1L << markerBitLength) - 1); int oneBit = (int) this.bsR(1); if (oneBit != -1) { bytes = bytes | oneBit; - } else + } else { + this.reportedBytesReadFromCompressedStream = + this.bytesReadFromCompressedStream; return false; + } } } } catch (IOException ex) { + this.reportedBytesReadFromCompressedStream = + this.bytesReadFromCompressedStream; return false; } } @@ -302,7 +313,6 @@ private CBZip2InputStream(final InputStream in, READ_MODE readMode, boolean skip } else if (readMode == READ_MODE.BYBLOCK) { this.currentState = STATE.NO_PROCESS_STATE; skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER,DELIMITER_BIT_LENGTH); - this.reportedBytesReadFromCompressedStream = this.bytesReadFromCompressedStream; if(!skipDecompression){ changeStateToProcessABlock(); } @@ -419,8 +429,6 @@ public int read(final byte[] dest, final int offs, final int len) result = b; skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER, DELIMITER_BIT_LENGTH); - //Exactly when we are about to start a new block, we advertise the stream position. - this.reportedBytesReadFromCompressedStream = this.bytesReadFromCompressedStream; changeStateToProcessABlock(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index bc7105b30ba72..5eeb0d0066e8e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -18,10 +18,11 @@ package org.apache.hadoop.net; import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.TreeMap; import java.util.concurrent.locks.ReadWriteLock; @@ -81,6 +82,7 @@ public static NetworkTopology getInstance(Configuration conf){ */ static class InnerNode extends NodeBase { protected List children=new ArrayList(); + private Map childrenMap = new HashMap(); private int numOfLeaves; /** Construct an InnerNode from a path-like string */ @@ -172,10 +174,13 @@ boolean add(Node n) { // this node is the parent of n; add n directly n.setParent(this); n.setLevel(this.level+1); - for(int i=0; i= 0) { - // excluded node is one of the children so adjust the leaf index - leafIndex = leafIndex>=excludedIndex ? leafIndex+1 : leafIndex; + if (excludedNode != null && + childrenMap.containsKey(excludedNode.getName())) { + int excludedIndex = children.indexOf(excludedNode); + if (excludedIndex != -1 && leafIndex >= 0) { + // excluded node is one of the children so adjust the leaf index + leafIndex = leafIndex>=excludedIndex ? leafIndex+1 : leafIndex; + } } } // range check @@ -387,7 +389,7 @@ int getNumOfLeaves() { private boolean clusterEverBeenMultiRack = false; /** the lock used to manage access */ - protected ReadWriteLock netlock = new ReentrantReadWriteLock(); + protected ReadWriteLock netlock = new ReentrantReadWriteLock(true); public NetworkTopology() { clusterMap = new InnerNode(InnerNode.ROOT); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java index 65d2211af53da..ca221f5b3dcd2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java @@ -29,9 +29,8 @@ import java.util.Map; /** - * Initializes {@link AuthenticationWithProxyUserFilter} - * which provides support for Kerberos HTTP SPNEGO authentication - * and proxy user authentication. + * Initializes hadoop-auth AuthenticationFilter which provides support for + * Kerberos HTTP SPNEGO authentication. *

* It enables anonymous access, simple/speudo and Kerberos HTTP SPNEGO * authentication for Hadoop JobTracker, NameNode, DataNodes and @@ -59,10 +58,8 @@ public class AuthenticationFilterInitializer extends FilterInitializer { public void initFilter(FilterContainer container, Configuration conf) { Map filterConfig = getFilterConfigMap(conf, PREFIX); - // extend AuthenticationFilter's feature to - // support proxy user operation. container.addFilter("authentication", - AuthenticationWithProxyUserFilter.class.getName(), + AuthenticationFilter.class.getName(), filterConfig); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationWithProxyUserFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationWithProxyUserFilter.java deleted file mode 100644 index ea9b282ab8999..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationWithProxyUserFilter.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.security; - -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; -import org.apache.hadoop.security.authorize.AuthorizationException; -import org.apache.hadoop.security.authorize.ProxyUsers; -import org.apache.hadoop.util.HttpExceptionUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; - -import javax.servlet.FilterChain; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.List; - -/** - * Extend the function of {@link AuthenticationFilter} to - * support authorizing proxy user. If the query string - * contains doAs parameter, then check the proxy user, - * otherwise do the next filter. - */ -public class AuthenticationWithProxyUserFilter extends AuthenticationFilter { - - /** - * Constant used in URL's query string to perform a proxy user request, the - * value of the DO_AS parameter is the user the request will be - * done on behalf of. - */ - private static final String DO_AS = "doAs"; - - private static final Charset UTF8_CHARSET = Charset.forName("UTF-8"); - - - /** - * This method provide the ability to do pre/post tasks - * in filter chain. Override this method to authorize - * proxy user between AuthenticationFilter and next filter. - * @param filterChain the filter chain object. - * @param request the request object. - * @param response the response object. - * - * @throws IOException - * @throws ServletException - */ - @Override - protected void doFilter(FilterChain filterChain, HttpServletRequest request, - HttpServletResponse response) throws IOException, ServletException { - - // authorize proxy user before calling next filter. - String proxyUser = getDoAs(request); - if (proxyUser != null) { - UserGroupInformation realUser = - UserGroupInformation.createRemoteUser(request.getRemoteUser()); - UserGroupInformation proxyUserInfo = - UserGroupInformation.createProxyUser(proxyUser, realUser); - - try { - ProxyUsers.authorize(proxyUserInfo, request.getRemoteAddr()); - } catch (AuthorizationException ex) { - HttpExceptionUtils.createServletExceptionResponse(response, - HttpServletResponse.SC_FORBIDDEN, ex); - // stop filter chain if there is an Authorization Exception. - return; - } - - final UserGroupInformation finalProxyUser = proxyUserInfo; - // Change the remote user after proxy user is authorized. - request = new HttpServletRequestWrapper(request) { - @Override - public String getRemoteUser() { - return finalProxyUser.getUserName(); - } - }; - - } - filterChain.doFilter(request, response); - } - - /** - * Get proxy user from query string. - * @param request the request object - * @return proxy user - */ - public static String getDoAs(HttpServletRequest request) { - String queryString = request.getQueryString(); - if (queryString == null) { - return null; - } - List list = URLEncodedUtils.parse(queryString, UTF8_CHARSET); - if (list != null) { - for (NameValuePair nv : list) { - if (DO_AS.equalsIgnoreCase(nv.getName())) { - return nv.getValue(); - } - } - } - return null; - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index 9fd39b09abac0..7380b1846060a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -18,14 +18,22 @@ package org.apache.hadoop.security; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicLong; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ticker; @@ -33,6 +41,13 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -42,7 +57,6 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Timer; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -62,13 +76,24 @@ public class Groups { private final GroupMappingServiceProvider impl; private final LoadingCache> cache; - private final Map> staticUserToGroupsMap = - new HashMap>(); + private final AtomicReference>> staticMapRef = + new AtomicReference<>(); private final long cacheTimeout; private final long negativeCacheTimeout; private final long warningDeltaMs; private final Timer timer; private Set negativeCache; + private final boolean reloadGroupsInBackground; + private final int reloadGroupsThreadCount; + + private final AtomicLong backgroundRefreshSuccess = + new AtomicLong(0); + private final AtomicLong backgroundRefreshException = + new AtomicLong(0); + private final AtomicLong backgroundRefreshQueued = + new AtomicLong(0); + private final AtomicLong backgroundRefreshRunning = + new AtomicLong(0); public Groups(Configuration conf) { this(conf, new Timer()); @@ -91,6 +116,18 @@ public Groups(Configuration conf, final Timer timer) { warningDeltaMs = conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS, CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT); + reloadGroupsInBackground = + conf.getBoolean( + CommonConfigurationKeys. + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD, + CommonConfigurationKeys. + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_DEFAULT); + reloadGroupsThreadCount = + conf.getInt( + CommonConfigurationKeys. + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS, + CommonConfigurationKeys. + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS_DEFAULT); parseStaticMapping(conf); this.timer = timer; @@ -129,6 +166,8 @@ private void parseStaticMapping(Configuration conf) { CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT); Collection mappings = StringUtils.getStringCollection( staticMapping, ";"); + Map> staticUserToGroupsMap = + new HashMap>(); for (String users : mappings) { Collection userToGroups = StringUtils.getStringCollection(users, "="); @@ -147,6 +186,8 @@ private void parseStaticMapping(Configuration conf) { } staticUserToGroupsMap.put(user, groups); } + staticMapRef.set( + staticUserToGroupsMap.isEmpty() ? null : staticUserToGroupsMap); } private boolean isNegativeCacheEnabled() { @@ -166,9 +207,12 @@ private IOException noGroupsForUser(String user) { */ public List getGroups(final String user) throws IOException { // No need to lookup for groups of static users - List staticMapping = staticUserToGroupsMap.get(user); - if (staticMapping != null) { - return staticMapping; + Map> staticUserToGroupsMap = staticMapRef.get(); + if (staticUserToGroupsMap != null) { + List staticMapping = staticUserToGroupsMap.get(user); + if (staticMapping != null) { + return staticMapping; + } } // Check the negative cache first @@ -185,6 +229,22 @@ public List getGroups(final String user) throws IOException { } } + public long getBackgroundRefreshSuccess() { + return backgroundRefreshSuccess.get(); + } + + public long getBackgroundRefreshException() { + return backgroundRefreshException.get(); + } + + public long getBackgroundRefreshQueued() { + return backgroundRefreshQueued.get(); + } + + public long getBackgroundRefreshRunning() { + return backgroundRefreshRunning.get(); + } + /** * Convert millisecond times from hadoop's timer to guava's nanosecond ticker. */ @@ -206,11 +266,41 @@ public long read() { * Deals with loading data into the cache. */ private class GroupCacheLoader extends CacheLoader> { + + private ListeningExecutorService executorService; + + GroupCacheLoader() { + if (reloadGroupsInBackground) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("Group-Cache-Reload") + .setDaemon(true) + .build(); + // With coreThreadCount == maxThreadCount we effectively + // create a fixed size thread pool. As allowCoreThreadTimeOut + // has been set, all threads will die after 60 seconds of non use + ThreadPoolExecutor parentExecutor = new ThreadPoolExecutor( + reloadGroupsThreadCount, + reloadGroupsThreadCount, + 60, + TimeUnit.SECONDS, + new LinkedBlockingQueue(), + threadFactory); + parentExecutor.allowCoreThreadTimeOut(true); + executorService = MoreExecutors.listeningDecorator(parentExecutor); + } + } + /** * This method will block if a cache entry doesn't exist, and * any subsequent requests for the same user will wait on this * request to return. If a user already exists in the cache, - * this will be run in the background. + * and when the key expires, the first call to reload the key + * will block, but subsequent requests will return the old + * value until the blocking thread returns. + * If reloadGroupsInBackground is true, then the thread that + * needs to refresh an expired key will not block either. Instead + * it will return the old cache value and schedule a background + * refresh * @param user key of cache * @return List of groups belonging to user * @throws IOException to prevent caching negative entries @@ -228,7 +318,48 @@ public List load(String user) throws Exception { throw noGroupsForUser(user); } - return groups; + // return immutable de-duped list + return Collections.unmodifiableList( + new ArrayList<>(new LinkedHashSet<>(groups))); + } + + /** + * Override the reload method to provide an asynchronous implementation. If + * reloadGroupsInBackground is false, then this method defers to the super + * implementation, otherwise is arranges for the cache to be updated later + */ + @Override + public ListenableFuture> reload(final String key, + List oldValue) + throws Exception { + if (!reloadGroupsInBackground) { + return super.reload(key, oldValue); + } + + backgroundRefreshQueued.incrementAndGet(); + ListenableFuture> listenableFuture = + executorService.submit(new Callable>() { + @Override + public List call() throws Exception { + backgroundRefreshQueued.decrementAndGet(); + backgroundRefreshRunning.incrementAndGet(); + List results = load(key); + return results; + } + }); + Futures.addCallback(listenableFuture, new FutureCallback>() { + @Override + public void onSuccess(List result) { + backgroundRefreshSuccess.incrementAndGet(); + backgroundRefreshRunning.decrementAndGet(); + } + @Override + public void onFailure(Throwable t) { + backgroundRefreshException.incrementAndGet(); + backgroundRefreshRunning.decrementAndGet(); + } + }); + return listenableFuture; } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java index e97106e1eb732..5b36553b708e7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java @@ -102,6 +102,27 @@ public class LdapGroupsMapping public static final String LDAP_KEYSTORE_PASSWORD_FILE_KEY = LDAP_KEYSTORE_PASSWORD_KEY + ".file"; public static final String LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT = ""; + + /** + * File path to the location of the SSL truststore to use + */ + public static final String LDAP_TRUSTSTORE_KEY = LDAP_CONFIG_PREFIX + + ".ssl.truststore"; + + /** + * The key of the credential entry containing the password for + * the LDAP SSL truststore + */ + public static final String LDAP_TRUSTSTORE_PASSWORD_KEY = + LDAP_CONFIG_PREFIX +".ssl.truststore.password"; + + /** + * The path to a file containing the password for + * the LDAP SSL truststore + */ + public static final String LDAP_TRUSTSTORE_PASSWORD_FILE_KEY = + LDAP_TRUSTSTORE_PASSWORD_KEY + ".file"; + /* * User to bind to the LDAP server with */ @@ -147,6 +168,21 @@ public class LdapGroupsMapping public static final String GROUP_NAME_ATTR_KEY = LDAP_CONFIG_PREFIX + ".search.attr.group.name"; public static final String GROUP_NAME_ATTR_DEFAULT = "cn"; + /* + * LDAP attribute names to use when doing posix-like lookups + */ + public static final String POSIX_UID_ATTR_KEY = LDAP_CONFIG_PREFIX + ".posix.attr.uid.name"; + public static final String POSIX_UID_ATTR_DEFAULT = "uidNumber"; + + public static final String POSIX_GID_ATTR_KEY = LDAP_CONFIG_PREFIX + ".posix.attr.gid.name"; + public static final String POSIX_GID_ATTR_DEFAULT = "gidNumber"; + + /* + * Posix attributes + */ + public static final String POSIX_GROUP = "posixGroup"; + public static final String POSIX_ACCOUNT = "posixAccount"; + /* * LDAP {@link SearchControls} attribute to set the time limit * for an invoked directory search. Prevents infinite wait cases. @@ -155,6 +191,13 @@ public class LdapGroupsMapping LDAP_CONFIG_PREFIX + ".directory.search.timeout"; public static final int DIRECTORY_SEARCH_TIMEOUT_DEFAULT = 10000; // 10s + public static final String CONNECTION_TIMEOUT = + LDAP_CONFIG_PREFIX + ".connection.timeout.ms"; + public static final int CONNECTION_TIMEOUT_DEFAULT = 60 * 1000; // 60 seconds + public static final String READ_TIMEOUT = + LDAP_CONFIG_PREFIX + ".read.timeout.ms"; + public static final int READ_TIMEOUT_DEFAULT = 60 * 1000; // 60 seconds + private static final Log LOG = LogFactory.getLog(LdapGroupsMapping.class); private static final SearchControls SEARCH_CONTROLS = new SearchControls(); @@ -169,6 +212,8 @@ public class LdapGroupsMapping private boolean useSsl; private String keystore; private String keystorePass; + private String truststore; + private String truststorePass; private String bindUser; private String bindPassword; private String baseDN; @@ -176,6 +221,9 @@ public class LdapGroupsMapping private String userSearchFilter; private String groupMemberAttr; private String groupNameAttr; + private String posixUidAttr; + private String posixGidAttr; + private boolean isPosix; public static final int RECONNECT_RETRY_COUNT = 3; @@ -226,15 +274,40 @@ List doGetGroups(String user) throws NamingException { SearchResult result = results.nextElement(); String userDn = result.getNameInNamespace(); - NamingEnumeration groupResults = - ctx.search(baseDN, - "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))", - new Object[]{userDn}, - SEARCH_CONTROLS); - while (groupResults.hasMoreElements()) { - SearchResult groupResult = groupResults.nextElement(); - Attribute groupName = groupResult.getAttributes().get(groupNameAttr); - groups.add(groupName.get().toString()); + NamingEnumeration groupResults = null; + + if (isPosix) { + String gidNumber = null; + String uidNumber = null; + Attribute gidAttribute = result.getAttributes().get(posixGidAttr); + Attribute uidAttribute = result.getAttributes().get(posixUidAttr); + if (gidAttribute != null) { + gidNumber = gidAttribute.get().toString(); + } + if (uidAttribute != null) { + uidNumber = uidAttribute.get().toString(); + } + if (uidNumber != null && gidNumber != null) { + groupResults = + ctx.search(baseDN, + "(&"+ groupSearchFilter + "(|(" + posixGidAttr + "={0})" + + "(" + groupMemberAttr + "={1})))", + new Object[] { gidNumber, uidNumber }, + SEARCH_CONTROLS); + } + } else { + groupResults = + ctx.search(baseDN, + "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))", + new Object[]{userDn}, + SEARCH_CONTROLS); + } + if (groupResults != null) { + while (groupResults.hasMoreElements()) { + SearchResult groupResult = groupResults.nextElement(); + Attribute groupName = groupResult.getAttributes().get(groupNameAttr); + groups.add(groupName.get().toString()); + } } } @@ -256,13 +329,29 @@ DirContext getDirContext() throws NamingException { // Set up SSL security, if necessary if (useSsl) { env.put(Context.SECURITY_PROTOCOL, "ssl"); - System.setProperty("javax.net.ssl.keyStore", keystore); - System.setProperty("javax.net.ssl.keyStorePassword", keystorePass); + if (!keystore.isEmpty()) { + System.setProperty("javax.net.ssl.keyStore", keystore); + } + if (!keystorePass.isEmpty()) { + System.setProperty("javax.net.ssl.keyStorePassword", keystorePass); + } + if (!truststore.isEmpty()) { + System.setProperty("javax.net.ssl.trustStore", truststore); + } + if (!truststorePass.isEmpty()) { + System.setProperty("javax.net.ssl.trustStorePassword", + truststorePass); + } } env.put(Context.SECURITY_PRINCIPAL, bindUser); env.put(Context.SECURITY_CREDENTIALS, bindPassword); + env.put("com.sun.jndi.ldap.connect.timeout", conf.get(CONNECTION_TIMEOUT, + String.valueOf(CONNECTION_TIMEOUT_DEFAULT))); + env.put("com.sun.jndi.ldap.read.timeout", conf.get(READ_TIMEOUT, + String.valueOf(READ_TIMEOUT_DEFAULT))); + ctx = new InitialDirContext(env); } @@ -298,15 +387,10 @@ public synchronized void setConf(Configuration conf) { if (ldapUrl == null || ldapUrl.isEmpty()) { throw new RuntimeException("LDAP URL is not configured"); } - + useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT); - keystore = conf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT); - - keystorePass = getPassword(conf, LDAP_KEYSTORE_PASSWORD_KEY, - LDAP_KEYSTORE_PASSWORD_DEFAULT); - if (keystorePass.isEmpty()) { - keystorePass = extractPassword(conf.get(LDAP_KEYSTORE_PASSWORD_FILE_KEY, - LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT)); + if (useSsl) { + loadSslConf(conf); } bindUser = conf.get(BIND_USER_KEY, BIND_USER_DEFAULT); @@ -321,19 +405,68 @@ public synchronized void setConf(Configuration conf) { conf.get(GROUP_SEARCH_FILTER_KEY, GROUP_SEARCH_FILTER_DEFAULT); userSearchFilter = conf.get(USER_SEARCH_FILTER_KEY, USER_SEARCH_FILTER_DEFAULT); + isPosix = groupSearchFilter.contains(POSIX_GROUP) && userSearchFilter + .contains(POSIX_ACCOUNT); groupMemberAttr = conf.get(GROUP_MEMBERSHIP_ATTR_KEY, GROUP_MEMBERSHIP_ATTR_DEFAULT); groupNameAttr = conf.get(GROUP_NAME_ATTR_KEY, GROUP_NAME_ATTR_DEFAULT); + posixUidAttr = + conf.get(POSIX_UID_ATTR_KEY, POSIX_UID_ATTR_DEFAULT); + posixGidAttr = + conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT); int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT); SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout); - // Limit the attributes returned to only those required to speed up the search. See HADOOP-10626 for more details. - SEARCH_CONTROLS.setReturningAttributes(new String[] {groupNameAttr}); + // Limit the attributes returned to only those required to speed up the search. + // See HADOOP-10626 and HADOOP-12001 for more details. + SEARCH_CONTROLS.setReturningAttributes( + new String[] {groupNameAttr, posixUidAttr, posixGidAttr}); this.conf = conf; } + private void loadSslConf(Configuration sslConf) { + keystore = sslConf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT); + keystorePass = getPassword(sslConf, LDAP_KEYSTORE_PASSWORD_KEY, + LDAP_KEYSTORE_PASSWORD_DEFAULT); + if (keystorePass.isEmpty()) { + keystorePass = extractPassword(sslConf.get( + LDAP_KEYSTORE_PASSWORD_FILE_KEY, + LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT)); + } + + truststore = sslConf.get(LDAP_TRUSTSTORE_KEY, ""); + truststorePass = getPasswordFromCredentialProviders( + sslConf, LDAP_TRUSTSTORE_PASSWORD_KEY, ""); + if (truststorePass.isEmpty()) { + truststorePass = extractPassword( + sslConf.get(LDAP_TRUSTSTORE_PASSWORD_FILE_KEY, "")); + } + } + + String getPasswordFromCredentialProviders( + Configuration conf, String alias, String defaultPass) { + String password = defaultPass; + try { + char[] passchars = conf.getPasswordFromCredentialProviders(alias); + if (passchars != null) { + password = new String(passchars); + } + } catch (IOException ioe) { + LOG.warn("Exception while trying to get password for alias " + alias + + ": ", ioe); + } + return password; + } + + /** + * Passwords should not be stored in configuration. Use + * {@link #getPasswordFromCredentialProviders( + * Configuration, String, String)} + * to avoid reading passwords from a configuration file. + */ + @Deprecated String getPassword(Configuration conf, String alias, String defaultPass) { String password = null; try { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 611b9066b51f4..93e2da5c37096 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -37,7 +37,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -251,14 +250,18 @@ public boolean logout() throws LoginException { public static final String HADOOP_TOKEN_FILE_LOCATION = "HADOOP_TOKEN_FILE_LOCATION"; + public static boolean isInitialized() { + return conf != null; + } + /** * A method to initialize the fields that depend on a configuration. * Must be called before useKerberos or groups is used. */ private static void ensureInitialized() { - if (conf == null) { + if (!isInitialized()) { synchronized(UserGroupInformation.class) { - if (conf == null) { // someone might have beat us + if (!isInitialized()) { // someone might have beat us initialize(new Configuration(), false); } } @@ -1048,10 +1051,14 @@ private void fixKerberosTicketOrder() { Object cred = iter.next(); if (cred instanceof KerberosTicket) { KerberosTicket ticket = (KerberosTicket) cred; - if (!ticket.getServer().getName().startsWith("krbtgt")) { - LOG.warn("The first kerberos ticket is not TGT(the server" + - " principal is " + ticket.getServer() + "), remove" + - " and destroy it."); + if (ticket.isDestroyed() || ticket.getServer() == null) { + LOG.warn("Ticket is already destroyed, remove it."); + iter.remove(); + } else if (!ticket.getServer().getName().startsWith("krbtgt")) { + LOG.warn( + "The first kerberos ticket is not TGT" + + "(the server principal is " + ticket.getServer() + + ")), remove and destroy it."); iter.remove(); try { ticket.destroy(); @@ -1475,11 +1482,11 @@ public String getShortUserName() { } public String getPrimaryGroupName() throws IOException { - String[] groups = getGroupNames(); - if (groups.length == 0) { + List groups = getGroups(); + if (groups.isEmpty()) { throw new IOException("There is no primary group for UGI " + this); } - return groups[0]; + return groups.get(0); } /** @@ -1591,27 +1598,36 @@ private synchronized Credentials getCredentialsInternal() { return credentials; } + /** + * Get the group names for this user. {@ #getGroups(String)} is less + * expensive alternative when checking for a contained element. + * @return the list of users with the primary group first. If the command + * fails, it returns an empty list. + */ + public String[] getGroupNames() { + List groups = getGroups(); + return groups.toArray(new String[groups.size()]); + } + /** * Get the group names for this user. * @return the list of users with the primary group first. If the command * fails, it returns an empty list. */ - public synchronized String[] getGroupNames() { + public List getGroups() { ensureInitialized(); try { - Set result = new LinkedHashSet - (groups.getGroups(getShortUserName())); - return result.toArray(new String[result.size()]); + return groups.getGroups(getShortUserName()); } catch (IOException ie) { if (LOG.isDebugEnabled()) { LOG.debug("Failed to get groups for user " + getShortUserName() + " by " + ie); LOG.trace("TRACE", ie); } - return StringUtils.emptyStringArray; + return Collections.emptyList(); } } - + /** * Return the username. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java index b1b474b5a8df7..a32a5fe74b0c0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java @@ -231,7 +231,7 @@ public final boolean isUserInList(UserGroupInformation ugi) { if (allAllowed || users.contains(ugi.getShortUserName())) { return true; } else if (!groups.isEmpty()) { - for(String group: ugi.getGroupNames()) { + for (String group : ugi.getGroups()) { if (groups.contains(group)) { return true; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index 02b3feafbe5dc..ad063473c1350 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -22,6 +22,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.security.MessageDigest; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -597,6 +598,11 @@ private void removeExpiredToken() throws IOException { } } // don't hold lock on 'this' to avoid edit log updates blocking token ops + logExpireTokens(expiredTokens); + } + + protected void logExpireTokens( + Collection expiredTokens) throws IOException { for (TokenIdent ident : expiredTokens) { logExpireToken(ident); removeStoredToken(ident); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java index 4b26b7611d66a..a3b5b0bbd94e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java @@ -93,6 +93,7 @@ public static void unJar(File jarFile, File toDir, Pattern unpackRegex) throws IOException { JarFile jar = new JarFile(jarFile); try { + String targetDirPath = toDir.getCanonicalPath() + File.separator; Enumeration entries = jar.entries(); while (entries.hasMoreElements()) { final JarEntry entry = entries.nextElement(); @@ -102,6 +103,10 @@ public static void unJar(File jarFile, File toDir, Pattern unpackRegex) try { File file = new File(toDir, entry.getName()); ensureDirectory(file.getParentFile()); + if (!file.getCanonicalPath().startsWith(targetDirPath)) { + throw new IOException("expanding " + entry.getName() + + " would create file outside of " + toDir); + } OutputStream out = new FileOutputStream(file); try { IOUtils.copyBytes(in, out, 8192); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java index b9d0d0b6646d8..c0eedf6110d7f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java @@ -25,11 +25,22 @@ * A simplified StopWatch implementation which can measure times in nanoseconds. */ public class StopWatch implements Closeable { + private final Timer timer; private boolean isStarted; private long startNanos; private long currentElapsedNanos; public StopWatch() { + this(new Timer()); + } + + /** + * Used for tests to be able to create a StopWatch which does not follow real + * time. + * @param timer The timer to base this StopWatch's timekeeping off of. + */ + public StopWatch(Timer timer) { + this.timer = timer; } /** @@ -49,7 +60,7 @@ public StopWatch start() { throw new IllegalStateException("StopWatch is already running"); } isStarted = true; - startNanos = System.nanoTime(); + startNanos = timer.monotonicNowNanos(); return this; } @@ -61,7 +72,7 @@ public StopWatch stop() { if (!isStarted) { throw new IllegalStateException("StopWatch is already stopped"); } - long now = System.nanoTime(); + long now = timer.monotonicNowNanos(); isStarted = false; currentElapsedNanos += now - startNanos; return this; @@ -90,7 +101,7 @@ public long now(TimeUnit timeUnit) { */ public long now() { return isStarted ? - System.nanoTime() - startNanos + currentElapsedNanos : + timer.monotonicNowNanos() - startNanos + currentElapsedNanos : currentElapsedNanos; } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 354e9a693fc0d..30551b989f1ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -223,6 +223,52 @@ for ldap providers in the same way as above does. + + hadoop.security.groups.cache.background.reload + false + + Whether to reload expired user->group mappings using a background thread + pool. If set to true, a pool of + hadoop.security.groups.cache.background.reload.threads is created to + update the cache in the background. + + + + + hadoop.security.groups.cache.background.reload.threads + 3 + + Only relevant if hadoop.security.groups.cache.background.reload is true. + Controls the number of concurrent background user->group cache entry + refreshes. Pending refresh requests beyond this value are queued and + processed when a thread is free. + + + + + hadoop.security.group.mapping.ldap.connection.timeout.ms + 60000 + + This property is the connection timeout (in milliseconds) for LDAP + operations. If the LDAP provider doesn't establish a connection within the + specified period, it will abort the connect attempt. Non-positive value + means no LDAP connection timeout is specified in which case it waits for the + connection to establish until the underlying network times out. + + + + + hadoop.security.group.mapping.ldap.read.timeout.ms + 60000 + + This property is the read timeout (in milliseconds) for LDAP + operations. If the LDAP provider doesn't get a LDAP response within the + specified period, it will abort the read attempt. Non-positive value + means no read timeout is specified in which case it waits for the response + infinitely. + + + hadoop.security.group.mapping.ldap.url @@ -260,6 +306,27 @@ for ldap providers in the same way as above does. + + hadoop.security.group.mapping.ldap.ssl.truststore + + + File path to the SSL truststore that contains the root certificate used to + sign the LDAP server's certificate. Specify this if the LDAP server's + certificate is not signed by a well known certificate authority. + + + + + hadoop.security.group.mapping.ldap.ssl.truststore.password.file + + + The path to a file containing the password of the LDAP SSL truststore. + + IMPORTANT: This file should be readable only by the Unix user running + the daemons. + + + hadoop.security.group.mapping.ldap.bind.user @@ -298,6 +365,11 @@ for ldap providers in the same way as above does. an LDAP server with a non-AD schema, this should be replaced with (&(objectClass=inetOrgPerson)(uid={0}). {0} is a special string used to denote where the username fits into the filter. + + If the LDAP server supports posixGroups, Hadoop can enable the feature by + setting the value of this property to "posixAccount" and the value of + the hadoop.security.group.mapping.ldap.search.filter.group property to + "posixGroup". @@ -307,7 +379,9 @@ for ldap providers in the same way as above does. An additional filter to use when searching for LDAP groups. This should be changed when resolving groups against a non-Active Directory installation. - posixGroups are currently not a supported group class. + + See the description of hadoop.security.group.mapping.ldap.search.filter.user + to enable posixGroups support. @@ -330,6 +404,24 @@ for ldap providers in the same way as above does. + + hadoop.security.group.mapping.ldap.posix.attr.uid.name + uidNumber + + The attribute of posixAccount to use when groups for membership. + Mostly useful for schemas wherein groups have memberUids that use an + attribute other than uidNumber. + + + + + hadoop.security.group.mapping.ldap.posix.attr.gid.name + gidNumber + + The attribute of posixAccount indicating the group id. + + + hadoop.security.group.mapping.ldap.directory.search.timeout 10000 @@ -1751,6 +1843,29 @@ for ldap providers in the same way as above does. + + hadoop.security.crypto.jceks.key.serialfilter + + Enhanced KeyStore Mechanisms in JDK 8u171 introduced jceks.key.serialFilter. + If jceks.key.serialFilter is configured, the JCEKS KeyStore uses it during + the deserialization of the encrypted Key object stored inside a + SecretKeyEntry. + If jceks.key.serialFilter is not configured it will cause an error when + recovering keystore file in KeyProviderFactory when recovering key from + keystore file using JDK 8u171 or newer. The filter pattern uses the same + format as jdk.serialFilter. + + The value of this property will be used as the following: + 1. The value of jceks.key.serialFilter system property takes precedence + over the value of this property. + 2. In the absence of jceks.key.serialFilter system property the value of + this property will be set as the value of jceks.key.serialFilter. + 3. If the value of this property and jceks.key.serialFilter system + property has not been set, org.apache.hadoop.crypto.key.KeyProvider + sets a default value for jceks.key.serialFilter. + + + hadoop.security.crypto.buffer.size 8192 diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 19cc79a152830..81539a920d87b 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -192,6 +192,7 @@ Each metrics record contains tags such as ProcessName, SessionId, and Hostname a | `GetImageAvgTime` | Average fsimage download time in milliseconds | | `PutImageNumOps` | Total number of fsimage uploads to SecondaryNameNode | | `PutImageAvgTime` | Average fsimage upload time in milliseconds | +| `TotalFileOps` | Total number of file operations performed | FSNamesystem ------------ @@ -229,7 +230,15 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `PendingDataNodeMessageCourt` | (HA-only) Current number of pending block-related messages for later processing in the standby NameNode | | `MillisSinceLastLoadedEdits` | (HA-only) Time in milliseconds since the last time standby NameNode load edit log. In active NameNode, set to 0 | | `BlockCapacity` | Current number of block capacity | +| `NumLiveDataNodes` | Number of datanodes which are currently live | +| `NumDeadDataNodes` | Number of datanodes which are currently dead | +| `NumDecomLiveDataNodes` | Number of datanodes which have been decommissioned and are now live | +| `NumDecomDeadDataNodes` | Number of datanodes which have been decommissioned and are now dead | +| `NumDecommissioningDataNodes` | Number of datanodes in decommissioning state | +| `VolumeFailuresTotal` | Total number of volume failures across all Datanodes | +| `EstimatedCapacityLostTotal` | An estimate of the total capacity lost due to volume failures | | `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat | +| `NumStaleStorages` | Number of storages marked as content stale (after NameNode restart/failover before first block report is received) | | `TotalFiles` | Current number of files and directories (same as FilesTotal) | | `LockQueueLength` | Number of threads waiting to acquire FSNameSystem lock | @@ -334,6 +343,10 @@ Each metrics record contains tags such as SessionId and Hostname as additional i | `SendDataPacketBlockedOnNetworkNanosAvgTime` | Average waiting time of sending packets in nanoseconds | | `SendDataPacketTransferNanosNumOps` | Total number of sending packets | | `SendDataPacketTransferNanosAvgTime` | Average transfer time of sending packets in nanoseconds | +| `TotalWriteTime` | Total number of milliseconds spent on write operation | +| `TotalReadTime` | Total number of milliseconds spent on read operation | +| `RemoteBytesRead` | Number of bytes read by remote clients | +| `RemoteBytesWritten` | Number of bytes written by remote clients | yarn context ============ @@ -454,7 +467,7 @@ MetricsSystem shows the statistics for metrics snapshots and publishes. Each met | `Sink_`*instance*`NumOps` | Total number of sink operations for the *instance* | | `Sink_`*instance*`AvgTime` | Average time in milliseconds of sink operations for the *instance* | | `Sink_`*instance*`Dropped` | Total number of dropped sink operations for the *instance* | -| `Sink_`*instance*`Qsize` | Current queue length of sink operations  (BUT always set to 0 because nothing to increment this metrics, see [HADOOP-9941](https://issues.apache.org/jira/browse/HADOOP-9941)) | +| `Sink_`*instance*`Qsize` | Current queue length of the sink | default context =============== diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md b/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md index adb7db91bb537..1c40aa95feb91 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md @@ -87,13 +87,27 @@ A special value of `*` implies that all users are allowed to access the service. If access control list is not defined for a service, the value of `security.service.authorization.default.acl` is applied. If `security.service.authorization.default.acl` is not defined, `*` is applied. -* Blocked Access Control ListsIn some cases, it is required to specify blocked access control list for a service. This specifies the list of users and groups who are not authorized to access the service. The format of the blocked access control list is same as that of access control list. The blocked access control list can be specified via `$HADOOP_CONF_DIR/hadoop-policy.xml`. The property name is derived by suffixing with ".blocked". +### Blocked Access Control Lists - Example: The property name of blocked access control list for `security.client.protocol.acl>> will be <<; umask -S - String [][] symbolic = new String [][] { - {"a+rw", "111",}, - {"u=rwx,g=rwx,o=rwx", "0",}, - {"u=rwx,g=rwx,o=rw", "1",}, - {"u=rwx,g=rwx,o=rx", "2",}, - {"u=rwx,g=rwx,o=r", "3",}, - {"u=rwx,g=rwx,o=wx", "4",}, - {"u=rwx,g=rwx,o=w", "5",}, - {"u=rwx,g=rwx,o=x", "6",}, - {"u=rwx,g=rwx,o=", "7",}, - {"u=rwx,g=rw,o=rwx", "10",}, - {"u=rwx,g=rw,o=rw", "11",}, - {"u=rwx,g=rw,o=rx", "12",}, - {"u=rwx,g=rw,o=r", "13",}, - {"u=rwx,g=rw,o=wx", "14",}, - {"u=rwx,g=rw,o=w", "15",}, - {"u=rwx,g=rw,o=x", "16",}, - {"u=rwx,g=rw,o=", "17",}, - {"u=rwx,g=rx,o=rwx", "20",}, - {"u=rwx,g=rx,o=rw", "21",}, - {"u=rwx,g=rx,o=rx", "22",}, - {"u=rwx,g=rx,o=r", "23",}, - {"u=rwx,g=rx,o=wx", "24",}, - {"u=rwx,g=rx,o=w", "25",}, - {"u=rwx,g=rx,o=x", "26",}, - {"u=rwx,g=rx,o=", "27",}, - {"u=rwx,g=r,o=rwx", "30",}, - {"u=rwx,g=r,o=rw", "31",}, - {"u=rwx,g=r,o=rx", "32",}, - {"u=rwx,g=r,o=r", "33",}, - {"u=rwx,g=r,o=wx", "34",}, - {"u=rwx,g=r,o=w", "35",}, - {"u=rwx,g=r,o=x", "36",}, - {"u=rwx,g=r,o=", "37",}, - {"u=rwx,g=wx,o=rwx", "40",}, - {"u=rwx,g=wx,o=rw", "41",}, - {"u=rwx,g=wx,o=rx", "42",}, - {"u=rwx,g=wx,o=r", "43",}, - {"u=rwx,g=wx,o=wx", "44",}, - {"u=rwx,g=wx,o=w", "45",}, - {"u=rwx,g=wx,o=x", "46",}, - {"u=rwx,g=wx,o=", "47",}, - {"u=rwx,g=w,o=rwx", "50",}, - {"u=rwx,g=w,o=rw", "51",}, - {"u=rwx,g=w,o=rx", "52",}, - {"u=rwx,g=w,o=r", "53",}, - {"u=rwx,g=w,o=wx", "54",}, - {"u=rwx,g=w,o=w", "55",}, - {"u=rwx,g=w,o=x", "56",}, - {"u=rwx,g=w,o=", "57",}, - {"u=rwx,g=x,o=rwx", "60",}, - {"u=rwx,g=x,o=rw", "61",}, - {"u=rwx,g=x,o=rx", "62",}, - {"u=rwx,g=x,o=r", "63",}, - {"u=rwx,g=x,o=wx", "64",}, - {"u=rwx,g=x,o=w", "65",}, - {"u=rwx,g=x,o=x", "66",}, - {"u=rwx,g=x,o=", "67",}, - {"u=rwx,g=,o=rwx", "70",}, - {"u=rwx,g=,o=rw", "71",}, - {"u=rwx,g=,o=rx", "72",}, - {"u=rwx,g=,o=r", "73",}, - {"u=rwx,g=,o=wx", "74",}, - {"u=rwx,g=,o=w", "75",}, - {"u=rwx,g=,o=x", "76",}, - {"u=rwx,g=,o=", "77",}, - {"u=rw,g=rwx,o=rwx", "100",}, - {"u=rw,g=rwx,o=rw", "101",}, - {"u=rw,g=rwx,o=rx", "102",}, - {"u=rw,g=rwx,o=r", "103",}, - {"u=rw,g=rwx,o=wx", "104",}, - {"u=rw,g=rwx,o=w", "105",}, - {"u=rw,g=rwx,o=x", "106",}, - {"u=rw,g=rwx,o=", "107",}, - {"u=rw,g=rw,o=rwx", "110",}, - {"u=rw,g=rw,o=rw", "111",}, - {"u=rw,g=rw,o=rx", "112",}, - {"u=rw,g=rw,o=r", "113",}, - {"u=rw,g=rw,o=wx", "114",}, - {"u=rw,g=rw,o=w", "115",}, - {"u=rw,g=rw,o=x", "116",}, - {"u=rw,g=rw,o=", "117",}, - {"u=rw,g=rx,o=rwx", "120",}, - {"u=rw,g=rx,o=rw", "121",}, - {"u=rw,g=rx,o=rx", "122",}, - {"u=rw,g=rx,o=r", "123",}, - {"u=rw,g=rx,o=wx", "124",}, - {"u=rw,g=rx,o=w", "125",}, - {"u=rw,g=rx,o=x", "126",}, - {"u=rw,g=rx,o=", "127",}, - {"u=rw,g=r,o=rwx", "130",}, - {"u=rw,g=r,o=rw", "131",}, - {"u=rw,g=r,o=rx", "132",}, - {"u=rw,g=r,o=r", "133",}, - {"u=rw,g=r,o=wx", "134",}, - {"u=rw,g=r,o=w", "135",}, - {"u=rw,g=r,o=x", "136",}, - {"u=rw,g=r,o=", "137",}, - {"u=rw,g=wx,o=rwx", "140",}, - {"u=rw,g=wx,o=rw", "141",}, - {"u=rw,g=wx,o=rx", "142",}, - {"u=rw,g=wx,o=r", "143",}, - {"u=rw,g=wx,o=wx", "144",}, - {"u=rw,g=wx,o=w", "145",}, - {"u=rw,g=wx,o=x", "146",}, - {"u=rw,g=wx,o=", "147",}, - {"u=rw,g=w,o=rwx", "150",}, - {"u=rw,g=w,o=rw", "151",}, - {"u=rw,g=w,o=rx", "152",}, - {"u=rw,g=w,o=r", "153",}, - {"u=rw,g=w,o=wx", "154",}, - {"u=rw,g=w,o=w", "155",}, - {"u=rw,g=w,o=x", "156",}, - {"u=rw,g=w,o=", "157",}, - {"u=rw,g=x,o=rwx", "160",}, - {"u=rw,g=x,o=rw", "161",}, - {"u=rw,g=x,o=rx", "162",}, - {"u=rw,g=x,o=r", "163",}, - {"u=rw,g=x,o=wx", "164",}, - {"u=rw,g=x,o=w", "165",}, - {"u=rw,g=x,o=x", "166",}, - {"u=rw,g=x,o=", "167",}, - {"u=rw,g=,o=rwx", "170",}, - {"u=rw,g=,o=rw", "171",}, - {"u=rw,g=,o=rx", "172",}, - {"u=rw,g=,o=r", "173",}, - {"u=rw,g=,o=wx", "174",}, - {"u=rw,g=,o=w", "175",}, - {"u=rw,g=,o=x", "176",}, - {"u=rw,g=,o=", "177",}, - {"u=rx,g=rwx,o=rwx", "200",}, - {"u=rx,g=rwx,o=rw", "201",}, - {"u=rx,g=rwx,o=rx", "202",}, - {"u=rx,g=rwx,o=r", "203",}, - {"u=rx,g=rwx,o=wx", "204",}, - {"u=rx,g=rwx,o=w", "205",}, - {"u=rx,g=rwx,o=x", "206",}, - {"u=rx,g=rwx,o=", "207",}, - {"u=rx,g=rw,o=rwx", "210",}, - {"u=rx,g=rw,o=rw", "211",}, - {"u=rx,g=rw,o=rx", "212",}, - {"u=rx,g=rw,o=r", "213",}, - {"u=rx,g=rw,o=wx", "214",}, - {"u=rx,g=rw,o=w", "215",}, - {"u=rx,g=rw,o=x", "216",}, - {"u=rx,g=rw,o=", "217",}, - {"u=rx,g=rx,o=rwx", "220",}, - {"u=rx,g=rx,o=rw", "221",}, - {"u=rx,g=rx,o=rx", "222",}, - {"u=rx,g=rx,o=r", "223",}, - {"u=rx,g=rx,o=wx", "224",}, - {"u=rx,g=rx,o=w", "225",}, - {"u=rx,g=rx,o=x", "226",}, - {"u=rx,g=rx,o=", "227",}, - {"u=rx,g=r,o=rwx", "230",}, - {"u=rx,g=r,o=rw", "231",}, - {"u=rx,g=r,o=rx", "232",}, - {"u=rx,g=r,o=r", "233",}, - {"u=rx,g=r,o=wx", "234",}, - {"u=rx,g=r,o=w", "235",}, - {"u=rx,g=r,o=x", "236",}, - {"u=rx,g=r,o=", "237",}, - {"u=rx,g=wx,o=rwx", "240",}, - {"u=rx,g=wx,o=rw", "241",}, - {"u=rx,g=wx,o=rx", "242",}, - {"u=rx,g=wx,o=r", "243",}, - {"u=rx,g=wx,o=wx", "244",}, - {"u=rx,g=wx,o=w", "245",}, - {"u=rx,g=wx,o=x", "246",}, - {"u=rx,g=wx,o=", "247",}, - {"u=rx,g=w,o=rwx", "250",}, - {"u=rx,g=w,o=rw", "251",}, - {"u=rx,g=w,o=rx", "252",}, - {"u=rx,g=w,o=r", "253",}, - {"u=rx,g=w,o=wx", "254",}, - {"u=rx,g=w,o=w", "255",}, - {"u=rx,g=w,o=x", "256",}, - {"u=rx,g=w,o=", "257",}, - {"u=rx,g=x,o=rwx", "260",}, - {"u=rx,g=x,o=rw", "261",}, - {"u=rx,g=x,o=rx", "262",}, - {"u=rx,g=x,o=r", "263",}, - {"u=rx,g=x,o=wx", "264",}, - {"u=rx,g=x,o=w", "265",}, - {"u=rx,g=x,o=x", "266",}, - {"u=rx,g=x,o=", "267",}, - {"u=rx,g=,o=rwx", "270",}, - {"u=rx,g=,o=rw", "271",}, - {"u=rx,g=,o=rx", "272",}, - {"u=rx,g=,o=r", "273",}, - {"u=rx,g=,o=wx", "274",}, - {"u=rx,g=,o=w", "275",}, - {"u=rx,g=,o=x", "276",}, - {"u=rx,g=,o=", "277",}, - {"u=r,g=rwx,o=rwx", "300",}, - {"u=r,g=rwx,o=rw", "301",}, - {"u=r,g=rwx,o=rx", "302",}, - {"u=r,g=rwx,o=r", "303",}, - {"u=r,g=rwx,o=wx", "304",}, - {"u=r,g=rwx,o=w", "305",}, - {"u=r,g=rwx,o=x", "306",}, - {"u=r,g=rwx,o=", "307",}, - {"u=r,g=rw,o=rwx", "310",}, - {"u=r,g=rw,o=rw", "311",}, - {"u=r,g=rw,o=rx", "312",}, - {"u=r,g=rw,o=r", "313",}, - {"u=r,g=rw,o=wx", "314",}, - {"u=r,g=rw,o=w", "315",}, - {"u=r,g=rw,o=x", "316",}, - {"u=r,g=rw,o=", "317",}, - {"u=r,g=rx,o=rwx", "320",}, - {"u=r,g=rx,o=rw", "321",}, - {"u=r,g=rx,o=rx", "322",}, - {"u=r,g=rx,o=r", "323",}, - {"u=r,g=rx,o=wx", "324",}, - {"u=r,g=rx,o=w", "325",}, - {"u=r,g=rx,o=x", "326",}, - {"u=r,g=rx,o=", "327",}, - {"u=r,g=r,o=rwx", "330",}, - {"u=r,g=r,o=rw", "331",}, - {"u=r,g=r,o=rx", "332",}, - {"u=r,g=r,o=r", "333",}, - {"u=r,g=r,o=wx", "334",}, - {"u=r,g=r,o=w", "335",}, - {"u=r,g=r,o=x", "336",}, - {"u=r,g=r,o=", "337",}, - {"u=r,g=wx,o=rwx", "340",}, - {"u=r,g=wx,o=rw", "341",}, - {"u=r,g=wx,o=rx", "342",}, - {"u=r,g=wx,o=r", "343",}, - {"u=r,g=wx,o=wx", "344",}, - {"u=r,g=wx,o=w", "345",}, - {"u=r,g=wx,o=x", "346",}, - {"u=r,g=wx,o=", "347",}, - {"u=r,g=w,o=rwx", "350",}, - {"u=r,g=w,o=rw", "351",}, - {"u=r,g=w,o=rx", "352",}, - {"u=r,g=w,o=r", "353",}, - {"u=r,g=w,o=wx", "354",}, - {"u=r,g=w,o=w", "355",}, - {"u=r,g=w,o=x", "356",}, - {"u=r,g=w,o=", "357",}, - {"u=r,g=x,o=rwx", "360",}, - {"u=r,g=x,o=rw", "361",}, - {"u=r,g=x,o=rx", "362",}, - {"u=r,g=x,o=r", "363",}, - {"u=r,g=x,o=wx", "364",}, - {"u=r,g=x,o=w", "365",}, - {"u=r,g=x,o=x", "366",}, - {"u=r,g=x,o=", "367",}, - {"u=r,g=,o=rwx", "370",}, - {"u=r,g=,o=rw", "371",}, - {"u=r,g=,o=rx", "372",}, - {"u=r,g=,o=r", "373",}, - {"u=r,g=,o=wx", "374",}, - {"u=r,g=,o=w", "375",}, - {"u=r,g=,o=x", "376",}, - {"u=r,g=,o=", "377",}, - {"u=wx,g=rwx,o=rwx", "400",}, - {"u=wx,g=rwx,o=rw", "401",}, - {"u=wx,g=rwx,o=rx", "402",}, - {"u=wx,g=rwx,o=r", "403",}, - {"u=wx,g=rwx,o=wx", "404",}, - {"u=wx,g=rwx,o=w", "405",}, - {"u=wx,g=rwx,o=x", "406",}, - {"u=wx,g=rwx,o=", "407",}, - {"u=wx,g=rw,o=rwx", "410",}, - {"u=wx,g=rw,o=rw", "411",}, - {"u=wx,g=rw,o=rx", "412",}, - {"u=wx,g=rw,o=r", "413",}, - {"u=wx,g=rw,o=wx", "414",}, - {"u=wx,g=rw,o=w", "415",}, - {"u=wx,g=rw,o=x", "416",}, - {"u=wx,g=rw,o=", "417",}, - {"u=wx,g=rx,o=rwx", "420",}, - {"u=wx,g=rx,o=rw", "421",}, - {"u=wx,g=rx,o=rx", "422",}, - {"u=wx,g=rx,o=r", "423",}, - {"u=wx,g=rx,o=wx", "424",}, - {"u=wx,g=rx,o=w", "425",}, - {"u=wx,g=rx,o=x", "426",}, - {"u=wx,g=rx,o=", "427",}, - {"u=wx,g=r,o=rwx", "430",}, - {"u=wx,g=r,o=rw", "431",}, - {"u=wx,g=r,o=rx", "432",}, - {"u=wx,g=r,o=r", "433",}, - {"u=wx,g=r,o=wx", "434",}, - {"u=wx,g=r,o=w", "435",}, - {"u=wx,g=r,o=x", "436",}, - {"u=wx,g=r,o=", "437",}, - {"u=wx,g=wx,o=rwx", "440",}, - {"u=wx,g=wx,o=rw", "441",}, - {"u=wx,g=wx,o=rx", "442",}, - {"u=wx,g=wx,o=r", "443",}, - {"u=wx,g=wx,o=wx", "444",}, - {"u=wx,g=wx,o=w", "445",}, - {"u=wx,g=wx,o=x", "446",}, - {"u=wx,g=wx,o=", "447",}, - {"u=wx,g=w,o=rwx", "450",}, - {"u=wx,g=w,o=rw", "451",}, - {"u=wx,g=w,o=rx", "452",}, - {"u=wx,g=w,o=r", "453",}, - {"u=wx,g=w,o=wx", "454",}, - {"u=wx,g=w,o=w", "455",}, - {"u=wx,g=w,o=x", "456",}, - {"u=wx,g=w,o=", "457",}, - {"u=wx,g=x,o=rwx", "460",}, - {"u=wx,g=x,o=rw", "461",}, - {"u=wx,g=x,o=rx", "462",}, - {"u=wx,g=x,o=r", "463",}, - {"u=wx,g=x,o=wx", "464",}, - {"u=wx,g=x,o=w", "465",}, - {"u=wx,g=x,o=x", "466",}, - {"u=wx,g=x,o=", "467",}, - {"u=wx,g=,o=rwx", "470",}, - {"u=wx,g=,o=rw", "471",}, - {"u=wx,g=,o=rx", "472",}, - {"u=wx,g=,o=r", "473",}, - {"u=wx,g=,o=wx", "474",}, - {"u=wx,g=,o=w", "475",}, - {"u=wx,g=,o=x", "476",}, - {"u=wx,g=,o=", "477",}, - {"u=w,g=rwx,o=rwx", "500",}, - {"u=w,g=rwx,o=rw", "501",}, - {"u=w,g=rwx,o=rx", "502",}, - {"u=w,g=rwx,o=r", "503",}, - {"u=w,g=rwx,o=wx", "504",}, - {"u=w,g=rwx,o=w", "505",}, - {"u=w,g=rwx,o=x", "506",}, - {"u=w,g=rwx,o=", "507",}, - {"u=w,g=rw,o=rwx", "510",}, - {"u=w,g=rw,o=rw", "511",}, - {"u=w,g=rw,o=rx", "512",}, - {"u=w,g=rw,o=r", "513",}, - {"u=w,g=rw,o=wx", "514",}, - {"u=w,g=rw,o=w", "515",}, - {"u=w,g=rw,o=x", "516",}, - {"u=w,g=rw,o=", "517",}, - {"u=w,g=rx,o=rwx", "520",}, - {"u=w,g=rx,o=rw", "521",}, - {"u=w,g=rx,o=rx", "522",}, - {"u=w,g=rx,o=r", "523",}, - {"u=w,g=rx,o=wx", "524",}, - {"u=w,g=rx,o=w", "525",}, - {"u=w,g=rx,o=x", "526",}, - {"u=w,g=rx,o=", "527",}, - {"u=w,g=r,o=rwx", "530",}, - {"u=w,g=r,o=rw", "531",}, - {"u=w,g=r,o=rx", "532",}, - {"u=w,g=r,o=r", "533",}, - {"u=w,g=r,o=wx", "534",}, - {"u=w,g=r,o=w", "535",}, - {"u=w,g=r,o=x", "536",}, - {"u=w,g=r,o=", "537",}, - {"u=w,g=wx,o=rwx", "540",}, - {"u=w,g=wx,o=rw", "541",}, - {"u=w,g=wx,o=rx", "542",}, - {"u=w,g=wx,o=r", "543",}, - {"u=w,g=wx,o=wx", "544",}, - {"u=w,g=wx,o=w", "545",}, - {"u=w,g=wx,o=x", "546",}, - {"u=w,g=wx,o=", "547",}, - {"u=w,g=w,o=rwx", "550",}, - {"u=w,g=w,o=rw", "551",}, - {"u=w,g=w,o=rx", "552",}, - {"u=w,g=w,o=r", "553",}, - {"u=w,g=w,o=wx", "554",}, - {"u=w,g=w,o=w", "555",}, - {"u=w,g=w,o=x", "556",}, - {"u=w,g=w,o=", "557",}, - {"u=w,g=x,o=rwx", "560",}, - {"u=w,g=x,o=rw", "561",}, - {"u=w,g=x,o=rx", "562",}, - {"u=w,g=x,o=r", "563",}, - {"u=w,g=x,o=wx", "564",}, - {"u=w,g=x,o=w", "565",}, - {"u=w,g=x,o=x", "566",}, - {"u=w,g=x,o=", "567",}, - {"u=w,g=,o=rwx", "570",}, - {"u=w,g=,o=rw", "571",}, - {"u=w,g=,o=rx", "572",}, - {"u=w,g=,o=r", "573",}, - {"u=w,g=,o=wx", "574",}, - {"u=w,g=,o=w", "575",}, - {"u=w,g=,o=x", "576",}, - {"u=w,g=,o=", "577",}, - {"u=x,g=rwx,o=rwx", "600",}, - {"u=x,g=rwx,o=rw", "601",}, - {"u=x,g=rwx,o=rx", "602",}, - {"u=x,g=rwx,o=r", "603",}, - {"u=x,g=rwx,o=wx", "604",}, - {"u=x,g=rwx,o=w", "605",}, - {"u=x,g=rwx,o=x", "606",}, - {"u=x,g=rwx,o=", "607",}, - {"u=x,g=rw,o=rwx", "610",}, - {"u=x,g=rw,o=rw", "611",}, - {"u=x,g=rw,o=rx", "612",}, - {"u=x,g=rw,o=r", "613",}, - {"u=x,g=rw,o=wx", "614",}, - {"u=x,g=rw,o=w", "615",}, - {"u=x,g=rw,o=x", "616",}, - {"u=x,g=rw,o=", "617",}, - {"u=x,g=rx,o=rwx", "620",}, - {"u=x,g=rx,o=rw", "621",}, - {"u=x,g=rx,o=rx", "622",}, - {"u=x,g=rx,o=r", "623",}, - {"u=x,g=rx,o=wx", "624",}, - {"u=x,g=rx,o=w", "625",}, - {"u=x,g=rx,o=x", "626",}, - {"u=x,g=rx,o=", "627",}, - {"u=x,g=r,o=rwx", "630",}, - {"u=x,g=r,o=rw", "631",}, - {"u=x,g=r,o=rx", "632",}, - {"u=x,g=r,o=r", "633",}, - {"u=x,g=r,o=wx", "634",}, - {"u=x,g=r,o=w", "635",}, - {"u=x,g=r,o=x", "636",}, - {"u=x,g=r,o=", "637",}, - {"u=x,g=wx,o=rwx", "640",}, - {"u=x,g=wx,o=rw", "641",}, - {"u=x,g=wx,o=rx", "642",}, - {"u=x,g=wx,o=r", "643",}, - {"u=x,g=wx,o=wx", "644",}, - {"u=x,g=wx,o=w", "645",}, - {"u=x,g=wx,o=x", "646",}, - {"u=x,g=wx,o=", "647",}, - {"u=x,g=w,o=rwx", "650",}, - {"u=x,g=w,o=rw", "651",}, - {"u=x,g=w,o=rx", "652",}, - {"u=x,g=w,o=r", "653",}, - {"u=x,g=w,o=wx", "654",}, - {"u=x,g=w,o=w", "655",}, - {"u=x,g=w,o=x", "656",}, - {"u=x,g=w,o=", "657",}, - {"u=x,g=x,o=rwx", "660",}, - {"u=x,g=x,o=rw", "661",}, - {"u=x,g=x,o=rx", "662",}, - {"u=x,g=x,o=r", "663",}, - {"u=x,g=x,o=wx", "664",}, - {"u=x,g=x,o=w", "665",}, - {"u=x,g=x,o=x", "666",}, - {"u=x,g=x,o=", "667",}, - {"u=x,g=,o=rwx", "670",}, - {"u=x,g=,o=rw", "671",}, - {"u=x,g=,o=rx", "672",}, - {"u=x,g=,o=r", "673",}, - {"u=x,g=,o=wx", "674",}, - {"u=x,g=,o=w", "675",}, - {"u=x,g=,o=x", "676",}, - {"u=x,g=,o=", "677",}, - {"u=,g=rwx,o=rwx", "700",}, - {"u=,g=rwx,o=rw", "701",}, - {"u=,g=rwx,o=rx", "702",}, - {"u=,g=rwx,o=r", "703",}, - {"u=,g=rwx,o=wx", "704",}, - {"u=,g=rwx,o=w", "705",}, - {"u=,g=rwx,o=x", "706",}, - {"u=,g=rwx,o=", "707",}, - {"u=,g=rw,o=rwx", "710",}, - {"u=,g=rw,o=rw", "711",}, - {"u=,g=rw,o=rx", "712",}, - {"u=,g=rw,o=r", "713",}, - {"u=,g=rw,o=wx", "714",}, - {"u=,g=rw,o=w", "715",}, - {"u=,g=rw,o=x", "716",}, - {"u=,g=rw,o=", "717",}, - {"u=,g=rx,o=rwx", "720",}, - {"u=,g=rx,o=rw", "721",}, - {"u=,g=rx,o=rx", "722",}, - {"u=,g=rx,o=r", "723",}, - {"u=,g=rx,o=wx", "724",}, - {"u=,g=rx,o=w", "725",}, - {"u=,g=rx,o=x", "726",}, - {"u=,g=rx,o=", "727",}, - {"u=,g=r,o=rwx", "730",}, - {"u=,g=r,o=rw", "731",}, - {"u=,g=r,o=rx", "732",}, - {"u=,g=r,o=r", "733",}, - {"u=,g=r,o=wx", "734",}, - {"u=,g=r,o=w", "735",}, - {"u=,g=r,o=x", "736",}, - {"u=,g=r,o=", "737",}, - {"u=,g=wx,o=rwx", "740",}, - {"u=,g=wx,o=rw", "741",}, - {"u=,g=wx,o=rx", "742",}, - {"u=,g=wx,o=r", "743",}, - {"u=,g=wx,o=wx", "744",}, - {"u=,g=wx,o=w", "745",}, - {"u=,g=wx,o=x", "746",}, - {"u=,g=wx,o=", "747",}, - {"u=,g=w,o=rwx", "750",}, - {"u=,g=w,o=rw", "751",}, - {"u=,g=w,o=rx", "752",}, - {"u=,g=w,o=r", "753",}, - {"u=,g=w,o=wx", "754",}, - {"u=,g=w,o=w", "755",}, - {"u=,g=w,o=x", "756",}, - {"u=,g=w,o=", "757",}, - {"u=,g=x,o=rwx", "760",}, - {"u=,g=x,o=rw", "761",}, - {"u=,g=x,o=rx", "762",}, - {"u=,g=x,o=r", "763",}, - {"u=,g=x,o=wx", "764",}, - {"u=,g=x,o=w", "765",}, - {"u=,g=x,o=x", "766",}, - {"u=,g=x,o=", "767",}, - {"u=,g=,o=rwx", "770",}, - {"u=,g=,o=rw", "771",}, - {"u=,g=,o=rx", "772",}, - {"u=,g=,o=r", "773",}, - {"u=,g=,o=wx", "774",}, - {"u=,g=,o=w", "775",}, - {"u=,g=,o=x", "776",}, - {"u=,g=,o=", "777"} - }; - - for(int i = 0; i < symbolic.length; i += 2) { - conf.set(FsPermission.UMASK_LABEL, symbolic[i][0]); - short val = Short.valueOf(symbolic[i][1], 8); + for (int i = 0; i < SYMBOLIC.length; ++i) { + conf.set(FsPermission.UMASK_LABEL, SYMBOLIC[i][0]); + short val = Short.valueOf(SYMBOLIC[i][1], 8); assertEquals(val, FsPermission.getUMask(conf).toShort()); } + conf.set(FsPermission.UMASK_LABEL, "a+rw"); + assertEquals(0111, FsPermission.getUMask(conf).toShort()); } public void testBadUmasks() { @@ -695,7 +196,7 @@ private boolean isCorrectExceptionMessage(String msg, String umask) { msg.contains(umask) && msg.contains("octal or symbolic"); } - + // Ensure that when the deprecated decimal umask key is used, it is correctly // parsed as such and converted correctly to an FsPermission value public void testDeprecatedUmask() { @@ -705,4 +206,522 @@ public void testDeprecatedUmask() { assertEquals(0456, umask.toShort()); } + + // Symbolic umask list is generated in linux shell using by the command: + // umask 0; umask ; umask -S + static final String[][] SYMBOLIC = new String[][] { + {"u=rwx,g=rwx,o=rwx", "0"}, + {"u=rwx,g=rwx,o=rw", "1"}, + {"u=rwx,g=rwx,o=rx", "2"}, + {"u=rwx,g=rwx,o=r", "3"}, + {"u=rwx,g=rwx,o=wx", "4"}, + {"u=rwx,g=rwx,o=w", "5"}, + {"u=rwx,g=rwx,o=x", "6"}, + {"u=rwx,g=rwx,o=", "7"}, + {"u=rwx,g=rw,o=rwx", "10"}, + {"u=rwx,g=rw,o=rw", "11"}, + {"u=rwx,g=rw,o=rx", "12"}, + {"u=rwx,g=rw,o=r", "13"}, + {"u=rwx,g=rw,o=wx", "14"}, + {"u=rwx,g=rw,o=w", "15"}, + {"u=rwx,g=rw,o=x", "16"}, + {"u=rwx,g=rw,o=", "17"}, + {"u=rwx,g=rx,o=rwx", "20"}, + {"u=rwx,g=rx,o=rw", "21"}, + {"u=rwx,g=rx,o=rx", "22"}, + {"u=rwx,g=rx,o=r", "23"}, + {"u=rwx,g=rx,o=wx", "24"}, + {"u=rwx,g=rx,o=w", "25"}, + {"u=rwx,g=rx,o=x", "26"}, + {"u=rwx,g=rx,o=", "27"}, + {"u=rwx,g=r,o=rwx", "30"}, + {"u=rwx,g=r,o=rw", "31"}, + {"u=rwx,g=r,o=rx", "32"}, + {"u=rwx,g=r,o=r", "33"}, + {"u=rwx,g=r,o=wx", "34"}, + {"u=rwx,g=r,o=w", "35"}, + {"u=rwx,g=r,o=x", "36"}, + {"u=rwx,g=r,o=", "37"}, + {"u=rwx,g=wx,o=rwx", "40"}, + {"u=rwx,g=wx,o=rw", "41"}, + {"u=rwx,g=wx,o=rx", "42"}, + {"u=rwx,g=wx,o=r", "43"}, + {"u=rwx,g=wx,o=wx", "44"}, + {"u=rwx,g=wx,o=w", "45"}, + {"u=rwx,g=wx,o=x", "46"}, + {"u=rwx,g=wx,o=", "47"}, + {"u=rwx,g=w,o=rwx", "50"}, + {"u=rwx,g=w,o=rw", "51"}, + {"u=rwx,g=w,o=rx", "52"}, + {"u=rwx,g=w,o=r", "53"}, + {"u=rwx,g=w,o=wx", "54"}, + {"u=rwx,g=w,o=w", "55"}, + {"u=rwx,g=w,o=x", "56"}, + {"u=rwx,g=w,o=", "57"}, + {"u=rwx,g=x,o=rwx", "60"}, + {"u=rwx,g=x,o=rw", "61"}, + {"u=rwx,g=x,o=rx", "62"}, + {"u=rwx,g=x,o=r", "63"}, + {"u=rwx,g=x,o=wx", "64"}, + {"u=rwx,g=x,o=w", "65"}, + {"u=rwx,g=x,o=x", "66"}, + {"u=rwx,g=x,o=", "67"}, + {"u=rwx,g=,o=rwx", "70"}, + {"u=rwx,g=,o=rw", "71"}, + {"u=rwx,g=,o=rx", "72"}, + {"u=rwx,g=,o=r", "73"}, + {"u=rwx,g=,o=wx", "74"}, + {"u=rwx,g=,o=w", "75"}, + {"u=rwx,g=,o=x", "76"}, + {"u=rwx,g=,o=", "77"}, + {"u=rw,g=rwx,o=rwx", "100"}, + {"u=rw,g=rwx,o=rw", "101"}, + {"u=rw,g=rwx,o=rx", "102"}, + {"u=rw,g=rwx,o=r", "103"}, + {"u=rw,g=rwx,o=wx", "104"}, + {"u=rw,g=rwx,o=w", "105"}, + {"u=rw,g=rwx,o=x", "106"}, + {"u=rw,g=rwx,o=", "107"}, + {"u=rw,g=rw,o=rwx", "110"}, + {"u=rw,g=rw,o=rw", "111"}, + {"u=rw,g=rw,o=rx", "112"}, + {"u=rw,g=rw,o=r", "113"}, + {"u=rw,g=rw,o=wx", "114"}, + {"u=rw,g=rw,o=w", "115"}, + {"u=rw,g=rw,o=x", "116"}, + {"u=rw,g=rw,o=", "117"}, + {"u=rw,g=rx,o=rwx", "120"}, + {"u=rw,g=rx,o=rw", "121"}, + {"u=rw,g=rx,o=rx", "122"}, + {"u=rw,g=rx,o=r", "123"}, + {"u=rw,g=rx,o=wx", "124"}, + {"u=rw,g=rx,o=w", "125"}, + {"u=rw,g=rx,o=x", "126"}, + {"u=rw,g=rx,o=", "127"}, + {"u=rw,g=r,o=rwx", "130"}, + {"u=rw,g=r,o=rw", "131"}, + {"u=rw,g=r,o=rx", "132"}, + {"u=rw,g=r,o=r", "133"}, + {"u=rw,g=r,o=wx", "134"}, + {"u=rw,g=r,o=w", "135"}, + {"u=rw,g=r,o=x", "136"}, + {"u=rw,g=r,o=", "137"}, + {"u=rw,g=wx,o=rwx", "140"}, + {"u=rw,g=wx,o=rw", "141"}, + {"u=rw,g=wx,o=rx", "142"}, + {"u=rw,g=wx,o=r", "143"}, + {"u=rw,g=wx,o=wx", "144"}, + {"u=rw,g=wx,o=w", "145"}, + {"u=rw,g=wx,o=x", "146"}, + {"u=rw,g=wx,o=", "147"}, + {"u=rw,g=w,o=rwx", "150"}, + {"u=rw,g=w,o=rw", "151"}, + {"u=rw,g=w,o=rx", "152"}, + {"u=rw,g=w,o=r", "153"}, + {"u=rw,g=w,o=wx", "154"}, + {"u=rw,g=w,o=w", "155"}, + {"u=rw,g=w,o=x", "156"}, + {"u=rw,g=w,o=", "157"}, + {"u=rw,g=x,o=rwx", "160"}, + {"u=rw,g=x,o=rw", "161"}, + {"u=rw,g=x,o=rx", "162"}, + {"u=rw,g=x,o=r", "163"}, + {"u=rw,g=x,o=wx", "164"}, + {"u=rw,g=x,o=w", "165"}, + {"u=rw,g=x,o=x", "166"}, + {"u=rw,g=x,o=", "167"}, + {"u=rw,g=,o=rwx", "170"}, + {"u=rw,g=,o=rw", "171"}, + {"u=rw,g=,o=rx", "172"}, + {"u=rw,g=,o=r", "173"}, + {"u=rw,g=,o=wx", "174"}, + {"u=rw,g=,o=w", "175"}, + {"u=rw,g=,o=x", "176"}, + {"u=rw,g=,o=", "177"}, + {"u=rx,g=rwx,o=rwx", "200"}, + {"u=rx,g=rwx,o=rw", "201"}, + {"u=rx,g=rwx,o=rx", "202"}, + {"u=rx,g=rwx,o=r", "203"}, + {"u=rx,g=rwx,o=wx", "204"}, + {"u=rx,g=rwx,o=w", "205"}, + {"u=rx,g=rwx,o=x", "206"}, + {"u=rx,g=rwx,o=", "207"}, + {"u=rx,g=rw,o=rwx", "210"}, + {"u=rx,g=rw,o=rw", "211"}, + {"u=rx,g=rw,o=rx", "212"}, + {"u=rx,g=rw,o=r", "213"}, + {"u=rx,g=rw,o=wx", "214"}, + {"u=rx,g=rw,o=w", "215"}, + {"u=rx,g=rw,o=x", "216"}, + {"u=rx,g=rw,o=", "217"}, + {"u=rx,g=rx,o=rwx", "220"}, + {"u=rx,g=rx,o=rw", "221"}, + {"u=rx,g=rx,o=rx", "222"}, + {"u=rx,g=rx,o=r", "223"}, + {"u=rx,g=rx,o=wx", "224"}, + {"u=rx,g=rx,o=w", "225"}, + {"u=rx,g=rx,o=x", "226"}, + {"u=rx,g=rx,o=", "227"}, + {"u=rx,g=r,o=rwx", "230"}, + {"u=rx,g=r,o=rw", "231"}, + {"u=rx,g=r,o=rx", "232"}, + {"u=rx,g=r,o=r", "233"}, + {"u=rx,g=r,o=wx", "234"}, + {"u=rx,g=r,o=w", "235"}, + {"u=rx,g=r,o=x", "236"}, + {"u=rx,g=r,o=", "237"}, + {"u=rx,g=wx,o=rwx", "240"}, + {"u=rx,g=wx,o=rw", "241"}, + {"u=rx,g=wx,o=rx", "242"}, + {"u=rx,g=wx,o=r", "243"}, + {"u=rx,g=wx,o=wx", "244"}, + {"u=rx,g=wx,o=w", "245"}, + {"u=rx,g=wx,o=x", "246"}, + {"u=rx,g=wx,o=", "247"}, + {"u=rx,g=w,o=rwx", "250"}, + {"u=rx,g=w,o=rw", "251"}, + {"u=rx,g=w,o=rx", "252"}, + {"u=rx,g=w,o=r", "253"}, + {"u=rx,g=w,o=wx", "254"}, + {"u=rx,g=w,o=w", "255"}, + {"u=rx,g=w,o=x", "256"}, + {"u=rx,g=w,o=", "257"}, + {"u=rx,g=x,o=rwx", "260"}, + {"u=rx,g=x,o=rw", "261"}, + {"u=rx,g=x,o=rx", "262"}, + {"u=rx,g=x,o=r", "263"}, + {"u=rx,g=x,o=wx", "264"}, + {"u=rx,g=x,o=w", "265"}, + {"u=rx,g=x,o=x", "266"}, + {"u=rx,g=x,o=", "267"}, + {"u=rx,g=,o=rwx", "270"}, + {"u=rx,g=,o=rw", "271"}, + {"u=rx,g=,o=rx", "272"}, + {"u=rx,g=,o=r", "273"}, + {"u=rx,g=,o=wx", "274"}, + {"u=rx,g=,o=w", "275"}, + {"u=rx,g=,o=x", "276"}, + {"u=rx,g=,o=", "277"}, + {"u=r,g=rwx,o=rwx", "300"}, + {"u=r,g=rwx,o=rw", "301"}, + {"u=r,g=rwx,o=rx", "302"}, + {"u=r,g=rwx,o=r", "303"}, + {"u=r,g=rwx,o=wx", "304"}, + {"u=r,g=rwx,o=w", "305"}, + {"u=r,g=rwx,o=x", "306"}, + {"u=r,g=rwx,o=", "307"}, + {"u=r,g=rw,o=rwx", "310"}, + {"u=r,g=rw,o=rw", "311"}, + {"u=r,g=rw,o=rx", "312"}, + {"u=r,g=rw,o=r", "313"}, + {"u=r,g=rw,o=wx", "314"}, + {"u=r,g=rw,o=w", "315"}, + {"u=r,g=rw,o=x", "316"}, + {"u=r,g=rw,o=", "317"}, + {"u=r,g=rx,o=rwx", "320"}, + {"u=r,g=rx,o=rw", "321"}, + {"u=r,g=rx,o=rx", "322"}, + {"u=r,g=rx,o=r", "323"}, + {"u=r,g=rx,o=wx", "324"}, + {"u=r,g=rx,o=w", "325"}, + {"u=r,g=rx,o=x", "326"}, + {"u=r,g=rx,o=", "327"}, + {"u=r,g=r,o=rwx", "330"}, + {"u=r,g=r,o=rw", "331"}, + {"u=r,g=r,o=rx", "332"}, + {"u=r,g=r,o=r", "333"}, + {"u=r,g=r,o=wx", "334"}, + {"u=r,g=r,o=w", "335"}, + {"u=r,g=r,o=x", "336"}, + {"u=r,g=r,o=", "337"}, + {"u=r,g=wx,o=rwx", "340"}, + {"u=r,g=wx,o=rw", "341"}, + {"u=r,g=wx,o=rx", "342"}, + {"u=r,g=wx,o=r", "343"}, + {"u=r,g=wx,o=wx", "344"}, + {"u=r,g=wx,o=w", "345"}, + {"u=r,g=wx,o=x", "346"}, + {"u=r,g=wx,o=", "347"}, + {"u=r,g=w,o=rwx", "350"}, + {"u=r,g=w,o=rw", "351"}, + {"u=r,g=w,o=rx", "352"}, + {"u=r,g=w,o=r", "353"}, + {"u=r,g=w,o=wx", "354"}, + {"u=r,g=w,o=w", "355"}, + {"u=r,g=w,o=x", "356"}, + {"u=r,g=w,o=", "357"}, + {"u=r,g=x,o=rwx", "360"}, + {"u=r,g=x,o=rw", "361"}, + {"u=r,g=x,o=rx", "362"}, + {"u=r,g=x,o=r", "363"}, + {"u=r,g=x,o=wx", "364"}, + {"u=r,g=x,o=w", "365"}, + {"u=r,g=x,o=x", "366"}, + {"u=r,g=x,o=", "367"}, + {"u=r,g=,o=rwx", "370"}, + {"u=r,g=,o=rw", "371"}, + {"u=r,g=,o=rx", "372"}, + {"u=r,g=,o=r", "373"}, + {"u=r,g=,o=wx", "374"}, + {"u=r,g=,o=w", "375"}, + {"u=r,g=,o=x", "376"}, + {"u=r,g=,o=", "377"}, + {"u=wx,g=rwx,o=rwx", "400"}, + {"u=wx,g=rwx,o=rw", "401"}, + {"u=wx,g=rwx,o=rx", "402"}, + {"u=wx,g=rwx,o=r", "403"}, + {"u=wx,g=rwx,o=wx", "404"}, + {"u=wx,g=rwx,o=w", "405"}, + {"u=wx,g=rwx,o=x", "406"}, + {"u=wx,g=rwx,o=", "407"}, + {"u=wx,g=rw,o=rwx", "410"}, + {"u=wx,g=rw,o=rw", "411"}, + {"u=wx,g=rw,o=rx", "412"}, + {"u=wx,g=rw,o=r", "413"}, + {"u=wx,g=rw,o=wx", "414"}, + {"u=wx,g=rw,o=w", "415"}, + {"u=wx,g=rw,o=x", "416"}, + {"u=wx,g=rw,o=", "417"}, + {"u=wx,g=rx,o=rwx", "420"}, + {"u=wx,g=rx,o=rw", "421"}, + {"u=wx,g=rx,o=rx", "422"}, + {"u=wx,g=rx,o=r", "423"}, + {"u=wx,g=rx,o=wx", "424"}, + {"u=wx,g=rx,o=w", "425"}, + {"u=wx,g=rx,o=x", "426"}, + {"u=wx,g=rx,o=", "427"}, + {"u=wx,g=r,o=rwx", "430"}, + {"u=wx,g=r,o=rw", "431"}, + {"u=wx,g=r,o=rx", "432"}, + {"u=wx,g=r,o=r", "433"}, + {"u=wx,g=r,o=wx", "434"}, + {"u=wx,g=r,o=w", "435"}, + {"u=wx,g=r,o=x", "436"}, + {"u=wx,g=r,o=", "437"}, + {"u=wx,g=wx,o=rwx", "440"}, + {"u=wx,g=wx,o=rw", "441"}, + {"u=wx,g=wx,o=rx", "442"}, + {"u=wx,g=wx,o=r", "443"}, + {"u=wx,g=wx,o=wx", "444"}, + {"u=wx,g=wx,o=w", "445"}, + {"u=wx,g=wx,o=x", "446"}, + {"u=wx,g=wx,o=", "447"}, + {"u=wx,g=w,o=rwx", "450"}, + {"u=wx,g=w,o=rw", "451"}, + {"u=wx,g=w,o=rx", "452"}, + {"u=wx,g=w,o=r", "453"}, + {"u=wx,g=w,o=wx", "454"}, + {"u=wx,g=w,o=w", "455"}, + {"u=wx,g=w,o=x", "456"}, + {"u=wx,g=w,o=", "457"}, + {"u=wx,g=x,o=rwx", "460"}, + {"u=wx,g=x,o=rw", "461"}, + {"u=wx,g=x,o=rx", "462"}, + {"u=wx,g=x,o=r", "463"}, + {"u=wx,g=x,o=wx", "464"}, + {"u=wx,g=x,o=w", "465"}, + {"u=wx,g=x,o=x", "466"}, + {"u=wx,g=x,o=", "467"}, + {"u=wx,g=,o=rwx", "470"}, + {"u=wx,g=,o=rw", "471"}, + {"u=wx,g=,o=rx", "472"}, + {"u=wx,g=,o=r", "473"}, + {"u=wx,g=,o=wx", "474"}, + {"u=wx,g=,o=w", "475"}, + {"u=wx,g=,o=x", "476"}, + {"u=wx,g=,o=", "477"}, + {"u=w,g=rwx,o=rwx", "500"}, + {"u=w,g=rwx,o=rw", "501"}, + {"u=w,g=rwx,o=rx", "502"}, + {"u=w,g=rwx,o=r", "503"}, + {"u=w,g=rwx,o=wx", "504"}, + {"u=w,g=rwx,o=w", "505"}, + {"u=w,g=rwx,o=x", "506"}, + {"u=w,g=rwx,o=", "507"}, + {"u=w,g=rw,o=rwx", "510"}, + {"u=w,g=rw,o=rw", "511"}, + {"u=w,g=rw,o=rx", "512"}, + {"u=w,g=rw,o=r", "513"}, + {"u=w,g=rw,o=wx", "514"}, + {"u=w,g=rw,o=w", "515"}, + {"u=w,g=rw,o=x", "516"}, + {"u=w,g=rw,o=", "517"}, + {"u=w,g=rx,o=rwx", "520"}, + {"u=w,g=rx,o=rw", "521"}, + {"u=w,g=rx,o=rx", "522"}, + {"u=w,g=rx,o=r", "523"}, + {"u=w,g=rx,o=wx", "524"}, + {"u=w,g=rx,o=w", "525"}, + {"u=w,g=rx,o=x", "526"}, + {"u=w,g=rx,o=", "527"}, + {"u=w,g=r,o=rwx", "530"}, + {"u=w,g=r,o=rw", "531"}, + {"u=w,g=r,o=rx", "532"}, + {"u=w,g=r,o=r", "533"}, + {"u=w,g=r,o=wx", "534"}, + {"u=w,g=r,o=w", "535"}, + {"u=w,g=r,o=x", "536"}, + {"u=w,g=r,o=", "537"}, + {"u=w,g=wx,o=rwx", "540"}, + {"u=w,g=wx,o=rw", "541"}, + {"u=w,g=wx,o=rx", "542"}, + {"u=w,g=wx,o=r", "543"}, + {"u=w,g=wx,o=wx", "544"}, + {"u=w,g=wx,o=w", "545"}, + {"u=w,g=wx,o=x", "546"}, + {"u=w,g=wx,o=", "547"}, + {"u=w,g=w,o=rwx", "550"}, + {"u=w,g=w,o=rw", "551"}, + {"u=w,g=w,o=rx", "552"}, + {"u=w,g=w,o=r", "553"}, + {"u=w,g=w,o=wx", "554"}, + {"u=w,g=w,o=w", "555"}, + {"u=w,g=w,o=x", "556"}, + {"u=w,g=w,o=", "557"}, + {"u=w,g=x,o=rwx", "560"}, + {"u=w,g=x,o=rw", "561"}, + {"u=w,g=x,o=rx", "562"}, + {"u=w,g=x,o=r", "563"}, + {"u=w,g=x,o=wx", "564"}, + {"u=w,g=x,o=w", "565"}, + {"u=w,g=x,o=x", "566"}, + {"u=w,g=x,o=", "567"}, + {"u=w,g=,o=rwx", "570"}, + {"u=w,g=,o=rw", "571"}, + {"u=w,g=,o=rx", "572"}, + {"u=w,g=,o=r", "573"}, + {"u=w,g=,o=wx", "574"}, + {"u=w,g=,o=w", "575"}, + {"u=w,g=,o=x", "576"}, + {"u=w,g=,o=", "577"}, + {"u=x,g=rwx,o=rwx", "600"}, + {"u=x,g=rwx,o=rw", "601"}, + {"u=x,g=rwx,o=rx", "602"}, + {"u=x,g=rwx,o=r", "603"}, + {"u=x,g=rwx,o=wx", "604"}, + {"u=x,g=rwx,o=w", "605"}, + {"u=x,g=rwx,o=x", "606"}, + {"u=x,g=rwx,o=", "607"}, + {"u=x,g=rw,o=rwx", "610"}, + {"u=x,g=rw,o=rw", "611"}, + {"u=x,g=rw,o=rx", "612"}, + {"u=x,g=rw,o=r", "613"}, + {"u=x,g=rw,o=wx", "614"}, + {"u=x,g=rw,o=w", "615"}, + {"u=x,g=rw,o=x", "616"}, + {"u=x,g=rw,o=", "617"}, + {"u=x,g=rx,o=rwx", "620"}, + {"u=x,g=rx,o=rw", "621"}, + {"u=x,g=rx,o=rx", "622"}, + {"u=x,g=rx,o=r", "623"}, + {"u=x,g=rx,o=wx", "624"}, + {"u=x,g=rx,o=w", "625"}, + {"u=x,g=rx,o=x", "626"}, + {"u=x,g=rx,o=", "627"}, + {"u=x,g=r,o=rwx", "630"}, + {"u=x,g=r,o=rw", "631"}, + {"u=x,g=r,o=rx", "632"}, + {"u=x,g=r,o=r", "633"}, + {"u=x,g=r,o=wx", "634"}, + {"u=x,g=r,o=w", "635"}, + {"u=x,g=r,o=x", "636"}, + {"u=x,g=r,o=", "637"}, + {"u=x,g=wx,o=rwx", "640"}, + {"u=x,g=wx,o=rw", "641"}, + {"u=x,g=wx,o=rx", "642"}, + {"u=x,g=wx,o=r", "643"}, + {"u=x,g=wx,o=wx", "644"}, + {"u=x,g=wx,o=w", "645"}, + {"u=x,g=wx,o=x", "646"}, + {"u=x,g=wx,o=", "647"}, + {"u=x,g=w,o=rwx", "650"}, + {"u=x,g=w,o=rw", "651"}, + {"u=x,g=w,o=rx", "652"}, + {"u=x,g=w,o=r", "653"}, + {"u=x,g=w,o=wx", "654"}, + {"u=x,g=w,o=w", "655"}, + {"u=x,g=w,o=x", "656"}, + {"u=x,g=w,o=", "657"}, + {"u=x,g=x,o=rwx", "660"}, + {"u=x,g=x,o=rw", "661"}, + {"u=x,g=x,o=rx", "662"}, + {"u=x,g=x,o=r", "663"}, + {"u=x,g=x,o=wx", "664"}, + {"u=x,g=x,o=w", "665"}, + {"u=x,g=x,o=x", "666"}, + {"u=x,g=x,o=", "667"}, + {"u=x,g=,o=rwx", "670"}, + {"u=x,g=,o=rw", "671"}, + {"u=x,g=,o=rx", "672"}, + {"u=x,g=,o=r", "673"}, + {"u=x,g=,o=wx", "674"}, + {"u=x,g=,o=w", "675"}, + {"u=x,g=,o=x", "676"}, + {"u=x,g=,o=", "677"}, + {"u=,g=rwx,o=rwx", "700"}, + {"u=,g=rwx,o=rw", "701"}, + {"u=,g=rwx,o=rx", "702"}, + {"u=,g=rwx,o=r", "703"}, + {"u=,g=rwx,o=wx", "704"}, + {"u=,g=rwx,o=w", "705"}, + {"u=,g=rwx,o=x", "706"}, + {"u=,g=rwx,o=", "707"}, + {"u=,g=rw,o=rwx", "710"}, + {"u=,g=rw,o=rw", "711"}, + {"u=,g=rw,o=rx", "712"}, + {"u=,g=rw,o=r", "713"}, + {"u=,g=rw,o=wx", "714"}, + {"u=,g=rw,o=w", "715"}, + {"u=,g=rw,o=x", "716"}, + {"u=,g=rw,o=", "717"}, + {"u=,g=rx,o=rwx", "720"}, + {"u=,g=rx,o=rw", "721"}, + {"u=,g=rx,o=rx", "722"}, + {"u=,g=rx,o=r", "723"}, + {"u=,g=rx,o=wx", "724"}, + {"u=,g=rx,o=w", "725"}, + {"u=,g=rx,o=x", "726"}, + {"u=,g=rx,o=", "727"}, + {"u=,g=r,o=rwx", "730"}, + {"u=,g=r,o=rw", "731"}, + {"u=,g=r,o=rx", "732"}, + {"u=,g=r,o=r", "733"}, + {"u=,g=r,o=wx", "734"}, + {"u=,g=r,o=w", "735"}, + {"u=,g=r,o=x", "736"}, + {"u=,g=r,o=", "737"}, + {"u=,g=wx,o=rwx", "740"}, + {"u=,g=wx,o=rw", "741"}, + {"u=,g=wx,o=rx", "742"}, + {"u=,g=wx,o=r", "743"}, + {"u=,g=wx,o=wx", "744"}, + {"u=,g=wx,o=w", "745"}, + {"u=,g=wx,o=x", "746"}, + {"u=,g=wx,o=", "747"}, + {"u=,g=w,o=rwx", "750"}, + {"u=,g=w,o=rw", "751"}, + {"u=,g=w,o=rx", "752"}, + {"u=,g=w,o=r", "753"}, + {"u=,g=w,o=wx", "754"}, + {"u=,g=w,o=w", "755"}, + {"u=,g=w,o=x", "756"}, + {"u=,g=w,o=", "757"}, + {"u=,g=x,o=rwx", "760"}, + {"u=,g=x,o=rw", "761"}, + {"u=,g=x,o=rx", "762"}, + {"u=,g=x,o=r", "763"}, + {"u=,g=x,o=wx", "764"}, + {"u=,g=x,o=w", "765"}, + {"u=,g=x,o=x", "766"}, + {"u=,g=x,o=", "767"}, + {"u=,g=,o=rwx", "770"}, + {"u=,g=,o=rw", "771"}, + {"u=,g=,o=rx", "772"}, + {"u=,g=,o=r", "773"}, + {"u=,g=,o=wx", "774"}, + {"u=,g=,o=w", "775"}, + {"u=,g=,o=x", "776"}, + {"u=,g=,o=", "777"} + }; + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java deleted file mode 100644 index cbdda907d029a..0000000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java +++ /dev/null @@ -1,239 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.http; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.minikdc.MiniKdc; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.AuthenticationFilterInitializer; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.KerberosTestUtils; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; -import org.apache.hadoop.security.authentication.util.Signer; -import org.apache.hadoop.security.authentication.util.SignerSecretProvider; -import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator; -import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.security.authorize.ProxyUsers; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.Assert; - -import java.io.File; -import java.io.FileWriter; -import java.io.Writer; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.util.Properties; -import static org.junit.Assert.assertTrue; - -/** - * This class is tested for http server with SPENGO authentication. - */ -public class TestHttpServerWithSpengo { - - static final Log LOG = LogFactory.getLog(TestHttpServerWithSpengo.class); - - private static final String SECRET_STR = "secret"; - private static final String HTTP_USER = "HTTP"; - private static final String PREFIX = "hadoop.http.authentication."; - private static final long TIMEOUT = 20000; - - private static File httpSpnegoKeytabFile = new File( - KerberosTestUtils.getKeytabFile()); - private static String httpSpnegoPrincipal = - KerberosTestUtils.getServerPrincipal(); - private static String realm = KerberosTestUtils.getRealm(); - - private static File testRootDir = new File("target", - TestHttpServerWithSpengo.class.getName() + "-root"); - private static MiniKdc testMiniKDC; - private static File secretFile = new File(testRootDir, SECRET_STR); - - @BeforeClass - public static void setUp() throws Exception { - try { - testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); - testMiniKDC.start(); - testMiniKDC.createPrincipal( - httpSpnegoKeytabFile, HTTP_USER + "/localhost"); - } catch (Exception e) { - assertTrue("Couldn't setup MiniKDC", false); - } - Writer w = new FileWriter(secretFile); - w.write("secret"); - w.close(); - } - - @AfterClass - public static void tearDown() { - if (testMiniKDC != null) { - testMiniKDC.stop(); - } - } - - /** - * groupA - * - userA - * groupB - * - userA, userB - * groupC - * - userC - * SPNEGO filter has been enabled. - * userA has the privilege to impersonate users in groupB. - * userA has admin access to all default servlets, but userB - * and userC don't have. So "/logs" can only be accessed by userA. - * @throws Exception - */ - @Test - public void testAuthenticationWithProxyUser() throws Exception { - - Configuration spengoConf = getSpengoConf(new Configuration()); - - //setup logs dir - System.setProperty("hadoop.log.dir", testRootDir.getAbsolutePath()); - - // Setup user group - UserGroupInformation.createUserForTesting("userA", - new String[]{"groupA", "groupB"}); - UserGroupInformation.createUserForTesting("userB", - new String[]{"groupB"}); - UserGroupInformation.createUserForTesting("userC", - new String[]{"groupC"}); - - // Make userA impersonate users in groupB - spengoConf.set("hadoop.proxyuser.userA.hosts", "*"); - spengoConf.set("hadoop.proxyuser.userA.groups", "groupB"); - ProxyUsers.refreshSuperUserGroupsConfiguration(spengoConf); - - HttpServer2 httpServer = null; - try { - // Create http server to test. - httpServer = getCommonBuilder() - .setConf(spengoConf) - .setACL(new AccessControlList("userA groupA")) - .build(); - httpServer.start(); - - // Get signer to encrypt token - Signer signer = getSignerToEncrypt(); - - // setup auth token for userA - AuthenticatedURL.Token token = getEncryptedAuthToken(signer, "userA"); - - String serverURL = "http://" + - NetUtils.getHostPortString(httpServer.getConnectorAddress(0)) + "/"; - - // The default authenticator is kerberos. - AuthenticatedURL authUrl = new AuthenticatedURL(); - - // userA impersonates userB, it's allowed. - for (String servlet : - new String[]{"stacks", "jmx", "conf"}) { - HttpURLConnection conn = authUrl - .openConnection(new URL(serverURL + servlet + "?doAs=userB"), - token); - Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - } - - // userA cannot impersonate userC, it fails. - for (String servlet : - new String[]{"stacks", "jmx", "conf"}){ - HttpURLConnection conn = authUrl - .openConnection(new URL(serverURL + servlet + "?doAs=userC"), - token); - Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN, - conn.getResponseCode()); - } - - // "/logs" and "/logLevel" require admin authorization, - // only userA has the access. - for (String servlet : - new String[]{"logLevel", "logs"}) { - HttpURLConnection conn = authUrl - .openConnection(new URL(serverURL + servlet), token); - Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); - } - - // Setup token for userB - token = getEncryptedAuthToken(signer, "userB"); - - // userB cannot access these servlets. - for (String servlet : - new String[]{"logLevel", "logs"}) { - HttpURLConnection conn = authUrl - .openConnection(new URL(serverURL + servlet), token); - Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN, - conn.getResponseCode()); - } - - } finally { - if (httpServer != null) { - httpServer.stop(); - } - } - } - - - private AuthenticatedURL.Token getEncryptedAuthToken(Signer signer, - String user) throws Exception { - AuthenticationToken token = - new AuthenticationToken(user, user, "kerberos"); - token.setExpires(System.currentTimeMillis() + TIMEOUT); - return new AuthenticatedURL.Token(signer.sign(token.toString())); - } - - private Signer getSignerToEncrypt() throws Exception { - SignerSecretProvider secretProvider = - StringSignerSecretProviderCreator.newStringSignerSecretProvider(); - Properties secretProviderProps = new Properties(); - secretProviderProps.setProperty( - AuthenticationFilter.SIGNATURE_SECRET, SECRET_STR); - secretProvider.init(secretProviderProps, null, TIMEOUT); - return new Signer(secretProvider); - } - - private Configuration getSpengoConf(Configuration conf) { - conf = new Configuration(); - conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, - AuthenticationFilterInitializer.class.getName()); - conf.set(PREFIX + "type", "kerberos"); - conf.setBoolean(PREFIX + "simple.anonymous.allowed", false); - conf.set(PREFIX + "signature.secret.file", - secretFile.getAbsolutePath()); - conf.set(PREFIX + "kerberos.keytab", - httpSpnegoKeytabFile.getAbsolutePath()); - conf.set(PREFIX + "kerberos.principal", httpSpnegoPrincipal); - conf.set(PREFIX + "cookie.domain", realm); - conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, - true); - return conf; - } - - private HttpServer2.Builder getCommonBuilder() throws Exception { - return new HttpServer2.Builder().setName("test") - .addEndpoint(new URI("http://localhost:0")) - .setFindPort(true); - } -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java index 3ab663f3dfa22..72fc5cb7037ca 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java @@ -18,8 +18,10 @@ package org.apache.hadoop.net; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import org.apache.commons.math3.stat.inference.ChiSquareTest; import org.junit.Assert; import org.junit.Test; @@ -79,12 +81,14 @@ public void setLevel(int i) { public void testCountNumNodes() throws Exception { // create the topology NetworkTopology cluster = new NetworkTopology(); - cluster.add(getNewNode("node1", "/d1/r1")); + NodeElement node1 = getNewNode("node1", "/d1/r1"); + cluster.add(node1); NodeElement node2 = getNewNode("node2", "/d1/r2"); cluster.add(node2); - cluster.add(getNewNode("node3", "/d1/r3")); - NodeElement node3 = getNewNode("node4", "/d1/r4"); + NodeElement node3 = getNewNode("node3", "/d1/r3"); cluster.add(node3); + NodeElement node4 = getNewNode("node4", "/d1/r4"); + cluster.add(node4); // create exclude list List excludedNodes = new ArrayList(); @@ -95,7 +99,7 @@ public void testCountNumNodes() throws Exception { assertEquals("4 nodes should be available with extra excluded Node", 4, cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes)); // add one existing node to exclude list - excludedNodes.add(node3); + excludedNodes.add(node4); assertEquals("excluded nodes with ROOT scope should be considered", 3, cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes)); assertEquals("excluded nodes without ~ scope should be considered", 2, @@ -112,6 +116,69 @@ public void testCountNumNodes() throws Exception { // getting count with non-exist scope. assertEquals("No nodes should be considered for non-exist scope", 0, cluster.countNumOfAvailableNodes("/non-exist", excludedNodes)); + // remove a node from the cluster + cluster.remove(node1); + assertEquals("1 node should be available", 1, + cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes)); + } + + /** + * Test how well we pick random nodes. + */ + @Test + public void testChooseRandom() { + // create the topology + NetworkTopology cluster = new NetworkTopology(); + NodeElement node1 = getNewNode("node1", "/d1/r1"); + cluster.add(node1); + NodeElement node2 = getNewNode("node2", "/d1/r2"); + cluster.add(node2); + NodeElement node3 = getNewNode("node3", "/d1/r3"); + cluster.add(node3); + NodeElement node4 = getNewNode("node4", "/d1/r3"); + cluster.add(node4); + + // Number of iterations to do the test + int numIterations = 100; + + // Pick random nodes + HashMap histogram = new HashMap(); + for (int i=0; i(); + for (int i=0; i conf = (Map) args[2]; assertEquals("/", conf.get("cookie.path")); @@ -60,8 +66,9 @@ public Object answer(InvocationOnMock invocationOnMock) assertEquals("bar", conf.get("foo")); return null; - }} - ).when(container).addFilter(Mockito.anyObject(), + } + } + ).when(container).addFilter(Mockito.anyObject(), Mockito.anyObject(), Mockito.>anyObject()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationWithProxyUserFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationWithProxyUserFilter.java deleted file mode 100644 index 504f5a19f0d8d..0000000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestAuthenticationWithProxyUserFilter.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.security; - - -import junit.framework.TestCase; -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.FilterContainer; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import java.util.Map; - -/** - * This class is tested for {@link AuthenticationWithProxyUserFilter} - * to verify configurations of this filter. - */ -public class TestAuthenticationWithProxyUserFilter extends TestCase { - - @SuppressWarnings("unchecked") - public void testConfiguration() throws Exception { - Configuration conf = new Configuration(); - conf.set("hadoop.http.authentication.foo", "bar"); - - conf.set(HttpServer2.BIND_ADDRESS, "barhost"); - - FilterContainer container = Mockito.mock(FilterContainer.class); - Mockito.doAnswer( - new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) - throws Throwable { - Object[] args = invocationOnMock.getArguments(); - - assertEquals("authentication", args[0]); - - assertEquals( - AuthenticationWithProxyUserFilter.class.getName(), args[1]); - - Map conf = (Map) args[2]; - assertEquals("/", conf.get("cookie.path")); - - assertEquals("simple", conf.get("type")); - assertEquals("36000", conf.get("token.validity")); - assertNull(conf.get("cookie.domain")); - assertEquals("true", conf.get("simple.anonymous.allowed")); - assertEquals("HTTP/barhost@LOCALHOST", - conf.get("kerberos.principal")); - assertEquals(System.getProperty("user.home") + - "/hadoop.keytab", conf.get("kerberos.keytab")); - assertEquals("bar", conf.get("foo")); - - return null; - } - } - ).when(container).addFilter(Mockito.anyObject(), - Mockito.anyObject(), - Mockito.>anyObject()); - - new AuthenticationFilterInitializer().initFilter(container, conf); - } - -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java index 36866946eaa72..2b47d41a625af 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java @@ -25,11 +25,16 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeoutException; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.FakeTimer; import org.junit.Before; import org.junit.Test; + +import com.google.common.base.Supplier; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; @@ -50,8 +55,8 @@ public class TestGroupsCaching { private Configuration conf; @Before - public void setup() { - FakeGroupMapping.resetRequestCount(); + public void setup() throws IOException { + FakeGroupMapping.clearAll(); ExceptionalGroupMapping.resetRequestCount(); conf = new Configuration(); @@ -66,13 +71,19 @@ public static class FakeGroupMapping extends ShellBasedUnixGroupsMapping { private static Set blackList = new HashSet(); private static int requestCount = 0; private static long getGroupsDelayMs = 0; + private static boolean throwException; + private static volatile CountDownLatch latch = null; @Override public List getGroups(String user) throws IOException { LOG.info("Getting groups for " + user); + delayIfNecessary(); + requestCount++; - delayIfNecessary(); + if (throwException) { + throw new IOException("For test"); + } if (blackList.contains(user)) { return new LinkedList(); @@ -80,7 +91,19 @@ public List getGroups(String user) throws IOException { return new LinkedList(allGroups); } + /** + * Delay returning on a latch or a specific amount of time. + */ private void delayIfNecessary() { + // cause current method to pause + // resume until get notified + if (latch != null) { + try { + latch.await(); + return; + } catch (InterruptedException e) {} + } + if (getGroupsDelayMs > 0) { try { Thread.sleep(getGroupsDelayMs); @@ -102,6 +125,16 @@ public static void clearBlackList() throws IOException { blackList.clear(); } + public static void clearAll() throws IOException { + LOG.info("Resetting FakeGroupMapping"); + blackList.clear(); + allGroups.clear(); + requestCount = 0; + getGroupsDelayMs = 0; + throwException = false; + latch = null; + } + @Override public void cacheGroupsAdd(List groups) throws IOException { LOG.info("Adding " + groups + " to groups."); @@ -124,6 +157,35 @@ public static void resetRequestCount() { public static void setGetGroupsDelayMs(long delayMs) { getGroupsDelayMs = delayMs; } + + public static void setThrowException(boolean throwIfTrue) { + throwException = throwIfTrue; + } + + /** + * Hold on returning the group names unless being notified, + * ensure this method is called before {@link #getGroups(String)}. + * Call {@link #resume()} will resume the process. + */ + public static void pause() { + // Set a static latch, multiple background refresh threads + // share this instance. So when await is called, all the + // threads will pause until the it decreases the count of + // the latch. + latch = new CountDownLatch(1); + } + + /** + * Resume the background refresh thread and return the value + * of group names. + */ + public static void resume() { + // if latch is null, it means pause was not called and it is + // safe to ignore. + if (latch != null) { + latch.countDown(); + } + } } public static class ExceptionalGroupMapping extends ShellBasedUnixGroupsMapping { @@ -403,6 +465,270 @@ public void run() { assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount()); } + @Test + public void testThreadNotBlockedWhenExpiredEntryExistsWithBackgroundRefresh() + throws Exception { + conf.setLong( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD, + true); + FakeTimer timer = new FakeTimer(); + final Groups groups = new Groups(conf, timer); + groups.cacheGroupsAdd(Arrays.asList(myGroups)); + groups.refresh(); + FakeGroupMapping.clearBlackList(); + + // We make an initial request to populate the cache + groups.getGroups("me"); + // Further lookups will have a delay + FakeGroupMapping.setGetGroupsDelayMs(100); + // add another groups + groups.cacheGroupsAdd(Arrays.asList("grp3")); + int startingRequestCount = FakeGroupMapping.getRequestCount(); + + // Then expire that entry + timer.advance(4 * 1000); + + // Now get the cache entry - it should return immediately + // with the old value and the cache will not have completed + // a request to getGroups yet. + assertEquals(groups.getGroups("me").size(), 2); + assertEquals(startingRequestCount, FakeGroupMapping.getRequestCount()); + + // Now sleep for over the delay time and the request count should + // have completed + Thread.sleep(110); + assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount()); + // Another call to get groups should give 3 groups instead of 2 + assertEquals(groups.getGroups("me").size(), 3); + } + + @Test + public void testThreadBlockedWhenExpiredEntryExistsWithoutBackgroundRefresh() + throws Exception { + conf.setLong( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD, + false); + FakeTimer timer = new FakeTimer(); + final Groups groups = new Groups(conf, timer); + groups.cacheGroupsAdd(Arrays.asList(myGroups)); + groups.refresh(); + FakeGroupMapping.clearBlackList(); + + // We make an initial request to populate the cache + groups.getGroups("me"); + // Further lookups will have a delay + FakeGroupMapping.setGetGroupsDelayMs(100); + // add another group + groups.cacheGroupsAdd(Arrays.asList("grp3")); + int startingRequestCount = FakeGroupMapping.getRequestCount(); + + // Then expire that entry + timer.advance(4 * 1000); + + // Now get the cache entry - it should block and return the new + // 3 group value + assertEquals(groups.getGroups("me").size(), 3); + assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount()); + } + + @Test + public void testExceptionOnBackgroundRefreshHandled() throws Exception { + conf.setLong( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD, + true); + FakeTimer timer = new FakeTimer(); + final Groups groups = new Groups(conf, timer); + groups.cacheGroupsAdd(Arrays.asList(myGroups)); + groups.refresh(); + FakeGroupMapping.clearBlackList(); + + // We make an initial request to populate the cache + groups.getGroups("me"); + + // add another group + groups.cacheGroupsAdd(Arrays.asList("grp3")); + int startingRequestCount = FakeGroupMapping.getRequestCount(); + // Arrange for an exception to occur only on the + // second call + FakeGroupMapping.setThrowException(true); + + // Then expire that entry + timer.advance(4 * 1000); + + // Now get the cache entry - it should return immediately + // with the old value and the cache will not have completed + // a request to getGroups yet. + assertEquals(groups.getGroups("me").size(), 2); + assertEquals(startingRequestCount, FakeGroupMapping.getRequestCount()); + + // Now sleep for a short time and re-check the request count. It should have + // increased, but the exception means the cache will not have updated + Thread.sleep(50); + FakeGroupMapping.setThrowException(false); + assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount()); + assertEquals(groups.getGroups("me").size(), 2); + + // Now sleep another short time - the 3rd call to getGroups above + // will have kicked off another refresh that updates the cache + Thread.sleep(50); + assertEquals(startingRequestCount + 2, FakeGroupMapping.getRequestCount()); + assertEquals(groups.getGroups("me").size(), 3); + } + + + @Test + public void testEntriesExpireIfBackgroundRefreshFails() throws Exception { + conf.setLong( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD, + true); + FakeTimer timer = new FakeTimer(); + final Groups groups = new Groups(conf, timer); + groups.cacheGroupsAdd(Arrays.asList(myGroups)); + groups.refresh(); + FakeGroupMapping.clearBlackList(); + + // We make an initial request to populate the cache + groups.getGroups("me"); + + // Now make all calls to the FakeGroupMapper throw exceptions + FakeGroupMapping.setThrowException(true); + + // The cache entry expires for refresh after 1 second + // It expires for eviction after 1 * 10 seconds after it was last written + // So if we call getGroups repeatedly over 9 seconds, 9 refreshes should + // be triggered which will fail to update the key, but the keys old value + // will be retrievable until it is evicted after about 10 seconds. + for(int i=0; i<9; i++) { + assertEquals(groups.getGroups("me").size(), 2); + timer.advance(1 * 1000); + } + // Wait until the 11th second. The call to getGroups should throw + // an exception as the key will have been evicted and FakeGroupMapping + // will throw IO Exception when it is asked for new groups. In this case + // load must be called synchronously as there is no key present + timer.advance(2 * 1000); + try { + groups.getGroups("me"); + fail("Should have thrown an exception here"); + } catch (Exception e) { + // pass + } + + // Finally check groups are retrieve again after FakeGroupMapping + // stops throw exceptions + FakeGroupMapping.setThrowException(false); + assertEquals(groups.getGroups("me").size(), 2); + } + + @Test + public void testBackgroundRefreshCounters() + throws IOException, InterruptedException { + conf.setLong( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD, + true); + conf.setInt( + CommonConfigurationKeys. + HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS, + 2); + FakeTimer timer = new FakeTimer(); + final Groups groups = new Groups(conf, timer); + groups.cacheGroupsAdd(Arrays.asList(myGroups)); + groups.refresh(); + FakeGroupMapping.clearBlackList(); + + // populate the cache + String[] grps = {"one", "two", "three", "four", "five"}; + for (String g: grps) { + groups.getGroups(g); + } + + // expire the cache + timer.advance(2*1000); + FakeGroupMapping.pause(); + + // Request all groups again, as there are 2 threads to process them + // 3 should get queued and 2 should be running + for (String g: grps) { + groups.getGroups(g); + } + waitForGroupCounters(groups, 3, 2, 0, 0); + FakeGroupMapping.resume(); + + // Once resumed, all results should be returned immediately + waitForGroupCounters(groups, 0, 0, 5, 0); + + // Now run again, this time throwing exceptions but no delay + timer.advance(2*1000); + FakeGroupMapping.setGetGroupsDelayMs(0); + FakeGroupMapping.setThrowException(true); + for (String g: grps) { + groups.getGroups(g); + } + waitForGroupCounters(groups, 0, 0, 5, 5); + } + + private void waitForGroupCounters(final Groups groups, long expectedQueued, + long expectedRunning, long expectedSuccess, long expectedExpection) + throws InterruptedException { + final long[] expected = {expectedQueued, expectedRunning, + expectedSuccess, expectedExpection}; + final long[] actual = new long[expected.length]; + // wait for a certain time until the counters reach + // to expected values. Check values in 20 ms interval. + try { + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + actual[0] = groups.getBackgroundRefreshQueued(); + actual[1] = groups.getBackgroundRefreshRunning(); + actual[2] = groups.getBackgroundRefreshSuccess(); + actual[3] = groups.getBackgroundRefreshException(); + return Arrays.equals(actual, expected); + } + }, 20, 1000); + } catch (TimeoutException e) { + fail("Excepted group counter values are not reached in given time," + + " expecting (Queued, Running, Success, Exception) : " + + Arrays.toString(expected) + " but actual : " + + Arrays.toString(actual)); + } + } + + @Test + public void testExceptionCallingLoadWithoutBackgroundRefreshReturnsOldValue() + throws Exception { + conf.setLong( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD, + false); + FakeTimer timer = new FakeTimer(); + final Groups groups = new Groups(conf, timer); + groups.cacheGroupsAdd(Arrays.asList(myGroups)); + groups.refresh(); + FakeGroupMapping.clearBlackList(); + + // First populate the cash + assertEquals(groups.getGroups("me").size(), 2); + + // Advance the timer so a refresh is required + timer.advance(2 * 1000); + + // This call should throw an exception + FakeGroupMapping.setThrowException(true); + assertEquals(groups.getGroups("me").size(), 2); + } + @Test public void testCacheEntriesExpire() throws Exception { conf.setLong( diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java index 7ab2dbd021c62..0a448b4de1eff 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java @@ -17,78 +17,68 @@ */ package org.apache.hadoop.security; +import static org.apache.hadoop.security.LdapGroupsMapping.CONNECTION_TIMEOUT; +import static org.apache.hadoop.security.LdapGroupsMapping.READ_TIMEOUT; +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; import static org.mockito.Mockito.*; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Writer; +import java.net.ServerSocket; +import java.net.Socket; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CountDownLatch; import javax.naming.CommunicationException; -import javax.naming.NamingEnumeration; import javax.naming.NamingException; -import javax.naming.directory.Attribute; -import javax.naming.directory.Attributes; -import javax.naming.directory.BasicAttribute; -import javax.naming.directory.BasicAttributes; -import javax.naming.directory.DirContext; import javax.naming.directory.SearchControls; import javax.naming.directory.SearchResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + @SuppressWarnings("unchecked") -public class TestLdapGroupsMapping { - private DirContext mockContext; - - private LdapGroupsMapping mappingSpy = spy(new LdapGroupsMapping()); - private NamingEnumeration mockUserNamingEnum = mock(NamingEnumeration.class); - private NamingEnumeration mockGroupNamingEnum = mock(NamingEnumeration.class); - private String[] testGroups = new String[] {"group1", "group2"}; - +public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase { + + private static final Logger LOG = LoggerFactory.getLogger( + TestLdapGroupsMapping.class); + + /** + * To construct a LDAP InitialDirContext object, it will firstly initiate a + * protocol session to server for authentication. After a session is + * established, a method of authentication is negotiated between the server + * and the client. When the client is authenticated, the LDAP server will send + * a bind response, whose message contents are bytes as the + * {@link #AUTHENTICATE_SUCCESS_MSG}. After receiving this bind response + * message, the LDAP context is considered connected to the server and thus + * can issue query requests for determining group membership. + */ + private static final byte[] AUTHENTICATE_SUCCESS_MSG = + {48, 12, 2, 1, 1, 97, 7, 10, 1, 0, 4, 0, 4, 0}; + @Before public void setupMocks() throws NamingException { - mockContext = mock(DirContext.class); - doReturn(mockContext).when(mappingSpy).getDirContext(); - SearchResult mockUserResult = mock(SearchResult.class); - // We only ever call hasMoreElements once for the user NamingEnum, so - // we can just have one return value - when(mockUserNamingEnum.hasMoreElements()).thenReturn(true); when(mockUserNamingEnum.nextElement()).thenReturn(mockUserResult); when(mockUserResult.getNameInNamespace()).thenReturn("CN=some_user,DC=test,DC=com"); - - SearchResult mockGroupResult = mock(SearchResult.class); - // We're going to have to define the loop here. We want two iterations, - // to get both the groups - when(mockGroupNamingEnum.hasMoreElements()).thenReturn(true, true, false); - when(mockGroupNamingEnum.nextElement()).thenReturn(mockGroupResult); - - // Define the attribute for the name of the first group - Attribute group1Attr = new BasicAttribute("cn"); - group1Attr.add(testGroups[0]); - Attributes group1Attrs = new BasicAttributes(); - group1Attrs.put(group1Attr); - - // Define the attribute for the name of the second group - Attribute group2Attr = new BasicAttribute("cn"); - group2Attr.add(testGroups[1]); - Attributes group2Attrs = new BasicAttributes(); - group2Attrs.put(group2Attr); - - // This search result gets reused, so return group1, then group2 - when(mockGroupResult.getAttributes()).thenReturn(group1Attrs, group2Attrs); } @Test @@ -214,4 +204,114 @@ public void testConfGetPassword() throws Exception { // extract password Assert.assertEquals("", mapping.getPassword(conf,"invalid-alias", "")); } + + /** + * Test that if the {@link LdapGroupsMapping#CONNECTION_TIMEOUT} is set in the + * configuration, the LdapGroupsMapping connection will timeout by this value + * if it does not get a LDAP response from the server. + * @throws IOException + * @throws InterruptedException + */ + @Test (timeout = 30000) + public void testLdapConnectionTimeout() + throws IOException, InterruptedException { + final int connectionTimeoutMs = 3 * 1000; // 3s + try (ServerSocket serverSock = new ServerSocket(0)) { + final CountDownLatch finLatch = new CountDownLatch(1); + + // Below we create a LDAP server which will accept a client request; + // but it will never reply to the bind (connect) request. + // Client of this LDAP server is expected to get a connection timeout. + final Thread ldapServer = new Thread(new Runnable() { + @Override + public void run() { + try { + try (Socket ignored = serverSock.accept()) { + finLatch.await(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + ldapServer.start(); + + final LdapGroupsMapping mapping = new LdapGroupsMapping(); + final Configuration conf = new Configuration(); + conf.set(LdapGroupsMapping.LDAP_URL_KEY, + "ldap://localhost:" + serverSock.getLocalPort()); + conf.setInt(CONNECTION_TIMEOUT, connectionTimeoutMs); + mapping.setConf(conf); + + try { + mapping.doGetGroups("hadoop"); + fail("The LDAP query should have timed out!"); + } catch (NamingException ne) { + LOG.debug("Got the exception while LDAP querying: ", ne); + assertExceptionContains("LDAP response read timed out, timeout used:" + + connectionTimeoutMs + "ms", ne); + assertFalse(ne.getMessage().contains("remaining name")); + } finally { + finLatch.countDown(); + } + ldapServer.join(); + } + } + + /** + * Test that if the {@link LdapGroupsMapping#READ_TIMEOUT} is set in the + * configuration, the LdapGroupsMapping query will timeout by this value if + * it does not get a LDAP response from the server. + * + * @throws IOException + * @throws InterruptedException + */ + @Test(timeout = 30000) + public void testLdapReadTimeout() throws IOException, InterruptedException { + final int readTimeoutMs = 4 * 1000; // 4s + try (ServerSocket serverSock = new ServerSocket(0)) { + final CountDownLatch finLatch = new CountDownLatch(1); + + // Below we create a LDAP server which will accept a client request, + // authenticate it successfully; but it will never reply to the following + // query request. + // Client of this LDAP server is expected to get a read timeout. + final Thread ldapServer = new Thread(new Runnable() { + @Override + public void run() { + try { + try (Socket clientSock = serverSock.accept()) { + IOUtils.skipFully(clientSock.getInputStream(), 1); + clientSock.getOutputStream().write(AUTHENTICATE_SUCCESS_MSG); + finLatch.await(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + ldapServer.start(); + + final LdapGroupsMapping mapping = new LdapGroupsMapping(); + final Configuration conf = new Configuration(); + conf.set(LdapGroupsMapping.LDAP_URL_KEY, + "ldap://localhost:" + serverSock.getLocalPort()); + conf.setInt(READ_TIMEOUT, readTimeoutMs); + mapping.setConf(conf); + + try { + mapping.doGetGroups("hadoop"); + fail("The LDAP query should have timed out!"); + } catch (NamingException ne) { + LOG.debug("Got the exception while LDAP querying: ", ne); + assertExceptionContains("LDAP response read timed out, timeout used:" + + readTimeoutMs + "ms", ne); + assertExceptionContains("remaining name", ne); + } finally { + finLatch.countDown(); + } + ldapServer.join(); + } + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java new file mode 100644 index 0000000000000..c54ac4c7367d4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttribute; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.SearchResult; + +import org.junit.Before; + +public class TestLdapGroupsMappingBase { + protected DirContext mockContext; + + protected LdapGroupsMapping mappingSpy = spy(new LdapGroupsMapping()); + protected NamingEnumeration mockUserNamingEnum = + mock(NamingEnumeration.class); + protected NamingEnumeration mockGroupNamingEnum = + mock(NamingEnumeration.class); + protected String[] testGroups = new String[] {"group1", "group2"}; + + @Before + public void setupMocksBase() throws NamingException { + mockContext = mock(DirContext.class); + doReturn(mockContext).when(mappingSpy).getDirContext(); + + // We only ever call hasMoreElements once for the user NamingEnum, so + // we can just have one return value + when(mockUserNamingEnum.hasMoreElements()).thenReturn(true); + + SearchResult mockGroupResult = mock(SearchResult.class); + // We're going to have to define the loop here. We want two iterations, + // to get both the groups + when(mockGroupNamingEnum.hasMoreElements()).thenReturn(true, true, false); + when(mockGroupNamingEnum.nextElement()).thenReturn(mockGroupResult); + + // Define the attribute for the name of the first group + Attribute group1Attr = new BasicAttribute("cn"); + group1Attr.add(testGroups[0]); + Attributes group1Attrs = new BasicAttributes(); + group1Attrs.put(group1Attr); + + // Define the attribute for the name of the second group + Attribute group2Attr = new BasicAttribute("cn"); + group2Attr.add(testGroups[1]); + Attributes group2Attrs = new BasicAttributes(); + group2Attrs.put(group2Attr); + + // This search result gets reused, so return group1, then group2 + when(mockGroupResult.getAttributes()).thenReturn(group1Attrs, group2Attrs); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java new file mode 100644 index 0000000000000..247f6c440db58 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security; + +import static org.mockito.Matchers.anyString; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.contains; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; +import javax.naming.directory.SearchControls; +import javax.naming.directory.SearchResult; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +@SuppressWarnings("unchecked") +public class TestLdapGroupsMappingWithPosixGroup + extends TestLdapGroupsMappingBase { + + @Before + public void setupMocks() throws NamingException { + SearchResult mockUserResult = mock(SearchResult.class); + when(mockUserNamingEnum.nextElement()).thenReturn(mockUserResult); + + Attribute mockUidNumberAttr = mock(Attribute.class); + Attribute mockGidNumberAttr = mock(Attribute.class); + Attribute mockUidAttr = mock(Attribute.class); + Attributes mockAttrs = mock(Attributes.class); + + when(mockUidAttr.get()).thenReturn("some_user"); + when(mockUidNumberAttr.get()).thenReturn("700"); + when(mockGidNumberAttr.get()).thenReturn("600"); + when(mockAttrs.get(eq("uid"))).thenReturn(mockUidAttr); + when(mockAttrs.get(eq("uidNumber"))).thenReturn(mockUidNumberAttr); + when(mockAttrs.get(eq("gidNumber"))).thenReturn(mockGidNumberAttr); + + when(mockUserResult.getAttributes()).thenReturn(mockAttrs); + } + + @Test + public void testGetGroups() throws IOException, NamingException { + // The search functionality of the mock context is reused, so we will + // return the user NamingEnumeration first, and then the group + when(mockContext.search(anyString(), contains("posix"), + any(Object[].class), any(SearchControls.class))) + .thenReturn(mockUserNamingEnum, mockGroupNamingEnum); + + doTestGetGroups(Arrays.asList(testGroups), 2); + } + + private void doTestGetGroups(List expectedGroups, int searchTimes) + throws IOException, NamingException { + Configuration conf = new Configuration(); + // Set this, so we don't throw an exception + conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test"); + conf.set(LdapGroupsMapping.GROUP_SEARCH_FILTER_KEY, + "(objectClass=posixGroup)(cn={0})"); + conf.set(LdapGroupsMapping.USER_SEARCH_FILTER_KEY, + "(objectClass=posixAccount)"); + conf.set(LdapGroupsMapping.GROUP_MEMBERSHIP_ATTR_KEY, "memberUid"); + conf.set(LdapGroupsMapping.POSIX_UID_ATTR_KEY, "uidNumber"); + conf.set(LdapGroupsMapping.POSIX_GID_ATTR_KEY, "gidNumber"); + conf.set(LdapGroupsMapping.GROUP_NAME_ATTR_KEY, "cn"); + + mappingSpy.setConf(conf); + // Username is arbitrary, since the spy is mocked to respond the same, + // regardless of input + List groups = mappingSpy.getGroups("some_user"); + + Assert.assertEquals(expectedGroups, groups); + + mappingSpy.getConf().set(LdapGroupsMapping.POSIX_UID_ATTR_KEY, "uid"); + + Assert.assertEquals(expectedGroups, groups); + + // We should have searched for a user, and then two groups + verify(mockContext, times(searchTimes)).search(anyString(), + anyString(), + any(Object[].class), + any(SearchControls.class)); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index d09a730bf2776..ea91af304955a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -281,10 +281,15 @@ public void testConstructorWithRules() throws Exception { UserGroupInformation.setConfiguration(conf); testConstructorSuccess("user1", "user1"); testConstructorSuccess("user4@OTHER.REALM", "other-user4"); - // failure test - testConstructorFailures("user2@DEFAULT.REALM"); - testConstructorFailures("user3/cron@DEFAULT.REALM"); - testConstructorFailures("user5/cron@OTHER.REALM"); + + // pass through test, no transformation + testConstructorSuccess("user2@DEFAULT.REALM", "user2@DEFAULT.REALM"); + testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3/cron@DEFAULT.REALM"); + testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM"); + + // failures + testConstructorFailures("user6@example.com@OTHER.REALM"); + testConstructorFailures("user7@example.com@DEFAULT.REALM"); testConstructorFailures(null); testConstructorFailures(""); } @@ -298,10 +303,13 @@ public void testConstructorWithKerberos() throws Exception { testConstructorSuccess("user1", "user1"); testConstructorSuccess("user2@DEFAULT.REALM", "user2"); - testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3"); + testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3"); + + // no rules applied, local name remains the same + testConstructorSuccess("user4@OTHER.REALM", "user4@OTHER.REALM"); + testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM"); + // failure test - testConstructorFailures("user4@OTHER.REALM"); - testConstructorFailures("user5/cron@OTHER.REALM"); testConstructorFailures(null); testConstructorFailures(""); } @@ -342,8 +350,9 @@ private void testConstructorFailures(String userName) { } catch (IllegalArgumentException e) { String expect = (userName == null || userName.isEmpty()) ? "Null user" : "Illegal principal name "+userName; - assertTrue("Did not find "+ expect + " in " + e, - e.toString().contains(expect)); + String expect2 = "Malformed Kerberos name: "+userName; + assertTrue("Did not find "+ expect + " or " + expect2 + " in " + e, + e.toString().contains(expect) || e.toString().contains(expect2)); } } @@ -432,8 +441,10 @@ public void testGettingGroups() throws Exception { UserGroupInformation uugi = UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES); assertEquals(USER_NAME, uugi.getUserName()); - assertArrayEquals(new String[]{GROUP1_NAME, GROUP2_NAME, GROUP3_NAME}, - uugi.getGroupNames()); + String[] expected = new String[]{GROUP1_NAME, GROUP2_NAME, GROUP3_NAME}; + assertArrayEquals(expected, uugi.getGroupNames()); + assertArrayEquals(expected, uugi.getGroups().toArray(new String[0])); + assertEquals(GROUP1_NAME, uugi.getPrimaryGroupName()); } @SuppressWarnings("unchecked") // from Mockito mocks diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index 84f79e68cdb98..39d840295c5a8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -141,12 +141,32 @@ public static void assertGlobEquals(File dir, String pattern, Joiner.on(",").join(expectedSet), Joiner.on(",").join(found)); } - + + protected static String E_NULL_THROWABLE = "Null Throwable"; + protected static String E_NULL_THROWABLE_STRING = + "Null Throwable.toString() value"; + protected static String E_UNEXPECTED_EXCEPTION = + "but got unexpected exception"; + + /** + * Assert that an exception's toString() value + * contained the expected text. + * @param string expected string + * @param t thrown exception + * @throws AssertionError if the expected string is not found + */ public static void assertExceptionContains(String string, Throwable t) { - String msg = t.getMessage(); - Assert.assertTrue( - "Expected to find '" + string + "' but got unexpected exception:" - + StringUtils.stringifyException(t), msg.contains(string)); + Assert.assertNotNull(E_NULL_THROWABLE, t); + String msg = t.toString(); + if (msg == null) { + throw new AssertionError(E_NULL_THROWABLE_STRING, t); + } + if (!msg.contains(string)) { + throw new AssertionError("Expected to find '" + string + "' " + + E_UNEXPECTED_EXCEPTION + ":" + + StringUtils.stringifyException(t), + t); + } } public static void waitFor(Supplier check, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java new file mode 100644 index 0000000000000..8a7b5f67a8e43 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.test; + +import org.junit.Test; + +public class TestGenericTestUtils extends GenericTestUtils { + + @Test + public void testAssertExceptionContainsNullEx() throws Throwable { + try { + assertExceptionContains("", null); + } catch (AssertionError e) { + if (!e.toString().contains(E_NULL_THROWABLE)) { + throw e; + } + } + } + + @Test + public void testAssertExceptionContainsNullString() throws Throwable { + try { + assertExceptionContains("", new BrokenException()); + } catch (AssertionError e) { + if (!e.toString().contains(E_NULL_THROWABLE_STRING)) { + throw e; + } + } + } + + @Test + public void testAssertExceptionContainsWrongText() throws Throwable { + try { + assertExceptionContains("Expected", new Exception("(actual)")); + } catch (AssertionError e) { + String s = e.toString(); + if (!s.contains(E_UNEXPECTED_EXCEPTION) + || !s.contains("(actual)") ) { + throw e; + } + if (e.getCause() == null) { + throw new AssertionError("No nested cause in assertion", e); + } + } + } + + @Test + public void testAssertExceptionContainsWorking() throws Throwable { + assertExceptionContains("Expected", new Exception("Expected")); + } + + private static class BrokenException extends Exception { + public BrokenException() { + } + + @Override + public String toString() { + return null; + } + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java index f592d0400a432..b2a65379edae9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.util; +import static org.junit.Assert.fail; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -25,6 +26,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; import java.util.regex.Pattern; import java.util.zip.ZipEntry; @@ -32,6 +35,7 @@ import junit.framework.TestCase; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -169,4 +173,37 @@ private File makeClassLoaderTestJar(String... clsNames) throws IOException { return jarFile; } -} \ No newline at end of file + + @Test + public void testUnJar2() throws IOException { + // make a simple zip + File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_NAME); + JarOutputStream jstream = + new JarOutputStream(new FileOutputStream(jarFile)); + JarEntry je = new JarEntry("META-INF/MANIFEST.MF"); + byte[] data = "Manifest-Version: 1.0\nCreated-By: 1.8.0_1 (Manual)" + .getBytes(StandardCharsets.UTF_8); + je.setSize(data.length); + jstream.putNextEntry(je); + jstream.write(data); + jstream.closeEntry(); + je = new JarEntry("../outside.path"); + data = "any data here".getBytes(StandardCharsets.UTF_8); + je.setSize(data.length); + jstream.putNextEntry(je); + jstream.write(data); + jstream.closeEntry(); + jstream.close(); + + File unjarDir = new File(TEST_ROOT_DIR, "unjar-path"); + + // Unjar everything + try { + RunJar.unJar(jarFile, unjarDir); + fail("unJar should throw IOException."); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains( + "would create file outside of", e); + } + } +} diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 4d8b7287bfcf4..67fa972af645c 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -22,12 +22,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-kms - 2.7.4-SNAPSHOT + 2.7.7 war Apache Hadoop KMS diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml index 2f2a0e5ca8824..07ac683a425a8 100644 --- a/hadoop-common-project/hadoop-minikdc/pom.xml +++ b/hadoop-common-project/hadoop-minikdc/pom.xml @@ -18,13 +18,13 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project 4.0.0 org.apache.hadoop hadoop-minikdc - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop MiniKDC Apache Hadoop MiniKDC jar diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index d6482fcee3511..c679314a411c1 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-nfs - 2.7.4-SNAPSHOT + 2.7.7 jar Apache Hadoop NFS diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml index aa3ad56a76f4d..72b8f359ab5ad 100644 --- a/hadoop-common-project/pom.xml +++ b/hadoop-common-project/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-common-project - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Common Project Apache Hadoop Common Project pom diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 8d79c6cf9e559..11bc1bd7b4d56 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-dist - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Distribution Apache Hadoop Distribution jar diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index 4f7089cb57819..f67e276dd8ee6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -22,12 +22,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-hdfs-httpfs - 2.7.4-SNAPSHOT + 2.7.7 war Apache Hadoop HttpFS diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index 06b08613a67a8..12d3e1f0f2de3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-hdfs-nfs - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop HDFS-NFS Apache Hadoop HDFS-NFS jar diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 51a11c117b46e..6d2d2c80d87ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1,6 +1,143 @@ Hadoop HDFS Change Log -Release 2.7.4 - UNRELEASED +Release 2.7.7 - 2018-07-18 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + HDFS-13602. Add checkOperation(WRITE) checks in FSNamesystem. + Contributed by Chao Sun. + + OPTIMIZATIONS + + BUG FIXES + + HDFS-13486. Backport HDFS-11817 (A faulty node can cause a lease + leak and NPE on accessing data) to branch-2.7. + Contributed by Kihwal Lee. + + HDFS-12156. TestFSImage fails without -Pnative + Contributed Akira Ajisaka. + +Release 2.7.6 - 2018-04-16 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + HDFS-11003. Expose XmitsInProgress through DataNodeMXBean. + (Brahma Reddy Battula) + + HDFS-11187. Optimize disk access for last partial chunk checksum of + Finalized replica. (Gabor Bota, Wei-Chiu Chuang via Xiao Chen) + + HDFS-12884. BlockUnderConstructionFeature.truncateBlock should be of type + BlockInfo. (chencan via shv) + + OPTIMIZATIONS + + BUG FIXES + + HDFS-12881. Output streams closed with IOUtils suppressing write errors. + (Ajay Kumar via jlowe) + + HDFS-12347. TestBalancerRPCDelay#testBalancerRPCDelay fails very frequently. + (Bharat Viswanadham via Tsz-Wo Nicholas Sze) + + HDFS-12371. BlockVerificationFailures and BlocksVerified show up as 0 + in Datanode JMX. (Hanisha Koneru via Kihwal Lee, shv). + + HDFS-13120. Snapshot diff could be corrupted after concat. (xyao) + + HDFS-7959. WebHdfs logging is missing on Datanode (Kihwal Lee via sjlee) + Backport HDFS-13126 by Erik Krogen. + + HDFS-10453. ReplicationMonitor thread could stuck for long time + due to the race between replication and delete of same file in a + large cluster. Contributed by He Xiaoqiao. + + HDFS-13112. Token expiration edits may cause log corruption or deadlock. + (daryn via kihwal) + + HDFS-4210. Throw helpful exception when DNS entry for JournalNode cannot be + resolved. (Charles Lamb and John Zhuge) + + HDFS-13195. DataNode conf page cannot display the current value after + reconfig. (Mao Baolong via kihwal) + + HDFS-12299. Race Between update pipeline and DN Re-Registration. + (Brahma Reddy Battula. Backport by Wei-Chiu Chuang, Konstantin Shvachko) + +Release 2.7.5 - 2017-12-14 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + HDFS-9153. Pretty-format the output for DFSIO. (Kai Zheng via wheat9) + + HDFS-8797. WebHdfsFileSystem creates too many connections for pread. (jing9) + + HDFS-12131. Add some of the FSNamesystem JMX values as metrics. + (Erik Krogen via wang, shv) + + HDFS-12420. Add an option to disallow 'namenode format -force'. + (Ajay Kumar via Arpit Agarwal, zhz) + + HDFS-10984. Expose nntop output as metrics. (Siddharth Wagle via xyao, zhz) + + HDFS-9259. Make SO_SNDBUF size configurable at DFSClient side for hdfs write + scenario. (original patch Mingliang Liu via Ming Ma, branch-2.7 backport + done under HDFS-12823, Erik Krogen via zhz). + + HDFS-12596. Add TestFsck#testFsckCorruptWhenOneReplicaIsCorrupt + back to branch-2.7. (Xiao Chen) + + OPTIMIZATIONS + + HDFS-10711. Optimize FSPermissionChecker group membership check. + (Daryn Sharp via kihwal) + + HDFS-12323. NameNode terminates after full GC thinking QJM unresponsive + if full GC is much longer than timeout. (Erik Krogen via shv) + + HDFS-8829. Make SO_RCVBUF and SO_SNDBUF size configurable for + DataTransferProtocol sockets and allow configuring auto-tuning (He Tianyi + via Colin P. McCabe) + + BUG FIXES + + HDFS-12157. Do fsyncDirectory(..) outside of FSDataset lock. + (Vinayakumar B. via kihwal) + + HDFS-9107. Prevent NN's unrecoverable death spiral after full GC (Daryn + Sharp via Colin P. McCabe) + + HDFS-10738. Fix + TestRefreshUserMappings.testRefreshSuperUserGroupsConfiguration + test failure. (Rakesh R via kihwal) + + HDFS-8865. Improve quota initialization performance. Contributed by Kihwal Lee. + + HDFS-9003. ForkJoin thread pool leaks. (Kihwal Lee via jing9) + + HDFS-12578. TestDeadDatanode#testNonDFSUsedONDeadNodeReReg failing + in branch-2.7. (Ajay Kumar via Xiao Chen) + + HDFS-12832. INode.getFullPathName may throw ArrayIndexOutOfBoundsException + lead to NameNode exit. (shv) + + HDFS-12638. Delete copy-on-truncate block along with the original block, + when deleting a file being truncated. (shv) + +Release 2.7.4 - 2017-08-04 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 0d5e1df709825..006ffd4b979b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop hadoop-project-dist - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project-dist org.apache.hadoop hadoop-hdfs - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop HDFS Apache Hadoop HDFS jar diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml index 7fd5d8e657124..fea94efbb3e43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml @@ -20,13 +20,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../../../../hadoop-project org.apache.hadoop.contrib hadoop-hdfs-bkjournal - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop HDFS BookKeeper Journal Apache Hadoop HDFS BookKeeper Journal jar diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 1a6a96bb9cf5f..2cdfc20d2543d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -51,6 +51,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT; @@ -299,6 +301,7 @@ public static class Conf { final int writeMaxPackets; final ByteArrayManager.Conf writeByteArrayManagerConf; final int socketTimeout; + private final int socketSendBufferSize; final int socketCacheCapacity; final long socketCacheExpiry; final long excludedNodesCacheExpiry; @@ -369,6 +372,8 @@ public Conf(Configuration conf) { defaultChecksumOpt = getChecksumOptFromConf(conf); socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, HdfsServerConstants.READ_TIMEOUT); + socketSendBufferSize = conf.getInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, + DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT); /** dfs.write.packet.size is an internal config variable */ writePacketSize = conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT); @@ -511,6 +516,10 @@ public Conf(Configuration conf) { DFSConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT); } + public int getSocketSendBufferSize() { + return socketSendBufferSize; + } + public boolean isUseLegacyBlockReaderLocal() { return useLegacyBlockReaderLocal; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index f6a7364bc29b4..c9f16141c377e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker; import org.apache.hadoop.http.HttpConfig; @@ -249,6 +250,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = "supergroup"; public static final String DFS_NAMENODE_ACLS_ENABLED_KEY = "dfs.namenode.acls.enabled"; public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false; + public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled"; + public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false; public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled"; public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true; public static final String DFS_ADMIN = "dfs.cluster.administrators"; @@ -272,6 +275,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_EDITS_DIR_MINIMUM_KEY = "dfs.namenode.edits.dir.minimum"; public static final int DFS_NAMENODE_EDITS_DIR_MINIMUM_DEFAULT = 1; + public static final String DFS_NAMENODE_QUOTA_INIT_THREADS_KEY = "dfs.namenode.quota.init-threads"; + public static final int DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT = 4; public static final String DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD = "dfs.namenode.edit.log.autoroll.multiplier.threshold"; public static final float DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD_DEFAULT = 2.0f; @@ -421,6 +426,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_HOSTS_KEY = "dfs.namenode.hosts"; public static final String DFS_NAMENODE_HOSTS_EXCLUDE_KEY = "dfs.namenode.hosts.exclude"; public static final String DFS_CLIENT_SOCKET_TIMEOUT_KEY = "dfs.client.socket-timeout"; + public static final String DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY = + "dfs.client.socket.send.buffer.size"; + public static final int DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT = + HdfsConstants.DEFAULT_DATA_SOCKET_SIZE; public static final String DFS_NAMENODE_CHECKPOINT_DIR_KEY = "dfs.namenode.checkpoint.dir"; public static final String DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY = "dfs.namenode.checkpoint.edits.dir"; public static final String DFS_HOSTS = "dfs.hosts"; @@ -867,4 +876,18 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.block-pinning.enabled"; public static final boolean DFS_DATANODE_BLOCK_PINNING_ENABLED_DEFAULT = false; + + public static final String + DFS_DATANODE_TRANSFER_SOCKET_SEND_BUFFER_SIZE_KEY = + "dfs.datanode.transfer.socket.send.buffer.size"; + public static final int + DFS_DATANODE_TRANSFER_SOCKET_SEND_BUFFER_SIZE_DEFAULT = + HdfsConstants.DEFAULT_DATA_SOCKET_SIZE; + + public static final String + DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_KEY = + "dfs.datanode.transfer.socket.recv.buffer.size"; + public static final int + DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_DEFAULT = + HdfsConstants.DEFAULT_DATA_SOCKET_SIZE; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index f26b883747283..89df16c3fa997 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -1345,6 +1345,21 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException { return false; // do not sleep, continue processing } + void updateBlockGS(final long newGS) { + block.setGenerationStamp(newGS); + } + + /** update pipeline at the namenode */ + @VisibleForTesting + public void updatePipeline(long newGS) throws IOException { + final ExtendedBlock oldBlock = block.getCurrentBlock(); + // the new GS has been propagated to all DN, it should be ok to update the + // local block state + updateBlockGS(newGS); + dfsClient.namenode.updatePipeline(dfsClient.clientName, oldBlock, + block.getCurrentBlock(), nodes, storageIDs); + } + DatanodeInfo[] getExcludedNodes() { return excludedNodes.getAllPresent(excludedNodes.asMap().keySet()) .keySet().toArray(new DatanodeInfo[0]); @@ -1701,7 +1716,9 @@ static Socket createSocketForPipeline(final DatanodeInfo first, final int timeout = client.getDatanodeReadTimeout(length); NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), client.getConf().socketTimeout); sock.setSoTimeout(timeout); - sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); + if (client.getConf().getSocketSendBufferSize() > 0) { + sock.setSendBufferSize(client.getConf().getSocketSendBufferSize()); + } if(DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Send buf size " + sock.getSendBufferSize()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DomainPeerServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DomainPeerServer.java index 95a138894d92e..5425bd5af5af4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DomainPeerServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DomainPeerServer.java @@ -49,6 +49,11 @@ public void setReceiveBufferSize(int size) throws IOException { sock.setAttribute(DomainSocket.RECEIVE_BUFFER_SIZE, size); } + @Override + public int getReceiveBufferSize() throws IOException { + return sock.getAttribute(DomainSocket.RECEIVE_BUFFER_SIZE); + } + @Override public Peer accept() throws IOException, SocketTimeoutException { DomainSocket connSock = sock.accept(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/PeerServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/PeerServer.java index c7b6b14df495f..72974e2cdacba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/PeerServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/PeerServer.java @@ -32,7 +32,14 @@ public interface PeerServer extends Closeable { public void setReceiveBufferSize(int size) throws IOException; /** - * Listens for a connection to be made to this server and accepts + * Get the receive buffer size of the PeerServer. + * + * @return The receive buffer size. + */ + int getReceiveBufferSize() throws IOException; + + /** + * Listens for a connection to be made to this server and accepts * it. The method blocks until a connection is made. * * @exception IOException if an I/O error occurs when waiting for a diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/TcpPeerServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/TcpPeerServer.java index c914e0276371a..483b48ef93da7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/TcpPeerServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/TcpPeerServer.java @@ -136,6 +136,11 @@ public void setReceiveBufferSize(int size) throws IOException { this.serverSocket.setReceiveBufferSize(size); } + @Override + public int getReceiveBufferSize() throws IOException { + return this.serverSocket.getReceiveBufferSize(); + } + @Override public Peer accept() throws IOException, SocketTimeoutException { Peer peer = peerFromSocket(serverSocket.accept()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java index dc3231827e2f6..dee74e6fcfd5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java @@ -24,7 +24,7 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.StopWatch; -import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.Timer; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -35,6 +35,7 @@ import com.google.protobuf.Message; import com.google.protobuf.TextFormat; + /** * Represents a set of calls for which a quorum of results is needed. * @param a key used to identify each of the outgoing calls @@ -60,11 +61,12 @@ class QuorumCall { * fraction of the configured timeout for any call. */ private static final float WAIT_PROGRESS_WARN_THRESHOLD = 0.7f; - private final StopWatch quorumStopWatch = new StopWatch(); + private final StopWatch quorumStopWatch; + private final Timer timer; static QuorumCall create( - Map> calls) { - final QuorumCall qr = new QuorumCall(); + Map> calls, Timer timer) { + final QuorumCall qr = new QuorumCall(timer); for (final Entry> e : calls.entrySet()) { Preconditions.checkArgument(e.getValue() != null, "null future for key: " + e.getKey()); @@ -82,18 +84,53 @@ public void onSuccess(RESULT res) { } return qr; } - + + static QuorumCall create( + Map> calls) { + return create(calls, new Timer()); + } + + /** + * Not intended for outside use. + */ private QuorumCall() { + this(new Timer()); + } + + private QuorumCall(Timer timer) { // Only instantiated from factory method above + this.timer = timer; + this.quorumStopWatch = new StopWatch(timer); } + /** + * Used in conjunction with {@link #getQuorumTimeoutIncreaseMillis(long, int)} + * to check for pauses. + */ private void restartQuorumStopWatch() { quorumStopWatch.reset().start(); } - private boolean shouldIncreaseQuorumTimeout(long offset, int millis) { + /** + * Check for a pause (e.g. GC) since the last time + * {@link #restartQuorumStopWatch()} was called. If detected, return the + * length of the pause; else, -1. + * @param offset Offset the elapsed time by this amount; use if some amount + * of pause was expected + * @param millis Total length of timeout in milliseconds + * @return Length of pause, if detected, else -1 + */ + private long getQuorumTimeoutIncreaseMillis(long offset, int millis) { long elapsed = quorumStopWatch.now(TimeUnit.MILLISECONDS); - return elapsed + offset > (millis * WAIT_PROGRESS_INFO_THRESHOLD); + long pauseTime = elapsed + offset; + if (pauseTime > (millis * WAIT_PROGRESS_INFO_THRESHOLD)) { + QuorumJournalManager.LOG.info("Pause detected while waiting for " + + "QuorumCall response; increasing timeout threshold by pause time " + + "of " + pauseTime + " ms."); + return pauseTime; + } else { + return -1; + } } @@ -119,7 +156,7 @@ public synchronized void waitFor( int minResponses, int minSuccesses, int maxExceptions, int millis, String operationName) throws InterruptedException, TimeoutException { - long st = Time.monotonicNow(); + long st = timer.monotonicNow(); long nextLogTime = st + (long)(millis * WAIT_PROGRESS_INFO_THRESHOLD); long et = st + millis; while (true) { @@ -128,7 +165,7 @@ public synchronized void waitFor( if (minResponses > 0 && countResponses() >= minResponses) return; if (minSuccesses > 0 && countSuccesses() >= minSuccesses) return; if (maxExceptions >= 0 && countExceptions() > maxExceptions) return; - long now = Time.monotonicNow(); + long now = timer.monotonicNow(); if (now > nextLogTime) { long waited = now - st; @@ -154,8 +191,9 @@ public synchronized void waitFor( long rem = et - now; if (rem <= 0) { // Increase timeout if a full GC occurred after restarting stopWatch - if (shouldIncreaseQuorumTimeout(0, millis)) { - et = et + millis; + long timeoutIncrease = getQuorumTimeoutIncreaseMillis(0, millis); + if (timeoutIncrease > 0) { + et += timeoutIncrease; } else { throw new TimeoutException(); } @@ -165,8 +203,9 @@ public synchronized void waitFor( rem = Math.max(rem, 1); wait(rem); // Increase timeout if a full GC occurred after restarting stopWatch - if (shouldIncreaseQuorumTimeout(-rem, millis)) { - et = et + millis; + long timeoutIncrease = getQuorumTimeoutIncreaseMillis(-rem, millis); + if (timeoutIncrease > 0) { + et += timeoutIncrease; } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index 1b849641c08ab..4123204690cf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -21,6 +21,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; +import java.net.UnknownHostException; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -387,8 +388,12 @@ private static List getLoggerAddresses(URI uri) List addrs = Lists.newArrayList(); for (String addr : parts) { - addrs.add(NetUtils.createSocketAddr( - addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT)); + InetSocketAddress isa = NetUtils.createSocketAddr( + addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT); + if (isa.isUnresolved()) { + throw new UnknownHostException(addr); + } + addrs.add(isa); } return addrs; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java index 8af7ebaa0bfcf..193e8ceeabc09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java @@ -21,7 +21,6 @@ import java.io.DataInput; import java.io.DataOutputStream; import java.io.IOException; -import java.io.InterruptedIOException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Iterator; @@ -365,34 +364,58 @@ public synchronized int getNumberOfKeys() { @Override //AbstractDelegationTokenManager protected void logUpdateMasterKey(DelegationKey key) throws IOException { - synchronized (noInterruptsLock) { + try { // The edit logging code will fail catastrophically if it // is interrupted during a logSync, since the interrupt // closes the edit log files. Doing this inside the - // above lock and then checking interruption status - // prevents this bug. - if (Thread.interrupted()) { - throw new InterruptedIOException( - "Interrupted before updating master key"); + // fsn lock will prevent being interrupted when stopping + // the secret manager. + namesystem.readLockInterruptibly(); + try { + // this monitor isn't necessary if stopped while holding write lock + // but for safety, guard against a stop with read lock. + synchronized (noInterruptsLock) { + if (Thread.currentThread().isInterrupted()) { + return; // leave flag set so secret monitor exits. + } + namesystem.logUpdateMasterKey(key); + } + } finally { + namesystem.readUnlock(); } - namesystem.logUpdateMasterKey(key); + } catch (InterruptedException ie) { + // AbstractDelegationTokenManager may crash if an exception is thrown. + // The interrupt flag will be detected when it attempts to sleep. + Thread.currentThread().interrupt(); } } @Override //AbstractDelegationTokenManager protected void logExpireToken(final DelegationTokenIdentifier dtId) throws IOException { - synchronized (noInterruptsLock) { + try { // The edit logging code will fail catastrophically if it // is interrupted during a logSync, since the interrupt // closes the edit log files. Doing this inside the - // above lock and then checking interruption status - // prevents this bug. - if (Thread.interrupted()) { - throw new InterruptedIOException( - "Interrupted before expiring delegation token"); + // fsn lock will prevent being interrupted when stopping + // the secret manager. + namesystem.readLockInterruptibly(); + try { + // this monitor isn't necessary if stopped while holding write lock + // but for safety, guard against a stop with read lock. + synchronized (noInterruptsLock) { + if (Thread.currentThread().isInterrupted()) { + return; // leave flag set so secret monitor exits. + } + namesystem.logExpireDelegationToken(dtId); + } + } finally { + namesystem.readUnlock(); } - namesystem.logExpireDelegationToken(dtId); + } catch (InterruptedException ie) { + // AbstractDelegationTokenManager may crash if an exception is thrown. + // The interrupt flag will be detected when it attempts to sleep. + Thread.currentThread().interrupt(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java index 1342c843e4eeb..6b9b7c22f3bce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java @@ -57,7 +57,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous { /** * The block source to use in the event of copy-on-write truncate. */ - private Block truncateBlock; + private BlockInfoContiguous truncateBlock; /** * ReplicaUnderConstruction contains information about replicas while @@ -190,11 +190,23 @@ assert getBlockUCState() != BlockUCState.COMPLETE : /** Set expected locations */ public void setExpectedLocations(DatanodeStorageInfo[] targets) { - int numLocations = targets == null ? 0 : targets.length; + if (targets == null) { + return; + } + int numLocations = 0; + for (DatanodeStorageInfo target : targets) { + if (target != null) { + numLocations++; + } + } + this.replicas = new ArrayList(numLocations); - for(int i = 0; i < numLocations; i++) - replicas.add( - new ReplicaUnderConstruction(this, targets[i], ReplicaState.RBW)); + for(int i = 0; i < targets.length; i++) { + // Only store non-null DatanodeStorageInfo. + if (targets[i] != null) { + replicas.add(new ReplicaUnderConstruction(this, targets[i], ReplicaState.RBW)); + } + } } /** @@ -233,11 +245,11 @@ public long getBlockRecoveryId() { } /** Get recover block */ - public Block getTruncateBlock() { + public BlockInfoContiguous getTruncateBlock() { return truncateBlock; } - public void setTruncateBlock(Block recoveryBlock) { + public void setTruncateBlock(BlockInfoContiguous recoveryBlock) { this.truncateBlock = recoveryBlock; } @@ -285,10 +297,15 @@ List commitBlock(Block block) throws IOException { * Initialize lease recovery for this block. * Find the first alive data-node starting from the previous primary and * make it primary. + * @param recoveryId Recovery ID (new gen stamp) + * @param startRecovery Issue recovery command to datanode if true. */ - public void initializeBlockRecovery(long recoveryId) { + public void initializeBlockRecovery(long recoveryId, boolean startRecovery) { setBlockUCState(BlockUCState.UNDER_RECOVERY); blockRecoveryId = recoveryId; + if (!startRecovery) { + return; + } if (replicas.size() == 0) { NameNode.blockStateChangeLog.warn("BLOCK*" + " BlockInfoUnderConstruction.initLeaseRecovery:" @@ -336,27 +353,33 @@ public void initializeBlockRecovery(long recoveryId) { void addReplicaIfNotPresent(DatanodeStorageInfo storage, Block block, ReplicaState rState) { - Iterator it = replicas.iterator(); - while (it.hasNext()) { - ReplicaUnderConstruction r = it.next(); - DatanodeStorageInfo expectedLocation = r.getExpectedStorageLocation(); - if(expectedLocation == storage) { - // Record the gen stamp from the report - r.setGenerationStamp(block.getGenerationStamp()); - return; - } else if (expectedLocation != null && - expectedLocation.getDatanodeDescriptor() == - storage.getDatanodeDescriptor()) { - - // The Datanode reported that the block is on a different storage - // than the one chosen by BlockPlacementPolicy. This can occur as - // we allow Datanodes to choose the target storage. Update our - // state by removing the stale entry and adding a new one. - it.remove(); - break; + if (replicas == null) { + replicas = new ArrayList(1); + replicas.add(new ReplicaUnderConstruction(block, storage, + rState)); + } else { + Iterator it = replicas.iterator(); + while (it.hasNext()) { + ReplicaUnderConstruction r = it.next(); + DatanodeStorageInfo expectedLocation = r.getExpectedStorageLocation(); + if (expectedLocation == storage) { + // Record the gen stamp from the report + r.setGenerationStamp(block.getGenerationStamp()); + return; + } else if (expectedLocation != null + && expectedLocation.getDatanodeDescriptor() == + storage.getDatanodeDescriptor()) { + + // The Datanode reported that the block is on a different storage + // than the one chosen by BlockPlacementPolicy. This can occur as + // we allow Datanodes to choose the target storage. Update our + // state by removing the stale entry and adding a new one. + it.remove(); + break; + } } + replicas.add(new ReplicaUnderConstruction(block, storage, rState)); } - replicas.add(new ReplicaUnderConstruction(block, storage, rState)); } @Override // BlockInfo diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index da15397185a3d..797215c2bde0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -95,7 +95,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1475,8 +1474,6 @@ int computeReplicationWorkForBlocks(List> blocksToReplicate) { } // choose replication targets: NOT HOLDING THE GLOBAL LOCK - // It is costly to extract the filename for which chooseTargets is called, - // so for now we pass in the block collection itself. rw.chooseTargets(blockplacement, storagePolicySuite, excludedNodes); } @@ -3708,17 +3705,16 @@ public void clearQueues() { private static class ReplicationWork { - private final Block block; - private final BlockCollection bc; - + private final String srcPath; + private final long blockSize; + private final byte storagePolicyID; private final DatanodeDescriptor srcNode; + private final int additionalReplRequired; + private final int priority; private final List containingNodes; private final List liveReplicaStorages; - private final int additionalReplRequired; - private DatanodeStorageInfo targets[]; - private final int priority; public ReplicationWork(Block block, BlockCollection bc, @@ -3728,7 +3724,9 @@ public ReplicationWork(Block block, int additionalReplRequired, int priority) { this.block = block; - this.bc = bc; + this.srcPath = bc.getName(); + this.blockSize = block.getNumBytes(); + this.storagePolicyID = bc.getStoragePolicyID(); this.srcNode = srcNode; this.srcNode.incrementPendingReplicationWithoutTargets(); this.containingNodes = containingNodes; @@ -3742,14 +3740,22 @@ private void chooseTargets(BlockPlacementPolicy blockplacement, BlockStoragePolicySuite storagePolicySuite, Set excludedNodes) { try { - targets = blockplacement.chooseTarget(bc.getName(), + targets = blockplacement.chooseTarget(getSrcPath(), additionalReplRequired, srcNode, liveReplicaStorages, false, - excludedNodes, block.getNumBytes(), - storagePolicySuite.getPolicy(bc.getStoragePolicyID())); + excludedNodes, blockSize, + storagePolicySuite.getPolicy(getStoragePolicyID())); } finally { srcNode.decrementPendingReplicationWithoutTargets(); } } + + private String getSrcPath() { + return srcPath; + } + + private byte getStoragePolicyID() { + return storagePolicyID; + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 5d821864c165f..e449585bc169f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -505,10 +505,11 @@ public DatanodeStorageInfo[] getDatanodeStorageInfos( DatanodeID[] datanodeID, String[] storageIDs, String format, Object... args) throws UnregisteredNodeException { if (datanodeID.length != storageIDs.length) { + // Error for pre-2.0.0-alpha clients. final String err = (storageIDs.length == 0? "Missing storageIDs: It is likely that the HDFS client," + " who made this call, is running in an older version of Hadoop" - + " which does not support storageIDs." + + "(pre-2.0.0-alpha) which does not support storageIDs." : "Length mismatched: storageIDs.length=" + storageIDs.length + " != " ) + " datanodeID.length=" + datanodeID.length; throw new HadoopIllegalArgumentException( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index dfc3c60b34d46..93e9b3251650e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -28,10 +29,13 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; + /** * Manage the heartbeats received from datanodes. * The datanode list and statistics are synchronized @@ -56,8 +60,8 @@ class HeartbeatManager implements DatanodeStatistics { private final long heartbeatRecheckInterval; /** Heartbeat monitor thread */ private final Daemon heartbeatThread = new Daemon(new Monitor()); + private final StopWatch heartbeatStopWatch = new StopWatch(); - final Namesystem namesystem; final BlockManager blockManager; @@ -245,7 +249,18 @@ synchronized void stopDecommission(final DatanodeDescriptor node) { stats.add(node); } } - + + @VisibleForTesting + void restartHeartbeatStopWatch() { + heartbeatStopWatch.reset().start(); + } + + @VisibleForTesting + boolean shouldAbortHeartbeatCheck(long offset) { + long elapsed = heartbeatStopWatch.now(TimeUnit.MILLISECONDS); + return elapsed + offset > heartbeatRecheckInterval; + } + /** * Check if there are any expired heartbeats, and if so, * whether any blocks have to be re-replicated. @@ -292,6 +307,10 @@ void heartbeatCheck() { int numOfStaleStorages = 0; synchronized(this) { for (DatanodeDescriptor d : datanodes) { + // check if an excessive GC pause has occurred + if (shouldAbortHeartbeatCheck(0)) { + return; + } if (dead == null && dm.isDatanodeDead(d)) { stats.incrExpiredHeartbeats(); dead = d; @@ -360,6 +379,7 @@ private class Monitor implements Runnable { @Override public void run() { while(namesystem.isRunning()) { + restartHeartbeatStopWatch(); try { final long now = Time.monotonicNow(); if (lastHeartbeatCheck + heartbeatRecheckInterval < now) { @@ -381,6 +401,12 @@ public void run() { Thread.sleep(5000); // 5 seconds } catch (InterruptedException ie) { } + // avoid declaring nodes dead for another cycle if a GC pause lasts + // longer than the node recheck interval + if (shouldAbortHeartbeatCheck(-5000)) { + LOG.warn("Skipping next heartbeat scan due to excessive pause"); + lastHeartbeatCheck = Time.monotonicNow(); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index c042190efbcc2..71e48044c2ebf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -170,8 +170,13 @@ class BlockSender implements java.io.Closeable { * See {{@link BlockSender#isLongRead()} */ private static final long LONG_READ_THRESHOLD_BYTES = 256 * 1024; - + // The number of bytes per checksum here determines the alignment + // of reads: we always start reading at a checksum chunk boundary, + // even if the checksum type is NULL. So, choosing too big of a value + // would risk sending too much unnecessary data. 512 (1 disk sector) + // is likely to result in minimal extra IO. + private static final long CHUNK_SIZE = 512; /** * Constructor * @@ -241,12 +246,6 @@ class BlockSender implements java.io.Closeable { synchronized(datanode.data) { replica = getReplica(block, datanode); replicaVisibleLength = replica.getVisibleLength(); - if (replica instanceof FinalizedReplica) { - // Load last checksum in case the replica is being written - // concurrently - final FinalizedReplica frep = (FinalizedReplica) replica; - chunkChecksum = frep.getLastChecksumAndDataLen(); - } } // if there is a write in progress if (replica instanceof ReplicaBeingWritten) { @@ -254,6 +253,10 @@ class BlockSender implements java.io.Closeable { waitForMinLength(rbw, startOffset + length); chunkChecksum = rbw.getLastChecksumAndDataLen(); } + if (replica instanceof FinalizedReplica) { + chunkChecksum = getPartialChunkChecksumForFinalized( + (FinalizedReplica)replica); + } if (replica.getGenerationStamp() < block.getGenerationStamp()) { throw new IOException("Replica gen stamp < block genstamp, block=" @@ -329,12 +332,8 @@ class BlockSender implements java.io.Closeable { } } if (csum == null) { - // The number of bytes per checksum here determines the alignment - // of reads: we always start reading at a checksum chunk boundary, - // even if the checksum type is NULL. So, choosing too big of a value - // would risk sending too much unnecessary data. 512 (1 disk sector) - // is likely to result in minimal extra IO. - csum = DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 512); + csum = DataChecksum.newDataChecksum(DataChecksum.Type.NULL, + (int)CHUNK_SIZE); } /* @@ -410,6 +409,37 @@ class BlockSender implements java.io.Closeable { } } + private ChunkChecksum getPartialChunkChecksumForFinalized( + FinalizedReplica finalized) throws IOException { + // There are a number of places in the code base where a finalized replica + // object is created. If last partial checksum is loaded whenever a + // finalized replica is created, it would increase latency in DataNode + // initialization. Therefore, the last partial chunk checksum is loaded + // lazily. + + // Load last checksum in case the replica is being written concurrently + final long replicaVisibleLength = replica.getVisibleLength(); + if (replicaVisibleLength % CHUNK_SIZE != 0 && + finalized.getLastPartialChunkChecksum() == null) { + // the finalized replica does not have precomputed last partial + // chunk checksum. Recompute now. + try { + finalized.loadLastPartialChunkChecksum(); + return new ChunkChecksum(finalized.getVisibleLength(), + finalized.getLastPartialChunkChecksum()); + } catch (FileNotFoundException e) { + // meta file is lost. Continue anyway to preserve existing behavior. + DataNode.LOG.warn( + "meta file " + finalized.getMetaFile() + " is missing!"); + return null; + } + } else { + // If the checksum is null, BlockSender will use on-disk checksum. + return new ChunkChecksum(finalized.getVisibleLength(), + finalized.getLastPartialChunkChecksum()); + } + } + /** * close opened files. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index 29ecb59b2f300..1c03ef6668664 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -70,7 +70,9 @@ public class DNConf { final int socketTimeout; final int socketWriteTimeout; final int socketKeepaliveTimeout; - + private final int transferSocketSendBufferSize; + private final int transferSocketRecvBufferSize; + final boolean transferToAllowed; final boolean dropCacheBehindWrites; final boolean syncBehindWrites; @@ -113,8 +115,14 @@ public DNConf(Configuration conf) { socketKeepaliveTimeout = conf.getInt( DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY, DFSConfigKeys.DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT); - - /* Based on results on different platforms, we might need set the default + this.transferSocketSendBufferSize = conf.getInt( + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_SEND_BUFFER_SIZE_KEY, + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_SEND_BUFFER_SIZE_DEFAULT); + this.transferSocketRecvBufferSize = conf.getInt( + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_KEY, + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_DEFAULT); + + /* Based on results on different platforms, we might need set the default * to false on some of them. */ transferToAllowed = conf.getBoolean( DFS_DATANODE_TRANSFERTO_ALLOWED_KEY, @@ -280,4 +288,12 @@ public boolean getIgnoreSecurePortsForTesting() { public long getBpReadyTimeout() { return bpReadyTimeout; } + + public int getTransferSocketRecvBufferSize() { + return transferSocketRecvBufferSize; + } + + public int getTransferSocketSendBufferSize() { + return transferSocketSendBufferSize; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index f23580cd7767f..5c17e7ab47a54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -905,7 +905,10 @@ private void initDataXceiver(Configuration conf) throws IOException { tcpPeerServer = new TcpPeerServer(dnConf.socketWriteTimeout, DataNode.getStreamingAddr(conf), backlogLength); } - tcpPeerServer.setReceiveBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); + if (dnConf.getTransferSocketRecvBufferSize() > 0) { + tcpPeerServer.setReceiveBufferSize( + dnConf.getTransferSocketRecvBufferSize()); + } streamingAddr = tcpPeerServer.getStreamingAddr(); LOG.info("Opened streaming server at " + streamingAddr); this.threadGroup = new ThreadGroup("dataXceiverServer"); @@ -953,8 +956,12 @@ static DomainPeerServer getDomainPeerServer(Configuration conf, } DomainPeerServer domainPeerServer = new DomainPeerServer(domainSocketPath, port); - domainPeerServer.setReceiveBufferSize( - HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); + int recvBufferSize = conf.getInt( + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_KEY, + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_DEFAULT); + if (recvBufferSize > 0) { + domainPeerServer.setReceiveBufferSize(recvBufferSize); + } return domainPeerServer; } @@ -1875,8 +1882,9 @@ void incrDatanodeNetworkErrors(String host) { } } } - - int getXmitsInProgress() { + + @Override //DataNodeMXBean + public int getXmitsInProgress() { return xmitsInProgress.get(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java index 92abd886fee28..eb7bd6e077440 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java @@ -79,6 +79,12 @@ public interface DataNodeMXBean { */ public int getXceiverCount(); + /** + * Returns an estimate of the number of data replication/reconstruction tasks + * running currently. + */ + public int getXmitsInProgress(); + /** * Gets the network error counts on a per-Datanode basis. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index f5fed8864fe60..9e948d02675fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -707,8 +707,11 @@ public void writeBlock(final ExtendedBlock block, (HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * targets.length); NetUtils.connect(mirrorSock, mirrorTarget, timeoutValue); mirrorSock.setSoTimeout(timeoutValue); - mirrorSock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE); - + if (dnConf.getTransferSocketSendBufferSize() > 0) { + mirrorSock.setSendBufferSize( + dnConf.getTransferSocketSendBufferSize()); + } + OutputStream unbufMirrorOut = NetUtils.getOutputStream(mirrorSock, writeTimeout); InputStream unbufMirrorIn = NetUtils.getInputStream(mirrorSock); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index caf6eaaa92023..8d312a8952031 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -278,7 +278,12 @@ synchronized int getNumPeers() { synchronized int getNumPeersXceiver() { return peersXceiver.size(); } - + + @VisibleForTesting + PeerServer getPeerServer() { + return peerServer; + } + synchronized void releasePeer(Peer peer) { peers.remove(peer); peersXceiver.remove(peer); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java index d8fa60a303113..cf4f9b986300b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import org.apache.hadoop.hdfs.protocol.Block; @@ -30,6 +29,7 @@ */ public class FinalizedReplica extends ReplicaInfo { private boolean unlinked; // copy-on-write done for block + private byte[] lastPartialChunkChecksum; /** * Constructor @@ -41,9 +41,24 @@ public class FinalizedReplica extends ReplicaInfo { */ public FinalizedReplica(long blockId, long len, long genStamp, FsVolumeSpi vol, File dir) { + this(blockId, len, genStamp, vol, dir, null); + } + + /** + * Constructor. + * @param blockId block id + * @param len replica length + * @param genStamp replica generation stamp + * @param vol volume where replica is located + * @param dir directory path where block and meta files are located + * @param checksum the last partial chunk checksum + */ + public FinalizedReplica(long blockId, long len, long genStamp, + FsVolumeSpi vol, File dir, byte[] checksum) { super(blockId, len, genStamp, vol, dir); + this.setLastPartialChunkChecksum(checksum); } - + /** * Constructor * @param block a block @@ -51,7 +66,20 @@ public FinalizedReplica(long blockId, long len, long genStamp, * @param dir directory path where block and meta files are located */ public FinalizedReplica(Block block, FsVolumeSpi vol, File dir) { + this(block, vol, dir, null); + } + + /** + * Constructor. + * @param block a block + * @param vol volume where replica is located + * @param dir directory path where block and meta files are located + * @param checksum the last partial chunk checksum + */ + public FinalizedReplica(Block block, FsVolumeSpi vol, File dir, + byte[] checksum) { super(block, vol, dir); + this.setLastPartialChunkChecksum(checksum); } /** @@ -61,6 +89,7 @@ public FinalizedReplica(Block block, FsVolumeSpi vol, File dir) { public FinalizedReplica(FinalizedReplica from) { super(from); this.unlinked = from.isUnlinked(); + this.setLastPartialChunkChecksum(from.getLastPartialChunkChecksum()); } @Override // ReplicaInfo @@ -104,30 +133,18 @@ public String toString() { + "\n unlinked =" + unlinked; } - /** - * gets the last chunk checksum and the length of the block corresponding - * to that checksum. - * Note, need to be called with the FsDataset lock acquired. May improve to - * lock only the FsVolume in the future. - * @throws IOException - */ - public ChunkChecksum getLastChecksumAndDataLen() throws IOException { - ChunkChecksum chunkChecksum = null; - try { - byte[] lastChecksum = getVolume().loadLastPartialChunkChecksum( - getBlockFile(), getMetaFile()); - if (lastChecksum != null) { - chunkChecksum = - new ChunkChecksum(getVisibleLength(), lastChecksum); - } - } catch (FileNotFoundException e) { - // meta file is lost. Try to continue anyway. - DataNode.LOG.warn("meta file " + getMetaFile() + - " is missing!"); - } catch (IOException ioe) { - DataNode.LOG.warn("Unable to read checksum from meta file " + - getMetaFile(), ioe); - } - return chunkChecksum; + public byte[] getLastPartialChunkChecksum() { + return lastPartialChunkChecksum; + } + + public void setLastPartialChunkChecksum(byte[] checksum) { + lastPartialChunkChecksum = checksum; + } + + public void loadLastPartialChunkChecksum() + throws IOException { + byte[] lastChecksum = getVolume().loadLastPartialChunkChecksum( + getBlockFile(), getMetaFile()); + setLastPartialChunkChecksum(lastChecksum); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java index a40cbc8374b39..b63af29c8158e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi.BlockIterator; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Time; @@ -81,6 +82,8 @@ void setConf(Conf conf) { */ private final DataNode datanode; + private final DataNodeMetrics metrics; + /** * A reference to the volume that we're scanning. */ @@ -301,6 +304,7 @@ public void handle(ExtendedBlock block, IOException e) { VolumeScanner(Conf conf, DataNode datanode, FsVolumeReference ref) { this.conf = conf; this.datanode = datanode; + this.metrics = datanode.getMetrics(); this.ref = ref; this.volume = ref.getVolume(); ScanResultHandler handler; @@ -445,12 +449,14 @@ private long scanBlock(ExtendedBlock cblock, long bytesPerSec) { throttler.setBandwidth(bytesPerSec); long bytesRead = blockSender.sendBlock(nullStream, null, throttler); resultHandler.handle(block, null); + metrics.incrBlocksVerified(); return bytesRead; } catch (IOException e) { resultHandler.handle(block, e); } finally { IOUtils.cleanup(null, blockSender); } + metrics.incrBlockVerificationFailures(); return -1; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index da8a6283cdb8a..7211a4dde760f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -71,7 +71,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; -import org.apache.hadoop.hdfs.server.datanode.BlockScanner; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataStorage; import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil; @@ -928,8 +927,7 @@ public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block, targetVolume, blockFiles[0].getParentFile(), 0); newReplicaInfo.setNumBytes(blockFiles[1].length()); // Finalize the copied files - newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo, - false); + newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo); removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile, oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId()); @@ -997,8 +995,10 @@ static void computeChecksum(File srcMeta, File dstMeta, File blockFile) // calculate and write the last crc checksum.calculateChunkedSums(data, 0, offset, crcs, 0); metaOut.write(crcs, 0, 4); + metaOut.close(); + metaOut = null; } finally { - IOUtils.cleanup(LOG, metaOut); + IOUtils.closeStream(metaOut); } } @@ -1127,10 +1127,8 @@ private synchronized ReplicaBeingWritten append(String bpid, v, newBlkFile.getParentFile(), Thread.currentThread(), bytesReserved); // load last checksum and datalen - byte[] lastChunkChecksum = v.loadLastPartialChunkChecksum( - replicaInfo.getBlockFile(), replicaInfo.getMetaFile()); newReplicaInfo.setLastChecksumAndDataLen( - replicaInfo.getNumBytes(), lastChunkChecksum); + replicaInfo.getNumBytes(), replicaInfo.getLastPartialChunkChecksum()); File newmeta = newReplicaInfo.getMetaFile(); @@ -1254,7 +1252,7 @@ public synchronized Replica recoverClose(ExtendedBlock b, long newGS, bumpReplicaGS(replicaInfo, newGS); // finalize the replica if RBW if (replicaInfo.getState() == ReplicaState.RBW) { - finalizeReplica(b.getBlockPoolId(), replicaInfo, false); + finalizeReplica(b.getBlockPoolId(), replicaInfo); } return replicaInfo; } @@ -1583,34 +1581,50 @@ public void adjustCrcChannelPosition(ExtendedBlock b, ReplicaOutputStreams strea * Complete the block write! */ @Override // FsDatasetSpi - public synchronized void finalizeBlock(ExtendedBlock b, boolean fsyncDir) + public void finalizeBlock(ExtendedBlock b, boolean fsyncDir) throws IOException { - if (Thread.interrupted()) { - // Don't allow data modifications from interrupted threads - throw new IOException("Cannot finalize block from Interrupted Thread"); + ReplicaInfo replicaInfo = null; + ReplicaInfo finalizedReplicaInfo = null; + synchronized (this) { + if (Thread.interrupted()) { + // Don't allow data modifications from interrupted threads + throw new IOException("Cannot finalize block from Interrupted Thread"); + } + replicaInfo = getReplicaInfo(b); + if (replicaInfo.getState() == ReplicaState.FINALIZED) { + // this is legal, when recovery happens on a file that has + // been opened for append but never modified + return; + } + finalizedReplicaInfo = finalizeReplica(b.getBlockPoolId(), replicaInfo); } - ReplicaInfo replicaInfo = getReplicaInfo(b); - if (replicaInfo.getState() == ReplicaState.FINALIZED) { - // this is legal, when recovery happens on a file that has - // been opened for append but never modified - return; + /* + * Sync the directory after rename from tmp/rbw to Finalized if + * configured. Though rename should be atomic operation, sync on both + * dest and src directories are done because IOUtils.fsync() calls + * directory's channel sync, not the journal itself. + */ + if (fsyncDir) { + File f = replicaInfo.getBlockFile(); + File dest = finalizedReplicaInfo.getBlockFile(); + fsyncDirectory(dest.getParentFile(), f.getParentFile()); } - finalizeReplica(b.getBlockPoolId(), replicaInfo, fsyncDir); } - + private synchronized FinalizedReplica finalizeReplica(String bpid, - ReplicaInfo replicaInfo, boolean fsyncDir) throws IOException { + ReplicaInfo replicaInfo) throws IOException { FinalizedReplica newReplicaInfo = null; if (replicaInfo.getState() == ReplicaState.RUR && ((ReplicaUnderRecovery)replicaInfo).getOriginalReplica().getState() == ReplicaState.FINALIZED) { newReplicaInfo = (FinalizedReplica) ((ReplicaUnderRecovery)replicaInfo).getOriginalReplica(); + newReplicaInfo.loadLastPartialChunkChecksum(); } else { FsVolumeImpl v = (FsVolumeImpl)replicaInfo.getVolume(); File f = replicaInfo.getBlockFile(); if (v == null) { - throw new IOException("No volume for temporary file " + f + + throw new IOException("No volume for temporary file " + f + " for block " + replicaInfo); } @@ -1618,15 +1632,18 @@ private synchronized FinalizedReplica finalizeReplica(String bpid, bpid, replicaInfo, f, replicaInfo.getBytesReserved()); newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile()); - /* - * Sync the directory after rename from tmp/rbw to Finalized if - * configured. Though rename should be atomic operation, sync on both - * dest and src directories are done because IOUtils.fsync() calls - * directory's channel sync, not the journal itself. - */ - if (fsyncDir) { - fsyncDirectory(dest.getParentFile(), f.getParentFile()); + byte[] checksum = null; + // copy the last partial checksum if the replica is originally + // in finalized or rbw state. + if (replicaInfo.getState() == ReplicaState.FINALIZED) { + FinalizedReplica finalized = (FinalizedReplica)replicaInfo; + checksum = finalized.getLastPartialChunkChecksum(); + } else if (replicaInfo.getState() == ReplicaState.RBW) { + ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo; + checksum = rbw.getLastChecksumAndDataLen().getChecksum(); } + newReplicaInfo.setLastPartialChunkChecksum(checksum); + if (v.isTransientStorage()) { ramDiskReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v); datanode.getMetrics().addRamDiskBytesWrite(replicaInfo.getNumBytes()); @@ -2553,12 +2570,12 @@ private FinalizedReplica updateReplicaUnderRecovery( // but it is immediately converted to finalized state within the same // lock, so no need to update it. volumeMap.add(bpid, newReplicaInfo); - finalizeReplica(bpid, newReplicaInfo, false); + finalizeReplica(bpid, newReplicaInfo); } } // finalize the block - return finalizeReplica(bpid, rur, false); + return finalizeReplica(bpid, rur); } private File[] copyReplicaWithNewBlockIdAndGS( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java index 86e2811a1c75b..16eb98fdc6526 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java @@ -91,6 +91,7 @@ public DatanodeHttpServer(final Configuration conf, .setFindPort(true); this.infoServer = builder.build(); + this.infoServer.setAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE, conf); this.infoServer.addInternalServlet(null, "/streamFile/*", StreamFile.class); this.infoServer.addInternalServlet(null, "/getFileChecksum/*", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java index 4d705b0dd9fe0..f510447644d5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java @@ -29,6 +29,7 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.QueryStringDecoder; import io.netty.handler.stream.ChunkedStream; +import java.net.InetSocketAddress; import org.apache.commons.io.Charsets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -71,11 +72,13 @@ import static io.netty.handler.codec.http.HttpResponseStatus.CREATED; import static io.netty.handler.codec.http.HttpResponseStatus.OK; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static io.netty.handler.codec.rtsp.RtspResponseStatuses.INTERNAL_SERVER_ERROR; import static org.apache.hadoop.hdfs.protocol.HdfsConstants.HDFS_URI_SCHEME; import static org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier.HDFS_DELEGATION_KIND; public class WebHdfsHandler extends SimpleChannelInboundHandler { static final Log LOG = LogFactory.getLog(WebHdfsHandler.class); + static final Log REQLOG = LogFactory.getLog("datanode.webhdfs"); public static final String WEBHDFS_PREFIX = WebHdfsFileSystem.PATH_PREFIX; public static final int WEBHDFS_PREFIX_LENGTH = WEBHDFS_PREFIX.length(); public static final String APPLICATION_OCTET_STREAM = @@ -89,6 +92,7 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler { private String path; private ParameterParser params; private UserGroupInformation ugi; + private DefaultHttpResponse resp = null; public WebHdfsHandler(Configuration conf, Configuration confForCreate) throws IOException { @@ -110,12 +114,30 @@ public void channelRead0(final ChannelHandlerContext ctx, ugi.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { - handle(ctx, req); + try { + handle(ctx, req); + } finally { + String host = null; + try { + host = ((InetSocketAddress)ctx.channel().remoteAddress()). + getAddress().getHostAddress(); + } catch (Exception e) { + LOG.warn("Error retrieving hostname: ", e); + host = "unknown"; + } + REQLOG.info(host + " " + req.getMethod() + " " + req.getUri() + " " + + getResponseCode()); + } return null; } }); } + int getResponseCode() { + return (resp == null) ? INTERNAL_SERVER_ERROR.code() : + resp.getStatus().code(); + } + public void handle(ChannelHandlerContext ctx, HttpRequest req) throws IOException, URISyntaxException { String op = params.op(); @@ -140,7 +162,7 @@ public void handle(ChannelHandlerContext ctx, HttpRequest req) @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { LOG.debug("Error ", cause); - DefaultHttpResponse resp = ExceptionHandler.exceptionCaught(cause); + resp = ExceptionHandler.exceptionCaught(cause); resp.headers().set(CONNECTION, CLOSE); ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE); } @@ -163,7 +185,7 @@ private void onCreate(ChannelHandlerContext ctx) OutputStream out = dfsClient.createWrappedOutputStream(dfsClient.create( path, permission, flags, replication, blockSize, null, bufferSize, null), null); - DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, CREATED); + resp = new DefaultHttpResponse(HTTP_1_1, CREATED); final URI uri = new URI(HDFS_URI_SCHEME, nnId, path, null, null); resp.headers().set(LOCATION, uri.toString()); @@ -180,7 +202,7 @@ private void onAppend(ChannelHandlerContext ctx) throws IOException { DFSClient dfsClient = newDfsClient(nnId, conf); OutputStream out = dfsClient.append(path, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); - DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, OK); + resp = new DefaultHttpResponse(HTTP_1_1, OK); resp.headers().set(CONTENT_LENGTH, 0); ctx.pipeline().replace(this, HdfsWriter.class.getSimpleName(), new HdfsWriter(dfsClient, out, resp)); @@ -192,8 +214,8 @@ private void onOpen(ChannelHandlerContext ctx) throws IOException { final long offset = params.offset(); final long length = params.length(); - DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); - HttpHeaders headers = response.headers(); + resp = new DefaultHttpResponse(HTTP_1_1, OK); + HttpHeaders headers = resp.headers(); // Allow the UI to access the file headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET); headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*"); @@ -217,7 +239,7 @@ private void onOpen(ChannelHandlerContext ctx) throws IOException { data = in; } - ctx.write(response); + ctx.write(resp); ctx.writeAndFlush(new ChunkedStream(data) { @Override public void close() throws Exception { @@ -239,7 +261,7 @@ private void onGetFileChecksum(ChannelHandlerContext ctx) throws IOException { IOUtils.cleanup(LOG, dfsclient); } final byte[] js = JsonUtil.toJsonString(checksum).getBytes(Charsets.UTF_8); - DefaultFullHttpResponse resp = + resp = new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.wrappedBuffer(js)); resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java index 6d80872e0d001..65a8a2a77d4fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; @@ -83,6 +84,8 @@ static enum BNState { private FSNamesystem namesystem; + private int quotaInitThreads; + /** * Construct a backup image. * @param conf Configuration @@ -92,6 +95,9 @@ static enum BNState { super(conf); storage.setDisablePreUpgradableLayoutCheck(true); bnState = BNState.DROP_UNTIL_NEXT_ROLL; + quotaInitThreads = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_KEY, + DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT); } synchronized FSNamesystem getNamesystem() { @@ -231,7 +237,7 @@ private synchronized void applyEdits(long firstTxId, int numTxns, byte[] data) FSImage.updateCountForQuota( getNamesystem().dir.getBlockStoragePolicySuite(), - getNamesystem().dir.rootDir); // inefficient! + getNamesystem().dir.rootDir, quotaInitThreads); } finally { backupInputStream.clear(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java index 1aef7dcada8d6..f7a3b975011be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java @@ -81,7 +81,7 @@ static HdfsFileStatus setOwner( throw new AccessControlException("User " + username + " is not a super user (non-super user cannot change owner)."); } - if (group != null && !pc.containsGroup(group)) { + if (group != null && !pc.isMemberOfGroup(group)) { throw new AccessControlException( "User " + username + " does not belong to " + group); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java index e7c9738152e8a..6f7bb4b69aa75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java @@ -230,7 +230,9 @@ static void unprotectedConcat(FSDirectory fsd, INodesInPath targetIIP, for (INodeFile nodeToRemove : srcList) { if(nodeToRemove != null) { nodeToRemove.setBlocks(null); - nodeToRemove.getParent().removeChild(nodeToRemove); + // Ensure the nodeToRemove is cleared from snapshot diff list + nodeToRemove.getParent().removeChild(nodeToRemove, + targetIIP.getLatestSnapshotId()); fsd.getINodeMap().remove(nodeToRemove); count++; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index cba3506e7e803..64d13f9f0fd53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -134,8 +134,7 @@ static List listXAttrs( final boolean isRawPath = FSDirectory.isReservedRawName(src); final INodesInPath iip = fsd.resolvePath(pc, src); if (fsd.isPermissionEnabled()) { - /* To access xattr names, you need EXECUTE in the owning directory. */ - fsd.checkParentAccess(pc, iip, FsAction.EXECUTE); + fsd.checkPathAccess(pc, iip, FsAction.READ); } final List all = FSDirXAttrOp.getXAttrs(fsd, iip); return XAttrPermissionFilter. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 8fa869bb574cb..15d03edc052fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -313,7 +313,7 @@ FSNamesystem getFSNamesystem() { return namesystem; } - private BlockManager getBlockManager() { + BlockManager getBlockManager() { return getFSNamesystem().getBlockManager(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index da3d8beb5e13f..c40a61087c8f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -27,6 +27,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.RecursiveAction; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -70,6 +72,7 @@ import org.apache.hadoop.hdfs.util.Canceler; import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.hdfs.util.MD5FileUtils; +import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.util.Time; @@ -100,6 +103,7 @@ public class FSImage implements Closeable { final private Configuration conf; protected NNStorageRetentionManager archivalManager; + private int quotaInitThreads; /** * The collection of newly added storage directories. These are partially @@ -153,6 +157,10 @@ protected FSImage(Configuration conf, storage.setRestoreFailedStorage(true); } + this.quotaInitThreads = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_KEY, + DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT); + this.editLog = new FSEditLog(conf, storage, editsDirs); archivalManager = new NNStorageRetentionManager(conf, storage, editLog); @@ -902,7 +910,7 @@ private long loadEdits(Iterable editStreams, FSEditLog.closeAllStreams(editStreams); // update the counts updateCountForQuota(target.getBlockManager().getStoragePolicySuite(), - target.dir.rootDir); + target.dir.rootDir, quotaInitThreads); } prog.endPhase(Phase.LOADING_EDITS); return lastAppliedTxId - prevLastAppliedTxId; @@ -917,65 +925,105 @@ private long loadEdits(Iterable editStreams, * throw QuotaExceededException. */ static void updateCountForQuota(BlockStoragePolicySuite bsps, - INodeDirectory root) { - updateCountForQuotaRecursively(bsps, root.getStoragePolicyID(), root, - new QuotaCounts.Builder().build()); - } - - private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps, - byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) { - final long parentNamespace = counts.getNameSpace(); - final long parentStoragespace = counts.getStorageSpace(); - final EnumCounters parentTypeSpaces = counts.getTypeSpaces(); - - dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, counts); - - for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) { - final byte childPolicyId = child.getStoragePolicyIDForQuota(blockStoragePolicyId); - if (child.isDirectory()) { - updateCountForQuotaRecursively(bsps, childPolicyId, - child.asDirectory(), counts); - } else { - // file or symlink: count here to reduce recursive calls. - child.computeQuotaUsage(bsps, childPolicyId, counts, false, - Snapshot.CURRENT_STATE_ID); - } - } - - if (dir.isQuotaSet()) { - // check if quota is violated. It indicates a software bug. - final QuotaCounts q = dir.getQuotaCounts(); - - final long namespace = counts.getNameSpace() - parentNamespace; - final long nsQuota = q.getNameSpace(); - if (Quota.isViolated(nsQuota, namespace)) { - LOG.warn("Namespace quota violation in image for " - + dir.getFullPathName() - + " quota = " + nsQuota + " < consumed = " + namespace); - } + INodeDirectory root, int threads) { + threads = (threads < 1) ? 1 : threads; + LOG.info("Initializing quota with " + threads + " thread(s)"); + long start = Time.now(); + QuotaCounts counts = new QuotaCounts.Builder().build(); + ForkJoinPool p = new ForkJoinPool(threads); + RecursiveAction task = new InitQuotaTask(bsps, root.getStoragePolicyID(), + root, counts); + p.execute(task); + task.join(); + p.shutdown(); + LOG.info("Quota initialization completed in " + (Time.now() - start) + + " milliseconds\n" + counts); + } - final long ssConsumed = counts.getStorageSpace() - parentStoragespace; - final long ssQuota = q.getStorageSpace(); - if (Quota.isViolated(ssQuota, ssConsumed)) { - LOG.warn("Storagespace quota violation in image for " - + dir.getFullPathName() - + " quota = " + ssQuota + " < consumed = " + ssConsumed); + /** + * parallel initialization using fork-join. + */ + private static class InitQuotaTask extends RecursiveAction { + private final INodeDirectory dir; + private final QuotaCounts counts; + private final BlockStoragePolicySuite bsps; + private final byte blockStoragePolicyId; + + public InitQuotaTask(BlockStoragePolicySuite bsps, + byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) { + this.dir = dir; + this.counts = counts; + this.bsps = bsps; + this.blockStoragePolicyId = blockStoragePolicyId; + } + + public void compute() { + QuotaCounts myCounts = new QuotaCounts.Builder().build(); + dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, + myCounts); + + ReadOnlyList children = + dir.getChildrenList(Snapshot.CURRENT_STATE_ID); + + if (children.size() > 0) { + List subtasks = new ArrayList(); + for (INode child : children) { + final byte childPolicyId = + child.getStoragePolicyIDForQuota(blockStoragePolicyId); + if (child.isDirectory()) { + subtasks.add(new InitQuotaTask(bsps, childPolicyId, + child.asDirectory(), myCounts)); + } else { + // file or symlink. count using the local counts variable + child.computeQuotaUsage(bsps, childPolicyId, myCounts, + false, Snapshot.CURRENT_STATE_ID); + } + } + // invoke and wait for completion + invokeAll(subtasks); } - final EnumCounters typeSpaces = counts.getTypeSpaces(); - for (StorageType t : StorageType.getTypesSupportingQuota()) { - final long typeSpace = typeSpaces.get(t) - parentTypeSpaces.get(t); - final long typeQuota = q.getTypeSpaces().get(t); - if (Quota.isViolated(typeQuota, typeSpace)) { - LOG.warn("Storage type quota violation in image for " + if (dir.isQuotaSet()) { + // check if quota is violated. It indicates a software bug. + final QuotaCounts q = dir.getQuotaCounts(); + + final long nsConsumed = myCounts.getNameSpace(); + final long nsQuota = q.getNameSpace(); + if (Quota.isViolated(nsQuota, nsConsumed)) { + LOG.warn("Namespace quota violation in image for " + + dir.getFullPathName() + + " quota = " + nsQuota + " < consumed = " + nsConsumed); + } + + final long ssConsumed = myCounts.getStorageSpace(); + final long ssQuota = q.getStorageSpace(); + if (Quota.isViolated(ssQuota, ssConsumed)) { + LOG.warn("Storagespace quota violation in image for " + dir.getFullPathName() - + " type = " + t.toString() + " quota = " - + typeQuota + " < consumed " + typeSpace); + + " quota = " + ssQuota + " < consumed = " + ssConsumed); } + + final EnumCounters tsConsumed = myCounts.getTypeSpaces(); + for (StorageType t : StorageType.getTypesSupportingQuota()) { + final long typeSpace = tsConsumed.get(t); + final long typeQuota = q.getTypeSpaces().get(t); + if (Quota.isViolated(typeQuota, typeSpace)) { + LOG.warn("Storage type quota violation in image for " + + dir.getFullPathName() + + " type = " + t.toString() + " quota = " + + typeQuota + " < consumed " + typeSpace); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Setting quota for " + dir + "\n" + myCounts); + } + dir.getDirectoryWithQuotaFeature().setSpaceConsumed(nsConsumed, + ssConsumed, tsConsumed); } - dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, ssConsumed, - typeSpaces); + synchronized(counts) { + counts.add(myCounts); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ce8878fbaeb68..b9162d0a94f7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -92,6 +92,7 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER; import static org.apache.hadoop.util.Time.now; import static org.apache.hadoop.util.Time.monotonicNow; +import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -953,6 +954,11 @@ private List initAuditLoggers(Configuration conf) { // Add audit logger to calculate top users if (topConf.isEnabled) { topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs); + if (DefaultMetricsSystem.instance().getSource( + TOPMETRICS_METRICS_SOURCE_NAME) == null) { + DefaultMetricsSystem.instance().register(TOPMETRICS_METRICS_SOURCE_NAME, + "Top N operations by user", topMetrics); + } auditLoggers.add(new TopAuditLogger(topMetrics)); } @@ -1479,6 +1485,10 @@ public void readLock() { this.fsLock.readLock(); } @Override + public void readLockInterruptibly() throws InterruptedException { + this.fsLock.readLockInterruptibly(); + } + @Override public void readUnlock() { this.fsLock.readUnlock(); } @@ -1732,6 +1742,7 @@ LocatedBlocks getBlockLocations(String clientMachine, String srcArg, if (res.updateAccessTime()) { String src = srcArg; + checkOperation(OperationCategory.WRITE); writeLock(); final long now = now(); try { @@ -1886,6 +1897,7 @@ void concat(String target, String [] srcs, boolean logRetryCache) final String operationName = "concat"; HdfsFileStatus stat = null; boolean success = false; + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -2182,7 +2194,8 @@ Block prepareFileForTruncate(INodesInPath iip, oldBlock = file.getLastBlock(); assert !oldBlock.isComplete() : "oldBlock should be under construction"; truncatedBlockUC = (BlockInfoContiguousUnderConstruction) oldBlock; - truncatedBlockUC.setTruncateBlock(new Block(oldBlock)); + truncatedBlockUC.setTruncateBlock(new BlockInfoContiguous(oldBlock, + file.getBlockReplication())); truncatedBlockUC.getTruncateBlock().setNumBytes( oldBlock.getNumBytes() - lastBlockDelta); truncatedBlockUC.getTruncateBlock().setGenerationStamp( @@ -2194,7 +2207,8 @@ Block prepareFileForTruncate(INodesInPath iip, truncatedBlockUC.getTruncateBlock().getNumBytes(), truncatedBlockUC); } if (shouldRecoverNow) { - truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp()); + truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp(), + true); } return newBlock; @@ -4059,7 +4073,7 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip, case UNDER_RECOVERY: final BlockInfoContiguousUnderConstruction uc = (BlockInfoContiguousUnderConstruction)lastBlock; // determine if last block was intended to be truncated - Block recoveryBlock = uc.getTruncateBlock(); + BlockInfoContiguous recoveryBlock = uc.getTruncateBlock(); boolean truncateRecovery = recoveryBlock != null; boolean copyOnTruncate = truncateRecovery && recoveryBlock.getBlockId() != uc.getBlockId(); @@ -4094,7 +4108,7 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip, } else if(truncateRecovery) { recoveryBlock.setGenerationStamp(blockRecoveryId); } - uc.initializeBlockRecovery(blockRecoveryId); + uc.initializeBlockRecovery(blockRecoveryId, true); leaseManager.renewLease(lease); // Cannot close file right now, since the last block requires recovery. // This may potentially cause infinite loop in lease recovery @@ -6030,16 +6044,20 @@ void shutdown() { } @Override // FSNamesystemMBean + @Metric({"NumLiveDataNodes", "Number of datanodes which are currently live"}) public int getNumLiveDataNodes() { return getBlockManager().getDatanodeManager().getNumLiveDataNodes(); } @Override // FSNamesystemMBean + @Metric({"NumDeadDataNodes", "Number of datanodes which are currently dead"}) public int getNumDeadDataNodes() { return getBlockManager().getDatanodeManager().getNumDeadDataNodes(); } @Override // FSNamesystemMBean + @Metric({"NumDecomLiveDataNodes", + "Number of datanodes which have been decommissioned and are now live"}) public int getNumDecomLiveDataNodes() { final List live = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false); @@ -6051,6 +6069,8 @@ public int getNumDecomLiveDataNodes() { } @Override // FSNamesystemMBean + @Metric({"NumDecomDeadDataNodes", + "Number of datanodes which have been decommissioned and are now dead"}) public int getNumDecomDeadDataNodes() { final List dead = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(null, dead, false); @@ -6062,6 +6082,8 @@ public int getNumDecomDeadDataNodes() { } @Override // FSNamesystemMBean + @Metric({"VolumeFailuresTotal", + "Total number of volume failures across all Datanodes"}) public int getVolumeFailuresTotal() { List live = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false); @@ -6073,6 +6095,8 @@ public int getVolumeFailuresTotal() { } @Override // FSNamesystemMBean + @Metric({"EstimatedCapacityLostTotal", + "An estimate of the total capacity lost due to volume failures"}) public long getEstimatedCapacityLostTotal() { List live = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false); @@ -6088,6 +6112,8 @@ public long getEstimatedCapacityLostTotal() { } @Override // FSNamesystemMBean + @Metric({"NumDecommissioningDataNodes", + "Number of datanodes in decommissioning state"}) public int getNumDecommissioningDataNodes() { return getBlockManager().getDatanodeManager().getDecommissioningNodes() .size(); @@ -6105,6 +6131,8 @@ public int getNumStaleDataNodes() { * before NN receives the first Heartbeat followed by the first Blockreport. */ @Override // FSNamesystemMBean + @Metric({"NumStaleStorages", + "Number of storages marked as content stale"}) public int getNumStaleStorages() { return getBlockManager().getDatanodeManager().getNumStaleStorages(); } @@ -6304,6 +6332,7 @@ void updatePipeline( String clientName, ExtendedBlock oldBlock, ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs, boolean logRetryCache) throws IOException { + checkOperation(OperationCategory.WRITE); LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + ", newGS=" + newBlock.getGenerationStamp() + ", newLength=" + newBlock.getNumBytes() @@ -6730,9 +6759,9 @@ public void logUpdateMasterKey(DelegationKey key) { assert !isInSafeMode() : "this should never be called while in safemode, since we stop " + "the DT manager before entering safemode!"; - // No need to hold FSN lock since we don't access any internal - // structures, and this is stopped before the FSN shuts itself - // down, etc. + // edit log rolling is not thread-safe and must be protected by the + // fsn lock. not updating namespace so read lock is sufficient. + assert hasReadLock(); getEditLog().logUpdateMasterKey(key); getEditLog().logSync(); } @@ -6746,9 +6775,10 @@ public void logExpireDelegationToken(DelegationTokenIdentifier id) { assert !isInSafeMode() : "this should never be called while in safemode, since we stop " + "the DT manager before entering safemode!"; - // No need to hold FSN lock since we don't access any internal - // structures, and this is stopped before the FSN shuts itself - // down, etc. + // edit log rolling is not thread-safe and must be protected by the + // fsn lock. not updating namespace so read lock is sufficient. + assert hasReadLock(); + // do not logSync so expiration edits are batched getEditLog().logCancelDelegationToken(id); } @@ -7435,6 +7465,7 @@ void deleteSnapshot(String snapshotRoot, String snapshotName, boolean logRetryCache) throws IOException { final String operationName = "deleteSnapshot"; boolean success = false; + checkOperation(OperationCategory.WRITE); writeLock(); BlocksMapUpdateInfo blocksToBeDeleted = null; try { @@ -7474,6 +7505,7 @@ RollingUpgradeInfo queryRollingUpgrade() throws IOException { checkOperation(OperationCategory.READ); readLock(); try { + checkOperation(OperationCategory.READ); if (!isRollingUpgrade()) { return null; } @@ -7667,6 +7699,7 @@ long addCacheDirective(CacheDirectiveInfo directive, if (!flags.contains(CacheFlag.FORCE)) { cacheManager.waitForRescanIfNeeded(); } + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -7698,6 +7731,7 @@ void modifyCacheDirective(CacheDirectiveInfo directive, if (!flags.contains(CacheFlag.FORCE)) { cacheManager.waitForRescanIfNeeded(); } + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -7722,6 +7756,7 @@ void modifyCacheDirective(CacheDirectiveInfo directive, void removeCacheDirective(long id, boolean logRetryCache) throws IOException { final String operationName = "removeCacheDirective"; boolean success = false; + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -7764,6 +7799,7 @@ BatchedListEntries listCacheDirectives( void addCachePool(CachePoolInfo req, boolean logRetryCache) throws IOException { final String operationName = "addCachePool"; + checkOperation(OperationCategory.WRITE); writeLock(); boolean success = false; String poolInfoStr = null; @@ -7788,6 +7824,7 @@ void addCachePool(CachePoolInfo req, boolean logRetryCache) void modifyCachePool(CachePoolInfo req, boolean logRetryCache) throws IOException { final String operationName = "modifyCachePool"; + checkOperation(OperationCategory.WRITE); writeLock(); boolean success = false; try { @@ -7812,6 +7849,7 @@ void modifyCachePool(CachePoolInfo req, boolean logRetryCache) void removeCachePool(String cachePoolName, boolean logRetryCache) throws IOException { final String operationName = "removeCachePool"; + checkOperation(OperationCategory.WRITE); writeLock(); boolean success = false; try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java index 8c60faac3fe59..eea7088323ead 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java @@ -145,6 +145,13 @@ public void readLock() { } } + public void readLockInterruptibly() throws InterruptedException { + coarseLock.readLock().lockInterruptibly(); + if (coarseLock.getReadHoldCount() == 1) { + readLockHeldTimeStampNanos.set(timer.monotonicNowNanos()); + } + } + public void readUnlock() { readUnlock(OP_NAME_OTHER); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java index 726319fa70563..c9b1c76b360d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java @@ -17,10 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.Collection; import java.util.Stack; import org.apache.commons.logging.Log; @@ -81,7 +78,7 @@ private String toAccessControlString(INodeAttributes inodeAttrib, private final UserGroupInformation callerUgi; private final String user; - private final Set groups; + private final Collection groups; private final boolean isSuper; private final INodeAttributeProvider attributeProvider; @@ -92,15 +89,13 @@ private String toAccessControlString(INodeAttributes inodeAttrib, this.fsOwner = fsOwner; this.supergroup = supergroup; this.callerUgi = callerUgi; - HashSet s = - new HashSet(Arrays.asList(callerUgi.getGroupNames())); - groups = Collections.unmodifiableSet(s); + this.groups = callerUgi.getGroups(); user = callerUgi.getShortUserName(); isSuper = user.equals(fsOwner) || groups.contains(supergroup); this.attributeProvider = attributeProvider; } - public boolean containsGroup(String group) { + public boolean isMemberOfGroup(String group) { return groups.contains(group); } @@ -108,10 +103,6 @@ public String getUser() { return user; } - public Set getGroups() { - return groups; - } - public boolean isSuperUser() { return isSuper; } @@ -337,7 +328,7 @@ private boolean hasPermission(INodeAttributes inode, FsAction access) { final FsAction checkAction; if (getUser().equals(inode.getUserName())) { //user class checkAction = mode.getUserAction(); - } else if (getGroups().contains(inode.getGroupName())) { //group class + } else if (isMemberOfGroup(inode.getGroupName())) { //group class checkAction = mode.getGroupAction(); } else { //other class checkAction = mode.getOtherAction(); @@ -407,7 +398,7 @@ private boolean hasAclPermission(INodeAttributes inode, // member of multiple groups that have entries that grant access, then // it doesn't matter which is chosen, so exit early after first match. String group = name == null ? inode.getGroupName() : name; - if (getGroups().contains(group)) { + if (isMemberOfGroup(group)) { FsAction masked = AclEntryStatusFormat.getPermission(entry).and( mode.getGroupAction()); if (masked.implies(access)) { @@ -470,7 +461,7 @@ public void checkPermission(CachePool pool, FsAction access) && mode.getUserAction().implies(access)) { return; } - if (getGroups().contains(pool.getGroupName()) + if (isMemberOfGroup(pool.getGroupName()) && mode.getGroupAction().implies(access)) { return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index e16fc7df72dcd..d4659ce97b20e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -30,6 +30,8 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.Block; @@ -869,9 +871,20 @@ public List getToDeleteList() { * {@link BlocksMapUpdateInfo#toDeleteList} * @param toDelete the to-be-deleted block */ - public void addDeleteBlock(Block toDelete) { + public void addDeleteBlock(BlockInfoContiguous toDelete) { assert toDelete != null : "toDelete is null"; toDeleteList.add(toDelete); + // If the file is being truncated + // the copy-on-truncate block should also be collected for deletion + if(!(toDelete instanceof BlockInfoContiguousUnderConstruction)) { + return; + } + BlockInfoContiguous truncateBlock = + ((BlockInfoContiguousUnderConstruction)toDelete).getTruncateBlock(); + if(truncateBlock == null || truncateBlock.equals(toDelete)) { + return; + } + addDeleteBlock(truncateBlock); } public void removeDeleteBlock(Block block) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java index fd7c42eb50c82..5270d0bf84e80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java @@ -491,8 +491,20 @@ synchronized boolean checkLeases() { try { INodesInPath iip = fsnamesystem.getFSDirectory().getINodesInPath(p, true); - boolean completed = fsnamesystem.internalReleaseLease(leaseToCheck, p, - iip, HdfsServerConstants.NAMENODE_LEASE_HOLDER); + // Sanity check to make sure the path is correct + if (!p.startsWith("/")) { + throw new IOException("Invalid path in the lease " + p); + } + boolean completed = false; + try { + completed = fsnamesystem.internalReleaseLease( + leaseToCheck, p, iip, + HdfsServerConstants.NAMENODE_LEASE_HOLDER); + } catch (IOException e) { + LOG.warn("Cannot release the path " + p + " in the lease " + + leaseToCheck + ". It will be retried.", e); + continue; + } if (LOG.isDebugEnabled()) { if (completed) { LOG.debug("Lease recovery for " + p + " is complete. File closed."); @@ -505,7 +517,7 @@ synchronized boolean checkLeases() { needSync = true; } } catch (IOException e) { - LOG.error("Cannot release the path " + p + " in the lease " + LOG.warn("Removing lease with an invalid path: " + p + "," + leaseToCheck, e); removing.add(p); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index b90be7c8d15a7..897df65be6193 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState; import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; @@ -992,6 +993,21 @@ private static boolean format(Configuration conf, boolean force, FSNamesystem fsn = new FSNamesystem(conf, fsImage); fsImage.getEditLog().initJournalsForWrite(); + // Abort NameNode format if reformat is disabled and if + // meta-dir already exists + if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, + DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) { + force = false; + isInteractive = false; + for (StorageDirectory sd : fsImage.storage.dirIterable(null)) { + if (sd.hasSomeData()) { + throw new NameNodeFormatException( + "NameNode format aborted as reformat is disabled for " + + "this cluster."); + } + } + } + if (!fsImage.confirmFormat(force, isInteractive)) { return true; // aborted } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java new file mode 100644 index 0000000000000..858d6e1e0f3ae --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Thrown when NameNode format fails. + */ +@InterfaceAudience.Private +public class NameNodeFormatException extends IOException { + + private static final long serialVersionUID = 1L; + + public NameNodeFormatException(String message, Throwable cause) { + super(message, cause); + } + + public NameNodeFormatException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java index a3b0448c24a1d..064a594266175 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java @@ -157,6 +157,13 @@ public boolean anyTypeSpaceCountGreaterOrEqual(long val) { return tsCounts.anyGreaterOrEqual(val); } + @Override + public String toString() { + return "name space=" + getNameSpace() + + "\nstorage space=" + getStorageSpace() + + "\nstorage types=" + getTypeSpaces(); + } + @Override public boolean equals(Object obj) { if (obj == this) { @@ -174,4 +181,5 @@ public int hashCode() { assert false : "hashCode not designed"; return 42; // any arbitrary constant will do } -} \ No newline at end of file + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java index 5c9e1219a5930..6e735d3e9df1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java @@ -20,7 +20,6 @@ import java.util.Collections; import java.util.List; -import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; @@ -130,7 +129,7 @@ void combineAndCollectSnapshotBlocks(BlockStoragePolicySuite bsps, INodeFile fil } // Check if last block is part of truncate recovery BlockInfoContiguous lastBlock = file.getLastBlock(); - Block dontRemoveBlock = null; + BlockInfoContiguous dontRemoveBlock = null; if(lastBlock != null && lastBlock.getBlockUCState().equals( HdfsServerConstants.BlockUCState.UNDER_RECOVERY)) { dontRemoveBlock = ((BlockInfoContiguousUnderConstruction) lastBlock) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java index ab55392886278..2719c8857ee00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java @@ -17,24 +17,32 @@ */ package org.apache.hadoop.hdfs.server.namenode.top.metrics; -import java.net.InetAddress; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.namenode.top.TopConf; import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager; +import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op; +import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.User; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.lib.Interns; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + import static org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.TopWindow; /** @@ -58,8 +66,11 @@ * Thread-safe: relies on thread-safety of RollingWindowManager */ @InterfaceAudience.Private -public class TopMetrics { +public class TopMetrics implements MetricsSource { public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class); + public static final String TOPMETRICS_METRICS_SOURCE_NAME = + "NNTopUserOpCounts"; + private final boolean isMetricsSourceEnabled; private static void logConf(Configuration conf) { LOG.info("NNTop conf: " + DFSConfigKeys.NNTOP_BUCKETS_PER_WINDOW_KEY + @@ -83,6 +94,8 @@ public TopMetrics(Configuration conf, int[] reportingPeriods) { rollingWindowManagers.put(reportingPeriods[i], new RollingWindowManager( conf, reportingPeriods[i])); } + isMetricsSourceEnabled = conf.getBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY, + DFSConfigKeys.NNTOP_ENABLED_DEFAULT); } /** @@ -128,4 +141,44 @@ public void report(long currTime, String userName, String cmd) { TopConf.ALL_CMDS, userName, 1); } } + + /** + * Flatten out the top window metrics into + * {@link org.apache.hadoop.metrics2.MetricsRecord}s for consumption by + * external metrics systems. Each metrics record added corresponds to the + * reporting period a.k.a window length of the configured rolling windows. + */ + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + if (!isMetricsSourceEnabled) { + return; + } + + for (final TopWindow window : getTopWindows()) { + MetricsRecordBuilder rb = collector.addRecord(buildOpRecordName(window)) + .setContext("dfs"); + for (final Op op: window.getOps()) { + rb.addCounter(buildOpTotalCountMetricsInfo(op), op.getTotalCount()); + for (User user : op.getTopUsers()) { + rb.addCounter(buildOpRecordMetricsInfo(op, user), user.getCount()); + } + } + } + } + + private String buildOpRecordName(TopWindow window) { + return TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=" + + window.getWindowLenMs(); + } + + private MetricsInfo buildOpTotalCountMetricsInfo(Op op) { + return Interns.info("op=" + StringUtils.deleteWhitespace(op.getOpType()) + + ".TotalCount", "Total operation count"); + } + + private MetricsInfo buildOpRecordMetricsInfo(Op op, User user) { + return Interns.info("op=" + StringUtils.deleteWhitespace(op.getOpType()) + + ".user=" + user.getUser() + + ".count", "Total operations performed by user"); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java index e36f0f7e0897b..deaeaa43247fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java @@ -21,7 +21,10 @@ public interface RwLock { /** Acquire read lock. */ public void readLock(); - + + /** Acquire read lock, unless interrupted while waiting */ + void readLockInterruptibly() throws InterruptedException; + /** Release read lock. */ public void readUnlock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java index 9e3b29a364d8d..8e21b77f1329d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.web; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -66,6 +67,16 @@ protected abstract HttpURLConnection connect(final long offset, final boolean resolved) throws IOException; } + static class InputStreamAndFileLength { + final Long length; + final InputStream in; + + InputStreamAndFileLength(Long length, InputStream in) { + this.length = length; + this.in = in; + } + } + enum StreamStatus { NORMAL, SEEK, CLOSED } @@ -102,7 +113,9 @@ protected InputStream getInputStream() throws IOException { if (in != null) { in.close(); } - in = openInputStream(); + InputStreamAndFileLength fin = openInputStream(startPos); + in = fin.in; + fileLength = fin.length; status = StreamStatus.NORMAL; break; case CLOSED: @@ -112,31 +125,33 @@ protected InputStream getInputStream() throws IOException { } @VisibleForTesting - protected InputStream openInputStream() throws IOException { + protected InputStreamAndFileLength openInputStream(long startOffset) + throws IOException { // Use the original url if no resolved url exists, eg. if // it's the first time a request is made. final boolean resolved = resolvedURL.getURL() != null; final URLOpener opener = resolved? resolvedURL: originalURL; - final HttpURLConnection connection = opener.connect(startPos, resolved); + final HttpURLConnection connection = opener.connect(startOffset, resolved); resolvedURL.setURL(getResolvedUrl(connection)); InputStream in = connection.getInputStream(); + final Long length; final Map> headers = connection.getHeaderFields(); if (isChunkedTransferEncoding(headers)) { // file length is not known - fileLength = null; + length = null; } else { // for non-chunked transfer-encoding, get content-length long streamlength = getStreamLength(connection, headers); - fileLength = startPos + streamlength; + length = startOffset + streamlength; // Java has a bug with >2GB request streams. It won't bounds check // the reads so the transfer blocks until the server times out in = new BoundedInputStream(in, streamlength); } - return in; + return new InputStreamAndFileLength(length, in); } private static long getStreamLength(HttpURLConnection connection, @@ -230,6 +245,36 @@ public void seek(long pos) throws IOException { } } + @Override + public int read(long position, byte[] buffer, int offset, int length) + throws IOException { + try (InputStream in = openInputStream(position).in) { + return in.read(buffer, offset, length); + } + } + + @Override + public void readFully(long position, byte[] buffer, int offset, int length) + throws IOException { + final InputStreamAndFileLength fin = openInputStream(position); + if (fin.length != null && length + position > fin.length) { + throw new EOFException("The length to read " + length + + " exceeds the file length " + fin.length); + } + try { + int nread = 0; + while (nread < length) { + int nbytes = fin.in.read(buffer, offset + nread, length - nread); + if (nbytes < 0) { + throw new EOFException("End of file reached before reading fully."); + } + nread += nbytes; + } + } finally { + fin.in.close(); + } + } + /** * Return the current offset from the start of the file */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 5b23da9dfb7c7..a9442d72d69ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2215,6 +2215,18 @@ + + dfs.client.socket.send.buffer.size + 131072 + + Socket send buffer size for a write pipeline in DFSClient side. + This may affect TCP connection throughput. + If it is set to zero or negative value, + no buffer size will be set explicitly, + thus enable tcp auto-tuning on some system. + + + dfs.domain.socket.path @@ -2619,4 +2631,47 @@ Instrumentation reporting long critical sections will suppress consecutive warnings within this interval. + + + dfs.namenode.quota.init-threads + 4 + + The number of concurrent threads to be used in quota initialization. The + speed of quota initialization also affects the namenode fail-over latency. + If the size of name space is big, try increasing this. + + + + + dfs.reformat.disabled + false + + Disable reformat of NameNode. If it's value is set to "true" + and metadata directories already exist then attempt to format NameNode + will throw NameNodeFormatException. + + + + + dfs.datanode.transfer.socket.send.buffer.size + 131072 + + Socket send buffer size for DataXceiver (mirroring packets to downstream + in pipeline). This may affect TCP connection throughput. + If it is set to zero or negative value, no buffer size will be set + explicitly, thus enable tcp auto-tuning on some system. + + + + + dfs.datanode.transfer.socket.recv.buffer.size + 131072 + + Socket receive buffer size for DataXceiver (receiving packets from client + during block writing). This may affect TCP connection throughput. + If it is set to zero or negative value, no buffer size will be set + explicitly, thus enable tcp auto-tuning on some system. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md index 8c15a55bb92d6..be4dbbf0a5c99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md @@ -126,7 +126,7 @@ Here is an example configuration with two Namenodes: nn-host1:http-port - dfs.namenode.secondaryhttp-address.ns1 + dfs.namenode.secondary.http-address.ns1 snn-host1:http-port @@ -138,7 +138,7 @@ Here is an example configuration with two Namenodes: nn-host2:http-port - dfs.namenode.secondaryhttp-address.ns2 + dfs.namenode.secondary.http-address.ns2 snn-host2:http-port diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 39af6dd945392..f71c6b0a527d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -433,7 +433,7 @@ Usage: |:---- |:---- | | `-backup` | Start backup node. | | `-checkpoint` | Start checkpoint node. | -| `-format` `[-clusterid cid]` `[-force]` `[-nonInteractive]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. -force option formats if the name directory exists. -nonInteractive option aborts if the name directory exists, unless -force option is specified. | +| `-format` `[-clusterid cid]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. Will throw NameNodeFormatException if name dir already exist and if reformat is disabled for cluster. | | `-upgrade` `[-clusterid cid]` [`-renameReserved` \] | Namenode should be started with upgrade option after the distribution of new Hadoop version. | | `-upgradeOnly` `[-clusterid cid]` [`-renameReserved` \] | Upgrade the specified NameNode and then shutdown it. | | `-rollback` | Rollback the NameNode to the previous version. This should be used after stopping the cluster and distributing the old Hadoop version. | diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 17398218d1188..44f7a0de1ccb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -142,6 +142,8 @@ public class MiniDFSCluster { public static final String HDFS_MINIDFS_BASEDIR = "hdfs.minidfs.basedir"; public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY = DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + ".testing"; + public static final String DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY + = DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY + ".testing"; // Changing this default may break some tests that assume it is 2. private static final int DEFAULT_STORAGES_PER_DATANODE = 2; @@ -788,7 +790,9 @@ private void initMiniDFSCluster( int safemodeExtension = conf.getInt( DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0); conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension); - conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 3); // 3 second + int decommissionInterval = conf.getInt( + DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY, 3); + conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, decommissionInterval); conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, StaticMapping.class, DNSToSwitchMapping.class); @@ -2633,6 +2637,29 @@ public static File getBlockFile(File storageDir, ExtendedBlock blk) { blk.getBlockPoolId()), blk.getBlockId()), blk.getBlockName()); } + /** + * Return all block files in given directory (recursive search). + */ + public static List getAllBlockFiles(File storageDir) { + List results = new ArrayList(); + File[] files = storageDir.listFiles(); + if (files == null) { + return null; + } + for (File f : files) { + if (f.getName().startsWith(Block.BLOCK_FILE_PREFIX) && + !f.getName().endsWith(Block.METADATA_EXTENSION)) { + results.add(f); + } else if (f.isDirectory()) { + List subdirResults = getAllBlockFiles(f); + if (subdirResults != null) { + results.addAll(subdirResults); + } + } + } + return results; + } + /** * Get the latest metadata file correpsonding to a block * @param storageDir storage directory diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java index 63a6f62babcea..29e3c8b85d00c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java @@ -32,6 +32,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector; import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException; @@ -527,4 +530,51 @@ public Boolean get() { } } } + + @Test + public void testUpdatePipeLineAfterDNReg()throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); + FileSystem fileSys = cluster.getFileSystem(); + + Path file = new Path("/testUpdatePipeLineAfterDNReg"); + FSDataOutputStream out = fileSys.create(file); + out.write(1); + out.hflush(); + //Get the First DN and disable the heartbeats and then put in Deadstate + DFSOutputStream dfsOut = (DFSOutputStream) out.getWrappedStream(); + DatanodeInfo[] pipeline = dfsOut.getPipeline(); + DataNode dn1 = cluster.getDataNode(pipeline[0].getIpcPort()); + dn1.setHeartbeatsDisabledForTests(true); + DatanodeDescriptor dn1Desc = cluster.getNamesystem(0).getBlockManager() + .getDatanodeManager().getDatanode(dn1.getDatanodeId()); + cluster.setDataNodeDead(dn1Desc); + //Re-register the DeadNode + DatanodeProtocolClientSideTranslatorPB dnp = + new DatanodeProtocolClientSideTranslatorPB( + cluster.getNameNode().getNameNodeAddress(), conf); + dnp.registerDatanode( + dn1.getDNRegistrationForBP(cluster.getNamesystem().getBlockPoolId())); + DFSOutputStream dfsO = (DFSOutputStream) out.getWrappedStream(); + String clientName = ((DistributedFileSystem) fileSys).getClient() + .getClientName(); + NamenodeProtocols namenode = cluster.getNameNodeRpc(); + //Update the genstamp and call updatepipeline + LocatedBlock newBlock = namenode + .updateBlockForPipeline(dfsO.getBlock(), clientName); + dfsO.getStreamer() + .updatePipeline(newBlock.getBlock().getGenerationStamp()); + newBlock = namenode.updateBlockForPipeline(dfsO.getBlock(), clientName); + //Should not throw any error Pipeline should be success + dfsO.getStreamer() + .updatePipeline(newBlock.getBlock().getGenerationStamp()); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java new file mode 100644 index 0000000000000..b3f370f44a136 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientSocketSize.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.log4j.Level; + +import org.junit.After; +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.Socket; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestDFSClientSocketSize { + private static final Logger LOG = LoggerFactory.getLogger( + TestDFSClientSocketSize.class); + static { + GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL); + } + + private final Configuration conf = new Configuration(); + private MiniDFSCluster cluster; + private Socket socket; + + @Test + public void testDefaultSendBufferSize() throws IOException { + socket = createSocket(); + assertEquals("Send buffer size should be the default value.", + DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT, + socket.getSendBufferSize()); + } + + @Test + public void testSpecifiedSendBufferSize() throws IOException { + final int mySendBufferSize = 64 * 1024; // 64 KB + conf.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, mySendBufferSize); + socket = createSocket(); + assertEquals("Send buffer size should be the customized value.", + mySendBufferSize, socket.getSendBufferSize()); + } + + @Test + public void testAutoTuningSendBufferSize() throws IOException { + conf.setInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY, 0); + socket = createSocket(); + LOG.info("The auto tuned send buffer size is: {}", + socket.getSendBufferSize()); + assertTrue("Send buffer size should be non-negative value which is " + + "determined by system (kernel).", socket.getSendBufferSize() > 0); + } + + @After + public void tearDown() throws Exception { + if (socket != null) { + LOG.info("Closing the DFSClient socket."); + } + if (cluster != null) { + LOG.info("Shutting down MiniDFSCluster."); + cluster.shutdown(); + } + } + + private Socket createSocket() throws IOException { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + LOG.info("MiniDFSCluster started."); + return DFSOutputStream.createSocketForPipeline( + new DatanodeInfo(cluster.dataNodes.get(0).datanode.getDatanodeId()), + 1, cluster.getFileSystem().getClient()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index 2e38d5fb406a9..3a441b5fb4684 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -27,7 +27,9 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; +import java.net.URI; import java.net.URISyntaxException; +import java.net.UnknownHostException; import java.util.List; import java.util.Map; import java.util.Random; @@ -53,7 +55,9 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -125,7 +129,10 @@ private static long determineMaxIpcNumber() throws Exception { } return ret; } - + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + /** * Sets up two of the nodes to each drop a single RPC, at all * possible combinations of RPCs. This may result in the @@ -185,6 +192,16 @@ public void testRecoverAfterDoubleFailures() throws Exception { } } + /** + * Expect {@link UnknownHostException} if a hostname can't be resolved. + */ + @Test + public void testUnresolvableHostName() throws Exception { + expectedException.expect(UnknownHostException.class); + new QuorumJournalManager(conf, + new URI("qjournal://" + "bogus:12345" + "/" + JID), FAKE_NSINFO); + } + /** * Test case in which three JournalNodes randomly flip flop between * up and down states every time they get an RPC. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumCall.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumCall.java index 506497e6ae4bd..97cf2f3c0684d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumCall.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumCall.java @@ -23,7 +23,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeoutException; -import org.apache.hadoop.hdfs.qjournal.client.QuorumCall; +import org.apache.hadoop.util.FakeTimer; import org.junit.Test; import com.google.common.base.Joiner; @@ -83,4 +83,33 @@ public void testQuorumFailsWithoutResponse() throws Exception { } } + @Test(timeout=10000) + public void testQuorumSucceedsWithLongPause() throws Exception { + final Map> futures = ImmutableMap.of( + "f1", SettableFuture.create()); + + FakeTimer timer = new FakeTimer() { + private int callCount = 0; + @Override + public long monotonicNowNanos() { + callCount++; + if (callCount == 1) { + long old = super.monotonicNowNanos(); + advance(1000000); + return old; + } else if (callCount == 10) { + futures.get("f1").set("first future"); + return super.monotonicNowNanos(); + } else { + return super.monotonicNowNanos(); + } + } + }; + + QuorumCall q = QuorumCall.create(futures, timer); + assertEquals(0, q.countResponses()); + + q.waitFor(1, 0, 0, 3000, "test"); // wait for 1 response + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index a884ecbce7719..26c808683ac5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -1679,7 +1679,7 @@ void testBalancerRPCDelay() throws Exception { initConf(conf); conf.setInt(DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_KEY, 30); - int numDNs = 40; + int numDNs = 20; long[] capacities = new long[numDNs]; String[] racks = new String[numDNs]; for(int i = 0; i < numDNs; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 497ad58a3c328..48c6953cfa9d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -233,7 +233,9 @@ public static void setWritingPrefersLocalNode( * @param bm the BlockManager to manipulate */ public static void checkHeartbeat(BlockManager bm) { - bm.getDatanodeManager().getHeartbeatManager().heartbeatCheck(); + HeartbeatManager hbm = bm.getDatanodeManager().getHeartbeatManager(); + hbm.restartHeartbeatStopWatch(); + hbm.heartbeatCheck(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java index a7ba29399dcad..c72a356085ca3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java @@ -50,7 +50,7 @@ public void testInitializeBlockRecovery() throws Exception { DFSTestUtil.resetLastUpdatesWithOffset(dd1, -3 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd2, -1 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd3, -2 * 1000); - blockInfo.initializeBlockRecovery(1); + blockInfo.initializeBlockRecovery(1, true); BlockInfoContiguousUnderConstruction[] blockInfoRecovery = dd2.getLeaseRecoveryCommand(1); assertEquals(blockInfoRecovery[0], blockInfo); @@ -58,7 +58,7 @@ public void testInitializeBlockRecovery() throws Exception { DFSTestUtil.resetLastUpdatesWithOffset(dd1, -2 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd2, -1 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd3, -3 * 1000); - blockInfo.initializeBlockRecovery(2); + blockInfo.initializeBlockRecovery(2, true); blockInfoRecovery = dd1.getLeaseRecoveryCommand(1); assertEquals(blockInfoRecovery[0], blockInfo); @@ -66,7 +66,7 @@ public void testInitializeBlockRecovery() throws Exception { DFSTestUtil.resetLastUpdatesWithOffset(dd1, -2 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd2, -1 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd3, -3 * 1000); - blockInfo.initializeBlockRecovery(3); + blockInfo.initializeBlockRecovery(3, true); blockInfoRecovery = dd3.getLeaseRecoveryCommand(1); assertEquals(blockInfoRecovery[0], blockInfo); @@ -75,7 +75,7 @@ public void testInitializeBlockRecovery() throws Exception { DFSTestUtil.resetLastUpdatesWithOffset(dd1, -2 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd2, -1 * 1000); DFSTestUtil.resetLastUpdatesWithOffset(dd3, 0); - blockInfo.initializeBlockRecovery(3); + blockInfo.initializeBlockRecovery(3, true); blockInfoRecovery = dd3.getLeaseRecoveryCommand(1); assertEquals(blockInfoRecovery[0], blockInfo); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java index 6fc30ba6d6314..c68ff37fb6414 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; @@ -33,6 +35,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; @@ -41,6 +44,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.util.Time; import org.junit.Test; +import org.mockito.Mockito; /** * Test if FSNamesystem handles heartbeat right @@ -241,4 +245,27 @@ public void testHeartbeatBlockRecovery() throws Exception { cluster.shutdown(); } } + + @Test + public void testHeartbeatStopWatch() throws Exception { + Namesystem ns = Mockito.mock(Namesystem.class); + BlockManager bm = Mockito.mock(BlockManager.class); + Configuration conf = new Configuration(); + long recheck = 2000; + conf.setLong( + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, recheck); + HeartbeatManager monitor = new HeartbeatManager(ns, bm, conf); + monitor.restartHeartbeatStopWatch(); + assertFalse(monitor.shouldAbortHeartbeatCheck(0)); + // sleep shorter than recheck and verify shouldn't abort + Thread.sleep(100); + assertFalse(monitor.shouldAbortHeartbeatCheck(0)); + // sleep longer than recheck and verify should abort unless ignore delay + Thread.sleep(recheck); + assertTrue(monitor.shouldAbortHeartbeatCheck(0)); + assertFalse(monitor.shouldAbortHeartbeatCheck(-recheck*3)); + // ensure it resets properly + monitor.restartHeartbeatStopWatch(); + assertFalse(monitor.shouldAbortHeartbeatCheck(0)); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java index b461e3a9f3a2f..250bbd3b717e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java @@ -70,6 +70,11 @@ public void testDataNodeMXBean() throws Exception { int xceiverCount = (Integer)mbs.getAttribute(mxbeanName, "XceiverCount"); Assert.assertEquals(datanode.getXceiverCount(), xceiverCount); + // Ensure mxbean's XmitsInProgress is same as the DataNode's + // live value. + int xmitsInProgress = + (Integer) mbs.getAttribute(mxbeanName, "XmitsInProgress"); + Assert.assertEquals(datanode.getXmitsInProgress(), xmitsInProgress); } finally { if (cluster != null) {cluster.shutdown();} } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeTransferSocketSize.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeTransferSocketSize.java new file mode 100644 index 0000000000000..0e98b86a45429 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeTransferSocketSize.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.Test; + +public class TestDataNodeTransferSocketSize { + + @Test + public void testSpecifiedDataSocketSize() throws Exception { + Configuration conf = new HdfsConfiguration(); + conf.setInt( + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_KEY, 4 * 1024); + SimulatedFSDataset.setFactory(conf); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + try { + List datanodes = cluster.getDataNodes(); + DataNode datanode = datanodes.get(0); + assertEquals("Receive buffer size should be 4K", + 4 * 1024, datanode.getXferServer().getPeerServer().getReceiveBufferSize()); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @Test + public void testAutoTuningDataSocketSize() throws Exception { + Configuration conf = new HdfsConfiguration(); + conf.setInt( + DFSConfigKeys.DFS_DATANODE_TRANSFER_SOCKET_RECV_BUFFER_SIZE_KEY, 0); + SimulatedFSDataset.setFactory(conf); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + try { + List datanodes = cluster.getDataNodes(); + DataNode datanode = datanodes.get(0); + assertTrue( + "Receive buffer size should be a default value (determined by kernel)", + datanode.getXferServer().getPeerServer().getReceiveBufferSize() > 0); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java index 32a272f6e3cda..3cac0c79673a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java @@ -106,6 +106,8 @@ public static String getImageFileMD5IgnoringTxId(File imageFile) try { raf.seek(IMAGE_TXID_POS); raf.writeLong(0); + raf.close(); + raf = null; } finally { IOUtils.closeStream(raf); } @@ -517,9 +519,11 @@ public static void corruptVersionFile(File versionFile, String key, String value out = new FileOutputStream(versionFile); props.store(out, null); - + out.close(); + out = null; } finally { - IOUtils.cleanup(null, fis, out); + IOUtils.closeStream(fis); + IOUtils.closeStream(out); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java index 75fd8dc9ce827..8533945ae1e9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java @@ -837,28 +837,37 @@ public Object run() throws Exception { } /* - * Check that execute/scan access to the parent dir is sufficient to get - * xattr names. + * Check that execute/scan access to the parent dir is not + * sufficient to get xattr names. */ fs.setPermission(path, new FsPermission((short) 0701)); user.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { + try { final FileSystem userFs = dfsCluster.getFileSystem(); userFs.listXAttrs(childDir); - return null; + fail("expected AccessControlException"); + } catch (AccessControlException ace) { + GenericTestUtils.assertExceptionContains("Permission denied", ace); } + return null; + } }); /* * Test that xattrs in the "trusted" namespace are filtered correctly. */ + // Allow the user to read child path. + fs.setPermission(childDir, new FsPermission((short) 0704)); fs.setXAttr(childDir, "trusted.myxattr", "1234".getBytes()); user.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { final FileSystem userFs = dfsCluster.getFileSystem(); - assertTrue(userFs.listXAttrs(childDir).size() == 1); + List xattrs = userFs.listXAttrs(childDir); + assertTrue(xattrs.size() == 1); + assertEquals(name1, xattrs.get(0)); return null; } }); @@ -1106,8 +1115,13 @@ public Object run() throws Exception { * and non-root can't do listXAttrs on /.reserved/raw. */ // non-raw path - final List xattrNames = userFs.listXAttrs(path); - assertTrue(xattrNames.size() == 0); + try { + userFs.listXAttrs(path); + fail("listXAttr should have thrown AccessControlException"); + } catch (AccessControlException ace) { + // expected + } + try { // raw path userFs.listXAttrs(rawPath); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java index 1fbe160a4711f..18215d6c8ad1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.junit.AfterClass; @@ -181,4 +182,45 @@ public void testGetBlockLocations() throws IOException { // close file out.close(); } + + /** + * A storage ID can be invalid if the storage failed or the node + * reregisters. When the node heart-beats, the storage report in it + * causes storage volumes to be added back. An invalid storage ID + * should not cause an NPE. + */ + @Test + public void testEmptyExpectedLocations() throws Exception { + final NamenodeProtocols namenode = cluster.getNameNodeRpc(); + final FSNamesystem fsn = cluster.getNamesystem(); + final BlockManager bm = fsn.getBlockManager(); + final Path p = new Path(BASE_DIR, "file2.dat"); + final String src = p.toString(); + final FSDataOutputStream out = TestFileCreation.createFile(hdfs, p, 1); + writeFile(p, out, 256); + out.hflush(); + + // make sure the block is readable + LocatedBlocks lbs = namenode.getBlockLocations(src, 0, 256); + LocatedBlock lastLB = lbs.getLocatedBlocks().get(0); + final Block b = lastLB.getBlock().getLocalBlock(); + + // fake a block recovery + long blockRecoveryId = fsn.getBlockIdManager().nextGenerationStamp(false); + BlockInfoContiguousUnderConstruction uc = bm.getStoredBlock(b). + convertToBlockUnderConstruction(BlockUCState.UNDER_CONSTRUCTION, null); + uc.initializeBlockRecovery(blockRecoveryId, false); + + try { + String[] storages = { "invalid-storage-id1" }; + fsn.commitBlockSynchronization(lastLB.getBlock(), blockRecoveryId, 256L, + true, false, lastLB.getLocations(), storages); + } catch (java.lang.IllegalStateException ise) { + // Although a failure is expected as of now, future commit policy + // changes may make it not fail. This is not critical to the test. + } + + // Invalid storage should not trigger an exception. + lbs = namenode.getBlockLocations(src, 0, 256); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java index 7c23dd55df5f0..8277a1d0820b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java @@ -39,9 +39,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ExitUtil.ExitException; @@ -446,4 +450,34 @@ public void testFormatWithoutForceEnterNo() throws IOException, File version = new File(hdfsDir, "current/VERSION"); assertFalse("Check version should not exist", version.exists()); } + + /** + * Test NameNode format failure when reformat is disabled and metadata + * directories exist. + */ + @Test + public void testNNFormatFailure() throws Exception { + NameNode.initMetrics(config, NamenodeRole.NAMENODE); + DFSTestUtil.formatNameNode(config); + config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true); + // Call to NameNode format will fail as name dir is not empty + try { + NameNode.format(config); + fail("NN format should fail."); + } catch (NameNodeFormatException e) { + GenericTestUtils.assertExceptionContains("NameNode format aborted as " + + "reformat is disabled for this cluster", e); + } + } + + /** + * Test NameNode format when reformat is disabled and metadata directories do + * not exist. + */ + @Test + public void testNNFormatSuccess() throws Exception { + NameNode.initMetrics(config, NamenodeRole.NAMENODE); + config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true); + DFSTestUtil.formatNameNode(config); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java index c29383a17a034..aa8ed31df317b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java @@ -68,7 +68,7 @@ private FSNamesystem makeNameSystemSpy(Block block, INodeFile file) block, (short) 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets); blockInfo.setBlockCollection(file); blockInfo.setGenerationStamp(genStamp); - blockInfo.initializeBlockRecovery(genStamp); + blockInfo.initializeBlockRecovery(genStamp, true); doReturn(true).when(file).removeLastBlock(any(Block.class)); doReturn(true).when(file).isUnderConstruction(); doReturn(new BlockInfoContiguous[1]).when(file).getBlocks(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java index 00ea58f86db54..777845a90b308 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java @@ -137,7 +137,8 @@ public void testDeadDatanode() throws Exception { public void testNonDFSUsedONDeadNodeReReg() throws Exception { Configuration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, + 3000); conf.setInt(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 6 * 1000); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java index 6edea6c6462f3..a1e63c5cb8eba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.ipc.RemoteException; @@ -354,6 +355,68 @@ public void testTruncateOverQuota() throws Exception { cluster.restartNameNode(true); } + /** + * Check whether the quota is initialized correctly. + */ + @Test + public void testQuotaInitialization() throws Exception { + final int size = 500; + Path testDir = new Path("/testDir"); + long expectedSize = 3 * BLOCKSIZE + BLOCKSIZE/2; + getDFS().mkdirs(testDir); + getDFS().setQuota(testDir, size*4, expectedSize*size*2); + + Path[] testDirs = new Path[size]; + for (int i = 0; i < size; i++) { + testDirs[i] = new Path(testDir, "sub" + i); + getDFS().mkdirs(testDirs[i]); + getDFS().setQuota(testDirs[i], 100, 1000000); + DFSTestUtil.createFile(getDFS(), new Path(testDirs[i], "a"), expectedSize, + (short)1, 1L); + } + + // Directly access the name system to obtain the current cached usage. + INodeDirectory root = getFSDirectory().getRoot(); + HashMap nsMap = new HashMap(); + HashMap dsMap = new HashMap(); + scanDirsWithQuota(root, nsMap, dsMap, false); + + FSImage.updateCountForQuota( + getFSDirectory().getBlockManager().getStoragePolicySuite(), root, 1); + scanDirsWithQuota(root, nsMap, dsMap, true); + + FSImage.updateCountForQuota( + getFSDirectory().getBlockManager().getStoragePolicySuite(), root, 2); + scanDirsWithQuota(root, nsMap, dsMap, true); + + FSImage.updateCountForQuota( + getFSDirectory().getBlockManager().getStoragePolicySuite(), root, 4); + scanDirsWithQuota(root, nsMap, dsMap, true); + } + + private void scanDirsWithQuota(INodeDirectory dir, + HashMap nsMap, + HashMap dsMap, boolean verify) { + if (dir.isQuotaSet()) { + // get the current consumption + QuotaCounts q = dir.getDirectoryWithQuotaFeature().getSpaceConsumed(); + String name = dir.getFullPathName(); + if (verify) { + assertEquals(nsMap.get(name).longValue(), q.getNameSpace()); + assertEquals(dsMap.get(name).longValue(), q.getStorageSpace()); + } else { + nsMap.put(name, Long.valueOf(q.getNameSpace())); + dsMap.put(name, Long.valueOf(q.getStorageSpace())); + } + } + + for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) { + if (child instanceof INodeDirectory) { + scanDirsWithQuota((INodeDirectory)child, nsMap, dsMap, verify); + } + } + } + /** * Test that the cached quota stays correct between the COMMIT * and COMPLETE block steps, even if the replication factor is diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 29116ba09720a..d3d039eabd629 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -25,6 +25,7 @@ import java.util.EnumSet; import org.junit.Assert; +import org.junit.Assume; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -46,6 +47,7 @@ import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.PathUtils; +import org.apache.hadoop.util.NativeCodeLoader; import org.junit.Test; public class TestFSImage { @@ -65,6 +67,13 @@ public void testCompression() throws IOException { setCompressCodec(conf, "org.apache.hadoop.io.compress.DefaultCodec"); setCompressCodec(conf, "org.apache.hadoop.io.compress.GzipCodec"); setCompressCodec(conf, "org.apache.hadoop.io.compress.BZip2Codec"); + } + + @Test + public void testNativeCompression() throws IOException { + Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded()); + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true); setCompressCodec(conf, "org.apache.hadoop.io.compress.Lz4Codec"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java index 741dd10c221da..61b7f7ca5fefe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java @@ -159,7 +159,7 @@ private void loadFSImageFromTempFile(File imageFile) throws IOException { try { loader.load(imageFile, false); FSImage.updateCountForQuota(fsn.getBlockManager().getStoragePolicySuite(), - INodeDirectory.valueOf(fsn.getFSDirectory().getINode("/"), "/")); + INodeDirectory.valueOf(fsn.getFSDirectory().getINode("/"), "/"), 4); } finally { fsn.getFSDirectory().writeUnlock(); fsn.writeUnlock(); @@ -509,4 +509,4 @@ public void testSaveLoadImageAfterSnapshotDeletion() fsn = cluster.getNamesystem(); hdfs = cluster.getFileSystem(); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java index fbcc73f5c369b..74586cceae1fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java @@ -31,6 +31,7 @@ import static org.junit.Assert.fail; import java.io.IOException; +import java.util.concurrent.ThreadLocalRandom; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -58,6 +59,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -1163,6 +1165,46 @@ public void testTruncate4Symlink() throws IOException { fs.delete(parent, true); } + /** + * While rolling upgrade is in-progress the test truncates a file + * such that copy-on-truncate is triggered, then deletes the file, + * and makes sure that no blocks involved in truncate are hanging around. + */ + @Test + public void testTruncateWithRollingUpgrade() throws Exception { + final DFSAdmin dfsadmin = new DFSAdmin(cluster.getConfiguration(0)); + DistributedFileSystem dfs = cluster.getFileSystem(); + //start rolling upgrade + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + int status = dfsadmin.run(new String[]{"-rollingUpgrade", "prepare"}); + assertEquals("could not prepare for rolling upgrade", 0, status); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + Path dir = new Path("/testTruncateWithRollingUpgrade"); + fs.mkdirs(dir); + final Path p = new Path(dir, "file"); + final byte[] data = new byte[3]; + ThreadLocalRandom.current().nextBytes(data); + writeContents(data, data.length, p); + + assertEquals("block num should 1", 1, + cluster.getNamesystem().getFSDirectory().getBlockManager() + .getTotalBlocks()); + + final boolean isReady = fs.truncate(p, 2); + assertFalse("should be copy-on-truncate", isReady); + assertEquals("block num should 2", 2, + cluster.getNamesystem().getFSDirectory().getBlockManager() + .getTotalBlocks()); + fs.delete(p, true); + + assertEquals("block num should 0", 0, + cluster.getNamesystem().getFSDirectory().getBlockManager() + .getTotalBlocks()); + status = dfsadmin.run(new String[]{"-rollingUpgrade", "finalize"}); + assertEquals("could not finalize rolling upgrade", 0, status); + } + static void writeContents(byte[] contents, int fileLength, Path p) throws IOException { FSDataOutputStream out = fs.create(p, true, BLOCK_SIZE, REPLICATION, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index bd48023928924..9575e2f0f64dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -1471,4 +1471,41 @@ public void testStoragePoliciesCK() throws Exception { } } } + + @Test(timeout = 300000) + public void testFsckCorruptWhenOneReplicaIsCorrupt() + throws Exception { + Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(2) + .build(); + try { + cluster.waitActive(); + FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); + cluster.transitionToActive(0); + String filePath = "/appendTest"; + Path fileName = new Path(filePath); + DFSTestUtil.createFile(fs, fileName, 512, (short) 2, 0); + DFSTestUtil.waitReplication(fs, fileName, (short) 2); + assertTrue("File not created", fs.exists(fileName)); + cluster.getDataNodes().get(1).shutdown(); + DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock"); + cluster.restartDataNode(1, true); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return ( + cluster.getNameNode(0).getNamesystem().getCorruptReplicaBlocks() + > 0); + } + }, 100, 5000); + + DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock"); + runFsck(cluster.getConfiguration(0), 0, true, "/"); + }finally { + if(cluster!=null){ + cluster.shutdown(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java index 92ea111495ebe..6a8d39fa53ef3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java @@ -89,7 +89,7 @@ public void testListCorruptFilesCorruptedBlock() throws Exception { File storageDir = cluster.getInstanceStorageDir(0, 1); File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); assertTrue("data directory does not exist", data_dir.exists()); - List metaFiles = MiniDFSCluster.getAllBlockMetadataFiles(data_dir); + List metaFiles = MiniDFSCluster.getAllBlockFiles(data_dir); assertTrue("Data directory does not contain any blocks or there was an " + "IO error", metaFiles != null && !metaFiles.isEmpty()); File metaFile = metaFiles.get(0); @@ -169,7 +169,7 @@ public void testListCorruptFileBlocksInSafeMode() throws Exception { File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, cluster.getNamesystem().getBlockPoolId()); assertTrue("data directory does not exist", data_dir.exists()); - List metaFiles = MiniDFSCluster.getAllBlockMetadataFiles(data_dir); + List metaFiles = MiniDFSCluster.getAllBlockFiles(data_dir); assertTrue("Data directory does not contain any blocks or there was an " + "IO error", metaFiles != null && !metaFiles.isEmpty()); File metaFile = metaFiles.get(0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java index 5aa19bba35955..c43c909c98a1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.URI; import java.util.Iterator; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -37,7 +38,11 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.junit.Assert; import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + import static org.mockito.Mockito.*; /** @@ -180,8 +185,25 @@ public void testEditsForCancelOnTokenExpire() throws IOException, Text renewer = new Text(UserGroupInformation.getCurrentUser().getUserName()); FSImage fsImage = mock(FSImage.class); FSEditLog log = mock(FSEditLog.class); - doReturn(log).when(fsImage).getEditLog(); + doReturn(log).when(fsImage).getEditLog(); + // verify that the namesystem read lock is held while logging token + // expirations. the namesystem is not updated, so write lock is not + // necessary, but the lock is required because edit log rolling is not + // thread-safe. + final AtomicReference fsnRef = new AtomicReference<>(); + doAnswer( + new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + // fsn claims read lock if either read or write locked. + Assert.assertTrue(fsnRef.get().hasReadLock()); + Assert.assertFalse(fsnRef.get().hasWriteLock()); + return null; + } + } + ).when(log).logCancelDelegationToken(any(DelegationTokenIdentifier.class)); FSNamesystem fsn = new FSNamesystem(conf, fsImage); + fsnRef.set(fsn); DelegationTokenSecretManager dtsm = fsn.getDelegationTokenSecretManager(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java index 1c270aa72e3b5..6c969912dc6e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; @@ -735,7 +736,13 @@ void invoke() throws Exception { DatanodeInfo[] newNodes = new DatanodeInfo[2]; newNodes[0] = nodes[0]; newNodes[1] = nodes[1]; - String[] storageIDs = {"s0", "s1"}; + final DatanodeManager dm = cluster.getNamesystem(0).getBlockManager() + .getDatanodeManager(); + final String storageID1 = dm.getDatanode(newNodes[0]).getStorageInfos()[0] + .getStorageID(); + final String storageID2 = dm.getDatanode(newNodes[1]).getStorageInfos()[0] + .getStorageID(); + String[] storageIDs = {storageID1, storageID2}; client.getNamenode().updatePipeline(client.getClientName(), oldBlock, newBlock, newNodes, storageIDs); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index ad4c17167cba9..8665834c6a3ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -24,10 +24,15 @@ import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertTrue; +import com.google.common.base.Joiner; import java.io.DataInputStream; +import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Random; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; @@ -46,6 +51,8 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger; @@ -84,6 +91,13 @@ public class TestNameNodeMetrics { DFS_REPLICATION_INTERVAL); CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, DFS_REPLICATION_INTERVAL); + // Set it long enough to essentially disable unless we manually call it + // Used for decommissioning DataNode metrics + CONF.setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY, + 9999999); + // For checking failed volume metrics + CONF.setInt(DFSConfigKeys.DFS_DF_INTERVAL_KEY, 1000); + CONF.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1); CONF.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + PERCENTILES_INTERVAL); // Enable stale DataNodes checking @@ -97,6 +111,8 @@ public class TestNameNodeMetrics { private final Random rand = new Random(); private FSNamesystem namesystem; private BlockManager bm; + // List of temporary files on local FileSystem to be cleaned up + private List tempFiles; private static Path getTestPath(String fileName) { return new Path(TEST_ROOT_DIR_PATH, fileName); @@ -109,6 +125,7 @@ public void setUp() throws Exception { namesystem = cluster.getNamesystem(); bm = namesystem.getBlockManager(); fs = cluster.getFileSystem(); + tempFiles = new ArrayList<>(); } @After @@ -120,6 +137,9 @@ public void tearDown() throws Exception { assertQuantileGauges("GetGroups1s", rb); } cluster.shutdown(); + for (Path p : tempFiles) { + FileUtils.deleteQuietly(new File(p.toUri().getPath())); + } } /** create a file with a length of fileLen */ @@ -196,7 +216,113 @@ public void testStaleNodes() throws Exception { .getBlockManager()); assertGauge("StaleDataNodes", 0, getMetrics(NS_METRICS)); } - + + /** + * Test metrics associated with volume failures. + */ + @Test + public void testVolumeFailures() throws Exception { + assertGauge("VolumeFailuresTotal", 0, getMetrics(NS_METRICS)); + assertGauge("EstimatedCapacityLostTotal", 0L, getMetrics(NS_METRICS)); + DataNode dn = cluster.getDataNodes().get(0); + FsVolumeSpi fsVolume = + DataNodeTestUtils.getFSDataset(dn).getVolumes().get(0); + File dataDir = new File(fsVolume.getBasePath()); + long capacity = ((FsVolumeImpl) fsVolume).getCapacity(); + DataNodeTestUtils.injectDataDirFailure(dataDir); + long lastDiskErrorCheck = dn.getLastDiskErrorCheck(); + dn.checkDiskErrorAsync(); + while (dn.getLastDiskErrorCheck() == lastDiskErrorCheck) { + Thread.sleep(100); + } + DataNodeTestUtils.triggerHeartbeat(dn); + BlockManagerTestUtil.checkHeartbeat(bm); + assertGauge("VolumeFailuresTotal", 1, getMetrics(NS_METRICS)); + assertGauge("EstimatedCapacityLostTotal", capacity, getMetrics(NS_METRICS)); + } + + /** + * Test metrics associated with liveness and decommission status of DataNodes. + */ + @Test + public void testDataNodeLivenessAndDecom() throws Exception { + Path hostFileDir = new Path(MiniDFSCluster.getBaseDirectory(), "hosts"); + FileSystem localFs = FileSystem.getLocal(CONF); + localFs.mkdirs(hostFileDir); + Path includeFile = new Path(hostFileDir, "include"); + Path excludeFile = new Path(hostFileDir, "exclude"); + tempFiles.add(includeFile); + tempFiles.add(excludeFile); + CONF.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath()); + CONF.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); + + List dataNodes = cluster.getDataNodes(); + DatanodeDescriptor[] dnDescriptors = new DatanodeDescriptor[DATANODE_COUNT]; + String[] dnAddresses = new String[DATANODE_COUNT]; + for (int i = 0; i < DATANODE_COUNT; i++) { + dnDescriptors[i] = bm.getDatanodeManager() + .getDatanode(dataNodes.get(i).getDatanodeId()); + dnAddresses[i] = dnDescriptors[i].getXferAddr(); + } + // First put all DNs into include + DFSTestUtil.writeFile(localFs, includeFile, + Joiner.on("\n").join(dnAddresses)); + DFSTestUtil.writeFile(localFs, excludeFile, ""); + bm.getDatanodeManager().refreshNodes(CONF); + assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", DATANODE_COUNT, getMetrics(NS_METRICS)); + + // Now decommission one DN + DFSTestUtil.writeFile(localFs, excludeFile, dnAddresses[0]); + bm.getDatanodeManager().refreshNodes(CONF); + assertGauge("NumDecommissioningDataNodes", 1, getMetrics(NS_METRICS)); + BlockManagerTestUtil.recheckDecommissionState(bm.getDatanodeManager()); + assertGauge("NumDecommissioningDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDecomLiveDataNodes", 1, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", DATANODE_COUNT, getMetrics(NS_METRICS)); + + // Now kill all DNs by expiring their heartbeats + for (int i = 0; i < DATANODE_COUNT; i++) { + DataNodeTestUtils.setHeartbeatsDisabledForTests(dataNodes.get(i), true); + long expireInterval = CONF.getLong( + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT) * 2L + + CONF.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, + DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 10 * 1000L; + DFSTestUtil.resetLastUpdatesWithOffset(dnDescriptors[i], + -(expireInterval + 1)); + } + BlockManagerTestUtil.checkHeartbeat(bm); + assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDecomDeadDataNodes", 1, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDeadDataNodes", DATANODE_COUNT, getMetrics(NS_METRICS)); + + // Now remove the decommissioned DN altogether + String[] includeHosts = new String[dnAddresses.length - 1]; + for (int i = 0; i < includeHosts.length; i++) { + includeHosts[i] = dnAddresses[i + 1]; + } + DFSTestUtil.writeFile(localFs, includeFile, + Joiner.on("\n").join(includeHosts)); + // Just init to a nonexistent host to clear out the previous exclusion + DFSTestUtil.writeFile(localFs, excludeFile, ""); + bm.getDatanodeManager().refreshNodes(CONF); + assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDecomDeadDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDeadDataNodes", DATANODE_COUNT - 1, getMetrics(NS_METRICS)); + + // Finally mark the remaining DNs as live again + for (int i = 1; i < dataNodes.size(); i++) { + DataNodeTestUtils.setHeartbeatsDisabledForTests(dataNodes.get(i), false); + DFSTestUtil.resetLastUpdatesWithOffset(dnDescriptors[i], 0); + } + BlockManagerTestUtil.checkHeartbeat(bm); + assertGauge("NumLiveDataNodes", DATANODE_COUNT - 1, getMetrics(NS_METRICS)); + assertGauge("NumDeadDataNodes", 0, getMetrics(NS_METRICS)); + } + /** Test metrics associated with addition of a file */ @Test public void testFileAdd() throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java new file mode 100644 index 0000000000000..4d3a4f030e2a0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.metrics; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.namenode.top.TopConf; +import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.Interns; +import org.junit.Test; + +import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Test for MetricsSource part of the {@link TopMetrics} impl. + */ +public class TestTopMetrics { + @Test + public void testPresence() { + Configuration conf = new Configuration(); + TopConf topConf = new TopConf(conf); + TopMetrics topMetrics = new TopMetrics(conf, + topConf.nntopReportingPeriodsMs); + // Dummy command + topMetrics.report("test", "listStatus"); + topMetrics.report("test", "listStatus"); + topMetrics.report("test", "listStatus"); + + MetricsRecordBuilder rb = getMetrics(topMetrics); + MetricsCollector mc = rb.parent(); + + verify(mc).addRecord(TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=60000"); + verify(mc).addRecord(TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=300000"); + verify(mc).addRecord(TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=1500000"); + + verify(rb, times(3)).addCounter(Interns.info("op=listStatus.TotalCount", + "Total operation count"), 3L); + verify(rb, times(3)).addCounter(Interns.info("op=*.TotalCount", + "Total operation count"), 3L); + + verify(rb, times(3)).addCounter(Interns.info("op=listStatus." + + "user=test.count", "Total operations performed by user"), 3L); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java index fc6c610801d36..658d0763907f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java @@ -25,8 +25,10 @@ import java.io.PrintStream; import java.security.PrivilegedAction; +import org.apache.commons.lang.math.RandomUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -38,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -60,11 +63,15 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Tests snapshot deletion. */ public class TestSnapshotDeletion { + private static final Logger LOG = + LoggerFactory.getLogger(TestSnapshotDeletion.class); protected static final long seed = 0; protected static final short REPLICATION = 3; protected static final short REPLICATION_1 = 2; @@ -1217,4 +1224,122 @@ public void testRenameAndDelete() throws IOException { // make sure bar has been cleaned from inodeMap Assert.assertNull(fsdir.getInode(fileId)); } + + @Test + public void testSnapshotWithConcatException() throws Exception { + final Path st = new Path("/st"); + hdfs.mkdirs(st); + hdfs.allowSnapshot(st); + + Path[] files = new Path[3]; + for (int i = 0; i < 3; i++) { + files[i] = new Path(st, i+ ".txt"); + } + + Path dest = new Path(st, "dest.txt"); + hdfs.createNewFile(dest); + hdfs.createSnapshot(st, "ss"); + + for (int j = 0; j < 3; j++) { + FileSystem fs = cluster.getFileSystem(); + DFSTestUtil.createFile(fs, files[j], false, 1024, + 1024, 512, (short) 1, RandomUtils.nextLong(), true); + } + + hdfs.createSnapshot(st, "s0"); + + // Verify the SnapshotException is thrown as expected for HDFS-4529 + exception.expect(RemoteException.class); + String error = "Concat: the source file /st/0.txt is in snapshot"; + exception.expectMessage(error); + hdfs.concat(dest, files); + + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.restartNameNodes(); + } + + @Test + public void testSnapshotDeleteWithConcat() throws Exception { + final Path st = new Path("/st"); + hdfs.mkdirs(st); + hdfs.allowSnapshot(st); + + Path[] files = new Path[3]; + for (int i = 0; i < 3; i++) { + files[i] = new Path(st, i+ ".txt"); + } + + Path dest = new Path(st, "dest.txt"); + hdfs.createNewFile(dest); + hdfs.createSnapshot(st, "ss"); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + FileSystem fs = cluster.getFileSystem(); + DFSTestUtil.createFile(fs, files[j], false, 1024, + 1024, 512, (short) 1, RandomUtils.nextLong(), true); + } + + hdfs.concat(dest, files); + + hdfs.createSnapshot(st, "s" + i); + } + + + hdfs.deleteSnapshot(st, "s1"); + + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.restartNameNodes(); + } + + @Test + public void testSnapshotDiffReportWithConcat() throws Exception { + final Path st = new Path("/st"); + hdfs.mkdirs(st); + hdfs.allowSnapshot(st); + + Path[] files = new Path[3]; + for (int i = 0; i < 3; i++) { + files[i] = new Path(st, i+ ".txt"); + } + + Path dest = new Path(st, "dest.txt"); + hdfs.createNewFile(dest); + hdfs.createSnapshot(st, "ss"); + + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + FileSystem fs = cluster.getFileSystem(); + DFSTestUtil.createFile(fs, files[j], false, 1024, + 1024, 512, (short) 1, RandomUtils.nextLong(), true); + } + + hdfs.concat(dest, files); + + hdfs.createSnapshot(st, "s" + i); + + SnapshotDiffReport sdr = hdfs.getSnapshotDiffReport(st, "s" + i, "ss"); + LOG.info("Snapshot Diff s{} to ss : {}", i, sdr); + Assert.assertEquals(sdr.getDiffList().size(), 1); + Assert.assertTrue(sdr.getDiffList().get(0).getType() == + SnapshotDiffReport.DiffType.MODIFY); + Assert.assertTrue(new Path(st, DFSUtil.bytes2String( + sdr.getDiffList().get(0).getSourcePath())).equals(dest)); + } + + hdfs.deleteSnapshot(st, "s1"); + + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + hdfs.saveNamespace(); + hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + cluster.restartNameNodes(); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 04eecf2f411b2..3c2ddc895f4f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -207,9 +207,11 @@ private void copyPartOfFile(File src, File dest) throws IOException { in = new FileInputStream(src); out = new FileOutputStream(dest); in.getChannel().transferTo(0, MAX_BYTES, out.getChannel()); + out.close(); + out = null; } finally { - IOUtils.cleanup(null, in); - IOUtils.cleanup(null, out); + IOUtils.closeStream(in); + IOUtils.closeStream(out); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java index 11deab8f8de81..40f2b9c15b45c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestByteRangeInputStream.java @@ -35,7 +35,9 @@ import java.net.URL; import com.google.common.net.HttpHeaders; +import org.apache.hadoop.hdfs.web.ByteRangeInputStream.InputStreamAndFileLength; import org.junit.Test; +import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; public class TestByteRangeInputStream { @@ -140,8 +142,9 @@ public void testByteRange() throws IOException { public void testPropagatedClose() throws IOException { ByteRangeInputStream bris = mock(ByteRangeInputStream.class, CALLS_REAL_METHODS); - InputStream mockStream = mock(InputStream.class); - doReturn(mockStream).when(bris).openInputStream(); + InputStreamAndFileLength mockStream = new InputStreamAndFileLength(1L, + mock(InputStream.class)); + doReturn(mockStream).when(bris).openInputStream(Mockito.anyLong()); Whitebox.setInternalState(bris, "status", ByteRangeInputStream.StreamStatus.SEEK); @@ -151,46 +154,46 @@ public void testPropagatedClose() throws IOException { // first open, shouldn't close underlying stream bris.getInputStream(); - verify(bris, times(++brisOpens)).openInputStream(); + verify(bris, times(++brisOpens)).openInputStream(Mockito.anyLong()); verify(bris, times(brisCloses)).close(); - verify(mockStream, times(isCloses)).close(); + verify(mockStream.in, times(isCloses)).close(); // stream is open, shouldn't close underlying stream bris.getInputStream(); - verify(bris, times(brisOpens)).openInputStream(); + verify(bris, times(brisOpens)).openInputStream(Mockito.anyLong()); verify(bris, times(brisCloses)).close(); - verify(mockStream, times(isCloses)).close(); + verify(mockStream.in, times(isCloses)).close(); // seek forces a reopen, should close underlying stream bris.seek(1); bris.getInputStream(); - verify(bris, times(++brisOpens)).openInputStream(); + verify(bris, times(++brisOpens)).openInputStream(Mockito.anyLong()); verify(bris, times(brisCloses)).close(); - verify(mockStream, times(++isCloses)).close(); + verify(mockStream.in, times(++isCloses)).close(); // verify that the underlying stream isn't closed after a seek // ie. the state was correctly updated bris.getInputStream(); - verify(bris, times(brisOpens)).openInputStream(); + verify(bris, times(brisOpens)).openInputStream(Mockito.anyLong()); verify(bris, times(brisCloses)).close(); - verify(mockStream, times(isCloses)).close(); + verify(mockStream.in, times(isCloses)).close(); // seeking to same location should be a no-op bris.seek(1); bris.getInputStream(); - verify(bris, times(brisOpens)).openInputStream(); + verify(bris, times(brisOpens)).openInputStream(Mockito.anyLong()); verify(bris, times(brisCloses)).close(); - verify(mockStream, times(isCloses)).close(); + verify(mockStream.in, times(isCloses)).close(); // close should of course close bris.close(); verify(bris, times(++brisCloses)).close(); - verify(mockStream, times(++isCloses)).close(); + verify(mockStream.in, times(++isCloses)).close(); // it's already closed, underlying stream should not close bris.close(); verify(bris, times(++brisCloses)).close(); - verify(mockStream, times(isCloses)).close(); + verify(mockStream.in, times(isCloses)).close(); // it's closed, don't reopen it boolean errored = false; @@ -202,9 +205,9 @@ public void testPropagatedClose() throws IOException { } finally { assertTrue("Read a closed steam", errored); } - verify(bris, times(brisOpens)).openInputStream(); + verify(bris, times(brisOpens)).openInputStream(Mockito.anyLong()); verify(bris, times(brisCloses)).close(); - verify(mockStream, times(isCloses)).close(); + verify(mockStream.in, times(isCloses)).close(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index f679f14312a85..a87f4c7697878 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -591,6 +591,45 @@ public void testWebHdfsOffsetAndLength() throws Exception{ } } + @Test + public void testWebHdfsPread() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .build(); + byte[] content = new byte[1024]; + RANDOM.nextBytes(content); + final Path foo = new Path("/foo"); + FSDataInputStream in = null; + try { + final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsFileSystem.SCHEME); + try (OutputStream os = fs.create(foo)) { + os.write(content); + } + + // pread + in = fs.open(foo, 1024); + byte[] buf = new byte[1024]; + try { + in.readFully(1020, buf, 0, 5); + Assert.fail("EOF expected"); + } catch (EOFException ignored) {} + + // mix pread with stateful read + int length = in.read(buf, 0, 512); + in.readFully(100, new byte[1024], 0, 100); + int preadLen = in.read(200, new byte[1024], 0, 200); + Assert.assertTrue(preadLen > 0); + IOUtils.readFully(in, buf, length, 1024 - length); + Assert.assertArrayEquals(content, buf); + } finally { + if (in != null) { + in.close(); + } + cluster.shutdown(); + } + } + @Test(timeout=90000) public void testWebHdfsReadRetries() throws Exception { // ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java index ca67245371b36..a924de74f4e07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java @@ -149,9 +149,13 @@ public void testGroupMappingRefresh() throws Exception { @Test public void testRefreshSuperUserGroupsConfiguration() throws Exception { final String SUPER_USER = "super_user"; - final String [] GROUP_NAMES1 = new String [] {"gr1" , "gr2"}; - final String [] GROUP_NAMES2 = new String [] {"gr3" , "gr4"}; - + final List groupNames1 = new ArrayList<>(); + groupNames1.add("gr1"); + groupNames1.add("gr2"); + final List groupNames2 = new ArrayList<>(); + groupNames2.add("gr3"); + groupNames2.add("gr4"); + //keys in conf String userKeyGroups = DefaultImpersonationProvider.getTestProvider(). getProxySuperuserGroupConfKey(SUPER_USER); @@ -176,12 +180,12 @@ public void testRefreshSuperUserGroupsConfiguration() throws Exception { when(ugi1.getUserName()).thenReturn("userL1"); when(ugi2.getUserName()).thenReturn("userL2"); - + // set groups for users - when(ugi1.getGroupNames()).thenReturn(GROUP_NAMES1); - when(ugi2.getGroupNames()).thenReturn(GROUP_NAMES2); - - + when(ugi1.getGroups()).thenReturn(groupNames1); + when(ugi2.getGroups()).thenReturn(groupNames2); + + // check before try { ProxyUsers.authorize(ugi1, "127.0.0.1"); diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 06c150f768471..0cc0ec4d1f237 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-hdfs-project - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop HDFS Project Apache Hadoop HDFS Project pom diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 40e13f1509a5e..c6e912b98cb5d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1,6 +1,74 @@ Hadoop MapReduce Change Log -Release 2.7.4 - UNRELEASED +Release 2.7.7 - 2018-07-18 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + +Release 2.7.6 - 2018-04-16 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + MAPREDUCE-5124. AM lacks flow control for task events. (Peter Bacsko via + jlowe) + + MAPREDUCE-7028. Concurrent task progress updates causing NPE in + Application Master. (Gergo Repas via jlowe) + + MAPREDUCE-7020. Task timeout in uber mode can crash AM. (Peter Bacsko + via jlowe) + + MAPREDUCE-7048. Uber AM can crash due to unknown task in statusUpdate. + (Peter Bacsko via jlowe) + + MAPREDUCE-7052. TestFixedLengthInputFormat#testFormatCompressedIn is + flaky. (Peter Bacsko via jlowe) + +Release 2.7.5 - 2017-12-14 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + MAPREDUCE-6937. Backport MAPREDUCE-6870 to branch-2 while preserving + compatibility. (Peter Bacsko via Haibo Chen) + + MAPREDUCE-6975. Logging task counters. (Prabhu Joseph via Naganarasimha) + + OPTIMIZATIONS + + BUG FIXES + + MAPREDUCE-6931. Remove TestDFSIO "Total Throughput" calculation. + (Dennis Huo via shv) + + MAPREDUCE-6750. Fix TestHSAdminServer#testRefreshSuperUserGroups + (Kihwal Lee via Varun Saxena) + + MAPREDUCE-6957. shuffle hangs after a node manager connection timeout. + (Jooseong Kim via jlowe) + + MAPREDUCE-6165. [JDK8] TestCombineFileInputFormat failed on JDK8. + Contributed by Akira AJISAKA. + +Release 2.7.4 - 2017-08-04 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index 640a323d82815..ee96274592661 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -19,12 +19,12 @@ hadoop-mapreduce-client org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-mapreduce-client-app - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client-app diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index 6627604989d1f..5f882ed7e492b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -22,9 +22,11 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,6 +39,7 @@ import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener; import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler; @@ -55,6 +58,8 @@ import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import com.google.common.annotations.VisibleForTesting; + /** * This class is responsible for talking to the task umblical. * It also converts all the old data structures @@ -80,6 +85,11 @@ public class TaskAttemptListenerImpl extends CompositeService private ConcurrentMap jvmIDToActiveAttemptMap = new ConcurrentHashMap(); + + private ConcurrentMap> attemptIdToStatus + = new ConcurrentHashMap<>(); + private Set launchedJVMs = Collections .newSetFromMap(new ConcurrentHashMap()); @@ -329,6 +339,15 @@ public boolean statusUpdate(TaskAttemptID taskAttemptID, TaskStatus taskStatus) throws IOException, InterruptedException { org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId yarnAttemptID = TypeConverter.toYarn(taskAttemptID); + + AtomicReference lastStatusRef = + attemptIdToStatus.get(yarnAttemptID); + if (lastStatusRef == null) { + LOG.error("Status update was called with illegal TaskAttemptId: " + + yarnAttemptID); + return false; + } + taskHeartbeatHandler.progressing(yarnAttemptID); TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus(); @@ -386,9 +405,8 @@ public boolean statusUpdate(TaskAttemptID taskAttemptID, // // isn't ever changed by the Task itself. // taskStatus.getIncludeCounters(); - context.getEventHandler().handle( - new TaskAttemptStatusUpdateEvent(taskAttemptStatus.id, - taskAttemptStatus)); + coalesceStatusUpdate(yarnAttemptID, taskAttemptStatus, lastStatusRef); + return true; } @@ -469,6 +487,9 @@ public void registerLaunchedTask( launchedJVMs.add(jvmId); taskHeartbeatHandler.register(attemptID); + + attemptIdToStatus.put(attemptID, + new AtomicReference()); } @Override @@ -490,6 +511,8 @@ public void unregister( //unregister this attempt taskHeartbeatHandler.unregister(attemptID); + + attemptIdToStatus.remove(attemptID); } @Override @@ -498,4 +521,52 @@ public ProtocolSignature getProtocolSignature(String protocol, return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion, clientMethodsHash); } + + private void coalesceStatusUpdate(TaskAttemptId yarnAttemptID, + TaskAttemptStatus taskAttemptStatus, + AtomicReference lastStatusRef) { + List fetchFailedMaps = taskAttemptStatus.fetchFailedMaps; + TaskAttemptStatus lastStatus = null; + boolean done = false; + while (!done) { + lastStatus = lastStatusRef.get(); + if (lastStatus != null && lastStatus.fetchFailedMaps != null) { + // merge fetchFailedMaps from the previous update + if (taskAttemptStatus.fetchFailedMaps == null) { + taskAttemptStatus.fetchFailedMaps = lastStatus.fetchFailedMaps; + } else { + taskAttemptStatus.fetchFailedMaps = + new ArrayList<>(lastStatus.fetchFailedMaps.size() + + fetchFailedMaps.size()); + taskAttemptStatus.fetchFailedMaps.addAll( + lastStatus.fetchFailedMaps); + taskAttemptStatus.fetchFailedMaps.addAll( + fetchFailedMaps); + } + } + + // lastStatusRef may be changed by either the AsyncDispatcher when + // it processes the update, or by another IPC server handler + done = lastStatusRef.compareAndSet(lastStatus, taskAttemptStatus); + if (!done) { + LOG.info("TaskAttempt " + yarnAttemptID + + ": lastStatusRef changed by another thread, retrying..."); + // let's revert taskAttemptStatus.fetchFailedMaps + taskAttemptStatus.fetchFailedMaps = fetchFailedMaps; + } + } + + boolean asyncUpdatedNeeded = (lastStatus == null); + if (asyncUpdatedNeeded) { + context.getEventHandler().handle( + new TaskAttemptStatusUpdateEvent(taskAttemptStatus.id, + lastStatusRef)); + } + } + + @VisibleForTesting + ConcurrentMap> getAttemptIdToStatus() { + return attemptIdToStatus; + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java index 715f63d1be556..cef4fd05093e9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.v2.app.job.event; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.v2.api.records.Phase; @@ -26,17 +27,16 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; public class TaskAttemptStatusUpdateEvent extends TaskAttemptEvent { - - private TaskAttemptStatus reportedTaskAttemptStatus; + private AtomicReference taskAttemptStatusRef; public TaskAttemptStatusUpdateEvent(TaskAttemptId id, - TaskAttemptStatus taskAttemptStatus) { + AtomicReference taskAttemptStatusRef) { super(id, TaskAttemptEventType.TA_UPDATE); - this.reportedTaskAttemptStatus = taskAttemptStatus; + this.taskAttemptStatusRef = taskAttemptStatusRef; } - public TaskAttemptStatus getReportedTaskAttemptStatus() { - return reportedTaskAttemptStatus; + public AtomicReference getTaskAttemptStatusRef() { + return taskAttemptStatusRef; } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 4553f10380d18..2476e7ae11f47 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -643,6 +643,11 @@ JobEventType.JOB_KILL, new KillTasksTransition()) private float reduceProgress; private float cleanupProgress; private boolean isUber = false; + /** Whether the job should be finished when all reducers are completed, + regardless of having running mappers */ + private boolean finishJobWhenReducersDone; + /** True if the job's mappers were already sent the T_KILL event */ + private boolean completingJob = false; private Credentials jobCredentials; private Token jobToken; @@ -714,6 +719,9 @@ public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId, this.maxFetchFailuresNotifications = conf.getInt( MRJobConfig.MAX_FETCH_FAILURES_NOTIFICATIONS, MRJobConfig.DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS); + this.finishJobWhenReducersDone = conf.getBoolean( + MRJobConfig.FINISH_JOB_WHEN_REDUCERS_DONE, + MRJobConfig.DEFAULT_FINISH_JOB_WHEN_REDUCERS_DONE); } protected StateMachine getStateMachine() { @@ -2014,7 +2022,9 @@ protected JobStateInternal checkJobAfterTaskCompletion(JobImpl job) { TimeUnit.MILLISECONDS); return JobStateInternal.FAIL_WAIT; } - + + checkReadyForCompletionWhenAllReducersDone(job); + return job.checkReadyForCommit(); } @@ -2045,6 +2055,34 @@ private void taskKilled(JobImpl job, Task task) { } job.metrics.killedTask(task); } + + /** Improvement: if all reducers have finished, we check if we have + restarted mappers that are still running. This can happen in a + situation when a node becomes UNHEALTHY and mappers are rescheduled. + See MAPREDUCE-6870 for details + @param job The MapReduce job + */ + private void checkReadyForCompletionWhenAllReducersDone(final JobImpl job) { + if (job.finishJobWhenReducersDone) { + int totalReduces = job.getTotalReduces(); + int completedReduces = job.getCompletedReduces(); + + if (totalReduces > 0 && totalReduces == completedReduces + && !job.completingJob) { + + for (TaskId mapTaskId : job.mapTasks) { + MapTaskImpl task = (MapTaskImpl) job.tasks.get(mapTaskId); + if (!task.isFinished()) { + LOG.info("Killing map task " + task.getID()); + job.eventHandler.handle( + new TaskEvent(task.getID(), TaskEventType.T_KILL)); + } + } + + job.completingJob = true; + } + } + } } // Transition class for handling jobs with no tasks diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 813010d045405..5a7e545dac43b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -33,6 +33,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -1662,6 +1663,7 @@ public void transition(TaskAttemptImpl taskAttempt, // register it to TaskAttemptListener so that it can start monitoring it. taskAttempt.taskAttemptListener .registerLaunchedTask(taskAttempt.attemptId, taskAttempt.jvmID); + //TODO Resolve to host / IP in case of a local address. InetSocketAddress nodeHttpInetAddr = // TODO: Costly to create sock-addr? NetUtils.createSocketAddr(taskAttempt.container.getNodeHttpAddress()); @@ -1957,15 +1959,20 @@ private void addDiagnosticInfo(String diag) { } private static class StatusUpdater - implements SingleArcTransition { + implements SingleArcTransition { @SuppressWarnings("unchecked") @Override public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { - // Status update calls don't really change the state of the attempt. + TaskAttemptStatusUpdateEvent statusEvent = + ((TaskAttemptStatusUpdateEvent)event); + + AtomicReference taskAttemptStatusRef = + statusEvent.getTaskAttemptStatusRef(); + TaskAttemptStatus newReportedStatus = - ((TaskAttemptStatusUpdateEvent) event) - .getReportedTaskAttemptStatus(); + taskAttemptStatusRef.getAndSet(null); + // Now switch the information in the reportedStatus taskAttempt.reportedStatus = newReportedStatus; taskAttempt.reportedStatus.taskState = taskAttempt.getState(); @@ -1974,12 +1981,10 @@ public void transition(TaskAttemptImpl taskAttempt, taskAttempt.eventHandler.handle (new SpeculatorEvent (taskAttempt.reportedStatus, taskAttempt.clock.getTime())); - taskAttempt.updateProgressSplits(); - //if fetch failures are present, send the fetch failure event to job //this only will happen in reduce attempt type - if (taskAttempt.reportedStatus.fetchFailedMaps != null && + if (taskAttempt.reportedStatus.fetchFailedMaps != null && taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) { taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent( taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java index d35d1e9bbb5e1..b2afdce3f7f34 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java @@ -31,14 +31,15 @@ import java.io.IOException; import java.util.Arrays; - -import junit.framework.Assert; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.api.records.Phase; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; @@ -46,15 +47,83 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler; import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent; +import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus; import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.Event; +import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.SystemClock; +import org.junit.After; +import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +/** + * Tests the behavior of TaskAttemptListenerImpl. + */ +@RunWith(MockitoJUnitRunner.class) public class TestTaskAttemptListenerImpl { - public static class MockTaskAttemptListenerImpl extends TaskAttemptListenerImpl { + private static final String ATTEMPT1_ID = + "attempt_123456789012_0001_m_000001_0"; + private static final String ATTEMPT2_ID = + "attempt_123456789012_0001_m_000002_0"; + + private static final TaskAttemptId TASKATTEMPTID1 = + TypeConverter.toYarn(TaskAttemptID.forName(ATTEMPT1_ID)); + private static final TaskAttemptId TASKATTEMPTID2 = + TypeConverter.toYarn(TaskAttemptID.forName(ATTEMPT2_ID)); + + @Mock + private AppContext appCtx; + + @Mock + private JobTokenSecretManager secret; + + @Mock + private RMHeartbeatHandler rmHeartbeatHandler; + + @Mock + private TaskHeartbeatHandler hbHandler; + + @Mock + private Dispatcher dispatcher; + + @Mock + private Task task; + + @SuppressWarnings("rawtypes") + @Mock + private EventHandler ea; + + @SuppressWarnings("rawtypes") + @Captor + private ArgumentCaptor eventCaptor; + + private JVMId id; + private WrappedJvmID wid; + private TaskAttemptID attemptID; + private TaskAttemptId attemptId; + private ReduceTaskStatus firstReduceStatus; + private ReduceTaskStatus secondReduceStatus; + private ReduceTaskStatus thirdReduceStatus; + + private MockTaskAttemptListenerImpl listener; + + /** + * Extension of the original TaskAttemptImpl + * for testing purposes + */ + public static class MockTaskAttemptListenerImpl + extends TaskAttemptListenerImpl { public MockTaskAttemptListenerImpl(AppContext context, JobTokenSecretManager jobTokenSecretManager, @@ -85,26 +154,24 @@ protected void stopRpcServer() { //Empty } } - + + @After + public void after() throws IOException { + if (listener != null) { + listener.close(); + listener = null; + } + } + @Test (timeout=5000) public void testGetTask() throws IOException { - AppContext appCtx = mock(AppContext.class); - JobTokenSecretManager secret = mock(JobTokenSecretManager.class); - RMHeartbeatHandler rmHeartbeatHandler = - mock(RMHeartbeatHandler.class); - TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class); - MockTaskAttemptListenerImpl listener = - new MockTaskAttemptListenerImpl(appCtx, secret, - rmHeartbeatHandler, hbHandler); - Configuration conf = new Configuration(); - listener.init(conf); - listener.start(); - JVMId id = new JVMId("foo",1, true, 1); - WrappedJvmID wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId()); + configureMocks(); + startListener(false); // Verify ask before registration. //The JVM ID has not been registered yet so we should kill it. JvmContext context = new JvmContext(); + context.jvmId = id; JvmTask result = listener.getTask(context); assertNotNull(result); @@ -112,20 +179,18 @@ public void testGetTask() throws IOException { // Verify ask after registration but before launch. // Don't kill, should be null. - TaskAttemptId attemptID = mock(TaskAttemptId.class); - Task task = mock(Task.class); //Now put a task with the ID listener.registerPendingTask(task, wid); result = listener.getTask(context); assertNull(result); // Unregister for more testing. - listener.unregister(attemptID, wid); + listener.unregister(attemptId, wid); // Verify ask after registration and launch //Now put a task with the ID listener.registerPendingTask(task, wid); - listener.registerLaunchedTask(attemptID, wid); - verify(hbHandler).register(attemptID); + listener.registerLaunchedTask(attemptId, wid); + verify(hbHandler).register(attemptId); result = listener.getTask(context); assertNotNull(result); assertFalse(result.shouldDie); @@ -136,15 +201,13 @@ public void testGetTask() throws IOException { assertNotNull(result); assertTrue(result.shouldDie); - listener.unregister(attemptID, wid); + listener.unregister(attemptId, wid); // Verify after unregistration. result = listener.getTask(context); assertNotNull(result); assertTrue(result.shouldDie); - listener.stop(); - // test JVMID JVMId jvmid = JVMId.forName("jvm_001_002_m_004"); assertNotNull(jvmid); @@ -190,14 +253,11 @@ public void testGetMapCompletionEvents() throws IOException { when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn( TypeConverter.fromYarn(empty)); - AppContext appCtx = mock(AppContext.class); + configureMocks(); when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob); - JobTokenSecretManager secret = mock(JobTokenSecretManager.class); - RMHeartbeatHandler rmHeartbeatHandler = - mock(RMHeartbeatHandler.class); - final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class); - TaskAttemptListenerImpl listener = - new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler) { + + listener = new MockTaskAttemptListenerImpl( + appCtx, secret, rmHeartbeatHandler, hbHandler) { @Override protected void registerHeartbeatHandler(Configuration conf) { taskHeartbeatHandler = hbHandler; @@ -238,20 +298,18 @@ private static TaskAttemptCompletionEvent createTce(int eventId, public void testCommitWindow() throws IOException { SystemClock clock = new SystemClock(); + configureMocks(); + org.apache.hadoop.mapreduce.v2.app.job.Task mockTask = mock(org.apache.hadoop.mapreduce.v2.app.job.Task.class); when(mockTask.canCommit(any(TaskAttemptId.class))).thenReturn(true); Job mockJob = mock(Job.class); when(mockJob.getTask(any(TaskId.class))).thenReturn(mockTask); - AppContext appCtx = mock(AppContext.class); when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob); when(appCtx.getClock()).thenReturn(clock); - JobTokenSecretManager secret = mock(JobTokenSecretManager.class); - RMHeartbeatHandler rmHeartbeatHandler = - mock(RMHeartbeatHandler.class); - final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class); - TaskAttemptListenerImpl listener = - new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler) { + + listener = new MockTaskAttemptListenerImpl( + appCtx, secret, rmHeartbeatHandler, hbHandler) { @Override protected void registerHeartbeatHandler(Configuration conf) { taskHeartbeatHandler = hbHandler; @@ -269,11 +327,121 @@ protected void registerHeartbeatHandler(Configuration conf) { verify(mockTask, never()).canCommit(any(TaskAttemptId.class)); // verify commit allowed when RM heartbeat is recent - when(rmHeartbeatHandler.getLastHeartbeatTime()).thenReturn(clock.getTime()); + when(rmHeartbeatHandler.getLastHeartbeatTime()) + .thenReturn(clock.getTime()); canCommit = listener.canCommit(tid); assertTrue(canCommit); verify(mockTask, times(1)).canCommit(any(TaskAttemptId.class)); + } + + @Test + public void testSingleStatusUpdate() + throws IOException, InterruptedException { + configureMocks(); + startListener(true); - listener.stop(); + listener.statusUpdate(attemptID, firstReduceStatus); + + verify(ea).handle(eventCaptor.capture()); + TaskAttemptStatusUpdateEvent updateEvent = + (TaskAttemptStatusUpdateEvent) eventCaptor.getValue(); + + TaskAttemptStatus status = updateEvent.getTaskAttemptStatusRef().get(); + assertTrue(status.fetchFailedMaps.contains(TASKATTEMPTID1)); + assertEquals(1, status.fetchFailedMaps.size()); + assertEquals(Phase.SHUFFLE, status.phase); + } + + @Test + public void testStatusUpdateEventCoalescing() + throws IOException, InterruptedException { + configureMocks(); + startListener(true); + + listener.statusUpdate(attemptID, firstReduceStatus); + listener.statusUpdate(attemptID, secondReduceStatus); + + verify(ea).handle(any(Event.class)); + ConcurrentMap> attemptIdToStatus = + listener.getAttemptIdToStatus(); + TaskAttemptStatus status = attemptIdToStatus.get(attemptId).get(); + + assertTrue(status.fetchFailedMaps.contains(TASKATTEMPTID1)); + assertTrue(status.fetchFailedMaps.contains(TASKATTEMPTID2)); + assertEquals(2, status.fetchFailedMaps.size()); + assertEquals(Phase.SORT, status.phase); + } + + @Test + public void testCoalescedStatusUpdatesCleared() + throws IOException, InterruptedException { + // First two events are coalesced, the third is not + configureMocks(); + startListener(true); + + listener.statusUpdate(attemptID, firstReduceStatus); + listener.statusUpdate(attemptID, secondReduceStatus); + ConcurrentMap> attemptIdToStatus = + listener.getAttemptIdToStatus(); + attemptIdToStatus.get(attemptId).set(null); + listener.statusUpdate(attemptID, thirdReduceStatus); + + verify(ea, times(2)).handle(eventCaptor.capture()); + TaskAttemptStatusUpdateEvent updateEvent = + (TaskAttemptStatusUpdateEvent) eventCaptor.getValue(); + + TaskAttemptStatus status = updateEvent.getTaskAttemptStatusRef().get(); + assertNull(status.fetchFailedMaps); + assertEquals(Phase.REDUCE, status.phase); + } + + @Test + public void testStatusUpdateFromUnregisteredTask() + throws IOException, InterruptedException{ + configureMocks(); + startListener(false); + + boolean taskFound = listener.statusUpdate(attemptID, firstReduceStatus); + + assertFalse(taskFound); + } + + private void configureMocks() { + firstReduceStatus = new ReduceTaskStatus(attemptID, 0.0f, 1, + TaskStatus.State.RUNNING, "", "RUNNING", "", TaskStatus.Phase.SHUFFLE, + new Counters()); + firstReduceStatus.addFetchFailedMap(TaskAttemptID.forName(ATTEMPT1_ID)); + + secondReduceStatus = new ReduceTaskStatus(attemptID, 0.0f, 1, + TaskStatus.State.RUNNING, "", "RUNNING", "", TaskStatus.Phase.SORT, + new Counters()); + secondReduceStatus.addFetchFailedMap(TaskAttemptID.forName(ATTEMPT2_ID)); + + thirdReduceStatus = new ReduceTaskStatus(attemptID, 0.0f, 1, + TaskStatus.State.RUNNING, "", "RUNNING", "", + TaskStatus.Phase.REDUCE, new Counters()); + + when(dispatcher.getEventHandler()).thenReturn(ea); + when(appCtx.getEventHandler()).thenReturn(ea); + listener = new MockTaskAttemptListenerImpl(appCtx, secret, + rmHeartbeatHandler, hbHandler); + id = new JVMId("foo", 1, true, 1); + wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId()); + attemptID = new TaskAttemptID("1", 1, TaskType.MAP, 1, 1); + attemptId = TypeConverter.toYarn(attemptID); + } + + private void startListener(boolean registerTask) { + Configuration conf = new Configuration(); + + listener.init(conf); + listener.start(); + + if (registerTask) { + listener.registerPendingTask(task, wid); + listener.registerLaunchedTask(attemptId, wid); + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java index 4e4e2e71f4c66..b50071228fc05 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.TaskCompletionEvent; @@ -425,7 +426,7 @@ private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) { status.stateString = "OK"; status.taskState = attempt.getState(); TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(attempt.getID(), - status); + new AtomicReference<>(status)); app.getContext().getEventHandler().handle(event); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java index 77f9a092a1b24..ca3c28cbaf5be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java @@ -24,6 +24,7 @@ import java.security.PrivilegedExceptionAction; import java.util.Iterator; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Assert; @@ -103,7 +104,8 @@ public void test() throws Exception { taskAttemptStatus.phase = Phase.MAP; // send the status update app.getContext().getEventHandler().handle( - new TaskAttemptStatusUpdateEvent(attempt.getID(), taskAttemptStatus)); + new TaskAttemptStatusUpdateEvent(attempt.getID(), + new AtomicReference<>(taskAttemptStatus))); //verify that all object are fully populated by invoking RPCs. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java index 1a2dcd03d747a..e7f4d799ac57b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java @@ -562,33 +562,13 @@ public void handle(TaskAttemptEvent event) { dispatcher.register(TaskAttemptEventType.class, taskAttemptEventHandler); // replace the tasks with spied versions to return the right attempts - Map spiedTasks = new HashMap(); - List nodeReports = new ArrayList(); - Map nodeReportsToTaskIds = - new HashMap(); - for (Map.Entry e: job.tasks.entrySet()) { - TaskId taskId = e.getKey(); - Task task = e.getValue(); - if (taskId.getTaskType() == TaskType.MAP) { - // add an attempt to the task to simulate nodes - NodeId nodeId = mock(NodeId.class); - TaskAttempt attempt = mock(TaskAttempt.class); - when(attempt.getNodeId()).thenReturn(nodeId); - TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); - when(attempt.getID()).thenReturn(attemptId); - // create a spied task - Task spied = spy(task); - doReturn(attempt).when(spied).getAttempt(any(TaskAttemptId.class)); - spiedTasks.put(taskId, spied); + Map spiedTasks = new HashMap<>(); + List nodeReports = new ArrayList<>(); + Map nodeReportsToTaskIds = new HashMap<>(); + + createSpiedMapTasks(nodeReportsToTaskIds, spiedTasks, job, + NodeState.UNHEALTHY, nodeReports); - // create a NodeReport based on the node id - NodeReport report = mock(NodeReport.class); - when(report.getNodeState()).thenReturn(NodeState.UNHEALTHY); - when(report.getNodeId()).thenReturn(nodeId); - nodeReports.add(report); - nodeReportsToTaskIds.put(report, taskId); - } - } // replace the tasks with the spied tasks job.tasks.putAll(spiedTasks); @@ -638,6 +618,82 @@ public void handle(TaskAttemptEvent event) { commitHandler.stop(); } + @Test + public void testJobNCompletedWhenAllReducersAreFinished() + throws Exception { + testJobCompletionWhenReducersAreFinished(true); + } + + @Test + public void testJobNotCompletedWhenAllReducersAreFinished() + throws Exception { + testJobCompletionWhenReducersAreFinished(false); + } + + private void testJobCompletionWhenReducersAreFinished(boolean killMappers) + throws InterruptedException, BrokenBarrierException { + Configuration conf = new Configuration(); + conf.setBoolean(MRJobConfig.FINISH_JOB_WHEN_REDUCERS_DONE, killMappers); + conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); + conf.setInt(MRJobConfig.NUM_REDUCES, 1); + DrainDispatcher dispatcher = new DrainDispatcher(); + dispatcher.init(conf); + final List killedEvents = + Collections.synchronizedList(new ArrayList()); + dispatcher.register(TaskEventType.class, new EventHandler() { + @Override + public void handle(TaskEvent event) { + if (event.getType() == TaskEventType.T_KILL) { + killedEvents.add(event); + } + } + }); + dispatcher.start(); + CyclicBarrier syncBarrier = new CyclicBarrier(2); + OutputCommitter committer = new TestingOutputCommitter(syncBarrier, true); + CommitterEventHandler commitHandler = + createCommitterEventHandler(dispatcher, committer); + commitHandler.init(conf); + commitHandler.start(); + + final JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null); + + // replace the tasks with spied versions to return the right attempts + Map spiedTasks = new HashMap<>(); + List nodeReports = new ArrayList<>(); + Map nodeReportsToTaskIds = new HashMap<>(); + + createSpiedMapTasks(nodeReportsToTaskIds, spiedTasks, job, + NodeState.RUNNING, nodeReports); + + // replace the tasks with the spied tasks + job.tasks.putAll(spiedTasks); + + // finish reducer + for (TaskId taskId: job.tasks.keySet()) { + if (taskId.getTaskType() == TaskType.REDUCE) { + job.handle(new JobTaskEvent(taskId, TaskState.SUCCEEDED)); + } + } + + dispatcher.await(); + + /* + * StubbedJob cannot finish in this test - we'd have to generate the + * necessary events in this test manually, but that wouldn't add too + * much value. Instead, we validate the T_KILL events. + */ + if (killMappers) { + Assert.assertEquals("Number of killed events", 2, killedEvents.size()); + Assert.assertEquals("AttemptID", "task_1234567890000_0001_m_000000", + killedEvents.get(0).getTaskID().toString()); + Assert.assertEquals("AttemptID", "task_1234567890000_0001_m_000001", + killedEvents.get(1).getTaskID().toString()); + } else { + Assert.assertEquals("Number of killed events", 0, killedEvents.size()); + } + } + public static void main(String[] args) throws Exception { TestJobImpl t = new TestJobImpl(); t.testJobNoTasks(); @@ -985,6 +1041,37 @@ private static void assertJobState(JobImpl job, JobStateInternal state) { Assert.assertEquals(state, job.getInternalState()); } + private void createSpiedMapTasks(Map + nodeReportsToTaskIds, Map spiedTasks, JobImpl job, + NodeState nodeState, List nodeReports) { + for (Map.Entry e: job.tasks.entrySet()) { + TaskId taskId = e.getKey(); + Task task = e.getValue(); + if (taskId.getTaskType() == TaskType.MAP) { + // add an attempt to the task to simulate nodes + NodeId nodeId = mock(NodeId.class); + TaskAttempt attempt = mock(TaskAttempt.class); + when(attempt.getNodeId()).thenReturn(nodeId); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + when(attempt.getID()).thenReturn(attemptId); + // create a spied task + Task spied = spy(task); + Map attemptMap = new HashMap<>(); + attemptMap.put(attemptId, attempt); + when(spied.getAttempts()).thenReturn(attemptMap); + doReturn(attempt).when(spied).getAttempt(any(TaskAttemptId.class)); + spiedTasks.put(taskId, spied); + + // create a NodeReport based on the node id + NodeReport report = mock(NodeReport.class); + when(report.getNodeState()).thenReturn(nodeState); + when(report.getNodeId()).thenReturn(nodeId); + nodeReports.add(report); + nodeReportsToTaskIds.put(report, taskId); + } + } + } + private static class JobSubmittedEventHandler implements EventHandler { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml index d7c4481e17c56..2e033c5db8f77 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml @@ -19,12 +19,12 @@ hadoop-mapreduce-client org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-mapreduce-client-common - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client-common diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index c08b104abf8f6..ab5190575dde0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -19,12 +19,12 @@ hadoop-mapreduce-client org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-mapreduce-client-core - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client-core diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java index e1bd085a525ef..670e558b959c7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java @@ -63,6 +63,7 @@ import org.apache.hadoop.mapreduce.task.ReduceContextImpl; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Progress; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; @@ -190,6 +191,7 @@ static synchronized String getOutputName(int partition) { protected SecretKey tokenSecret; protected SecretKey shuffleSecret; protected GcTimeUpdater gcUpdater; + private boolean uberized = false; //////////////////////////////////////////// // Constructors @@ -736,6 +738,7 @@ public void run() { int remainingRetries = MAX_RETRIES; // get current flag value and reset it as well boolean sendProgress = resetProgressFlag(); + while (!taskDone.get()) { synchronized (lock) { done = false; @@ -770,9 +773,14 @@ public void run() { // if Task Tracker is not aware of our task ID (probably because it died and // came back up), kill ourselves if (!taskFound) { - LOG.warn("Parent died. Exiting "+taskId); - resetDoneFlag(); - System.exit(66); + if (uberized) { + taskDone.set(true); + break; + } else { + LOG.warn("Parent died. Exiting "+taskId); + resetDoneFlag(); + System.exit(66); + } } sendProgress = resetProgressFlag(); @@ -1069,6 +1077,18 @@ public void done(TaskUmbilicalProtocol umbilical, sendLastUpdate(umbilical); //signal the tasktracker that we are done sendDone(umbilical); + LOG.info("Final Counters for " + taskId + ": " + + getCounters().toString()); + /** + * File System Counters + * FILE: Number of bytes read=0 + * FILE: Number of bytes written=146972 + * ... + * Map-Reduce Framework + * Map output records=6 + * Map output records=6 + * ... + */ } /** @@ -1095,11 +1115,17 @@ boolean isCommitRequired() throws IOException { public void statusUpdate(TaskUmbilicalProtocol umbilical) throws IOException { int retries = MAX_RETRIES; + while (true) { try { if (!umbilical.statusUpdate(getTaskID(), taskStatus)) { - LOG.warn("Parent died. Exiting "+taskId); - System.exit(66); + if (uberized) { + LOG.warn("Task no longer available: " + taskId); + break; + } else { + LOG.warn("Parent died. Exiting " + taskId); + ExitUtil.terminate(66); + } } taskStatus.clearStatus(); return; @@ -1312,6 +1338,8 @@ public void setConf(Configuration conf) { NetUtils.addStaticResolution(name, resolvedName); } } + + uberized = conf.getBoolean("mapreduce.task.uberized", false); } public Configuration getConf() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 09e1002d192b5..6636838acd0bc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -387,7 +387,7 @@ public interface MRJobConfig { public static final String JOB_ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job"; public static final String DEFAULT_JOB_ACL_MODIFY_JOB = " "; - + public static final String JOB_RUNNING_MAP_LIMIT = "mapreduce.job.running.map.limit"; public static final int DEFAULT_JOB_RUNNING_MAP_LIMIT = 0; @@ -920,4 +920,13 @@ public interface MRJobConfig { * A comma-separated list of properties whose value will be redacted. */ String MR_JOB_REDACTED_PROPERTIES = "mapreduce.job.redacted-properties"; + + /** + * Specifies whether the job should complete once all reducers + * have finished, regardless of whether there are still running mappers. + */ + String FINISH_JOB_WHEN_REDUCERS_DONE = + "mapreduce.job.finish-when-all-reducers-done"; + + boolean DEFAULT_FINISH_JOB_WHEN_REDUCERS_DONE = false; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java index 040c54be975f3..b2b7656063718 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java @@ -29,7 +29,6 @@ import java.util.Set; import java.util.Iterator; import java.util.Map; -import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -289,6 +288,26 @@ private void getMoreSplits(JobContext job, List stats, maxSize, minSizeNode, minSizeRack, splits); } + /** + * Process all the nodes and create splits that are local to a node. + * Generate one split per node iteration, and walk over nodes multiple times + * to distribute the splits across nodes. + *

+ * Note: The order of processing the nodes is undetermined because the + * implementation of nodeToBlocks is {@link java.util.HashMap} and its order + * of the entries is undetermined. + * @param nodeToBlocks Mapping from a node to the list of blocks that + * it contains. + * @param blockToNodes Mapping from a block to the nodes on which + * it has replicas. + * @param rackToBlocks Mapping from a rack name to the list of blocks it has. + * @param totLength Total length of the input files. + * @param maxSize Max size of each split. + * If set to 0, disable smoothing load. + * @param minSizeNode Minimum split size per node. + * @param minSizeRack Minimum split size per rack. + * @param splits New splits created by this method are added to the list. + */ @VisibleForTesting void createSplits(Map> nodeToBlocks, Map blockToNodes, @@ -309,11 +328,6 @@ void createSplits(Map> nodeToBlocks, Set completedNodes = new HashSet(); while(true) { - // it is allowed for maxSize to be 0. Disable smoothing load for such cases - - // process all nodes and create splits that are local to a node. Generate - // one split per node iteration, and walk over nodes multiple times to - // distribute the splits across nodes. for (Iterator>> iter = nodeToBlocks .entrySet().iterator(); iter.hasNext();) { Map.Entry> one = iter.next(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java index 2e255f884e774..3ed0e4d205ccb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java @@ -281,11 +281,6 @@ private DataInputStream openShuffleUrl(MapHost host, for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } - - // Add back all the remaining maps, WITHOUT marking them as failed - for(TaskAttemptID left: remaining) { - scheduler.putBackKnownMapOutput(host, left); - } } return input; @@ -320,12 +315,14 @@ protected void copyFromHost(MapHost host) throws IOException { // Construct the url and connect URL url = getMapOutputURL(host, maps); - DataInputStream input = openShuffleUrl(host, remaining, url); - if (input == null) { - return; - } + DataInputStream input = null; try { + input = openShuffleUrl(host, remaining, url); + if (input == null) { + return; + } + // Loop through available map-outputs and fetch them // On any error, faildTasks is not null and we exit // after putting back the remaining maps to the diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java index cce36dec499fc..2fd7609658d22 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java @@ -215,6 +215,9 @@ public synchronized void copySucceeded(TaskAttemptID mapId, reduceShuffleBytes.increment(bytes); lastProgressTime = Time.monotonicNow(); LOG.debug("map " + mapId + " done " + status.getStateString()); + } else { + LOG.warn("Aborting already-finished MapOutput for " + mapId); + output.abort(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index d01dd5fce2656..da4710eebdbdc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1461,6 +1461,14 @@ + + mapreduce.job.finish-when-all-reducers-done + false + Specifies whether the job should complete once all reducers + have finished, regardless of whether there are still running mappers. + + + mapreduce.job.token.tracking.ids.enabled false diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java new file mode 100644 index 0000000000000..6bf06017615aa --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTask.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.ExitUtil.ExitException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class TestTask { + @Mock + private TaskUmbilicalProtocol umbilical; + + private Task task; + + @Before + public void setup() { + task = new StubTask(); + ExitUtil.disableSystemExit(); + } + + @Test + public void testStatusUpdateDoesNotExitInUberMode() throws Exception { + setupTest(true); + + task.statusUpdate(umbilical); + } + + @Test(expected = ExitException.class) + public void testStatusUpdateExitsInNonUberMode() throws Exception { + setupTest(false); + + task.statusUpdate(umbilical); + } + + private void setupTest(boolean uberized) + throws IOException, InterruptedException { + Configuration conf = new Configuration(false); + conf.setBoolean("mapreduce.task.uberized", uberized); + task.setConf(conf); + // (false, true) to avoid possible infinite loop + when(umbilical.statusUpdate(any(TaskAttemptID.class), + any(TaskStatus.class))).thenReturn(false, true); + } + + public class StubTask extends Task { + @Override + public void run(JobConf job, TaskUmbilicalProtocol umbilical) + throws IOException, ClassNotFoundException, InterruptedException { + // nop + } + + @Override + public boolean isMapTask() { + return false; + } + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java index f31e160c76898..826256f65f64c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java @@ -455,7 +455,10 @@ public void testCopyFromHostWithRetryThenTimeout() throws Exception { underTest.copyFromHost(host); verify(allErrs).increment(1); - verify(ss).copyFailed(map1ID, host, false, false); + verify(ss, times(1)).copyFailed(map1ID, host, false, false); + verify(ss, times(1)).copyFailed(map2ID, host, false, false); + verify(ss, times(1)).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); + verify(ss, times(1)).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java index 654b7488b98c8..5d0a0270e0ae3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.mapreduce.task.reduce; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.io.compress.CompressionCodec; @@ -283,6 +285,84 @@ public void addFetchFailedMap(TaskAttemptID mapTaskId) { scheduler.copyFailed(failedAttemptID, host1, true, false); } + @Test + public void testDuplicateCopySucceeded() throws Exception { + JobConf job = new JobConf(); + job.setNumMapTasks(2); + //mock creation + TaskUmbilicalProtocol mockUmbilical = mock(TaskUmbilicalProtocol.class); + Reporter mockReporter = mock(Reporter.class); + FileSystem mockFileSystem = mock(FileSystem.class); + Class combinerClass = + job.getCombinerClass(); + @SuppressWarnings("unchecked") // needed for mock with generic + CombineOutputCollector mockCombineOutputCollector = + (CombineOutputCollector) mock(CombineOutputCollector.class); + org.apache.hadoop.mapreduce.TaskAttemptID mockTaskAttemptID = + mock(org.apache.hadoop.mapreduce.TaskAttemptID.class); + LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class); + CompressionCodec mockCompressionCodec = mock(CompressionCodec.class); + Counter mockCounter = mock(Counter.class); + TaskStatus mockTaskStatus = mock(TaskStatus.class); + Progress mockProgress = mock(Progress.class); + MapOutputFile mockMapOutputFile = mock(MapOutputFile.class); + Task mockTask = mock(Task.class); + @SuppressWarnings("unchecked") + MapOutput output1 = mock(MapOutput.class); + @SuppressWarnings("unchecked") + MapOutput output2 = mock(MapOutput.class); + @SuppressWarnings("unchecked") + MapOutput output3 = mock(MapOutput.class); + + ShuffleConsumerPlugin.Context context = + new ShuffleConsumerPlugin.Context( + mockTaskAttemptID, job, mockFileSystem, + mockUmbilical, mockLocalDirAllocator, + mockReporter, mockCompressionCodec, + combinerClass, mockCombineOutputCollector, + mockCounter, mockCounter, mockCounter, + mockCounter, mockCounter, mockCounter, + mockTaskStatus, mockProgress, mockProgress, + mockTask, mockMapOutputFile, null); + TaskStatus status = new TaskStatus() { + @Override + public boolean getIsMap() { + return false; + } + @Override + public void addFetchFailedMap(TaskAttemptID mapTaskId) { + } + }; + Progress progress = new Progress(); + ShuffleSchedulerImpl scheduler = new ShuffleSchedulerImpl(job, + status, null, null, progress, context.getShuffledMapsCounter(), + context.getReduceShuffleBytes(), context.getFailedShuffleCounter()); + + MapHost host1 = new MapHost("host1", null); + TaskAttemptID succeedAttempt1ID = new TaskAttemptID( + new org.apache.hadoop.mapred.TaskID( + new JobID("test", 0), TaskType.MAP, 0), 0); + TaskAttemptID succeedAttempt2ID = new TaskAttemptID( + new org.apache.hadoop.mapred.TaskID( + new JobID("test", 0), TaskType.MAP, 0), 1); + TaskAttemptID succeedAttempt3ID = new TaskAttemptID( + new org.apache.hadoop.mapred.TaskID( + new JobID("test", 0), TaskType.MAP, 1), 0); + + long bytes = (long)500 * 1024 * 1024; + //First successful copy for map 0 should commit output + scheduler.copySucceeded(succeedAttempt1ID, host1, bytes, 0, 1, output1); + verify(output1).commit(); + + //Second successful copy for map 0 should abort output + scheduler.copySucceeded(succeedAttempt2ID, host1, bytes, 0, 1, output2); + verify(output2).abort(); + + //First successful copy for map 1 should commit output + scheduler.copySucceeded(succeedAttempt3ID, host1, bytes, 0, 1, output3); + verify(output3).commit(); + } + private static String copyMessage(int attemptNo, double rate1, double rate2) { int attemptZero = attemptNo - 1; return String.format("copy task(attempt_test_0000_m_%06d_%d succeeded at %1.2f MB/s)" diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml index b37480726a31a..3385c45a74802 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml @@ -19,12 +19,12 @@ hadoop-mapreduce-client org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-mapreduce-client-hs-plugins - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client-hs-plugins diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index b42dc2b55c38c..baa8c584dd4c1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -19,12 +19,12 @@ hadoop-mapreduce-client org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-mapreduce-client-hs - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client-hs diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 0b22d13f1d12b..99b806e28163c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -497,7 +497,7 @@ public synchronized Path getConfFile() { public synchronized Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(confFile.toUri(), conf); Configuration jobConf = new Configuration(false); - jobConf.addResource(fc.open(confFile), confFile.toString()); + jobConf.addResource(fc.open(confFile), confFile.toString(), true); return jobConf; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java index d8313479f32d0..1eb1d1c58d369 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java @@ -187,7 +187,8 @@ public void testRefreshSuperUserGroups() throws Exception { when(ugi.getRealUser()).thenReturn(superUser); when(superUser.getShortUserName()).thenReturn("superuser"); when(superUser.getUserName()).thenReturn("superuser"); - when(ugi.getGroupNames()).thenReturn(new String[] { "group3" }); + when(ugi.getGroups()) + .thenReturn(Arrays.asList(new String[] { "group3" })); when(ugi.getUserName()).thenReturn("regularUser"); // Set super user groups not to include groups of regularUser diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index 0769d5340137a..7d7b532f10e44 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -19,12 +19,12 @@ hadoop-mapreduce-client org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-mapreduce-client-jobclient - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client-jobclient diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index 1db287e4b47b2..c3ca15874c6a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -28,6 +28,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintStream; +import java.text.DecimalFormat; import java.util.Date; import java.util.Random; import java.util.StringTokenizer; @@ -804,7 +805,7 @@ else if(testType == TestType.TEST_TYPE_READ_SKIP && skipSize == 0) long tStart = System.currentTimeMillis(); sequentialTest(fs, testType, nrBytes, nrFiles); long execTime = System.currentTimeMillis() - tStart; - String resultLine = "Seq Test exec time sec: " + (float)execTime / 1000; + String resultLine = "Seq Test exec time sec: " + msToSecs(execTime); LOG.info(resultLine); return 0; } @@ -868,6 +869,10 @@ static float toMB(long bytes) { return ((float)bytes)/MEGA; } + static float msToSecs(long timeMillis) { + return timeMillis / 1000.0f; + } + private void analyzeResult( FileSystem fs, TestType testType, long execTime, @@ -903,19 +908,20 @@ else if (attr.endsWith(":sqrate")) if(in != null) in.close(); if(lines != null) lines.close(); } - + double med = rate / 1000 / tasks; double stdDev = Math.sqrt(Math.abs(sqrate / 1000 / tasks - med*med)); + DecimalFormat df = new DecimalFormat("#.##"); String resultLines[] = { - "----- TestDFSIO ----- : " + testType, - " Date & time: " + new Date(System.currentTimeMillis()), - " Number of files: " + tasks, - "Total MBytes processed: " + toMB(size), - " Throughput mb/sec: " + size * 1000.0 / (time * MEGA), - "Average IO rate mb/sec: " + med, - " IO rate std deviation: " + stdDev, - " Test exec time sec: " + (float)execTime / 1000, - "" }; + "----- TestDFSIO ----- : " + testType, + " Date & time: " + new Date(System.currentTimeMillis()), + " Number of files: " + tasks, + " Total MBytes processed: " + df.format(toMB(size)), + " Throughput mb/sec: " + df.format(toMB(size) / msToSecs(time)), + " Average IO rate mb/sec: " + df.format(med), + " IO rate std deviation: " + df.format(stdDev), + " Test exec time sec: " + df.format(msToSecs(execTime)), + "" }; PrintStream res = null; try { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java index 8013feb1ba2f3..6281e71ef50c2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java @@ -300,7 +300,7 @@ private void runRandomTests(CompressionCodec codec) throws IOException { if (i > 0) { if (i == (MAX_TESTS-1)) { // Test a split size that is less than record len - numSplits = (int)(fileSize/Math.floor(recordLength/2)); + numSplits = (int)(fileSize/ Math.max(1, Math.floor(recordLength/2))); } else { if (MAX_TESTS % i == 0) { // Let us create a split size that is forced to be diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java index 5106c3843f294..a7e6832800422 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java @@ -184,6 +184,90 @@ public void testSplitableCodecs() throws IOException { // corner case when we have byte alignment and position of stream are same verifyPartitions(471507, 218, file, codec, conf); verifyPartitions(473608, 110, file, codec, conf); + + // corner case when split size is small and position of stream is before + // the first BZip2 block + verifyPartitions(100, 20, file, codec, conf); + verifyPartitions(100, 25, file, codec, conf); + verifyPartitions(100, 30, file, codec, conf); + verifyPartitions(100, 50, file, codec, conf); + verifyPartitions(100, 100, file, codec, conf); + } + + // Test a corner case when position of stream is right after BZip2 marker + @Test (timeout=900000) + public void testSplitableCodecs2() throws IOException { + JobConf conf = new JobConf(defaultConf); + // Create the codec + CompressionCodec codec = null; + try { + codec = (CompressionCodec) + ReflectionUtils.newInstance(conf.getClassByName("org.apache.hadoop.io.compress.BZip2Codec"), conf); + } catch (ClassNotFoundException cnfe) { + throw new IOException("Illegal codec!"); + } + Path file = new Path(workDir, "test"+codec.getDefaultExtension()); + + FileSystem localFs = FileSystem.getLocal(conf); + localFs.delete(workDir, true); + FileInputFormat.setInputPaths(conf, workDir); + + int length = 250000; + LOG.info("creating; entries = " + length); + // create a file with length entries + Writer writer = + new OutputStreamWriter(codec.createOutputStream(localFs.create(file))); + try { + for (int i = 0; i < length; i++) { + writer.write(Integer.toString(i)); + writer.write("\n"); + } + } finally { + writer.close(); + } + + // Test split positions around a block boundary where the block does + // not start on a byte boundary. + for (long splitpos = 203418; splitpos < 203430; ++splitpos) { + TextInputFormat format = new TextInputFormat(); + format.configure(conf); + LOG.info("setting block size of the input file to " + splitpos); + conf.setLong("mapreduce.input.fileinputformat.split.minsize", splitpos); + LongWritable key = new LongWritable(); + Text value = new Text(); + InputSplit[] splits = format.getSplits(conf, 2); + LOG.info("splitting: got = " + splits.length); + + // check each split + BitSet bits = new BitSet(length); + for (int j = 0; j < splits.length; j++) { + LOG.debug("split[" + j + "]= " + splits[j]); + RecordReader reader = + format.getRecordReader(splits[j], conf, Reporter.NULL); + try { + int counter = 0; + while (reader.next(key, value)) { + int v = Integer.parseInt(value.toString()); + LOG.debug("read " + v); + if (bits.get(v)) { + LOG.warn("conflict with " + v + " in split " + j + + " at position " + reader.getPos()); + } + assertFalse("Key in multiple partitions.", bits.get(v)); + bits.set(v); + counter++; + } + if (counter > 0) { + LOG.info("splits[" + j + "]=" + splits[j] + " count=" + counter); + } else { + LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + counter); + } + } finally { + reader.close(); + } + } + assertEquals("Some keys in no partition.", length, bits.cardinality()); + } } private void verifyPartitions(int length, int numSplits, Path file, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 2e8ba5e627025..4055b06dd7acd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -152,7 +152,9 @@ public void start() throws IOException, FileNotFoundException, URISyntaxException { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) - .numDataNodes(numDataNodes).startupOption(dfsOpts).build(); + .numDataNodes(numDataNodes) + .format(dfsOpts == StartupOption.FORMAT) + .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index 85c675c308fb7..b49f2d831ab1f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -22,6 +22,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -53,13 +54,22 @@ import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.OneBlockInfo; import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.OneFileInfo; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; + +import org.junit.Before; import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import com.google.common.collect.HashMultiset; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; public class TestCombineFileInputFormat { @@ -92,6 +102,14 @@ public class TestCombineFileInputFormat { static final int BLOCKSIZE = 1024; static final byte[] databuf = new byte[BLOCKSIZE]; + @Mock + private List mockList; + + @Before + public void initMocks() { + MockitoAnnotations.initMocks(this); + } + private static final String DUMMY_FS_URI = "dummyfs:///"; /** Dummy class to extend CombineFileInputFormat*/ @@ -299,7 +317,51 @@ public void testReinit() throws Exception { assertFalse(rr.nextKeyValue()); } + /** + * For testing each split has the expected name, length, and offset. + */ + private final class Split { + private String name; + private long length; + private long offset; + + public Split(String name, long length, long offset) { + this.name = name; + this.length = length; + this.offset = offset; + } + + public String getName() { + return name; + } + + public long getLength() { + return length; + } + + public long getOffset() { + return offset; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Split) { + Split split = ((Split) obj); + return split.name.equals(name) && split.length == length + && split.offset == offset; + } + return false; + } + } + + /** + * The test suppresses unchecked warnings in + * {@link org.mockito.Mockito#reset}. Although calling the method is + * a bad manner, we call the method instead of splitting the test + * (i.e. restarting MiniDFSCluster) to save time. + */ @Test + @SuppressWarnings("unchecked") public void testSplitPlacement() throws Exception { MiniDFSCluster dfs = null; FileSystem fileSys = null; @@ -326,10 +388,10 @@ public void testSplitPlacement() throws Exception { throw new IOException("Mkdirs failed to create " + inDir.toString()); } Path file1 = new Path(dir1 + "/file1"); - writeFile(conf, file1, (short)1, 1); + writeFile(conf, file1, (short) 1, 1); // create another file on the same datanode Path file5 = new Path(dir5 + "/file5"); - writeFile(conf, file5, (short)1, 1); + writeFile(conf, file5, (short) 1, 1); // split it using a CombinedFile input format DummyInputFormat inFormat = new DummyInputFormat(); Job job = Job.getInstance(conf); @@ -350,13 +412,13 @@ public void testSplitPlacement() throws Exception { assertEquals(0, fileSplit.getOffset(1)); assertEquals(BLOCKSIZE, fileSplit.getLength(1)); assertEquals(hosts1[0], fileSplit.getLocations()[0]); - + dfs.startDataNodes(conf, 1, true, null, rack2, hosts2, null); dfs.waitActive(); // create file on two datanodes. Path file2 = new Path(dir2 + "/file2"); - writeFile(conf, file2, (short)2, 2); + writeFile(conf, file2, (short) 2, 2); // split it using a CombinedFile input format inFormat = new DummyInputFormat(); @@ -365,34 +427,67 @@ public void testSplitPlacement() throws Exception { splits = inFormat.getSplits(job); System.out.println("Made splits(Test1): " + splits.size()); - // make sure that each split has different locations for (InputSplit split : splits) { System.out.println("File split(Test1): " + split); } - assertEquals(2, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file2.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + /** + * If rack1 is processed first by + * {@link CombineFileInputFormat#createSplits}, + * create only one split on rack1. Otherwise create two splits. + */ + if (splits.size() == 2) { + // first split is on rack2, contains file2 + if (split.equals(splits.get(0))) { + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + // second split is on rack1, contains file1 + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 1) { + // first split is on rack1, contains file1 and file2. + assertEquals(3, fileSplit.getNumPaths()); + Set expected = new HashSet<>(); + expected.add(new Split(file1.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file2.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file2.getName(), BLOCKSIZE, BLOCKSIZE)); + List actual = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + assertTrue(actual.containsAll(expected)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } else { + fail("Expected split size is 1 or 2, but actual size is " + + splits.size()); + } + } // create another file on 3 datanodes and 3 racks. dfs.startDataNodes(conf, 1, true, null, rack3, hosts3, null); dfs.waitActive(); Path file3 = new Path(dir3 + "/file3"); - writeFile(conf, new Path(dir3 + "/file3"), (short)3, 3); + writeFile(conf, new Path(dir3 + "/file3"), (short) 3, 3); inFormat = new DummyInputFormat(); FileInputFormat.setInputPaths(job, dir1 + "," + dir2 + "," + dir3); inFormat.setMinSplitSizeRack(BLOCKSIZE); @@ -400,37 +495,98 @@ public void testSplitPlacement() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test2): " + split); } - assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(3, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file3.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file3.getName(), fileSplit.getPath(2).getName()); - assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file2.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + Set expected = new HashSet<>(); + expected.add(new Split(file1.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file2.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file2.getName(), BLOCKSIZE, BLOCKSIZE)); + expected.add(new Split(file3.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file3.getName(), BLOCKSIZE, BLOCKSIZE)); + expected.add(new Split(file3.getName(), BLOCKSIZE, BLOCKSIZE * 2)); + List actual = new ArrayList<>(); + + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + /** + * If rack1 is processed first by + * {@link CombineFileInputFormat#createSplits}, + * create only one split on rack1. + * If rack2 or rack3 is processed first and rack1 is processed second, + * create one split on rack2 or rack3 and the other split is on rack1. + * Otherwise create 3 splits for each rack. + */ + if (splits.size() == 3) { + // first split is on rack3, contains file3 + if (split.equals(splits.get(0))) { + assertEquals(3, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file3.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file3.getName(), fileSplit.getPath(2).getName()); + assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); + } + // second split is on rack2, contains file2 + if (split.equals(splits.get(1))) { + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + // third split is on rack1, contains file1 + if (split.equals(splits.get(2))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 2) { + // first split is on rack2 or rack3, contains one or two files. + if (split.equals(splits.get(0))) { + assertEquals(1, fileSplit.getLocations().length); + if (fileSplit.getLocations()[0].equals(hosts2[0])) { + assertEquals(2, fileSplit.getNumPaths()); + } else if (fileSplit.getLocations()[0].equals(hosts3[0])) { + assertEquals(3, fileSplit.getNumPaths()); + } else { + fail("First split should be on rack2 or rack3."); + } + } + // second split is on rack1, contains the rest files. + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 1) { + // first split is rack1, contains all three files. + assertEquals(1, fileSplit.getLocations().length); + assertEquals(6, fileSplit.getNumPaths()); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } else { + fail("Split size should be 1, 2, or 3."); + } + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + } + + assertEquals(6, actual.size()); + assertTrue(actual.containsAll(expected)); // create file4 on all three racks Path file4 = new Path(dir4 + "/file4"); @@ -442,37 +598,85 @@ public void testSplitPlacement() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test3): " + split); } - assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(6, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file3.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file3.getName(), fileSplit.getPath(2).getName()); - assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file2.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + expected.add(new Split(file4.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file4.getName(), BLOCKSIZE, BLOCKSIZE)); + expected.add(new Split(file4.getName(), BLOCKSIZE, BLOCKSIZE * 2)); + actual.clear(); + + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + /** + * If rack1 is processed first by + * {@link CombineFileInputFormat#createSplits}, + * create only one split on rack1. + * If rack2 or rack3 is processed first and rack1 is processed second, + * create one split on rack2 or rack3 and the other split is on rack1. + * Otherwise create 3 splits for each rack. + */ + if (splits.size() == 3) { + // first split is on rack3, contains file3 and file4 + if (split.equals(splits.get(0))) { + assertEquals(6, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); + } + // second split is on rack2, contains file2 + if (split.equals(splits.get(1))) { + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + // third split is on rack1, contains file1 + if (split.equals(splits.get(2))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 2) { + // first split is on rack2 or rack3, contains two or three files. + if (split.equals(splits.get(0))) { + assertEquals(1, fileSplit.getLocations().length); + if (fileSplit.getLocations()[0].equals(hosts2[0])) { + assertEquals(5, fileSplit.getNumPaths()); + } else if (fileSplit.getLocations()[0].equals(hosts3[0])) { + assertEquals(6, fileSplit.getNumPaths()); + } else { + fail("First split should be on rack2 or rack3."); + } + } + // second split is on rack1, contains the rest files. + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 1) { + // first split is rack1, contains all four files. + assertEquals(1, fileSplit.getLocations().length); + assertEquals(9, fileSplit.getNumPaths()); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } else { + fail("Split size should be 1, 2, or 3."); + } + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + } + + assertEquals(9, actual.size()); + assertTrue(actual.containsAll(expected)); // maximum split size is 2 blocks inFormat = new DummyInputFormat(); @@ -485,34 +689,26 @@ public void testSplitPlacement() throws Exception { System.out.println("File split(Test4): " + split); } assertEquals(5, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file3.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file2.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals("host2.rack2.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file3.getName(), fileSplit.getPath(1).getName()); - assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals("host1.rack1.com", fileSplit.getLocations()[0]); + + actual.clear(); + reset(mockList); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + mockList.add(fileSplit.getLocations()[0]); + } + + assertEquals(9, actual.size()); + assertTrue(actual.containsAll(expected)); + // verify the splits are on all the racks + verify(mockList, atLeastOnce()).add(hosts1[0]); + verify(mockList, atLeastOnce()).add(hosts2[0]); + verify(mockList, atLeastOnce()).add(hosts3[0]); // maximum split size is 3 blocks inFormat = new DummyInputFormat(); @@ -524,44 +720,26 @@ public void testSplitPlacement() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test5): " + split); } + assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(3, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file3.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file3.getName(), fileSplit.getPath(2).getName()); - assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file2.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file4.getName(), fileSplit.getPath(2).getName()); - assertEquals(0, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals("host2.rack2.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(3, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file4.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file4.getName(), fileSplit.getPath(2).getName()); - assertEquals(2*BLOCKSIZE, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals("host1.rack1.com", fileSplit.getLocations()[0]); + + actual.clear(); + reset(mockList); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + mockList.add(fileSplit.getLocations()[0]); + } + + assertEquals(9, actual.size()); + assertTrue(actual.containsAll(expected)); + verify(mockList, atLeastOnce()).add(hosts1[0]); + verify(mockList, atLeastOnce()).add(hosts2[0]); // maximum split size is 4 blocks inFormat = new DummyInputFormat(); @@ -572,41 +750,23 @@ public void testSplitPlacement() throws Exception { System.out.println("File split(Test6): " + split); } assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(4, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file3.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file3.getName(), fileSplit.getPath(2).getName()); - assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(4, fileSplit.getNumPaths()); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file2.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file4.getName(), fileSplit.getPath(2).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals(file4.getName(), fileSplit.getPath(3).getName()); - assertEquals( 2 * BLOCKSIZE, fileSplit.getOffset(3)); - assertEquals(BLOCKSIZE, fileSplit.getLength(3)); - assertEquals("host2.rack2.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + actual.clear(); + reset(mockList); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + mockList.add(fileSplit.getLocations()[0]); + } + + assertEquals(9, actual.size()); + assertTrue(actual.containsAll(expected)); + verify(mockList, atLeastOnce()).add(hosts1[0]); // maximum split size is 7 blocks and min is 3 blocks inFormat = new DummyInputFormat(); @@ -619,20 +779,31 @@ public void testSplitPlacement() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test7): " + split); } + assertEquals(2, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(6, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(3, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals("host1.rack1.com", fileSplit.getLocations()[0]); + + actual.clear(); + reset(mockList); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + mockList.add(fileSplit.getLocations()[0]); + } + + assertEquals(9, actual.size()); + assertTrue(actual.containsAll(expected)); + verify(mockList, atLeastOnce()).add(hosts1[0]); // Rack 1 has file1, file2 and file3 and file4 // Rack 2 has file2 and file3 and file4 // Rack 3 has file3 and file4 - // setup a filter so that only file1 and file2 can be combined + // setup a filter so that only (file1 and file2) or (file3 and file4) + // can be combined inFormat = new DummyInputFormat(); FileInputFormat.addInputPath(job, inDir); inFormat.setMinSplitSizeRack(1); // everything is at least rack local @@ -642,19 +813,101 @@ public void testSplitPlacement() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test1): " + split); } - assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(6, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 + + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + if (splits.size() == 2) { + // first split is on rack1, contains file1 and file2. + if (split.equals(splits.get(0))) { + assertEquals(3, fileSplit.getNumPaths()); + expected.clear(); + expected.add(new Split(file1.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file2.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file2.getName(), BLOCKSIZE, BLOCKSIZE)); + actual.clear(); + for (int i = 0; i < 3; i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + assertTrue(actual.containsAll(expected)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + if (split.equals(splits.get(1))) { + // second split contains the file3 and file4, however, + // the locations is undetermined. + assertEquals(6, fileSplit.getNumPaths()); + expected.clear(); + expected.add(new Split(file3.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file3.getName(), BLOCKSIZE, BLOCKSIZE)); + expected.add(new Split(file3.getName(), BLOCKSIZE, BLOCKSIZE * 2)); + expected.add(new Split(file4.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file4.getName(), BLOCKSIZE, BLOCKSIZE)); + expected.add(new Split(file4.getName(), BLOCKSIZE, BLOCKSIZE * 2)); + actual.clear(); + for (int i = 0; i < 6; i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + assertTrue(actual.containsAll(expected)); + assertEquals(1, fileSplit.getLocations().length); + } + } else if (splits.size() == 3) { + if (split.equals(splits.get(0))) { + // first split is on rack2, contains file2 + assertEquals(2, fileSplit.getNumPaths()); + expected.clear(); + expected.add(new Split(file2.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file2.getName(), BLOCKSIZE, BLOCKSIZE)); + actual.clear(); + for (int i = 0; i < 2; i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + assertTrue(actual.containsAll(expected)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + if (split.equals(splits.get(1))) { + // second split is on rack1, contains file1 + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + if (split.equals(splits.get(2))) { + // third split contains file3 and file4, however, + // the locations is undetermined. + assertEquals(6, fileSplit.getNumPaths()); + expected.clear(); + expected.add(new Split(file3.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file3.getName(), BLOCKSIZE, BLOCKSIZE)); + expected.add(new Split(file3.getName(), BLOCKSIZE, BLOCKSIZE * 2)); + expected.add(new Split(file4.getName(), BLOCKSIZE, 0)); + expected.add(new Split(file4.getName(), BLOCKSIZE, BLOCKSIZE)); + expected.add(new Split(file4.getName(), BLOCKSIZE, BLOCKSIZE * 2)); + actual.clear(); + for (int i = 0; i < 6; i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + assertTrue(actual.containsAll(expected)); + assertEquals(1, fileSplit.getLocations().length); + } + } else { + fail("Split size should be 2 or 3."); + } + } // measure performance when there are multiple pools and // many files in each pool. @@ -844,7 +1097,14 @@ public void testNodeInputSplit() throws IOException, InterruptedException { assertEquals(3, nodeSplits.count(locations[1])); } + /** + * The test suppresses unchecked warnings in + * {@link org.mockito.Mockito#reset}. Although calling the method is + * a bad manner, we call the method instead of splitting the test + * (i.e. restarting MiniDFSCluster) to save time. + */ @Test + @SuppressWarnings("unchecked") public void testSplitPlacementForCompressedFiles() throws Exception { MiniDFSCluster dfs = null; FileSystem fileSys = null; @@ -915,21 +1175,55 @@ public void testSplitPlacementForCompressedFiles() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test1): " + split); } - assertEquals(2, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f2.getLen(), fileSplit.getLength(0)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f1.getLen(), fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + Set expected = new HashSet<>(); + expected.add(new Split(file1.getName(), f1.getLen(), 0)); + expected.add(new Split(file2.getName(), f2.getLen(), 0)); + List actual = new ArrayList<>(); + + /** + * If rack1 is processed first by + * {@link CombineFileInputFormat#createSplits}, + * create only one split on rack1. Otherwise create two splits. + */ + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + if (splits.size() == 2) { + if (split.equals(splits.get(0))) { + // first split is on rack2, contains file2. + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(f2.getLen(), fileSplit.getLength(0)); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + if (split.equals(splits.get(1))) { + // second split is on rack1, contains file1. + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(f1.getLen(), fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 1) { + // first split is on rack1, contains file1 and file2. + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } else { + fail("Split size should be 1 or 2."); + } + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + } + assertEquals(2, actual.size()); + assertTrue(actual.containsAll(expected)); // create another file on 3 datanodes and 3 racks. dfs.startDataNodes(conf, 1, true, null, rack3, hosts3, null); @@ -943,28 +1237,83 @@ public void testSplitPlacementForCompressedFiles() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test2): " + split); } - assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f3.getLen(), fileSplit.getLength(0)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f2.getLen(), fileSplit.getLength(0)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f1.getLen(), fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + expected.add(new Split(file3.getName(), f3.getLen(), 0)); + actual.clear(); + + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + /** + * If rack1 is processed first by + * {@link CombineFileInputFormat#createSplits}, + * create only one split on rack1. + * If rack2 or rack3 is processed first and rack1 is processed second, + * create one split on rack2 or rack3 and the other split is on rack1. + * Otherwise create 3 splits for each rack. + */ + if (splits.size() == 3) { + // first split is on rack3, contains file3 + if (split.equals(splits.get(0))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(f3.getLen(), fileSplit.getLength(0)); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); + } + // second split is on rack2, contains file2 + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(f2.getLen(), fileSplit.getLength(0)); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + // third split is on rack1, contains file1 + if (split.equals(splits.get(2))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(f1.getLen(), fileSplit.getLength(0)); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 2) { + // first split is on rack2 or rack3, contains one or two files. + if (split.equals(splits.get(0))) { + assertEquals(1, fileSplit.getLocations().length); + if (fileSplit.getLocations()[0].equals(hosts2[0])) { + assertEquals(2, fileSplit.getNumPaths()); + } else if (fileSplit.getLocations()[0].equals(hosts3[0])) { + assertEquals(1, fileSplit.getNumPaths()); + } else { + fail("First split should be on rack2 or rack3."); + } + } + // second split is on rack1, contains the rest files. + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 1) { + // first split is rack1, contains all three files. + assertEquals(1, fileSplit.getLocations().length); + assertEquals(3, fileSplit.getNumPaths()); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } else { + fail("Split size should be 1, 2, or 3."); + } + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + } + + assertEquals(3, actual.size()); + assertTrue(actual.containsAll(expected)); // create file4 on all three racks Path file4 = new Path(dir4 + "/file4.gz"); @@ -977,31 +1326,79 @@ public void testSplitPlacementForCompressedFiles() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test3): " + split); } - assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f3.getLen(), fileSplit.getLength(0)); - assertEquals(file4.getName(), fileSplit.getPath(1).getName()); - assertEquals(0, fileSplit.getOffset(1)); - assertEquals(f4.getLen(), fileSplit.getLength(1)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f2.getLen(), fileSplit.getLength(0)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f1.getLen(), fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + expected.add(new Split(file3.getName(), f3.getLen(), 0)); + actual.clear(); + + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + /** + * If rack1 is processed first by + * {@link CombineFileInputFormat#createSplits}, + * create only one split on rack1. + * If rack2 or rack3 is processed first and rack1 is processed second, + * create one split on rack2 or rack3 and the other split is on rack1. + * Otherwise create 3 splits for each rack. + */ + if (splits.size() == 3) { + // first split is on rack3, contains file3 and file4 + if (split.equals(splits.get(0))) { + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); + } + // second split is on rack2, contains file2 + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(f2.getLen(), fileSplit.getLength(0)); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + // third split is on rack1, contains file1 + if (split.equals(splits.get(2))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(f1.getLen(), fileSplit.getLength(0)); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 2) { + // first split is on rack2 or rack3, contains two or three files. + if (split.equals(splits.get(0))) { + assertEquals(1, fileSplit.getLocations().length); + if (fileSplit.getLocations()[0].equals(hosts2[0])) { + assertEquals(3, fileSplit.getNumPaths()); + } else if (fileSplit.getLocations()[0].equals(hosts3[0])) { + assertEquals(2, fileSplit.getNumPaths()); + } else { + fail("First split should be on rack2 or rack3."); + } + } + // second split is on rack1, contains the rest files. + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 1) { + // first split is rack1, contains all four files. + assertEquals(1, fileSplit.getLocations().length); + assertEquals(4, fileSplit.getNumPaths()); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } else { + fail("Split size should be 1, 2, or 3."); + } + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + } + + assertEquals(4, actual.size()); + assertTrue(actual.containsAll(expected)); // maximum split size is file1's length inFormat = new DummyInputFormat(); @@ -1014,32 +1411,24 @@ public void testSplitPlacementForCompressedFiles() throws Exception { System.out.println("File split(Test4): " + split); } assertEquals(4, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f3.getLen(), fileSplit.getLength(0)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f2.getLen(), fileSplit.getLength(0)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r3 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f1.getLen(), fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(3); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file4.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f4.getLen(), fileSplit.getLength(0)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r1 + + actual.clear(); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + mockList.add(fileSplit.getLocations()[0]); + } + + assertEquals(4, actual.size()); + assertTrue(actual.containsAll(expected)); + verify(mockList, atLeastOnce()).add(hosts1[0]); + verify(mockList, atLeastOnce()).add(hosts2[0]); + verify(mockList, atLeastOnce()).add(hosts3[0]); // maximum split size is twice file1's length inFormat = new DummyInputFormat(); @@ -1051,31 +1440,33 @@ public void testSplitPlacementForCompressedFiles() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test5): " + split); } - assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f3.getLen(), fileSplit.getLength(0)); - assertEquals(file4.getName(), fileSplit.getPath(1).getName()); - assertEquals(0, fileSplit.getOffset(1)); - assertEquals(f4.getLen(), fileSplit.getLength(1)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file2.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f2.getLen(), fileSplit.getLength(0)); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f1.getLen(), fileSplit.getLength(0)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + actual.clear(); + reset(mockList); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + mockList.add(fileSplit.getLocations()[0]); + } + assertEquals(4, actual.size()); + assertTrue(actual.containsAll(expected)); + + if (splits.size() == 3) { + // splits are on all the racks + verify(mockList, times(1)).add(hosts1[0]); + verify(mockList, times(1)).add(hosts2[0]); + verify(mockList, times(1)).add(hosts3[0]); + } else if (splits.size() == 2) { + // one split is on rack1, another split is on rack2 or rack3 + verify(mockList, times(1)).add(hosts1[0]); + } else { + fail("Split size should be 2 or 3."); + } // maximum split size is 4 times file1's length inFormat = new DummyInputFormat(); @@ -1087,26 +1478,29 @@ public void testSplitPlacementForCompressedFiles() throws Exception { for (InputSplit split : splits) { System.out.println("File split(Test6): " + split); } - assertEquals(2, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f3.getLen(), fileSplit.getLength(0)); - assertEquals(file4.getName(), fileSplit.getPath(1).getName()); - assertEquals(0, fileSplit.getOffset(1)); - assertEquals(f4.getLen(), fileSplit.getLength(1)); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(file1.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(f1.getLen(), fileSplit.getLength(0)); - assertEquals(file2.getName(), fileSplit.getPath(1).getName()); - assertEquals(0, fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(f2.getLen(), fileSplit.getLength(1)); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 + + /** + * If rack1 is processed first by + * {@link CombineFileInputFormat#createSplits}, + * create only one split on rack1. Otherwise create two splits. + */ + assertTrue("Split size should be 1 or 2.", + splits.size() == 1 || splits.size() == 2); + actual.clear(); + reset(mockList); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + mockList.add(fileSplit.getLocations()[0]); + } + assertEquals(4, actual.size()); + assertTrue(actual.containsAll(expected)); + verify(mockList, times(1)).add(hosts1[0]); // maximum split size and min-split-size per rack is 4 times file1's length inFormat = new DummyInputFormat(); @@ -1146,25 +1540,57 @@ public void testSplitPlacementForCompressedFiles() throws Exception { inFormat = new DummyInputFormat(); FileInputFormat.addInputPath(job, inDir); inFormat.setMinSplitSizeRack(1); // everything is at least rack local - inFormat.createPool(new TestFilter(dir1), - new TestFilter(dir2)); + inFormat.createPool(new TestFilter(dir1), + new TestFilter(dir2)); splits = inFormat.getSplits(job); for (InputSplit split : splits) { System.out.println("File split(Test9): " + split); } - assertEquals(3, splits.size()); - fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 - fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(1, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); - assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 + + actual.clear(); + for (InputSplit split : splits) { + fileSplit = (CombineFileSplit) split; + if (splits.size() == 3) { + // If rack2 is processed first + if (split.equals(splits.get(0))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); + } + if (split.equals(splits.get(1))) { + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + if (split.equals(splits.get(2))) { + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); + } + } else if (splits.size() == 2) { + // If rack1 is processed first + if (split.equals(splits.get(0))) { + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); + } + if (split.equals(splits.get(1))) { + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); + } + } else { + fail("Split size should be 2 or 3."); + } + for (int i = 0; i < fileSplit.getNumPaths(); i++) { + String name = fileSplit.getPath(i).getName(); + long length = fileSplit.getLength(i); + long offset = fileSplit.getOffset(i); + actual.add(new Split(name, length, offset)); + } + } + assertEquals(4, actual.size()); + assertTrue(actual.containsAll(expected)); // measure performance when there are multiple pools and // many files in each pool. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java index d2edd192df510..98bd03e559574 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecutionWithMRApp.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; @@ -84,7 +85,8 @@ public void testSpeculateSuccessfulWithoutUpdateEvents() throws Exception { createTaskAttemptStatus(taskAttempt.getKey(), (float) 0.8, TaskAttemptState.RUNNING); TaskAttemptStatusUpdateEvent event = - new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), status); + new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), + new AtomicReference<>(status)); appEventHandler.handle(event); } } @@ -155,7 +157,8 @@ public void testSepculateSuccessfulWithUpdateEvents() throws Exception { createTaskAttemptStatus(taskAttempt.getKey(), (float) 0.5, TaskAttemptState.RUNNING); TaskAttemptStatusUpdateEvent event = - new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), status); + new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), + new AtomicReference<>(status)); appEventHandler.handle(event); } } @@ -180,7 +183,8 @@ public void testSepculateSuccessfulWithUpdateEvents() throws Exception { TaskAttemptState.RUNNING); speculatedTask = task.getValue(); TaskAttemptStatusUpdateEvent event = - new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), status); + new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), + new AtomicReference<>(status)); appEventHandler.handle(event); } } @@ -195,7 +199,8 @@ public void testSepculateSuccessfulWithUpdateEvents() throws Exception { createTaskAttemptStatus(taskAttempt.getKey(), (float) 0.75, TaskAttemptState.RUNNING); TaskAttemptStatusUpdateEvent event = - new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), status); + new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), + new AtomicReference<>(status)); appEventHandler.handle(event); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml index 5c5a5b2445695..3c819d2ad7b4d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml @@ -19,12 +19,12 @@ hadoop-mapreduce-client org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-mapreduce-client-shuffle - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client-shuffle diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 27fe352ffdfd6..7e89be5087c48 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-mapreduce-client - 2.7.4-SNAPSHOT + 2.7.7 hadoop-mapreduce-client pom diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index 54c2c866cd1ac..6770b43653a91 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-mapreduce-examples - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop MapReduce Examples Apache Hadoop MapReduce Examples jar diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index 14292cdf5a0ea..b6e77ae563664 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -18,12 +18,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-mapreduce - 2.7.4-SNAPSHOT + 2.7.7 pom hadoop-mapreduce http://hadoop.apache.org/mapreduce/ diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml index 109859315df13..f1a2a2c868393 100644 --- a/hadoop-maven-plugins/pom.xml +++ b/hadoop-maven-plugins/pom.xml @@ -19,7 +19,7 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml index c3c8b0403f7ae..75cc4c69055e3 100644 --- a/hadoop-minicluster/pom.xml +++ b/hadoop-minicluster/pom.xml @@ -18,12 +18,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-minicluster - 2.7.4-SNAPSHOT + 2.7.7 jar Apache Hadoop Mini-Cluster diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 74043784529f3..266ef3daf0e39 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-project-dist - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Project Dist POM Apache Hadoop Project Dist POM pom diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index dca3d3753c1fa..639bc074bd3e8 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -20,18 +20,18 @@ org.apache.hadoop hadoop-main - 2.7.4-SNAPSHOT + 2.7.7 org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Project POM Apache Hadoop Project POM pom - 2017 + 2018 false true diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm index 1cbc181b3df90..36ed5a3578367 100644 --- a/hadoop-project/src/site/markdown/index.md.vm +++ b/hadoop-project/src/site/markdown/index.md.vm @@ -16,47 +16,31 @@ Apache Hadoop ${project.version} ================================ Apache Hadoop ${project.version} is a minor release in the 2.x.y release -line, building upon the previous stable release 2.4.1. +line, building upon the previous stable 2.7 release. Here is a short overview of the major features and improvements. * Common - * Authentication improvements when using an HTTP proxy server. This is - useful when accessing WebHDFS via a proxy server. + * Multiple unit test failures fixed across all subprojects. - * A new Hadoop metrics sink that allows writing directly to Graphite. + * Optimized UGI group handling. - * [Specification work](./hadoop-project-dist/hadoop-common/filesystem/index.html) - related to the Hadoop Compatible Filesystem (HCFS) effort. + * Improved reading of compressed splits. * HDFS - * Support for POSIX-style filesystem extended attributes. See the - [user documentation](./hadoop-project-dist/hadoop-hdfs/ExtendedAttributes.html) - for more details. + * Fixed bugs to prevent NameNode from crashing due to NPE and full GC. - * Using the OfflineImageViewer, clients can now browse an fsimage via - the WebHDFS API. + * NameNode performance improvements for path handling. - * The NFS gateway received a number of supportability improvements and - bug fixes. The Hadoop portmapper is no longer required to run the gateway, - and the gateway is now able to reject connections from unprivileged ports. - - * The SecondaryNameNode, JournalNode, and DataNode web UIs have been - modernized with HTML5 and Javascript. + * Namesystem metrics improvements. * YARN - * YARN's REST APIs now support write/modify operations. Users can submit and - kill applications through REST APIs. - - * The timeline store in YARN, used for storing generic and - application-specific information for applications, supports authentication - through Kerberos. + * Resource Manager improvements and bug fixes. - * The Fair Scheduler supports dynamic hierarchical user queues, user queues - are created dynamically at runtime under any specified parent-queue. + * Handling Node Manager timeouts during shuffle stage. Getting Started =============== diff --git a/hadoop-tools/hadoop-ant/pom.xml b/hadoop-tools/hadoop-ant/pom.xml index 177ca3f51ee84..8df8f17f34c5b 100644 --- a/hadoop-tools/hadoop-ant/pom.xml +++ b/hadoop-tools/hadoop-ant/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-ant - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Ant Tasks Apache Hadoop Ant Tasks jar diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml index e5c97ebd071d4..bd7e9f45922ea 100644 --- a/hadoop-tools/hadoop-archives/pom.xml +++ b/hadoop-tools/hadoop-archives/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-archives - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Archives Apache Hadoop Archives jar diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 1fa68d35feb9e..e152ca92f0961 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -19,11 +19,11 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project hadoop-aws - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Amazon Web Services support This module contains code to support integration with Amazon Web Services. diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 73a407ee8e229..aa7bbd42d9c2b 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -19,7 +19,7 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project hadoop-azure diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml index 28244612e3373..32b3102e0cf02 100644 --- a/hadoop-tools/hadoop-datajoin/pom.xml +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-datajoin - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Data Join Apache Hadoop Data Join jar diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 982bb5435deaa..65285d73d1362 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-distcp - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Distributed Copy Apache Hadoop Distributed Copy jar diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml index 3020f04645b6c..95ed7edc4a9a4 100644 --- a/hadoop-tools/hadoop-extras/pom.xml +++ b/hadoop-tools/hadoop-extras/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-extras - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Extras Apache Hadoop Extras jar diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index aefdb16b22bb0..907fa94c04f29 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-gridmix - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Gridmix Apache Hadoop Gridmix jar diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml index cf320ec7692af..409150741ae03 100644 --- a/hadoop-tools/hadoop-openstack/pom.xml +++ b/hadoop-tools/hadoop-openstack/pom.xml @@ -19,11 +19,11 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project hadoop-openstack - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop OpenStack support This module contains code to support integration with OpenStack. diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml index c8bdaa4851047..dfb39ea5222cf 100644 --- a/hadoop-tools/hadoop-pipes/pom.xml +++ b/hadoop-tools/hadoop-pipes/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-pipes - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Pipes Apache Hadoop Pipes pom diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index caf58f4f110ee..e69fb80867126 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-rumen - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Rumen Apache Hadoop Rumen jar diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml index f0efffd74a38e..ae053a34caa43 100644 --- a/hadoop-tools/hadoop-sls/pom.xml +++ b/hadoop-tools/hadoop-sls/pom.xml @@ -19,12 +19,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-sls - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Scheduler Load Simulator Apache Hadoop Scheduler Load Simulator jar diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index c926b68ef670c..c8349e564366d 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-streaming - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop MapReduce Streaming Apache Hadoop MapReduce Streaming jar diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 5413bbdae236a..be8336d600d74 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project-dist - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project-dist org.apache.hadoop hadoop-tools-dist - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Tools Dist Apache Hadoop Tools Dist jar diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 68f27a20753ed..4dbf67fe2c489 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-tools - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Tools Apache Hadoop Tools pom diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 22349a02f0783..82c80e976f2d7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1,6 +1,72 @@ Hadoop YARN Change Log -Release 2.7.4 - UNRELEASED +Release 2.7.7 - 2018-07-18 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + Addendum fix in container executor. + Contributed by Wilfred Spiegelenburg. + +Release 2.7.6 - 2018-04-16 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + YARN-7661. NodeManager metrics return wrong value after update node + resource (Yang Wang via jlowe) + + YARN-3425. NPE from RMNodeLabelsManager.serviceStop when + NodeLabelsManager.serviceInit failed. (Bibin A Chundatt via wangda) + Backport YARN-6632 by Inigo Goiri. + + YARN-4167. NPE on RMActiveServices#serviceStop when store is null. + (Bibin A Chundatt via rohithsharmaks) + Backport YARN-6633 by Inigo Goiri. + + YARN-7590. Improve container-executor validation check. + Contributed by Eric Yang. + + YARN-7249. Fix CapacityScheduler NPE issue when a container preempted while + the node is being removed. (Wangda Tan via Eric Payne) + +Release 2.7.5 - 2017-12-14 + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + + YARN-5195. RM intermittently crashed with NPE while handling + APP_ATTEMPT_REMOVED event when async-scheduling enabled in + CapacityScheduler. (sandflee and Jonathan Hung via jlowe) + + YARN-6959. RM may allocate wrong AM Container for new attempt. + (Yuqi Wang via Jian He) + + YARN-7084. TestSchedulingMonitor#testRMStarts fails sporadically. + (Jason Lowe) + +Release 2.7.4 - 2017-08-04 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 80b83ed9f84db..f61f660683917 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-api - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-api diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index 45251ae1b1d82..9457df5dbc5cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn-applications org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-applications-distributedshell - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-applications-distributedshell diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml index 79a8ffa362455..7bef45c43e26a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn-applications org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-applications-unmanaged-am-launcher - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-applications-unmanaged-am-launcher diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index a89b1970b92fb..78b6e641676f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-applications - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-applications pom diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 4c40dca6520bd..28e6484517282 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -17,11 +17,11 @@ hadoop-yarn org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 org.apache.hadoop hadoop-yarn-client - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-client diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index c9eca6a2f9021..64d02c3d44b56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-common - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-common diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index b9031d68c5fd3..49259760156de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -248,7 +248,9 @@ protected void serviceStart() throws Exception { // for UT purpose protected void stopDispatcher() { AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher; - asyncDispatcher.stop(); + if (null != asyncDispatcher) { + asyncDispatcher.stop(); + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml index f82f03b43c13e..0a951adb3175a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml @@ -19,11 +19,11 @@ hadoop-yarn org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 hadoop-yarn-registry - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-registry diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index bb15aa506671a..57261d1b41235 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -22,12 +22,12 @@ hadoop-yarn-server org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-server-applicationhistoryservice - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server-applicationhistoryservice diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index 1552b14f3a454..5477e32f5c431 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn-server org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-server-common - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server-common diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 9605c5ac56469..c8a907ec51e60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn-server org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-server-nodemanager - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server-nodemanager diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e194c95dd6a78..e4288fe380e52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -229,7 +229,7 @@ public void startLocalizer(Path nmPrivateContainerTokensPath, buildMainArgs(command, user, appId, locId, nmAddr, localDirs); String[] commandArray = command.toArray(new String[command.size()]); ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray, - null, null, 0L, true); + null, null, 0L, false); if (LOG.isDebugEnabled()) { LOG.debug("initApplication: " + Arrays.toString(commandArray)); } @@ -291,7 +291,8 @@ public int launchContainer(Container container, resourcesOptions)); String[] commandArray = command.toArray(new String[command.size()]); shExec = new ShellCommandExecutor(commandArray, null, // NM's cwd - container.getLaunchContext().getEnvironment()); // sanitized env + container.getLaunchContext().getEnvironment(), 0L, + false); // sanitized env if (LOG.isDebugEnabled()) { LOG.debug("launchContainer: " + Arrays.toString(commandArray)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index 3615feefa9315..de6746a6b65ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -117,7 +117,7 @@ public void releaseContainer(Resource res) { public void addResource(Resource res) { availableMB = availableMB + res.getMemory(); - availableGB.incr((int)Math.floor(availableMB/1024d)); + availableGB.set((int)Math.floor(availableMB/1024d)); availableVCores.incr(res.getVirtualCores()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index e7cdb13a84b0a..474900b3c6c83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -413,9 +413,31 @@ char *get_app_directory(const char * nm_root, const char *user, * Get the user directory of a particular user */ char *get_user_directory(const char *nm_root, const char *user) { + int result = check_nm_local_dir(nm_uid, nm_root); + if (result != 0) { + return NULL; + } return concatenate(USER_DIR_PATTERN, "user_dir_path", 2, nm_root, user); } +/** + * Check node manager local dir permission. + */ +int check_nm_local_dir(uid_t caller_uid, const char *nm_root) { + struct stat info; + errno = 0; + int err = stat(nm_root, &info); + if (err < 0) { + fprintf(LOGFILE, "Error checking file stats for %s %d %s.\n", nm_root, err, strerror(errno)); + return 1; + } + if (caller_uid != info.st_uid) { + fprintf(LOGFILE, "Permission mismatch for %s for caller uid: %d, owner uid: %d.\n", nm_root, caller_uid, info.st_uid); + return 1; + } + return 0; +} + /** * Get the container directory for the given container_id */ @@ -559,6 +581,11 @@ static int create_container_directories(const char* user, const char *app_id, for(local_dir_ptr = local_dir; *local_dir_ptr != NULL; ++local_dir_ptr) { char *container_dir = get_container_work_directory(*local_dir_ptr, user, app_id, container_id); + int check = check_nm_local_dir(nm_uid, *local_dir_ptr); + if (check != 0) { + free(container_dir); + continue; + } if (container_dir == NULL) { return -1; } @@ -585,6 +612,14 @@ static int create_container_directories(const char* user, const char *app_id, char* const* log_dir_ptr; for(log_dir_ptr = log_dir; *log_dir_ptr != NULL; ++log_dir_ptr) { char *container_log_dir = get_app_log_directory(*log_dir_ptr, combined_name); + int check = check_nm_local_dir(nm_uid, *log_dir_ptr); + if (check != 0) { + container_log_dir = NULL; + } + if (strstr(container_log_dir, "..") != 0) { + fprintf(LOGFILE, "Unsupported container log directory path detected.\n"); + container_log_dir = NULL; + } if (container_log_dir == NULL) { free(combined_name); return -1; @@ -899,6 +934,10 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) { char *any_one_app_log_dir = NULL; for(log_root=log_dirs; *log_root != NULL; ++log_root) { char *app_log_dir = get_app_log_directory(*log_root, app_id); + int result = check_nm_local_dir(nm_uid, *log_root); + if (result != 0) { + app_log_dir = NULL; + } if (app_log_dir == NULL) { // try the next one } else if (create_directory_for_user(app_log_dir) != 0) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 2645ddaa78164..50099627a0633 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -168,6 +168,11 @@ char *get_user_directory(const char *nm_root, const char *user); char *get_app_directory(const char * nm_root, const char *user, const char *app_id); +/** + * Check node manager local dir permission. + */ +int check_nm_local_dir(uid_t caller_uid, const char *nm_root); + char *get_container_work_directory(const char *nm_root, const char *user, const char *app_id, const char *container_id); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 468aa76c098fd..f71468927804c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -129,7 +129,7 @@ int main(int argc, char **argv) { } char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME; - char *conf_file = resolve_config_path(orig_conf_file, argv[0]); + char *conf_file = resolve_config_path(orig_conf_file, executable_file); char *local_dirs, *log_dirs; char *resources, *resources_key, *resources_value; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index bb52b4e8689dc..a8cc04171ea18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -51,6 +51,7 @@ static char* username = NULL; static char* yarn_username = NULL; static char** local_dirs = NULL; static char** log_dirs = NULL; +static uid_t nm_uid = -1; /** * Run the command using the effective user id. @@ -165,8 +166,8 @@ void check_pid_file(const char* pid_file, pid_t mypid) { } void test_get_user_directory() { - char *user_dir = get_user_directory(TMPDIR, "user"); - char *expected = TMPDIR "/usercache/user"; + char *user_dir = get_user_directory(TEST_ROOT, "user"); + char *expected = TEST_ROOT "/usercache/user"; if (strcmp(user_dir, expected) != 0) { printf("test_get_user_directory expected %s got %s\n", expected, user_dir); exit(1); @@ -174,9 +175,32 @@ void test_get_user_directory() { free(user_dir); } +void test_check_nm_local_dir() { + // check filesystem is same as running user. + int expected = 0; + char *local_path = TEST_ROOT "target"; + char *root_path = "/"; + if (mkdirs(local_path, 0700) != 0) { + printf("FAIL: unble to create node manager local directory: %s\n", local_path); + exit(1); + } + int actual = check_nm_local_dir(nm_uid, local_path); + if (expected != actual) { + printf("test_nm_local_dir expected %d got %d\n", expected, actual); + exit(1); + } + // check filesystem is different from running user. + expected = 1; + actual = check_nm_local_dir(nm_uid, root_path); + if (expected != actual && nm_uid != 0) { + printf("test_nm_local_dir expected %d got %d\n", expected, actual); + exit(1); + } +} + void test_get_app_directory() { - char *expected = TMPDIR "/usercache/user/appcache/app_200906101234_0001"; - char *app_dir = (char *) get_app_directory(TMPDIR, "user", + char *expected = TEST_ROOT "/usercache/user/appcache/app_200906101234_0001"; + char *app_dir = (char *) get_app_directory(TEST_ROOT, "user", "app_200906101234_0001"); if (strcmp(app_dir, expected) != 0) { printf("test_get_app_directory expected %s got %s\n", expected, app_dir); @@ -186,9 +210,9 @@ void test_get_app_directory() { } void test_get_container_directory() { - char *container_dir = get_container_work_directory(TMPDIR, "owen", "app_1", + char *container_dir = get_container_work_directory(TEST_ROOT, "owen", "app_1", "container_1"); - char *expected = TMPDIR"/usercache/owen/appcache/app_1/container_1"; + char *expected = TEST_ROOT "/usercache/owen/appcache/app_1/container_1"; if (strcmp(container_dir, expected) != 0) { printf("Fail get_container_work_directory got %s expected %s\n", container_dir, expected); @@ -198,9 +222,9 @@ void test_get_container_directory() { } void test_get_container_launcher_file() { - char *expected_file = (TMPDIR"/usercache/user/appcache/app_200906101234_0001" + char *expected_file = (TEST_ROOT "/usercache/user/appcache/app_200906101234_0001" "/launch_container.sh"); - char *app_dir = get_app_directory(TMPDIR, "user", + char *app_dir = get_app_directory(TEST_ROOT, "user", "app_200906101234_0001"); char *container_file = get_container_launcher_file(app_dir); if (strcmp(container_file, expected_file) != 0) { @@ -729,6 +753,9 @@ int main(int argc, char **argv) { LOGFILE = stdout; ERRORFILE = stderr; + nm_uid = getuid(); + + printf("Attempting to clean up from any previous runs\n"); // clean up any junk from previous run if (system("chmod -R u=rwx " TEST_ROOT "; rm -fr " TEST_ROOT)) { exit(1); @@ -772,6 +799,9 @@ int main(int argc, char **argv) { printf("\nTesting get_user_directory()\n"); test_get_user_directory(); + printf("\nTesting check_nm_local_dir()\n"); + test_check_nm_local_dir(); + printf("\nTesting get_app_directory()\n"); test_get_app_directory(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java index 4dc4648cf41a3..2af25c55d3395 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java @@ -74,6 +74,12 @@ public class TestNodeManagerMetrics { // allocatedGB: 3.5GB allocated memory is shown as 4GB // availableGB: 4.5GB available memory is shown as 4GB checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2); + + // Update resource and check available resource again + metrics.addResource(total); + MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics"); + assertGauge("AvailableGB", 12, rb); + assertGauge("AvailableVCores", 18, rb); } private void checkMetrics(int launched, int completed, int failed, int killed, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 74fc2d4d21761..f73909f527d65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn-server org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-server-resourcemanager - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server-resourcemanager diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 5faaab297ebda..1dd7120238f96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -592,7 +592,9 @@ protected void serviceStop() throws Exception { if (rmContext != null) { RMStateStore store = rmContext.getStateStore(); try { - store.close(); + if (null != store) { + store.close(); + } } catch (Exception e) { LOG.error("Error closing store.", e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index e61587d14864a..1e9a591d4ee04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -227,6 +227,7 @@ protected synchronized void containerLaunchedOnNode( application.containerLaunchedOnNode(containerId, node.getNodeID()); } + // TODO: Rename it to getCurrentApplicationAttempt public T getApplicationAttempt(ApplicationAttemptId applicationAttemptId) { SchedulerApplication app = applications.get(applicationAttemptId.getApplicationId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index c8ea5a5c1e0cb..561cdd74c8290 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -938,8 +938,19 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId, FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId); if (application == null) { - LOG.info("Calling allocate on removed " + - "or non existant application " + applicationAttemptId); + LOG.error("Calling allocate on removed or non existent application " + + applicationAttemptId.getApplicationId()); + return EMPTY_ALLOCATION; + } + + // The allocate may be the leftover from previous attempt, and it will + // impact current attempt, such as confuse the request and allocation for + // current attempt's AM container. + // Note outside precondition check for the attempt id may be + // outdated here, so double check it here is necessary. + if (!application.getApplicationAttemptId().equals(applicationAttemptId)) { + LOG.error("Calling allocate on previous or removed " + + "or non existent application attempt " + applicationAttemptId); return EMPTY_ALLOCATION; } @@ -1107,12 +1118,19 @@ private synchronized void updateLabelsOnNode(NodeId nodeId, node.updateLabels(newLabels); } - private synchronized void allocateContainersToNode(FiCaSchedulerNode node) { + @VisibleForTesting + public synchronized void allocateContainersToNode(FiCaSchedulerNode node) { if (rmContext.isWorkPreservingRecoveryEnabled() && !rmContext.isSchedulerReadyForAllocatingContainers()) { return; } + if (!nodes.containsKey(node.getNodeID())) { + LOG.info("Skipping scheduling as the node " + node.getNodeID() + + " has been removed"); + return; + } + // Assign new containers... // 1. Check for reserved applications // 2. Schedule if there are no reservations @@ -1410,6 +1428,12 @@ protected synchronized void completedContainer(RMContainer rmContainer, // Get the node on which the container was allocated FiCaSchedulerNode node = getNode(container.getNodeId()); + if (node == null) { + LOG.info("Container=" + container + " of application=" + appId + + " completed with event=" + event + " on a node=" + container + .getNodeId() + ". However the node might be already removed by RM."); + return; + } // Inform the queue LeafQueue queue = (LeafQueue)application.getQueue(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 4591f63749418..839bef544829b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -889,8 +889,19 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, // Make sure this application exists FSAppAttempt application = getSchedulerApp(appAttemptId); if (application == null) { - LOG.info("Calling allocate on removed " + - "or non existant application " + appAttemptId); + LOG.error("Calling allocate on removed or non existent application " + + appAttemptId.getApplicationId()); + return EMPTY_ALLOCATION; + } + + // The allocate may be the leftover from previous attempt, and it will + // impact current attempt, such as confuse the request and allocation for + // current attempt's AM container. + // Note outside precondition check for the attempt id may be + // outdated here, so double check it here is necessary. + if (!application.getApplicationAttemptId().equals(appAttemptId)) { + LOG.error("Calling allocate on previous or removed " + + "or non existent application attempt " + appAttemptId); return EMPTY_ALLOCATION; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 694b0610cb5c5..0666d6827504b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -300,8 +300,19 @@ public Allocation allocate( List release, List blacklistAdditions, List blacklistRemovals) { FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId); if (application == null) { - LOG.error("Calling allocate on removed " + - "or non existant application " + applicationAttemptId); + LOG.error("Calling allocate on removed or non existent application " + + applicationAttemptId.getApplicationId()); + return EMPTY_ALLOCATION; + } + + // The allocate may be the leftover from previous attempt, and it will + // impact current attempt, such as confuse the request and allocation for + // current attempt's AM container. + // Note outside precondition check for the attempt id may be + // outdated here, so double check it here is necessary. + if (!application.getApplicationAttemptId().equals(applicationAttemptId)) { + LOG.error("Calling allocate on previous or removed " + + "or non existent application attempt " + applicationAttemptId); return EMPTY_ALLOCATION; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java index d53282ed67f33..85ac20dc3ac78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java @@ -24,42 +24,30 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy; import org.junit.Test; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestSchedulingMonitor { @Test(timeout = 10000) - public void testRMStarts() { + public void testRMStarts() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, ProportionalCapacityPreemptionPolicy.class.getCanonicalName()); ResourceManager rm = new ResourceManager(); - try { - rm.init(conf); - } catch (Exception e) { - fail("ResourceManager does not start when " + - YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + " is set to true"); - } - + rm.init(conf); SchedulingEditPolicy mPolicy = mock(SchedulingEditPolicy.class); when(mPolicy.getMonitoringInterval()).thenReturn(1000L); SchedulingMonitor monitor = new SchedulingMonitor(rm.getRMContext(), mPolicy); - try { - monitor.serviceInit(conf); - monitor.serviceStart(); - } catch (Exception e) { - fail("SchedulingMonitor failes to start."); - } - verify(mPolicy, times(1)).editSchedule(); - try { - monitor.close(); - rm.close(); - } catch (Exception e) { - fail("Failed to close."); - } + monitor.serviceInit(conf); + monitor.serviceStart(); + verify(mPolicy, timeout(10000)).editSchedule(); + monitor.close(); + rm.close(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index e4920e889d2c2..97f40765cc03a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -116,6 +116,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -2848,4 +2849,42 @@ private void setMaxAllocVcores(CapacitySchedulerConfiguration conf, + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; conf.setInt(propName, maxAllocVcores); } + + @Test + public void testSchedulingOnRemovedNode() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setBoolean( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, + false); + + MockRM rm = new MockRM(conf); + rm.start(); + RMApp app = rm.submitApp(100); + rm.drainEvents(); + + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10); + MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1); + + //remove nm2 to keep am alive + MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10240, 10); + + am.allocate(ResourceRequest.ANY, 2048, 1, null); + + CapacityScheduler scheduler = + (CapacityScheduler) rm.getRMContext().getScheduler(); + FiCaSchedulerNode node = scheduler.getAllNodes().get(nm2.getNodeId()); + scheduler.handle(new NodeRemovedSchedulerEvent( + rm.getRMContext().getRMNodes().get(nm2.getNodeId()))); + // schedulerNode is removed, try allocate a container + scheduler.allocateContainersToNode(node); + + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent( + am.getApplicationAttemptId(), + RMAppAttemptState.FINISHED, false); + scheduler.handle(appRemovedEvent1); + rm.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 7b3c17f700c64..6f0de0782e791 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1387,45 +1387,45 @@ public void testQueueDemandCalculation() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); + int minReqSize = + FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; + // First ask, queue1 requests 1 large (minReqSize * 2). ApplicationAttemptId id11 = createAppAttemptId(1, 1); createMockRMApp(id11); - scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", false); + scheduler.addApplication(id11.getApplicationId(), + "root.queue1", "user1", false); scheduler.addApplicationAttempt(id11, false, false); - ApplicationAttemptId id21 = createAppAttemptId(2, 1); - createMockRMApp(id21); - scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1", false); - scheduler.addApplicationAttempt(id21, false, false); - ApplicationAttemptId id22 = createAppAttemptId(2, 2); - createMockRMApp(id22); - - scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1", false); - scheduler.addApplicationAttempt(id22, false, false); - - int minReqSize = - FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; - - // First ask, queue1 requests 1 large (minReqSize * 2). List ask1 = new ArrayList(); - ResourceRequest request1 = - createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true); + ResourceRequest request1 = createResourceRequest(minReqSize * 2, + ResourceRequest.ANY, 1, 1, true); ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList(), null, null); // Second ask, queue2 requests 1 large + (2 * minReqSize) + ApplicationAttemptId id21 = createAppAttemptId(2, 1); + createMockRMApp(id21); + scheduler.addApplication(id21.getApplicationId(), + "root.queue2", "user1", false); + scheduler.addApplicationAttempt(id21, false, false); List ask2 = new ArrayList(); - ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1, - false); - ResourceRequest request3 = createResourceRequest(minReqSize, "bar", 1, 2, - false); + ResourceRequest request2 = createResourceRequest(2 * minReqSize, + "foo", 1, 1, false); + ResourceRequest request3 = createResourceRequest(minReqSize, + ResourceRequest.ANY, 1, 2, false); ask2.add(request2); ask2.add(request3); scheduler.allocate(id21, ask2, new ArrayList(), null, null); // Third ask, queue2 requests 1 large + ApplicationAttemptId id22 = createAppAttemptId(2, 2); + createMockRMApp(id22); + scheduler.addApplication(id22.getApplicationId(), + "root.queue2", "user1", false); + scheduler.addApplicationAttempt(id22, false, false); List ask3 = new ArrayList(); - ResourceRequest request4 = - createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, true); + ResourceRequest request4 = createResourceRequest(2 * minReqSize, + ResourceRequest.ANY, 1, 1, true); ask3.add(request4); scheduler.allocate(id22, ask3, new ArrayList(), null, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml index af2f6700c8758..8264d8371b996 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml @@ -17,11 +17,11 @@ hadoop-yarn-server org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 org.apache.hadoop hadoop-yarn-server-sharedcachemanager - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server-sharedcachemanager diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index 52a4433799c01..498bd71267498 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -19,11 +19,11 @@ hadoop-yarn-server org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 org.apache.hadoop hadoop-yarn-server-tests - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server-tests diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml index 6be626219ced7..34119d9fce441 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn-server org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-server-web-proxy - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server-web-proxy diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index 23b2ea080fe64..81b516ba49ac1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-server - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-server pom diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml index 998392a76c6f7..45308e4a5880c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml @@ -19,12 +19,12 @@ hadoop-yarn org.apache.hadoop - 2.7.4-SNAPSHOT + 2.7.7 4.0.0 org.apache.hadoop hadoop-yarn-site - 2.7.4-SNAPSHOT + 2.7.7 hadoop-yarn-site pom diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml index c57bfe1e07c7c..c7a1120c22de7 100644 --- a/hadoop-yarn-project/hadoop-yarn/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/pom.xml @@ -16,12 +16,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../../hadoop-project org.apache.hadoop hadoop-yarn - 2.7.4-SNAPSHOT + 2.7.7 pom hadoop-yarn diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index c2c2d5d5268db..4e94df040c4fe 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -18,12 +18,12 @@ org.apache.hadoop hadoop-project - 2.7.4-SNAPSHOT + 2.7.7 ../hadoop-project org.apache.hadoop hadoop-yarn-project - 2.7.4-SNAPSHOT + 2.7.7 pom hadoop-yarn-project http://hadoop.apache.org/yarn/ diff --git a/pom.xml b/pom.xml index f8833db1bfdd7..e74596a5a0355 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 4.0.0 org.apache.hadoop hadoop-main - 2.7.4-SNAPSHOT + 2.7.7 Apache Hadoop Main Apache Hadoop Main pom