Refactored code (#103)

Added tony-site.xml configs into tonyConf
Moved tony.history.location out of tony-default.xml
Fixed util function + skips config check in test
Removed redundate else statement in startTHS.sh
Replaced history file path in application.example.conf
Used TonyConfigurationKeys instead of hardcoded string
Added all config for history server in tony-default.xml + TonyConfigurationKeys
Refactored Tony config intialization method

tony-core/src/main/java/com/linkedin/tony/Constants.java

@@ -52,6 +52,8 @@ public class Constants {
   public static final String HDFS_SITE_CONF = "hdfs-site.xml";
   public static final String CORE_SITE_CONF = YarnConfiguration.CORE_SITE_CONFIGURATION_FILE;
   public static final String HADOOP_CONF_DIR = ApplicationConstants.Environment.HADOOP_CONF_DIR.key();
+  public static final String TONY_SITE_CONF = "tony-site.xml";
+  public static final String TONY_CONF_DIR = "TONY_CONF_DIR";
 
   public static final String AM_NAME = "am";
   public static final String WORKER_JOB_NAME = "worker";
@@ -75,7 +77,6 @@ public class Constants {
   public static final String TONY_FOLDER = ".tony";
 
   // Configuration related constants
-
   // Name of the file containing all configuration keys and their default values
   public static final String TONY_DEFAULT_XML = "tony-default.xml";
   // Default file name of user-provided configuration file

tony-core/src/main/java/com/linkedin/tony/TonyClient.java

@@ -75,6 +75,8 @@
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 
+import static com.linkedin.tony.Constants.*;
+
 
 /**
  * User entry point to submit tensorflow job.
@@ -84,9 +86,10 @@ public class TonyClient implements AutoCloseable {
 
   private static final String APP_TYPE = "TENSORFLOW";
   private static final String ARCHIVE_SUFFIX = "tony_archive.zip";
-  private static final String CORE_SITE_CONF = YarnConfiguration.CORE_SITE_CONFIGURATION_FILE;
-  private static final String HADOOP_CONF_DIR = ApplicationConstants.Environment.HADOOP_CONF_DIR.key();
-  private static final String HDFS_SITE_CONF = "hdfs-site.xml";
+  private static final String TONY_CONF_DIR = Constants.TONY_CONF_DIR;
+  private static final String TONY_SITE_CONF = Constants.TONY_SITE_CONF;
+  private static final String TONY_DEFAULT_XML = Constants.TONY_DEFAULT_XML;
+  private static final String TONY_XML = Constants.TONY_XML;
 
   // Configurations
   private YarnClient yarnClient;
@@ -144,7 +147,6 @@ public ImmutableSet<TaskUrl> getTaskUrls() {
   private boolean run() throws IOException, InterruptedException, URISyntaxException, YarnException {
     LOG.info("Starting client..");
     yarnClient.start();
-
     YarnClientApplication app = yarnClient.createApplication();
     GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
 
@@ -226,10 +228,10 @@ private void createYarnClient() {
         YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS) * numRMConnectRetries;
     yarnConf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, rmMaxWaitMS);
 
-    if (System.getenv("HADOOP_CONF_DIR") != null) {
-      hdfsConf.addResource(new Path(System.getenv(HADOOP_CONF_DIR) + File.separatorChar + CORE_SITE_CONF));
-      yarnConf.addResource(new Path(System.getenv(HADOOP_CONF_DIR) + File.separatorChar + CORE_SITE_CONF));
-      hdfsConf.addResource(new Path(System.getenv(HADOOP_CONF_DIR) + File.separatorChar + HDFS_SITE_CONF));
+    if (System.getenv(Constants.HADOOP_CONF_DIR) != null) {
+      hdfsConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.CORE_SITE_CONF));
+      yarnConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.CORE_SITE_CONF));
+      hdfsConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.HDFS_SITE_CONF));
     }
     yarnClient = YarnClient.createYarnClient();
     yarnClient.init(yarnConf);
@@ -258,18 +260,7 @@ public boolean init(String[] args) throws ParseException {
       return false;
     }
 
-    tonyConf.addResource(Constants.TONY_DEFAULT_XML);
-    if (cliParser.hasOption("conf_file")) {
-      tonyConf.addResource(new Path(cliParser.getOptionValue("conf_file")));
-    } else {
-      tonyConf.addResource(Constants.TONY_XML);
-    }
-    if (cliParser.hasOption("conf")) {
-      String[] confs = cliParser.getOptionValues("conf");
-      for (Map.Entry<String, String> cliConf : Utils.parseKeyValue(confs).entrySet()) {
-        tonyConf.set(cliConf.getKey(), cliConf.getValue());
-      }
-    }
+    initTonyConf(tonyConf, cliParser);
 
     String amMemoryString = tonyConf.get(TonyConfigurationKeys.AM_MEMORY,
         TonyConfigurationKeys.DEFAULT_AM_MEMORY);
@@ -349,6 +340,29 @@ public boolean init(String[] args) throws ParseException {
     return true;
   }
 
+  /**
+   * Add resource if exist to {@code tonyConf}
+   * @param tonyConf Configuration object.
+   * @param cliParser CommandLine object that has all the command line arguments.
+   */
+  public static void initTonyConf(Configuration tonyConf, CommandLine cliParser) {
+    if (System.getenv(Constants.TONY_CONF_DIR) != null) {
+      tonyConf.addResource(new Path(System.getenv(Constants.TONY_CONF_DIR) + File.separatorChar + Constants.TONY_SITE_CONF));
+    }
+    tonyConf.addResource(Constants.TONY_DEFAULT_XML);
+    if (cliParser.hasOption("conf_file")) {
+      tonyConf.addResource(new Path(cliParser.getOptionValue("conf_file")));
+    } else {
+      tonyConf.addResource(Constants.TONY_XML);
+    }
+    if (cliParser.hasOption("conf")) {
+      String[] confs = cliParser.getOptionValues("conf");
+      for (Map.Entry<String, String> cliConf : Utils.parseKeyValue(confs).entrySet()) {
+        tonyConf.set(cliConf.getKey(), cliConf.getValue());
+      }
+    }
+  }
+
   public Configuration getTonyConf() {
     return this.tonyConf;
   }

tony-core/src/main/java/com/linkedin/tony/TonyConfigurationKeys.java

@@ -20,7 +20,37 @@ private TonyConfigurationKeys() {
 
   // History folder configuration
   public static final String TONY_HISTORY_LOCATION = TONY_PREFIX + "history.location";
-  public static final String DEFAULT_TONY_HISTORY_LOCATION = "/system/tony-history/";
+  public static final String DEFAULT_TONY_HISTORY_LOCATION = "/system/tony-history";
+
+  public static final String TONY_KEYTAB_USER = TONY_PREFIX + "keytab.user";
+  public static final String DEFAULT_TONY_KEYTAB_USER = "tony/[email protected]";
+
+  public static final String TONY_KEYTAB_LOCATION = TONY_PREFIX + "keytab.location";
+  public static final String DEFAULT_TONY_KEYTAB_LOCATION = "/export/apps/hadoop/keytabs/tony.keytab";
+
+  public static final String TONY_HTTPS_PORT = TONY_PREFIX + "https.port";
+  public static final String DEFAULT_TONY_HTTPS_PORT = "19886";
+
+  public static final String TONY_HTTPS_KEYSTORE_PATH = TONY_PREFIX + "https.keystore.path";
+  public static final String DEFAULT_TONY_HTTPS_KEYSTORE_PATH = "/export/apps/tony/ltx1-unojh01.grid.linkedin.com.jks";
+
+  public static final String TONY_HTTPS_KEYSTORE_TYPE = TONY_PREFIX + "https.keystore.type";
+  public static final String DEFAULT_TONY_HTTPS_KEYSTORE_TYPE = "JKS";
+
+  public static final String TONY_HTTPS_KEYSTORE_PASSWORD = TONY_PREFIX + "https.keystore.password";
+  public static final String DEFAULT_TONY_HTTPS_KEYSTORE_PASSWORD = "password";
+
+  public static final String TONY_HTTPS_KEYSTORE_ALGORITHM = TONY_PREFIX + "https.keystore.algorithm";
+  public static final String DEFAULT_TONY_HTTPS_KEYSTORE_ALGORITHM = "SunX509";
+
+  public static final String TONY_HTTP_PORT = TONY_PREFIX + "http.port";
+  public static final String DEFAULT_TONY_HTTP_PORT = "disabled";
+
+  public static final String TONY_SECRET_KEY = TONY_PREFIX + "secret.key";
+  public static final String DEFAULT_TONY_SECRET_KEY = "Prod";
+
+  public static final String TONY_INIT_MODULE = TONY_PREFIX + "init.module";
+  public static final String DEFAULT_TONY_INIT_MODULE = "Startup";
 
   // Application configurations
   public static final String YARN_QUEUE_NAME = TONY_PREFIX + "yarn.queue";

tony-core/src/main/java/com/linkedin/tony/Utils.java

@@ -25,6 +25,7 @@
 import java.util.stream.Collectors;
 import net.lingala.zip4j.core.ZipFile;
 import net.lingala.zip4j.exception.ZipException;
+import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Options;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
@@ -44,6 +45,10 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
+import static com.linkedin.tony.Constants.TONY_CONF_DIR;
+import static com.linkedin.tony.Constants.TONY_SITE_CONF;
+import static com.linkedin.tony.Constants.TONY_DEFAULT_XML;
+import static com.linkedin.tony.Constants.TONY_XML;
 import static org.apache.hadoop.yarn.api.records.ResourceInformation.*;
 
 

tony-core/src/main/resources/tony-default.xml

@@ -202,7 +202,7 @@
     <name>tony.application.yarn-conf-path</name>
   </property>
 
-  <!-- History folder configurations -->
+  <!--History Server configurations-->
   <property>
     <description>
       Location for Tony AM to write config and events on HDFS
@@ -211,4 +211,77 @@
     <value>/system/tony-history</value>
   </property>
 
+  <property>
+    <description>
+      Tony keytab principal
+    </description>
+    <name>tony.keytab.user</name>
+    <value>user1</value>
+  </property>
+
+  <property>
+    <description>
+      Location of Tony keytab file
+    </description>
+    <name>tony.keytab.location</name>
+    <value>/location/to/tony.keytab</value>
+  </property>
+
+  <property>
+    <name>tony.https.port</name>
+    <value>19886</value>
+  </property>
+
+  <property>
+    <description>
+      Location of keystore file. Usually as JKS.
+      For more info: https://www.playframework.com/documentation/2.6.x/ConfiguringHttps
+    </description>
+    <name>tony.https.keystore.path</name>
+    <value>/location/to/tony.jks</value>
+  </property>
+
+  <property>
+    <name>tony.https.keystore.type</name>
+    <value>JKS</value>
+  </property>
+
+  <property>
+    <name>tony.https.keystore.password</name>
+    <value>password</value>
+  </property>
+
+  <property>
+    <name>tony.https.keystore.algorithm</name>
+    <value>SunX509</value>
+  </property>
+
+  <property>
+    <description>
+      Whether or not http port is open for Tony history server.
+      In development, this should be set to enabled.
+      In production, this should always be disabled.
+    </description>
+    <name>tony.http.port</name>
+    <value>disabled</value>
+  </property>
+
+  <property>
+    <description>
+      Secret key for Play framework.
+      In development, this is optional.
+      In production, this is mandatory.
+    </description>
+    <name>tony.secret.key</name>
+    <value>Prod</value>
+  </property>
+
+  <property>
+    <description>
+      Name of module that binds other Singletons that
+      need to be run once before starting history server.
+    </description>
+    <name>tony.init.module</name>
+    <value>Startup</value>
+  </property>
 </configuration>

tony-history-server/app/controllers/JobConfigPageController.java

@@ -2,6 +2,7 @@
 
 import cache.CacheWrapper;
 import com.google.common.cache.Cache;
+import com.linkedin.tony.TonyConfigurationKeys;
 import com.typesafe.config.Config;
 import hadoop.Configuration;
 import java.util.List;
@@ -37,7 +38,7 @@ public Result index(String jobId) {
     }
 
     List<JobConfig> listOfConfigs;
-    String tonyHistoryFolder = config.getString("tony.history.location");
+    String tonyHistoryFolder = config.getString(TonyConfigurationKeys.TONY_HISTORY_LOCATION);
     Path xmlPath = new Path(tonyHistoryFolder + "/" + jobId + "/config.xml");
     listOfConfigs = cache.getIfPresent(jobId);
     if (listOfConfigs == null) {

tony-history-server/app/controllers/JobsMetadataPageController.java

@@ -2,6 +2,7 @@
 
 import cache.CacheWrapper;
 import com.google.common.cache.Cache;
+import com.linkedin.tony.TonyConfigurationKeys;
 import com.typesafe.config.Config;
 import hadoop.Configuration;
 import java.util.ArrayList;
@@ -32,6 +33,7 @@ public JobsMetadataPageController(Config config) {
   }
 
   public Result index() {
+
     HdfsConfiguration conf = Configuration.getHdfsConf();
     FileSystem myFs = HdfsUtils.getFileSystem(conf);
     Cache<String, JobMetadata> cache = CacheWrapper.getMetadataCache();
@@ -41,7 +43,7 @@ public Result index() {
     }
 
     List<JobMetadata> listOfMetadata = new ArrayList<>();
-    Path tonyHistoryFolder = new Path(config.getString("tony.history.location"));
+    Path tonyHistoryFolder = new Path(config.getString(TonyConfigurationKeys.TONY_HISTORY_LOCATION));
     JobMetadata tmpMetadata;
     String jobId;
 

tony-history-server/build.gradle

@@ -3,6 +3,7 @@ apply plugin: "idea"
 apply plugin: "distribution"
 
 model {
+
   components {
     play {
       platform play: playVersion, scala: scalaVersion, java: '1.8'
@@ -31,6 +32,7 @@ model {
 }
 
 dependencies {
+  play project(':tony-core')
   play deps.external.play_guice
   play deps.external.play_logback
   play deps.hadoop.hdfs_client
@@ -58,6 +60,7 @@ repositories {
 
 test.dependsOn('testPlayBinary')
 
+
 def ivyForArchives = tasks.findByPath('ivyForArchives')
 if (ivyForArchives) {
   ivyForArchives.dependsOn('dist')

tony-history-server/conf/application.example.conf

@@ -6,5 +6,5 @@ tony {
     user = "user1"
     location = "/some/path/to/tony.keytab"
   }
-  history.location = "/Users/phattran/.tony/system"
+  history.location = "/tmp/tony-history"
 }

tony-history-server/startTHS.sh

@@ -182,8 +182,6 @@ save () {
 if [[ -z "${TONY_CONF_DIR}" ]]; then
   echo "TONY_CONF_DIR not set. Default to /export/apps/tony"
   TONY_CONF_DIR="/export/apps/tony"
-else
-  TONY_CONF_DIR="${TONY_CONF_DIR}"
 fi
 
 if [ ! -f $TONY_CONF_DIR/tony-site.xml ]; then