Skip to content

Commit

Permalink
LIHADOOP-41391: Inject TonY version into TonY configuration (#113)
Browse files Browse the repository at this point in the history
* LIHADOOP-41391: Inject TonY version into TonY configuration

* Fix TestTonyConfigurationFields

* Remove /test/ from .gitignore, add documentation to version-info.gradle file

* Updated VersionInfo InputStream and error handling in constructor
  • Loading branch information
erwa authored and Keqiu Hu committed Dec 3, 2018
1 parent 42e2895 commit 7bc1d5e
Show file tree
Hide file tree
Showing 18 changed files with 265 additions and 17 deletions.
3 changes: 3 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ subprojects {
apply plugin: "idea"
apply plugin: "maven-publish"
apply plugin: "signing"

apply from: "$rootDir/gradle/version-info.gradle"

repositories {
mavenCentral()
}
Expand Down
83 changes: 83 additions & 0 deletions gradle/version-info.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* This file writes out a version-info.properties file during the processResources task, which gets added to the
* built jar file. This version-info.properties file is used by VersionInfo.java to return build information at runtime.
*/

import java.security.MessageDigest

ext.writeVersionInfo = { file ->
ant.propertyfile(file: file) {
entry(key: "version", value: getVersion())
entry(key: "revision", value: getRevision())
entry(key: "branch", value: getBranch())
entry(key: "user", value: getUser())
entry(key: "date", value: getDate())
entry(key: "url", value: getUrl())
entry(key: "srcChecksum", value: getSrcChecksum())
}
}

def getVersion() {
return project.version
}

def getRevision() {
return execCommand(["git", "log", "-1"]).split("\\s+")[1]
}

def getBranch() {
def gitBranchOutput = execCommand(["git", "branch"])
def lines = gitBranchOutput.split("\n")
for (line in lines) {
if (line.startsWith("*")) {
return line.split("\\s+")[1]
}
}
throw new GradleException("Could not determine git branch.")
}

def getUser() {
return System.getProperty("user.name")
}

def getDate() {
return new Date().toString()
}

def getUrl() {
def gitRemoteOutput = execCommand(["git", "remote", "-v"])
def lines = gitRemoteOutput.split("\n")
for (line in lines) {
if (line.startsWith("origin") && line.endsWith("(fetch)")) {
return line.split("\\s+")[1]
}
}
throw new GradleException("Could not determine git URI.")
}

def getSrcChecksum() {
MessageDigest digest = MessageDigest.getInstance("MD5")

sourceSets.main.java.each { file ->
digest.update(file.bytes)
}

sourceSets.main.resources.each { file ->
digest.update(file.bytes)
}

return new BigInteger(1, digest.digest()).toString(16).padLeft(32, "0")
}

def execCommand(commandList) {
def stdout = new ByteArrayOutputStream()
exec {
commandLine commandList
standardOutput = stdout
}
return stdout.toString()
}

processResources << {
writeVersionInfo(new File(sourceSets.main.output.resourcesDir, "version-info.properties"))
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import java.util.UUID;

import com.linkedin.tony.TonyClient;
import com.linkedin.tony.Utils;
import com.linkedin.tony.util.Utils;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,22 @@
import com.linkedin.tony.Constants;
import com.linkedin.tony.TonyClient;
import com.linkedin.tony.TonyConfigurationKeys;
import com.linkedin.tony.Utils;
import com.linkedin.tony.util.Utils;
import com.linkedin.tony.rpc.TaskUrl;
import com.linkedin.tonyproxy.ProxyServer;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.ServerSocket;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.UUID;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.exceptions.YarnException;

import static com.linkedin.tony.Constants.*;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
import com.google.common.annotations.VisibleForTesting;
import com.linkedin.tony.io.HdfsAvroFileSplitReader;
import com.linkedin.tony.rpc.impl.ApplicationRpcClient;
import com.linkedin.tony.util.Utils;
import java.io.File;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import com.linkedin.tony.tensorflow.TensorFlowContainerRequest;
import com.linkedin.tony.tensorflow.TonySession;
import com.linkedin.tony.tensorflow.TonySession.TonyTask;
import com.linkedin.tony.util.HistoryFileUtils;
import com.linkedin.tony.util.Utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
Expand Down
3 changes: 3 additions & 0 deletions tony-core/src/main/java/com/linkedin/tony/TonyClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import com.google.common.collect.ImmutableSet;
import com.linkedin.tony.rpc.TaskUrl;
import com.linkedin.tony.rpc.impl.ApplicationRpcClient;
import com.linkedin.tony.util.Utils;
import com.linkedin.tony.util.VersionInfo;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
Expand Down Expand Up @@ -138,6 +140,7 @@ public TonyClient() {
public TonyClient(Configuration conf) {
initOptions();
tonyConf = conf;
VersionInfo.injectVersionInfo(tonyConf);
}

public ImmutableSet<TaskUrl> getTaskUrls() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,16 @@ private TonyConfigurationKeys() {

public static final String TONY_PREFIX = "tony.";

// Version info configuration
public static final String TONY_VERSION_INFO_PREFIX = TONY_PREFIX + "version-info.";
public static final String TONY_VERSION_INFO_VERSION= TONY_VERSION_INFO_PREFIX + "version";
public static final String TONY_VERSION_INFO_REVISION = TONY_VERSION_INFO_PREFIX + "revision";
public static final String TONY_VERSION_INFO_BRANCH = TONY_VERSION_INFO_PREFIX + "branch";
public static final String TONY_VERSION_INFO_USER = TONY_VERSION_INFO_PREFIX + "user";
public static final String TONY_VERSION_INFO_DATE = TONY_VERSION_INFO_PREFIX + "date";
public static final String TONY_VERSION_INFO_URL = TONY_VERSION_INFO_PREFIX + "url";
public static final String TONY_VERSION_INFO_CHECKSUM = TONY_VERSION_INFO_PREFIX + "checksum";

public static final String OTHER_NAMENODES_TO_ACCESS = TONY_PREFIX + "other.namenodes";

// History folder configuration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/
package com.linkedin.tony;

import com.linkedin.tony.util.Utils;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
package com.linkedin.tony.io;

import com.google.common.annotations.VisibleForTesting;
import com.linkedin.tony.Utils;
import com.linkedin.tony.util.Utils;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.File;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
*/
package com.linkedin.tony.rpc.impl.pb;

import com.linkedin.tony.ProtoUtils;
import com.linkedin.tony.util.ProtoUtils;
import com.linkedin.tony.rpc.GetTaskUrlsResponse;
import com.linkedin.tony.rpc.TaskUrl;
import com.linkedin.tony.rpc.proto.YarnTensorFlowClusterProtos.GetTaskUrlsResponseProto;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import com.google.common.base.Preconditions;
import com.linkedin.tony.Constants;
import com.linkedin.tony.TonyConfigurationKeys;
import com.linkedin.tony.Utils;
import com.linkedin.tony.util.Utils;
import com.linkedin.tony.rpc.TaskUrl;
import java.net.URI;
import java.util.ArrayList;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
* Copyright 2018 LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
* See LICENSE in the project root for license information.
*/
package com.linkedin.tony;
package com.linkedin.tony.util;

import com.linkedin.tony.TonyJobMetadata;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright 2018 LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
* See LICENSE in the project root for license information.
*/
package com.linkedin.tony;
package com.linkedin.tony.util;

import com.linkedin.tony.rpc.TaskUrl;
import com.linkedin.tony.rpc.proto.YarnTensorFlowClusterProtos.GetTaskUrlsResponseProto.TaskUrlProto;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@
* Copyright 2018 LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
* See LICENSE in the project root for license information.
*/
package com.linkedin.tony;
package com.linkedin.tony.util;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.linkedin.tony.Constants;
import com.linkedin.tony.TFConfig;
import com.linkedin.tony.TonyConfigurationKeys;
import com.linkedin.tony.rpc.TaskUrl;
import com.linkedin.tony.tensorflow.TensorFlowContainerRequest;
import java.io.File;
Expand Down
Loading

0 comments on commit 7bc1d5e

Please sign in to comment.