Merge pull request #84 from bzz/migrate-to-sdk-v3

Migrate to SDK.v3 - protocol.v2 with decode

Author: bzz

.clang-format

@@ -0,0 +1,2 @@
+BasedOnStyle: Google
+AllowShortIfStatementsOnASingleLine: true

.travis.yml

@@ -1,57 +1,66 @@
 dist: trusty
-sudo: false
-
 language: scala
-services:
-  - docker
-
 jdk:
   - openjdk8
-
 scala:
    - 2.11.11
 
+services:
+  - docker
+
 cache:
   directories:
     - $HOME/.ivy2
     - $HOME/.sbt
 
+stages:
+  - name: test
+  - name: release
+    if: tag IS present
 
-before_install:
- - docker run --privileged -d -p 9432:9432 --name bblfsh bblfsh/bblfshd
- - docker exec -it bblfsh bblfshctl driver install --recommended
- - sudo apt-get update
- - sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils libxml2-dev
-
-env:
-  global:
-   - OSXCROSS_PATH="$HOME/osxcross"
-   - OSXCROSS_REV=3034f7149716d815bc473d0a7b35d17e4cf175aa
-   - SDK_VERSION=10.11
-   - DARWIN_VERSION=15
-   - OSX_VERSION_MIN=10.6
-   - OSXCROSS_SDK_URL="https://s3.dockerproject.org/darwin/v2/MacOSX10.11.sdk.tar.xz"
 
-install:
- - mkdir -p /tmp/osxcross
- - cd /tmp/osxcross
- - curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" | tar -C /tmp/osxcross --strip=1 -xzf -
- - curl -s -S -L -o tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz ${OSXCROSS_SDK_URL}
- - UNATTENDED=yes ./build.sh >/dev/null
- - mv target "${OSXCROSS_PATH}"
- - curl -S -L "https://github.com/karalabe/xgo/blob/647f256c447ee20f9bf13ebc42e612d55994a383/docker/base/patch.tar.xz?raw=true" | xz -dc - | tar -xf -
- - mv v1 "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/include/c++/v1"
- - rm -rf /tmp/osxcross "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man"
+jobs:
+  include:
+    - name: 'All tests'
+      stage: test
+      if: tag IS present # TODO(bzz): enable on PRs as soon as migrated to V2
+      install: &test_setup_anchor
+        - docker run --privileged -d -p 9432:9432 --name bblfsh bblfsh/bblfshd
+        - docker exec -it bblfsh bblfshctl driver install --recommended
+        - sudo apt-get update
+        - sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils
+      script:
+        - ./sbt assembly test
+      after_failure: &failure_logs_anchor
+        - docker logs bblfsh
 
-script:
- - cd $TRAVIS_BUILD_DIR
- - ./sbt assembly test
- - if [[ -z "$TRAVIS_TAG" ]]; then echo "Skipping this build for non-tag builds."; exit 0; fi
- - ./sbt publishLocal
- - openssl aes-256-cbc -K $encrypted_97aef7f4ae04_key -iv $encrypted_97aef7f4ae04_iv -in key.asc.enc -out key.asc -d
- - gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc
- - ./sbt publishSigned
- - ./sbt sonatypeRelease
+    - name: 'V2: passing tests' # TODO(#83): remove, after both tests sets converge
+      install: *test_setup_anchor
+      script:
+        - sudo apt-get install -y binutils
+        - ./sbt assembly
+        - ./sbt "testOnly *Close* *ClientVersion* *SupportedLanguages*"
+        - ./sbt "testOnly org.bblfsh.client.v2.BblfshClientParseTest -- -z \"Decoded UAST after parsing\""
+        - ./sbt "testOnly org.bblfsh.client.v2.BblfshClientParseTest -- -z \"Decoded UAST RootNode\""
+      after_failure: *failure_logs_anchor
 
-after_failure:
- - docker logs bblfsh
+    - name: 'Cross-compile, release & publish to Sonatype'
+      stage: release
+      before_install:
+        - mkdir -p /tmp/osxcross
+        - cd /tmp/osxcross
+        - curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" | tar -C /tmp/osxcross --strip=1 -xzf -
+        - curl -s -S -L -o tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz ${OSXCROSS_SDK_URL}
+        - UNATTENDED=yes ./build.sh >/dev/null
+        - mv target "${OSXCROSS_PATH}"
+        - curl -S -L "https://github.com/karalabe/xgo/blob/647f256c447ee20f9bf13ebc42e612d55994a383/docker/base/patch.tar.xz?raw=true" | xz -dc - | tar -xf -
+        - mv v1 "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/include/c++/v1"
+        - rm -rf /tmp/osxcross "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man"
+      script:
+        - cd $TRAVIS_BUILD_DIR
+        - ./sbt assembly
+        - ./sbt publishLocal
+        - openssl aes-256-cbc -K $encrypted_97aef7f4ae04_key -iv $encrypted_97aef7f4ae04_iv -in key.asc.enc -out key.asc -d
+        - gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc
+        - ./sbt publishSigned
+        - ./sbt sonatypeRelease

build.sbt

@@ -1,13 +1,13 @@
-import sys.process._
+
 
 name := "bblfsh-client"
 organization := "org.bblfsh"
-version := "1.11.0"
+version := "2.0.0-SNAPSHOT"
 
 scalaVersion := "2.11.11"
-val libuastVersion = "1.9.3"
-val sdkVersion = "v1.17.0"
-val sdkMajor = "v1"
+val libuastVersion = "3.3.1"
+val sdkMajor = "v3"
+val sdkVersion = s"${sdkMajor}.1.0"
 val protoDir = "src/main/proto"
 
 mainClass in Compile := Some("org.bblfsh.client.cli.ScalaClientCLI")
@@ -92,35 +92,48 @@ getProtoFiles := {
 
     println(s"Downloading and installing SDK$sdkMajor protocol buffer files...")
 
-    val bblfshProto = f"$protoDir%s/gopkg.in/bblfsh"
-    val sdkProto = f"$bblfshProto%s/sdk.$sdkMajor%s"
+    val bblfshProto = s"${protoDir}/github.com/bblfsh"
+    val sdkProto = s"${bblfshProto}/sdk/${sdkMajor}"
 
-    f"mkdir -p $sdkProto%s/protocol" !
+    s"mkdir -p ${sdkProto}/protocol" !
 
-    f"mkdir -p $sdkProto%s/uast" !
+    s"mkdir -p ${sdkProto}/uast/role" !
 
     val unzip_dir = "sdk-" + sdkVersion.substring(1)
 
-    f"curl -SL https://github.com/bblfsh/sdk/archive/$sdkVersion%s.tar.gz" #| "tar xz" #&&
-    f"cp $unzip_dir%s/protocol/generated.proto $sdkProto%s/protocol/" #&&
-    f"cp $unzip_dir%s/uast/generated.proto $sdkProto%s/uast/" #&&
-    f"rm -rf $unzip_dir%s" !
+    s"curl -SL https://github.com/bblfsh/sdk/archive/${sdkVersion}.tar.gz" #| "tar xz" #&&
+    s"cp ${unzip_dir}/protocol/driver.proto ${sdkProto}/protocol/" #&&
+    s"cp ${unzip_dir}/uast/role/generated.proto ${sdkProto}/uast/role" #&&
+    s"rm -rf ${unzip_dir}" !
+
+    println("Done unpacking SDK")
 }
 
 val getLibuast = TaskKey[Unit]("getLibuast", "Retrieve libuast")
 getLibuast := {
     import sys.process._
 
-    val releaseUrl = s"https://github.com/bblfsh/libuast/archive/v$libuastVersion.tar.gz"
-    println(s"Downloading libuast from $releaseUrl")
+    val ghUrl = "https://github.com/bblfsh/libuast"
+    val os = if (System.getProperty("os.name").toLowerCase.contains("mac os x")) "darwin" else "linux"
+    val binaryReleaseUrl = s"${ghUrl}/releases/download/v${libuastVersion}/libuast-${os}-amd64.tar.gz"
+    println(s"Downloading libuast binary from ${binaryReleaseUrl}")
+
+    s"curl -sL ${binaryReleaseUrl} -o libuast-bin.tar.gz" #&&
+    "tar xzf libuast-bin.tar.gz" #&&
+    s"mv ${os}-amd64 libuast" #&&
+    "mkdir -p src/main/resources" #&&
+    "rm -rf src/main/resources/libuast" #&&
+    "mv libuast src/main/resources" #&&
+    "rm src/main/resources/libuast/libuast.so" #&& // always a static build
+    "rm libuast-bin.tar.gz" !
+
+    "find src/main/resources"!
 
-    f"curl -SL $releaseUrl -o libuast.tar.gz" #&&
-    "tar zxf libuast.tar.gz" #&&
-    f"mv libuast-$libuastVersion libuast" #&&
-    "rm -rf src/libuast-native" #&&
-    "mv libuast/src/ src/libuast-native" #&&
-    "rm -rf libuast" #&&
-    "rm libuast.tar.gz" !
+    "nm src/main/resources/libuast/libuast.a" #| "grep -c UastDecode"!
+
+    "nm src/main/resources/libuast/libuast.a" #| "wc -l"!
+
+    println("Done unpacking libuast")
 }
 
 val compileScalaLibuast = TaskKey[Unit]("compileScalaLibuast", "Compile libScalaUast JNI library")
@@ -131,15 +144,11 @@ compileScalaLibuast := {
 
     "mkdir -p ./src/main/resources/lib/" !
 
-    val sourceFiles = "src/main/scala/org/bblfsh/client/libuast/org_bblfsh_client_libuast_Libuast.cc " +
-        "src/main/scala/org/bblfsh/client/libuast/jni_utils.cc " +
-        "src/main/scala/org/bblfsh/client/libuast/nodeiface.cc " +
-        "src/main/scala/org/bblfsh/client/libuast/memtracker.cc " +
-        "src/libuast-native/uast.cc " +
-        "src/libuast-native/roles.c "
+    val nativeSourceFiles = "src/main/native/org_bblfsh_client_v2_libuast_Libuast.cc " +
+        "src/main/native/jni_utils.cc "
 
-    compileUnix(sourceFiles)
-    crossCompileMacOS(sourceFiles)
+    compileUnix(nativeSourceFiles)
+    crossCompileMacOS(nativeSourceFiles)
 }
 
 def compileUnix(sourceFiles: String) = {
@@ -149,31 +158,32 @@ def compileUnix(sourceFiles: String) = {
   if (javaHome == null) {
     javaHome = "/usr/lib/jvm/java-8-openjdk-amd64"
   }
-  val xml2Conf = "xml2-config --cflags --libs" !!
 
   val osName = System.getProperty("os.name").toLowerCase()
-  if (osName.contains("mac os x")) {
-    val cmd:String = "g++ -shared -Wall -fPIC -O2 -std=c++11 " +
+  if (osName.contains("mac os x")) { // TODO(bzz): change back to '-fPIC -O2' for release
+    val cmd:String = "g++ -shared -Wall -g -std=c++11 " +
       "-I/usr/include " +
       "-I" + javaHome + "/include/ " +
       "-I" + javaHome + "/include/darwin " +
-      "-Isrc/libuast-native/  " +
-      "-o src/main/resources/lib/libscalauast.dylib " +
+      "-Isrc/main/resources/libuast " +
+      "-o src/main/resources/lib/libscalauast.dylib " + // sic, must be in the classpath for the test
       sourceFiles +
-      xml2Conf + " "
+      "src/main/resources/libuast/libuast.a "
 
     checkedProcess(cmd, "macOS build")
   } else {
     val cmd:String = "g++ -shared -Wall -fPIC -O2 -std=c++11 " +
       "-I/usr/include " +
       "-I" + javaHome + "/include/ " +
       "-I" + javaHome + "/include/linux " +
-      "-Isrc/libuast-native/  " +
+      "-Isrc/main/resources/libuast " +
       "-o src/main/resources/lib/libscalauast.so " +
       sourceFiles +
-      xml2Conf + " "
+      "src/main/resources/libuast/libuast.a "
 
     checkedProcess(cmd, "Linux build")
+
+    "nm src/main/resources/lib/libscalauast.so" #| "grep -c UastDecode"!
   }
 }
 
@@ -183,22 +193,21 @@ def crossCompileMacOS(sourceFiles: String): Unit = {
       println("Skipping cross-compilation for macOS on macOS")
       return
   }
-  val osxHome = System.getenv("OSXCROSS_PATH")
 
+  val osxHome = System.getenv("OSXCROSS_PATH")
   if (osxHome == null || osxHome.isEmpty) {
     println("OSXCROSS_PATH variable not defined, not cross-compiling for macOS")
     return
   }
 
   val cmd = osxHome + "/bin/o64-clang++-libc++ -shared -Wall -fPIC -O2 -lxml2 -std=c++11 " +
-      "-I" + osxHome + "/SDK/MacOSX10.11.sdk/usr/include/libxml2/ " +
-      "-I" + osxHome + "/SDK/src/libuast-native/roles.c " +
       "-I" + osxHome + "/SDK/MacOSX10.11.sdk/usr/include/ " +
       "-I/usr/lib/jvm/java-8-openjdk-amd64/include " +
       "-I/usr/lib/jvm/java-8-openjdk-amd64/include/linux " +
       "-Isrc/libuast-native/ " +
       "-o src/main/resources/lib/libscalauast.dylib " +
-      sourceFiles
+      sourceFiles +
+      "src/main/resources/libuast/libuast.a "
 
   checkedProcess(cmd, "macOS cross-compile build")
 }
@@ -214,4 +223,10 @@ def checkedProcess(cmd: String, name: String) {
   }
 }
 
+cleanFiles ++= Seq(
+  baseDirectory.value / "src/main/resources/libuast",
+  baseDirectory.value / "src/main/resources/lib",
+  baseDirectory.value / s"${protoDir}/github.com/bblfsh"
+)
+
 mainClass := Def.sequential(getProtoFiles, getLibuast, compileScalaLibuast, (mainClass in Compile)).value

src/main/native/jni_utils.cc

@@ -0,0 +1,58 @@
+#include "jni_utils.h"
+
+// Class fully qualified names
+const char *CLS_NODE = "org/bblfsh/client/v2/Node";
+const char *CLS_CTX = "org/bblfsh/client/v2/Context";
+
+extern JavaVM *jvm;  // FIXME(bzz): double-check and document
+
+JNIEnv *getJNIEnv() {
+  JNIEnv *pEnv = NULL;
+
+  switch (jvm->GetEnv((void **)&pEnv, JNI_VERSION_1_8)) {
+    case JNI_OK:  // Thread is ready to use, nothing to do
+      break;
+
+    case JNI_EDETACHED:  // Thread is detached, need to attach
+      jvm->AttachCurrentThread((void **)&pEnv, NULL);
+      break;
+  }
+
+  return pEnv;
+}
+
+jobject NewJavaObject(JNIEnv *env, const char *className, const char *initSign,
+                      ...) {
+  jclass cls = env->FindClass(className);
+  if (env->ExceptionOccurred() || !cls) {
+    return NULL;
+  }
+
+  jmethodID initId = env->GetMethodID(cls, "<init>", initSign);
+  if (env->ExceptionOccurred() || !initId) {
+    return NULL;
+  }
+
+  va_list varargs;
+  va_start(varargs, initSign);
+  jobject instance = env->NewObjectV(cls, initId, varargs);
+  va_end(varargs);
+  if (env->ExceptionOccurred() || !instance) {
+    return NULL;
+  }
+
+  return instance;
+}
+
+jfieldID getField(JNIEnv *env, jobject obj, const char *name) {
+  jclass cls = env->GetObjectClass(obj);
+  if (env->ExceptionOccurred() || !cls) {
+    return nullptr;
+  }
+
+  jfieldID jfid = env->GetFieldID(cls, name, "J");
+  if (env->ExceptionOccurred() || !jfid) {
+    return nullptr;
+  }
+  return jfid;
+}

src/main/native/jni_utils.h

@@ -0,0 +1,13 @@
+#ifndef _Included_org_bblfsh_client_libuast_Libuast_jni_utils
+#define _Included_org_bblfsh_client_libuast_Libuast_jni_utils
+
+#include <jni.h>
+
+extern const char *CLS_NODE;
+extern const char *CLS_CTX;
+
+JNIEnv *getJNIEnv();
+jobject NewJavaObject(JNIEnv *, const char *, const char *, ...);
+jfieldID getField(JNIEnv *env, jobject obj, const char *name);
+
+#endif