Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Migrate to SDK.v3 - protocol.v2 with decode #84

Merged
merged 13 commits into from
Jun 3, 2019
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .clang-format
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: true
91 changes: 50 additions & 41 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,57 +1,66 @@
dist: trusty
sudo: false

language: scala
services:
- docker

jdk:
- openjdk8

scala:
- 2.11.11

services:
- docker

cache:
directories:
- $HOME/.ivy2
- $HOME/.sbt

stages:
- name: test
- name: release
if: tag IS present

before_install:
- docker run --privileged -d -p 9432:9432 --name bblfsh bblfsh/bblfshd
- docker exec -it bblfsh bblfshctl driver install --recommended
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils libxml2-dev

env:
global:
- OSXCROSS_PATH="$HOME/osxcross"
- OSXCROSS_REV=3034f7149716d815bc473d0a7b35d17e4cf175aa
- SDK_VERSION=10.11
- DARWIN_VERSION=15
- OSX_VERSION_MIN=10.6
- OSXCROSS_SDK_URL="https://s3.dockerproject.org/darwin/v2/MacOSX10.11.sdk.tar.xz"

install:
- mkdir -p /tmp/osxcross
- cd /tmp/osxcross
- curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" | tar -C /tmp/osxcross --strip=1 -xzf -
- curl -s -S -L -o tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz ${OSXCROSS_SDK_URL}
- UNATTENDED=yes ./build.sh >/dev/null
- mv target "${OSXCROSS_PATH}"
- curl -S -L "https://github.com/karalabe/xgo/blob/647f256c447ee20f9bf13ebc42e612d55994a383/docker/base/patch.tar.xz?raw=true" | xz -dc - | tar -xf -
- mv v1 "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/include/c++/v1"
- rm -rf /tmp/osxcross "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man"
jobs:
include:
- name: 'All tests'
stage: test
if: tag IS present # TODO(bzz): enable on PRs as soon as migrated to V2
install: &test_setup_anchor
- docker run --privileged -d -p 9432:9432 --name bblfsh bblfsh/bblfshd
- docker exec -it bblfsh bblfshctl driver install --recommended
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils
script:
- ./sbt assembly test
after_failure: &failure_logs_anchor
- docker logs bblfsh

script:
- cd $TRAVIS_BUILD_DIR
- ./sbt assembly test
- if [[ -z "$TRAVIS_TAG" ]]; then echo "Skipping this build for non-tag builds."; exit 0; fi
- ./sbt publishLocal
- openssl aes-256-cbc -K $encrypted_97aef7f4ae04_key -iv $encrypted_97aef7f4ae04_iv -in key.asc.enc -out key.asc -d
- gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc
- ./sbt publishSigned
- ./sbt sonatypeRelease
- name: 'V2: passing tests' # TODO(#83): remove, after both tests sets converge
install: *test_setup_anchor
script:
- sudo apt-get install -y binutils
- ./sbt assembly
- ./sbt "testOnly *Close* *ClientVersion* *SupportedLanguages*"
- ./sbt "testOnly org.bblfsh.client.v2.BblfshClientParseTest -- -z \"Decoded UAST after parsing\""
- ./sbt "testOnly org.bblfsh.client.v2.BblfshClientParseTest -- -z \"Decoded UAST RootNode\""
after_failure: *failure_logs_anchor

after_failure:
- docker logs bblfsh
- name: 'Cross-compile, release & publish to Sonatype'
stage: release
before_install:
- mkdir -p /tmp/osxcross
- cd /tmp/osxcross
- curl -sSL "https://codeload.github.com/tpoechtrager/osxcross/tar.gz/${OSXCROSS_REV}" | tar -C /tmp/osxcross --strip=1 -xzf -
- curl -s -S -L -o tarballs/MacOSX${SDK_VERSION}.sdk.tar.xz ${OSXCROSS_SDK_URL}
- UNATTENDED=yes ./build.sh >/dev/null
- mv target "${OSXCROSS_PATH}"
- curl -S -L "https://github.com/karalabe/xgo/blob/647f256c447ee20f9bf13ebc42e612d55994a383/docker/base/patch.tar.xz?raw=true" | xz -dc - | tar -xf -
- mv v1 "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/include/c++/v1"
- rm -rf /tmp/osxcross "${OSXCROSS_PATH}/SDK/MacOSX${SDK_VERSION}.sdk/usr/share/man"
script:
- cd $TRAVIS_BUILD_DIR
- ./sbt assembly
- ./sbt publishLocal
- openssl aes-256-cbc -K $encrypted_97aef7f4ae04_key -iv $encrypted_97aef7f4ae04_iv -in key.asc.enc -out key.asc -d
- gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc
- ./sbt publishSigned
- ./sbt sonatypeRelease
99 changes: 57 additions & 42 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import sys.process._


name := "bblfsh-client"
organization := "org.bblfsh"
version := "1.11.0"
version := "2.0.0-SNAPSHOT"

scalaVersion := "2.11.11"
val libuastVersion = "1.9.3"
val sdkVersion = "v1.17.0"
val sdkMajor = "v1"
val libuastVersion = "3.3.1"
val sdkMajor = "v3"
val sdkVersion = s"${sdkMajor}.1.0"
val protoDir = "src/main/proto"

mainClass in Compile := Some("org.bblfsh.client.cli.ScalaClientCLI")
Expand Down Expand Up @@ -92,35 +92,48 @@ getProtoFiles := {

println(s"Downloading and installing SDK$sdkMajor protocol buffer files...")

val bblfshProto = f"$protoDir%s/gopkg.in/bblfsh"
val sdkProto = f"$bblfshProto%s/sdk.$sdkMajor%s"
val bblfshProto = s"${protoDir}/github.com/bblfsh"
val sdkProto = s"${bblfshProto}/sdk/${sdkMajor}"

f"mkdir -p $sdkProto%s/protocol" !
s"mkdir -p ${sdkProto}/protocol" !

f"mkdir -p $sdkProto%s/uast" !
s"mkdir -p ${sdkProto}/uast/role" !

val unzip_dir = "sdk-" + sdkVersion.substring(1)

f"curl -SL https://github.com/bblfsh/sdk/archive/$sdkVersion%s.tar.gz" #| "tar xz" #&&
f"cp $unzip_dir%s/protocol/generated.proto $sdkProto%s/protocol/" #&&
f"cp $unzip_dir%s/uast/generated.proto $sdkProto%s/uast/" #&&
f"rm -rf $unzip_dir%s" !
s"curl -SL https://github.com/bblfsh/sdk/archive/${sdkVersion}.tar.gz" #| "tar xz" #&&
s"cp ${unzip_dir}/protocol/driver.proto ${sdkProto}/protocol/" #&&
s"cp ${unzip_dir}/uast/role/generated.proto ${sdkProto}/uast/role" #&&
s"rm -rf ${unzip_dir}" !

println("Done unpacking SDK")
}

val getLibuast = TaskKey[Unit]("getLibuast", "Retrieve libuast")
getLibuast := {
import sys.process._

val releaseUrl = s"https://github.com/bblfsh/libuast/archive/v$libuastVersion.tar.gz"
println(s"Downloading libuast from $releaseUrl")
val ghUrl = "https://github.com/bblfsh/libuast"
val os = if (System.getProperty("os.name").toLowerCase.contains("mac os x")) "darwin" else "linux"
val binaryReleaseUrl = s"${ghUrl}/releases/download/v${libuastVersion}/libuast-${os}-amd64.tar.gz"
println(s"Downloading libuast binary from ${binaryReleaseUrl}")

s"curl -sL ${binaryReleaseUrl} -o libuast-bin.tar.gz" #&&
"tar xzf libuast-bin.tar.gz" #&&
s"mv ${os}-amd64 libuast" #&&
"mkdir -p src/main/resources" #&&
"rm -rf src/main/resources/libuast" #&&
"mv libuast src/main/resources" #&&
"rm src/main/resources/libuast/libuast.so" #&& // always a static build
"rm libuast-bin.tar.gz" !

"find src/main/resources"!

f"curl -SL $releaseUrl -o libuast.tar.gz" #&&
"tar zxf libuast.tar.gz" #&&
f"mv libuast-$libuastVersion libuast" #&&
"rm -rf src/libuast-native" #&&
"mv libuast/src/ src/libuast-native" #&&
"rm -rf libuast" #&&
"rm libuast.tar.gz" !
"nm src/main/resources/libuast/libuast.a" #| "grep -c UastDecode"!

"nm src/main/resources/libuast/libuast.a" #| "wc -l"!

println("Done unpacking libuast")
}

val compileScalaLibuast = TaskKey[Unit]("compileScalaLibuast", "Compile libScalaUast JNI library")
Expand All @@ -131,15 +144,11 @@ compileScalaLibuast := {

"mkdir -p ./src/main/resources/lib/" !

val sourceFiles = "src/main/scala/org/bblfsh/client/libuast/org_bblfsh_client_libuast_Libuast.cc " +
"src/main/scala/org/bblfsh/client/libuast/jni_utils.cc " +
"src/main/scala/org/bblfsh/client/libuast/nodeiface.cc " +
"src/main/scala/org/bblfsh/client/libuast/memtracker.cc " +
"src/libuast-native/uast.cc " +
"src/libuast-native/roles.c "
val nativeSourceFiles = "src/main/native/org_bblfsh_client_v2_libuast_Libuast.cc " +
"src/main/native/jni_utils.cc "

compileUnix(sourceFiles)
crossCompileMacOS(sourceFiles)
compileUnix(nativeSourceFiles)
crossCompileMacOS(nativeSourceFiles)
}

def compileUnix(sourceFiles: String) = {
Expand All @@ -149,31 +158,32 @@ def compileUnix(sourceFiles: String) = {
if (javaHome == null) {
javaHome = "/usr/lib/jvm/java-8-openjdk-amd64"
}
val xml2Conf = "xml2-config --cflags --libs" !!

val osName = System.getProperty("os.name").toLowerCase()
if (osName.contains("mac os x")) {
val cmd:String = "g++ -shared -Wall -fPIC -O2 -std=c++11 " +
if (osName.contains("mac os x")) { // TODO(bzz): change back to '-fPIC -O2' for release
val cmd:String = "g++ -shared -Wall -g -std=c++11 " +
"-I/usr/include " +
"-I" + javaHome + "/include/ " +
"-I" + javaHome + "/include/darwin " +
"-Isrc/libuast-native/ " +
"-o src/main/resources/lib/libscalauast.dylib " +
"-Isrc/main/resources/libuast " +
"-o src/main/resources/lib/libscalauast.dylib " + // sic, must be in the classpath for the test
sourceFiles +
xml2Conf + " "
"src/main/resources/libuast/libuast.a "

checkedProcess(cmd, "macOS build")
} else {
val cmd:String = "g++ -shared -Wall -fPIC -O2 -std=c++11 " +
"-I/usr/include " +
"-I" + javaHome + "/include/ " +
"-I" + javaHome + "/include/linux " +
"-Isrc/libuast-native/ " +
"-Isrc/main/resources/libuast " +
"-o src/main/resources/lib/libscalauast.so " +
sourceFiles +
xml2Conf + " "
"src/main/resources/libuast/libuast.a "

checkedProcess(cmd, "Linux build")

"nm src/main/resources/lib/libscalauast.so" #| "grep -c UastDecode"!
}
}

Expand All @@ -183,22 +193,21 @@ def crossCompileMacOS(sourceFiles: String): Unit = {
println("Skipping cross-compilation for macOS on macOS")
return
}
val osxHome = System.getenv("OSXCROSS_PATH")

val osxHome = System.getenv("OSXCROSS_PATH")
if (osxHome == null || osxHome.isEmpty) {
println("OSXCROSS_PATH variable not defined, not cross-compiling for macOS")
return
}

val cmd = osxHome + "/bin/o64-clang++-libc++ -shared -Wall -fPIC -O2 -lxml2 -std=c++11 " +
"-I" + osxHome + "/SDK/MacOSX10.11.sdk/usr/include/libxml2/ " +
"-I" + osxHome + "/SDK/src/libuast-native/roles.c " +
"-I" + osxHome + "/SDK/MacOSX10.11.sdk/usr/include/ " +
"-I/usr/lib/jvm/java-8-openjdk-amd64/include " +
"-I/usr/lib/jvm/java-8-openjdk-amd64/include/linux " +
"-Isrc/libuast-native/ " +
"-o src/main/resources/lib/libscalauast.dylib " +
sourceFiles
sourceFiles +
"src/main/resources/libuast/libuast.a "

checkedProcess(cmd, "macOS cross-compile build")
}
Expand All @@ -214,4 +223,10 @@ def checkedProcess(cmd: String, name: String) {
}
}

cleanFiles ++= Seq(
baseDirectory.value / "src/main/resources/libuast",
baseDirectory.value / "src/main/resources/lib",
baseDirectory.value / s"${protoDir}/github.com/bblfsh"
)

mainClass := Def.sequential(getProtoFiles, getLibuast, compileScalaLibuast, (mainClass in Compile)).value
58 changes: 58 additions & 0 deletions src/main/native/jni_utils.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#include "jni_utils.h"

// Class fully qualified names
const char *CLS_NODE = "org/bblfsh/client/v2/Node";
const char *CLS_CTX = "org/bblfsh/client/v2/Context";

extern JavaVM *jvm; // FIXME(bzz): double-check and document

JNIEnv *getJNIEnv() {
JNIEnv *pEnv = NULL;

switch (jvm->GetEnv((void **)&pEnv, JNI_VERSION_1_8)) {
case JNI_OK: // Thread is ready to use, nothing to do
break;

case JNI_EDETACHED: // Thread is detached, need to attach
jvm->AttachCurrentThread((void **)&pEnv, NULL);
break;
}

return pEnv;
}

jobject NewJavaObject(JNIEnv *env, const char *className, const char *initSign,
...) {
jclass cls = env->FindClass(className);
if (env->ExceptionOccurred() || !cls) {
return NULL;
}

jmethodID initId = env->GetMethodID(cls, "<init>", initSign);
if (env->ExceptionOccurred() || !initId) {
return NULL;
}

va_list varargs;
va_start(varargs, initSign);
jobject instance = env->NewObjectV(cls, initId, varargs);
va_end(varargs);
if (env->ExceptionOccurred() || !instance) {
return NULL;
}

return instance;
}

jfieldID getField(JNIEnv *env, jobject obj, const char *name) {
jclass cls = env->GetObjectClass(obj);
if (env->ExceptionOccurred() || !cls) {
return nullptr;
}

jfieldID jfid = env->GetFieldID(cls, name, "J");
if (env->ExceptionOccurred() || !jfid) {
return nullptr;
}
return jfid;
}
13 changes: 13 additions & 0 deletions src/main/native/jni_utils.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#ifndef _Included_org_bblfsh_client_libuast_Libuast_jni_utils
#define _Included_org_bblfsh_client_libuast_Libuast_jni_utils

#include <jni.h>

extern const char *CLS_NODE;
extern const char *CLS_CTX;

JNIEnv *getJNIEnv();
jobject NewJavaObject(JNIEnv *, const char *, const char *, ...);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I follow the naming convention in use here. Google style is MixedCaseLeadingCaps for classes and functions (with a carve-out for class accessors to follow snake_case style). But we seem to be mixing that and conventional camel.

(I'm not wedded to Google style, I only mentioned that because you included a clang-format config based on that style)

Copy link
Contributor Author

@bzz bzz Jun 3, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an old code, I belive it follow the same that was used in libuast.
TBH Google one from above was just placed randomly, to have at least something, and mostly because of 2-space indentation.

I'm not settled on this one either, but I think it's totally fine to use almost whatever in a the well-localized JNI wrappers themselves.

Will be happy to refactor it, if needed, and add a better formatter config to all the projects later on, as soon as we have something functional in scala-client. For now, I'm mostly not very worried about it because of the limited scope and no expectations for this native piece to grow.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, this doesn't need to be addressed right now, I just wanted to make a note of it while we're paying attention to this code. 🙂

jfieldID getField(JNIEnv *env, jobject obj, const char *name);

#endif
Loading