diff --git a/src/main/native/jni_utils.cc b/src/main/native/jni_utils.cc index 6d953c7..2a90e4c 100644 --- a/src/main/native/jni_utils.cc +++ b/src/main/native/jni_utils.cc @@ -28,7 +28,8 @@ JNIEnv *getJNIEnv() { const char CLS_NODE[] = "org/bblfsh/client/v2/NodeExt"; const char CLS_CTX_EXT[] = "org/bblfsh/client/v2/ContextExt"; const char CLS_CTX[] = "org/bblfsh/client/v2/Context"; -const char CLS_TO[] = "org/bblfsh/client/v2/TreeOrder"; +const char CLS_TO[] = "org/bblfsh/client/v2/libuast/Libuast$TreeOrder"; +const char CLS_ENCS[] = "org/bblfsh/client/v2/libuast/Libuast$UastFormat"; const char CLS_OBJ[] = "java/lang/Object"; const char CLS_RE[] = "java/lang/RuntimeException"; const char CLS_JNODE[] = "org/bblfsh/client/v2/JNode"; diff --git a/src/main/native/jni_utils.h b/src/main/native/jni_utils.h index 467791c..957a98b 100644 --- a/src/main/native/jni_utils.h +++ b/src/main/native/jni_utils.h @@ -12,6 +12,7 @@ extern const char CLS_CTX[]; extern const char CLS_OBJ[]; extern const char CLS_RE[]; extern const char CLS_TO[]; +extern const char CLS_ENCS[]; // Fully qualified class names for Bablefish UAST types extern const char CLS_JNODE[]; diff --git a/src/main/native/org_bblfsh_client_v2_Context.h b/src/main/native/org_bblfsh_client_v2_Context.h index 1c307e7..4b8eed6 100644 --- a/src/main/native/org_bblfsh_client_v2_Context.h +++ b/src/main/native/org_bblfsh_client_v2_Context.h @@ -25,11 +25,11 @@ JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_Context_filter /* * Class: org_bblfsh_client_v2_Context - * Method: encode - * Signature: (Lorg/bblfsh/client/v2/JNode;)Ljava/nio/ByteBuffer; + * Method: nativeEncode + * Signature: (Lorg/bblfsh/client/v2/JNode;I)Ljava/nio/ByteBuffer; */ -JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_Context_encode - (JNIEnv *, jobject, jobject); +JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_Context_nativeEncode + (JNIEnv *, jobject, jobject, jint); /* * Class: org_bblfsh_client_v2_Context diff --git a/src/main/native/org_bblfsh_client_v2_ContextExt.h b/src/main/native/org_bblfsh_client_v2_ContextExt.h index e3cc34d..e1d4341 100644 --- a/src/main/native/org_bblfsh_client_v2_ContextExt.h +++ b/src/main/native/org_bblfsh_client_v2_ContextExt.h @@ -25,11 +25,11 @@ JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_ContextExt_filter /* * Class: org_bblfsh_client_v2_ContextExt - * Method: encode - * Signature: (Lorg/bblfsh/client/v2/NodeExt;)Ljava/nio/ByteBuffer; + * Method: nativeEncode + * Signature: (Lorg/bblfsh/client/v2/NodeExt;I)Ljava/nio/ByteBuffer; */ -JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_ContextExt_encode - (JNIEnv *, jobject, jobject); +JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_ContextExt_nativeEncode + (JNIEnv *, jobject, jobject, jint); /* * Class: org_bblfsh_client_v2_ContextExt diff --git a/src/main/native/org_bblfsh_client_v2_libuast_Libuast.cc b/src/main/native/org_bblfsh_client_v2_libuast_Libuast.cc index 0467247..fbdcbd9 100644 --- a/src/main/native/org_bblfsh_client_v2_libuast_Libuast.cc +++ b/src/main/native/org_bblfsh_client_v2_libuast_Libuast.cc @@ -631,8 +631,8 @@ class Context { // ========================================== JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_libuast_Libuast_decode( - JNIEnv *env, jobject self, jobject directBuf) { - UastFormat format = UAST_BINARY; // TODO: make it arg + JNIEnv *env, jobject self, jobject directBuf, jint fmt) { + UastFormat format = (UastFormat) fmt; // works only with ByteBuffer.allocateDirect() void *buf = env->GetDirectBufferAddress(directBuf); @@ -824,12 +824,12 @@ JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_Context_filter( return iter; } -JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_Context_encode( - JNIEnv *env, jobject self, jobject jnode) { - UastFormat fmt = UAST_BINARY; // TODO(#107): make it argument +JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_Context_nativeEncode( + JNIEnv *env, jobject self, jobject jnode, jint fmt) { + UastFormat format = (UastFormat) fmt; // TODO(#107): make it argument Context *p = getHandle(env, self, nativeContext); - return p->Encode(jnode, fmt); + return p->Encode(jnode, format); } JNIEXPORT jlong JNICALL @@ -864,12 +864,12 @@ JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_ContextExt_filter( return filterUastIterExt(ctx, self, jquery, env); } -JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_ContextExt_encode( - JNIEnv *env, jobject self, jobject node) { - UastFormat fmt = UAST_BINARY; // TODO(#107): make it argument +JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_ContextExt_nativeEncode( + JNIEnv *env, jobject self, jobject node, jint fmt) { + UastFormat format = (UastFormat) fmt; ContextExt *p = getHandle(env, self, nativeContext); - return p->Encode(node, fmt); + return p->Encode(node, format); } JNIEXPORT void JNICALL @@ -924,6 +924,18 @@ JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_libuast_Libuast_getTreeOrder return jObj; } +// ========================================== +// UAST encoding formats +// ========================================== +JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_libuast_Libuast_getUastFormats(JNIEnv *env, + jobject self) { + jobject jObj = NewJavaObject(env, CLS_ENCS, "(II)V", + UAST_BINARY, + UAST_YAML); + return jObj; +} + + JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) { JNIEnv *env; diff --git a/src/main/native/org_bblfsh_client_v2_libuast_Libuast.h b/src/main/native/org_bblfsh_client_v2_libuast_Libuast.h index 4c69637..a5f74e2 100644 --- a/src/main/native/org_bblfsh_client_v2_libuast_Libuast.h +++ b/src/main/native/org_bblfsh_client_v2_libuast_Libuast.h @@ -10,19 +10,27 @@ extern "C" { /* * Class: org_bblfsh_client_v2_libuast_Libuast * Method: decode - * Signature: (Ljava/nio/ByteBuffer;)Lorg/bblfsh/client/v2/ContextExt; + * Signature: (Ljava/nio/ByteBuffer;I)Lorg/bblfsh/client/v2/ContextExt; */ JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_libuast_Libuast_decode - (JNIEnv *, jobject, jobject); + (JNIEnv *, jobject, jobject, jint); /* * Class: org_bblfsh_client_v2_libuast_Libuast * Method: getTreeOrders - * Signature: ()Lorg/bblfsh/client/v2/TreeOrder; + * Signature: ()Lorg/bblfsh/client/v2/libuast/Libuast/TreeOrder; */ JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_libuast_Libuast_getTreeOrders (JNIEnv *, jobject); +/* + * Class: org_bblfsh_client_v2_libuast_Libuast + * Method: getUastFormats + * Signature: ()Lorg/bblfsh/client/v2/libuast/Libuast/UastFormat; + */ +JNIEXPORT jobject JNICALL Java_org_bblfsh_client_v2_libuast_Libuast_getUastFormats + (JNIEnv *, jobject); + #ifdef __cplusplus } #endif diff --git a/src/main/native/org_bblfsh_client_v2_libuast_Libuast_TreeOrder.h b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_TreeOrder.h new file mode 100644 index 0000000..f1a925d --- /dev/null +++ b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_TreeOrder.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_bblfsh_client_v2_libuast_Libuast_TreeOrder */ + +#ifndef _Included_org_bblfsh_client_v2_libuast_Libuast_TreeOrder +#define _Included_org_bblfsh_client_v2_libuast_Libuast_TreeOrder +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/main/native/org_bblfsh_client_v2_libuast_Libuast_TreeOrder__.h b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_TreeOrder__.h new file mode 100644 index 0000000..e6d0c3c --- /dev/null +++ b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_TreeOrder__.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_bblfsh_client_v2_libuast_Libuast_TreeOrder__ */ + +#ifndef _Included_org_bblfsh_client_v2_libuast_Libuast_TreeOrder__ +#define _Included_org_bblfsh_client_v2_libuast_Libuast_TreeOrder__ +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/main/native/org_bblfsh_client_v2_libuast_Libuast_UastFormat.h b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_UastFormat.h new file mode 100644 index 0000000..edbd804 --- /dev/null +++ b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_UastFormat.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_bblfsh_client_v2_libuast_Libuast_UastFormat */ + +#ifndef _Included_org_bblfsh_client_v2_libuast_Libuast_UastFormat +#define _Included_org_bblfsh_client_v2_libuast_Libuast_UastFormat +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/main/native/org_bblfsh_client_v2_libuast_Libuast_UastFormat__.h b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_UastFormat__.h new file mode 100644 index 0000000..6e378d1 --- /dev/null +++ b/src/main/native/org_bblfsh_client_v2_libuast_Libuast_UastFormat__.h @@ -0,0 +1,13 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_bblfsh_client_v2_libuast_Libuast_UastFormat__ */ + +#ifndef _Included_org_bblfsh_client_v2_libuast_Libuast_UastFormat__ +#define _Included_org_bblfsh_client_v2_libuast_Libuast_UastFormat__ +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/main/scala/org/bblfsh/client/v2/BblfshClient.scala b/src/main/scala/org/bblfsh/client/v2/BblfshClient.scala index 9f12679..c6c2788 100644 --- a/src/main/scala/org/bblfsh/client/v2/BblfshClient.scala +++ b/src/main/scala/org/bblfsh/client/v2/BblfshClient.scala @@ -21,7 +21,7 @@ class BblfshClient(host: String, port: Int, maxMsgSize: Int) { private val stubInfo = DriverHostGrpc.blockingStub(channel) /** - * Parses file with a given name and content using + * Parses file with a given name and content using * the provided timeout. * * @param name file name @@ -110,16 +110,25 @@ object BblfshClient { val DEFAULT_MAX_MSG_SIZE = 100 * 1024 * 1024 // bytes private val libuast = new Libuast + private val orders = libuast.getTreeOrders + private val formats = libuast.getUastFormats - private val treeOrder = libuast.getTreeOrders + abstract class UastFormat(val toInt: Int) + abstract class TreeOrder(val toInt: Int) - val AnyOrder = treeOrder.AnyOrder - val PreOrder = treeOrder.PreOrder - val PostOrder = treeOrder.PostOrder - val LevelOrder = treeOrder.LevelOrder - val ChildrenOrder = treeOrder.ChildrenOrder - val PositionOrder = treeOrder.PositionOrder + // Lift tree order as constants + case object UastBinary extends UastFormat(formats.uastBinary) + case object UastYaml extends UastFormat(formats.uastYaml) + // Lift orders from libuast as types + case object AnyOrder extends TreeOrder(orders.anyOrder) + case object PreOrder extends TreeOrder(orders.preOrder) + case object PostOrder extends TreeOrder(orders.postOrder) + case object LevelOrder extends TreeOrder(orders.levelOrder) + case object ChildrenOrder extends TreeOrder(orders.childrenOrder) + case object PositionOrder extends TreeOrder(orders.positionOrder) + + /** Creates a BblfshClient with default parameters */ def apply( host: String, port: Int, maxMsgSize: Int = DEFAULT_MAX_MSG_SIZE @@ -127,15 +136,26 @@ object BblfshClient { /** * Decodes bytes from wired format of bblfsh protocol.v2. - * Requires a buffer in Direct mode. + * Requires a buffer in Direct mode, and the format + * to decode from * * Since v2. */ - def decode(buf: ByteBuffer): ContextExt = Libuast.synchronized { + def decode(buf: ByteBuffer, fmt: UastFormat): ContextExt = Libuast.synchronized { if (!buf.isDirect()) { throw new RuntimeException("Only directly-allocated buffer decoding is supported.") } - libuast.decode(buf) + libuast.decode(buf, fmt) + } + + /** + * Decodes bytes from wired binary format of bblfsh protocol.v2. + * Requires a buffer in Direct mode + * + * Since v2. + */ + def decode(buf: ByteBuffer): ContextExt = Libuast.synchronized { + decode(buf, UastBinary) } /** Enables API: resp.uast.decode() */ @@ -146,10 +166,10 @@ object BblfshClient { * Always copies memory to a new buffer in Direct mode, * to be able to pass it to JNI. */ - def decode(): ContextExt = { + def decode(fmt: UastFormat): ContextExt = { val bufDirectCopy = ByteBuffer.allocateDirect(buf.size) buf.copyTo(bufDirectCopy) - val result = BblfshClient.decode(bufDirectCopy) + val result = BblfshClient.decode(bufDirectCopy, fmt) // Sometimes the direct buffer can take a lot to deallocate, // causing Out of Memory, because it is not allocated in // in the JVM heap and will only be deallocated them when @@ -160,33 +180,46 @@ object BblfshClient { System.gc() result } + + /** + * Decodes in binary format + */ + def decode(): ContextExt = { + decode(UastBinary) + } } /** Enables API: resp.get() */ implicit class ResponseMethods(val resp: ParseResponse) { - def get(): JNode = { - val ctx = resp.uast.decode() + /** Gets the root decoding the tree in binary format */ + def get(fmt: UastFormat): JNode = { + val ctx = resp.uast.decode(fmt) val node = ctx.root().load() ctx.dispose() node } + + /** Gets the root node decoding the tree in binary format */ + def get(): JNode = { + get(UastBinary) + } } /** Enables API: client.filter and client.iterator for client an instance of BblfshClient */ implicit class BblfshClientMethods(val client: BblfshClient) { def filter(node: NodeExt, query: String) = BblfshClient.filter(node, query) def filter(node: JNode, query: String) = BblfshClient.filter(node, query) - def iterator(node: NodeExt, treeOrder: Int) = BblfshClient.iterator(node, treeOrder) - def iterator(node: JNode, treeOrder: Int) = BblfshClient.iterator(node, treeOrder) + def iterator(node: NodeExt, treeOrder: TreeOrder) = BblfshClient.iterator(node, treeOrder) + def iterator(node: JNode, treeOrder: TreeOrder) = BblfshClient.iterator(node, treeOrder) } /** Factory method for iterator over an external/native node */ - def iterator(node: NodeExt, treeOrder: Int): Libuast.UastIterExt = { + def iterator(node: NodeExt, treeOrder: TreeOrder): Libuast.UastIterExt = { Libuast.UastIterExt(node, treeOrder) } /** Factory method for iterator over an managed node */ - def iterator(node: JNode, treeOrder: Int): Libuast.UastIter = { + def iterator(node: JNode, treeOrder: TreeOrder): Libuast.UastIter = { Libuast.UastIter(node, treeOrder) } @@ -203,4 +236,3 @@ object BblfshClient { } } - diff --git a/src/main/scala/org/bblfsh/client/v2/ContextExt.scala b/src/main/scala/org/bblfsh/client/v2/ContextExt.scala index 3eff652..7cc8328 100644 --- a/src/main/scala/org/bblfsh/client/v2/ContextExt.scala +++ b/src/main/scala/org/bblfsh/client/v2/ContextExt.scala @@ -1,5 +1,5 @@ package org.bblfsh.client.v2 - + import java.nio.ByteBuffer import org.bblfsh.client.v2.libuast.Libuast.{UastIter, UastIterExt} @@ -10,31 +10,49 @@ import org.bblfsh.client.v2.libuast.Libuast.{UastIter, UastIterExt} * This is equivalent of pyuast.ContextExt API */ case class ContextExt(nativeContext: Long) { + import BblfshClient.{UastFormat, UastBinary} + // @native def load(): JNode // TODO(bzz): clarify when it's needed VS just .root().load() @native def root(): NodeExt @native def filter(query: String): UastIterExt - @native def encode(n: NodeExt): ByteBuffer + @native def nativeEncode(n: NodeExt, fmt: Int): ByteBuffer + def encode(n: NodeExt, fmt: UastFormat): ByteBuffer = { + nativeEncode(n, fmt) + } + // encode using binary format + def encode(n: NodeExt): ByteBuffer = { + encode(n, UastBinary) + } @native def dispose() override def finalize(): Unit = { this.dispose() } } - /** * Represents JVM-side constructed tree * * This is equivalent of pyuast.Context API */ case class Context(nativeContext: Long) { + import BblfshClient.{UastFormat, UastBinary} + @native def root(): JNode @native def filter(query: String, node: JNode): UastIter - @native def encode(n: JNode): ByteBuffer + @native def nativeEncode(n: JNode, fmt: Int): ByteBuffer + def encode(n: JNode, fmt: UastFormat): ByteBuffer = { + nativeEncode(n, fmt) + } + // encode using binary format + def encode(n: JNode): ByteBuffer = { + encode(n, UastBinary) + } @native def dispose() override def finalize(): Unit = { this.dispose() } } + object Context { @native def create(): Long def apply(): Context = new Context(create()) diff --git a/src/main/scala/org/bblfsh/client/v2/NodeExt.scala b/src/main/scala/org/bblfsh/client/v2/NodeExt.scala index f80749e..34c0e5e 100644 --- a/src/main/scala/org/bblfsh/client/v2/NodeExt.scala +++ b/src/main/scala/org/bblfsh/client/v2/NodeExt.scala @@ -28,21 +28,33 @@ case class NodeExt(ctx: ContextExt, handle: Long) { * This is equivalent of pyuast.Node API. */ sealed abstract class JNode { - def toByteArray: Array[Byte] = { - val buf = toByteBuffer + import BblfshClient.{UastFormat, UastBinary} + + def toByteArray(fmt: UastFormat): Array[Byte] = { + val buf = toByteBuffer(fmt) val arr = new Array[Byte](buf.capacity()) buf.get(arr) buf.rewind() arr } - def toByteBuffer: ByteBuffer = { + /** Use binary UAST format */ + def toByteArray: Array[Byte] = { + toByteArray(UastBinary) + } + + def toByteBuffer(fmt: UastFormat): ByteBuffer = { val ctx = Context() - val bb = ctx.encode(this) + val bb = ctx.encode(this, fmt) ctx.dispose() bb } + /** Use binary UAST format */ + def toByteBuffer: ByteBuffer = { + toByteBuffer(UastBinary) + } + /* Dynamic dispatch is a convenience to be called from JNI */ def children: Seq[JNode] = this match { case JObject(l) => l map (_._2) @@ -75,8 +87,10 @@ sealed abstract class JNode { } object JNode { - private def decodeFrom(bytes: ByteBuffer): JNode = { - val ctx = BblfshClient.decode(bytes) + import BblfshClient.{UastFormat, UastBinary} + + private def decodeFrom(bytes: ByteBuffer, fmt: UastFormat): JNode = { + val ctx = BblfshClient.decode(bytes, fmt) val node = ctx.root().load() ctx.dispose() node @@ -91,7 +105,7 @@ object JNode { * @param original UAST encoded in wire format of protocol.v2 * @return JNode of the UAST root */ - def parseFrom(original: ByteBuffer): JNode = { + def parseFrom(original: ByteBuffer, fmt: UastFormat): JNode = { val bufDirect = if (!original.isDirect) { val bufDirectCopy = ByteBuffer.allocateDirect(original.capacity()) original.rewind() @@ -102,7 +116,12 @@ object JNode { } else { original } - decodeFrom(bufDirect) + decodeFrom(bufDirect, fmt) + } + + /** Parse from a buffer using binary UAST format */ + def parseFrom(original: ByteBuffer): JNode = { + parseFrom(original, UastBinary) } /** @@ -114,11 +133,16 @@ object JNode { * @param bytes UAST encoded in wire format of protocol.v2 * @return JNode of the UAST root */ - def parseFrom(bytes: Array[Byte]): JNode = { + def parseFrom(bytes: Array[Byte], fmt: UastFormat): JNode = { val bufDirect = ByteBuffer.allocateDirect(bytes.size) bufDirect.put(bytes) bufDirect.flip() - decodeFrom(bufDirect) + decodeFrom(bufDirect, fmt) + } + + /** Parse from an array using binary UAST format */ + def parseFrom(bytes: Array[Byte]): JNode = { + parseFrom(bytes, UastBinary) } } @@ -134,7 +158,14 @@ case class JBool(value: Boolean) extends JNode case class JObject(obj: mutable.Buffer[JField]) extends JNode { def this() = this(mutable.Buffer[JField]()) - def filter(p: ((String, JNode)) => Boolean) = this.obj.filter(p) + def filter(p: JField => Boolean) = obj.filter(p) + def keys(): mutable.Buffer[String] = { + obj.map{ case (key, value) => key } + } + // Gets only the first ocurrence + def get(key: String): Option[JNode] = { + obj.collectFirst { case (k, v) if k == key => v } + } def add(k: String, v: JNode) = { obj += ((k, v)) } diff --git a/src/main/scala/org/bblfsh/client/v2/TreeOrder.scala b/src/main/scala/org/bblfsh/client/v2/TreeOrder.scala deleted file mode 100644 index 2a8e76e..0000000 --- a/src/main/scala/org/bblfsh/client/v2/TreeOrder.scala +++ /dev/null @@ -1,10 +0,0 @@ -package org.bblfsh.client.v2 - -case class TreeOrder( - AnyOrder: Int, - PreOrder: Int, - PostOrder: Int, - LevelOrder: Int, - ChildrenOrder: Int, - PositionOrder: Int -) diff --git a/src/main/scala/org/bblfsh/client/v2/libuast/Libuast.scala b/src/main/scala/org/bblfsh/client/v2/libuast/Libuast.scala index df0fbe3..cf36a25 100644 --- a/src/main/scala/org/bblfsh/client/v2/libuast/Libuast.scala +++ b/src/main/scala/org/bblfsh/client/v2/libuast/Libuast.scala @@ -1,6 +1,6 @@ package org.bblfsh.client.v2.libuast -import org.bblfsh.client.v2.{ContextExt, Context, JNode, NodeExt, TreeOrder} +import org.bblfsh.client.v2.{ContextExt, Context, JNode, NodeExt} import org.bblfsh.client.v2.libuast.Libuast.UastIterExt import scala.collection.Iterator @@ -18,6 +18,19 @@ object Libuast { Libuast.loadBinaryLib("libscalauast") } + case class UastFormat( + uastBinary: Int, + uastYaml: Int + ) + + case class TreeOrder( + anyOrder: Int, + preOrder: Int, + postOrder: Int, + levelOrder: Int, + childrenOrder: Int, + positionOrder: Int + ) /** * Skeletal Node iterator implementation that delegates to Libuast. @@ -137,9 +150,14 @@ object Libuast { class Libuast { Libuast - /** Decode UAST from a byte array */ - @native def decode(buf: ByteBuffer): ContextExt + /** Decode UAST from a byte array + * Receives the buffer to decode and the format it is encoded in + */ + @native def decode(buf: ByteBuffer, fmt: Int): ContextExt /** Lifts the tree order values from the libuast */ - @native def getTreeOrders(): TreeOrder + @native def getTreeOrders: Libuast.TreeOrder + + /** Lifts the uast decoding / encoding options from the libuast */ + @native def getUastFormats: Libuast.UastFormat } diff --git a/src/main/scala/org/bblfsh/client/v2/package.scala b/src/main/scala/org/bblfsh/client/v2/package.scala index 7a1ea1e..0d7b993 100644 --- a/src/main/scala/org/bblfsh/client/v2/package.scala +++ b/src/main/scala/org/bblfsh/client/v2/package.scala @@ -8,4 +8,46 @@ package org.bblfsh.client package object v2 { /** Key, Value representation of [[org.bblfsh.client.v2.JObject]] */ type JField = (String, JNode) + + import BblfshClient._ + + /** Allow to use methods + * f(fmt: UastFormat) as f(0) or f(1) + * g(fmt: Int) as g(UastBinary) or g(UastYaml) + */ + implicit def formatToInt(fmt: UastFormat): Int = { + fmt.toInt + } + + implicit def intToFormat(x: Int): UastFormat = { + x match { + case UastBinary.toInt => UastBinary + case UastYaml.toInt => UastYaml + case _ => + System.err.println("warning: not valid numeric format, using UastBinary") + UastBinary + } + } + + /** Allow to use methods + * f(order: TreeOrder) as f(0), f(1), f(2), ... + * g(order: Int) as g(AnyOrder), g(PreOrder), g(PostOrder) + */ + implicit def orderToInt(order: TreeOrder): Int = { + order.toInt + } + + implicit def intToOrder(x: Int): TreeOrder = { + x match { + case AnyOrder.toInt => AnyOrder + case PreOrder.toInt => PreOrder + case PostOrder.toInt => PostOrder + case LevelOrder.toInt => LevelOrder + case ChildrenOrder.toInt => ChildrenOrder + case PositionOrder.toInt => PositionOrder + case _ => + System.err.println("warning: not valid numeric order, using AnyOrder") + AnyOrder + } + } } diff --git a/src/test/scala/org/bblfsh/client/v2/BblfshClientParseTest.scala b/src/test/scala/org/bblfsh/client/v2/BblfshClientParseTest.scala index b699265..1195cf2 100644 --- a/src/test/scala/org/bblfsh/client/v2/BblfshClientParseTest.scala +++ b/src/test/scala/org/bblfsh/client/v2/BblfshClientParseTest.scala @@ -1,7 +1,6 @@ package org.bblfsh.client.v2 import java.nio.ByteBuffer - import scala.io.Source class BblfshClientParseTest extends BblfshClientBaseTest { @@ -72,5 +71,32 @@ class BblfshClientParseTest extends BblfshClientBaseTest { encodedBytes shouldEqual resp.uast.asReadOnlyByteBuffer } + "BblfshClient.decode" should "decode in binary format" in { + val defaultDecoded = resp.uast.decode() + val binaryDecoded = resp.uast.decode(UastBinary) + val default = defaultDecoded.root().load() + val binary = binaryDecoded.root().load() + + default shouldEqual binary + } + + "BblfshClient.decode" should "be the inverse for ContextExt.encode for binary format" in { + val fmt = UastBinary + val ctx: ContextExt = resp.uast.decode() + val tree = ctx.root() + val bytes = ctx.encode(tree, fmt) + val decoded = BblfshClient.decode(bytes, fmt) + + ctx.root().load() shouldEqual decoded.root().load() + } + + "BblfshClient.decode with invalid number" should "use binary format" in { + val invalidNumDec = resp.uast.decode(-1) + val binaryDecoded = resp.uast.decode(UastBinary) + val invalidNum = invalidNumDec.root().load() + val binary = binaryDecoded.root().load() + + invalidNum shouldEqual binary + } } diff --git a/src/test/scala/org/bblfsh/client/v2/libuast/IteratorManagedTest.scala b/src/test/scala/org/bblfsh/client/v2/libuast/IteratorManagedTest.scala index 43c4b02..b98c318 100644 --- a/src/test/scala/org/bblfsh/client/v2/libuast/IteratorManagedTest.scala +++ b/src/test/scala/org/bblfsh/client/v2/libuast/IteratorManagedTest.scala @@ -3,6 +3,16 @@ package org.bblfsh.client.v2.libuast import org.bblfsh.client.v2.{BblfshClient, JArray, JInt, JNode, JObject, JString} // TODO import org.bblfsh.client.v2.nodes._ import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FlatSpec, Matchers} +import org.scalatest.prop.TableDrivenPropertyChecks._ +import BblfshClient.{ + TreeOrder, + PreOrder, + PostOrder, + ChildrenOrder, + PositionOrder, + AnyOrder, + LevelOrder +} class IteratorManagedTest extends FlatSpec with Matchers @@ -23,7 +33,7 @@ class IteratorManagedTest extends FlatSpec } "Managed UAST iterator" should "return non-empty results on JVM objects" in { - iter = BblfshClient.iterator(mangedRootNode, BblfshClient.PreOrder) + iter = BblfshClient.iterator(mangedRootNode, PreOrder) iter.hasNext() should be(true) val nodes = iter.toList @@ -36,31 +46,70 @@ class IteratorManagedTest extends FlatSpec } "Managed UAST iterator" should "go though all nodes of small object" in { - iter = BblfshClient.iterator(mangedRootNode, BblfshClient.PreOrder) + iter = BblfshClient.iterator(mangedRootNode, PreOrder) val nodes = iter.toList nodes.size should be(3) // number of composite nodes } - val pyClientTestRoot = JObject( - "@type" -> JString("root"), - "children" -> JArray( + def testTree: JObject = { + // Helper to get a position encoded into JObject + def encodePosition(startOffset: Int, startLine: Int, startCol: Int, + endOffset: Int, endLine: Int, endCol: Int): JObject = { + JObject( - "@type" -> JString("son1"), - "children" -> JArray( - JObject("@type" -> JString("son1_1")), - JObject("@type" -> JString("son1_2")) + "@type" -> JString("uast:Positions"), + "start" -> JObject( + "@type" -> JString("uast:Position"), + "offset" -> JInt(startOffset), + "line" -> JInt(startLine), + "col" -> JInt(startCol) + ), + "end" -> JObject( + "@type" -> JString("uast:Position"), + "offset" -> JInt(endOffset), + "line" -> JInt(endLine), + "col" -> JInt(endCol) ) - ), - JObject( - "@type" -> JString("son2"), - "children" -> JArray( - JObject("@type" -> JString("son2_1")), - JObject("@type" -> JString("son2_2")) + ) + } + + // The actual tree + JObject( + "@type" -> JString("root"), + "@pos" -> encodePosition(0,1,1, 1,1,2), + "children" -> JArray( + JObject( + "@type" -> JString("son1"), + "@pos" -> encodePosition(2,2,2, 3,2,3), + "children" -> JArray( + JObject( + "@type" -> JString("son1_1"), + "@pos" -> encodePosition(10,10,1, 12,2,2) + ), + JObject( + "@type" -> JString("son1_2"), + "@pos" -> encodePosition(10,10,1, 12,2,2) + ) + ) + ), + JObject( + "@type" -> JString("son2"), + "@pos" -> encodePosition(100,100,1, 101,100,2), + "children" -> JArray( + JObject( + "@type" -> JString("son2_1"), + "@pos" -> encodePosition(5,5,1, 6,5,2) + ), + JObject( + "@type" -> JString("son2_2"), + "@pos" -> encodePosition(15,15,1, 16,15,2) + ) + ) ) ) ) - ) + } def getNodeTypes(iterator: Libuast.UastIter): List[String] = iterator @@ -68,30 +117,78 @@ class IteratorManagedTest extends FlatSpec .map(_ ("@type").asInstanceOf[JString].str) .toList - // Equivalent of the test.py#testIteratorPreOrder - // https://github.com/bblfsh/python-client/blob/15ffb98bfa09e6aae4d1580f0e4f02eb2a530205/bblfsh/test.py#L270 - "Managed UAST iterator" should "return nodes in PreOrder" in { - val preIter = BblfshClient.iterator(pyClientTestRoot, BblfshClient.PreOrder) - val nodes = getNodeTypes(preIter) + def getNodePositions(iterator: Libuast.UastIter): List[(Long, Long, Long)] = { + def positionToTuple(pos: JObject): Option[(Long, Long, Long)] = { + val offset = pos.get("offset") + val line = pos.get("line") + val col = pos.get("col") + val maybePos = (offset, line, col) + + maybePos match { + case (Some(o: JInt), Some(l: JInt), Some(c: JInt)) => + Some(o.num, l.num, c.num) + case _ => None + } + } + + // Filter only the nodes which correspond to + // positions and have a start field + iterator + .collect { case node: JObject => + node.get("@pos").collect { case pos: JObject => + pos.get("start").collect { case start: JObject => + positionToTuple(start) + // The result of this is an Option[Option]. Convert it to a single option + }.flatten + // Likewise + }.flatten + } + .collect { case Some(pos) => pos } + .toList + } + + val iterators = + Table( + ("order", "expected"), + (PreOrder, Seq("root", "son1", "son1_1", "son1_2", "son2", "son2_1", "son2_2")), + (PostOrder, Seq("son1_1", "son1_2", "son1", "son2_1", "son2_2", "son2", "root")), + (LevelOrder, Seq("root", "son1", "son2", "son1_1", "son1_2", "son2_1", "son2_2")), + (PositionOrder, Seq("root", "son1", "son2_1", "son1_1", "son1_2", "son2_2", "son2")), + (AnyOrder, Seq("root", "son1", "son2_1", "son1_1", "son1_2", "son2_2", "son2")), + (ChildrenOrder, Seq("son1", "son2")) + ) + + forAll (iterators) { (order: TreeOrder, expected: Seq[String]) => + val iter = BblfshClient.iterator(testTree, order) + val nodes = getNodeTypes(iter) - val poActual = Seq("root", "son1", "son1_1", "son1_2", "son2", "son2_1", "son2_2") - nodes should have size (poActual.size) - nodes shouldEqual poActual + order match { + case AnyOrder => + nodes.toSet shouldEqual expected.toSet + case _ => + nodes shouldEqual expected + } - preIter.close() + iter.close() } - "Managed UAST iterator" should "return nodes in PostOrder" in { - val postIter = BblfshClient.iterator(pyClientTestRoot, BblfshClient.PostOrder) - val nodes = getNodeTypes(postIter) + "Positions in PositionOrder" should "actually be ordered" in { + val posIter = BblfshClient.iterator(testTree, PositionOrder) + val positions = getNodePositions(posIter) + val expected = Seq((0,1,1), (2,2,2), (5,5,1), (10,10,1), (10,10,1), (15,15,1), (100,100,1)) + positions shouldEqual expected - val poActual = Seq("son1_1", "son1_2", "son1", "son2_1", "son2_2", "son2", "root") - nodes should have size (poActual.size) - nodes shouldEqual poActual - - postIter.close() + posIter.close() } - // TODO(#108) more tests coverage for other iteration orders, refactor to a table-driven test + "Managed UAST iterator with invalid numeric order" should "use AnyOrder" in { + val invalidNumIter = BblfshClient.iterator(testTree, -1) + val anyOrderIter = BblfshClient.iterator(testTree, AnyOrder) + val nodesNumIter = getNodeTypes(invalidNumIter) + val nodesAnyIter = getNodeTypes(anyOrderIter) + nodesNumIter shouldEqual nodesAnyIter + invalidNumIter.close() + anyOrderIter.close() + } }