Merge pull request bblfsh#111 from bzz/expose-parse-mode

v2: expose parse mode

Author: bzz

.travis.yml

@@ -39,6 +39,7 @@ jobs:
         - ./sbt assembly test
       after_failure: &failure_logs_anchor
         - docker logs bblfsh
+        - ls hs_* 1> /dev/null 2>&1 && cat hs_*
 
     - &release
       name: 'Cross-compile, release & publish to Sonatype'

README.md

@@ -79,24 +79,19 @@ API
 ```scala
 import scala.io.Source
 import org.bblfsh.client.BblfshClient
+import gopkg.in.bblfsh.sdk.v2.protocol.driver.Mode
 
-val client = BblfshClient("0.0.0.0", 9432)
+val client = BblfshClient("localhost", 9432)
 
 val filename = "/path/to/file.py" // client responsible for encoding it to utf-8
 val fileContent = Source.fromFile(filename).getLines.mkString("\n")
-val resp = client.parse(filename, fileContent)
+val resp = client.parse(filename, fileContent, Mode.SEMANTIC)
 
 // Full response
-println(resp.uast.get)
+println(resp.get)
 
 // Filtered response
-println(client.filter(resp.uast.get, "//Import[@roleImport]"))
-
-// Filtered responses using XPath functions returning types
-// other than NodeLists (Bool, Number, String):
-println(client.filterBool(resp.uast.get, "boolean(//*[@strtOffset or @endOffset])"))
-println(client.filterString(resp.uast.get, "name(//*[1])"))
-println(client.filterNumber(resp.uast.get, "count(//*)"))
+println(client.filter(resp.get, "//uast:Import"))
 ```
 
 Command line:

src/main/scala/org/bblfsh/client/v2/BblfshClient.scala

@@ -29,19 +29,22 @@ class BblfshClient(host: String, port: Int, maxMsgSize: Int) {
     * @param lang    (optional) language to parse, default auto-detect \w enry
     * @param timeout (disabled) bblfsh request timeout, seconds
     *                Right now this does not have any effect in v2.
+    * @param mode    (optional) mode to parse, default to bblfshd 'default' mode
     * @return UAST in parse response.
     */
   def parseWithOptions(
     name: String,
     content: String,
     lang: String,
-    timeout: Long
+    timeout: Long,
+    mode: Mode
   ): ParseResponse = {
-    // FIXME(bzz): make timout work in v2 again
+    // TODO(#100): make timeout work in v2 again
     val req = ParseRequest(
       filename = name,
       content = content,
-      language = lang
+      language = lang,
+      mode = mode
     )
     stub.parse(req)
   }
@@ -61,7 +64,13 @@ class BblfshClient(host: String, port: Int, maxMsgSize: Int) {
     name: String,
     content: String,
     lang: String = ""
-  ): ParseResponse = parseWithOptions(name, content, lang, DEFAULT_TIMEOUT_SEC)
+  ): ParseResponse = parseWithOptions(name, content, lang, DEFAULT_TIMEOUT_SEC, Mode.DEFAULT_MODE)
+
+  def parse(
+    name: String,
+    content: String,
+    mode: Mode
+  ): ParseResponse = parseWithOptions(name, content, "", DEFAULT_TIMEOUT_SEC, mode)
 
   /**
     * Parses the given file name and content,
@@ -80,7 +89,7 @@ class BblfshClient(host: String, port: Int, maxMsgSize: Int) {
     content: String,
     timeout: Long,
     lang: String = ""
-  ): ParseResponse = parseWithOptions(name, content, lang, timeout)
+  ): ParseResponse = parseWithOptions(name, content, lang, timeout, Mode.DEFAULT_MODE)
 
   def supportedLanguages(): SupportedLanguagesResponse = {
     val req = SupportedLanguagesRequest()

src/main/scala/org/bblfsh/client/v2/ContextExt.scala

@@ -31,11 +31,7 @@ case class Context(nativeContext: Long) {
     @native def root(): JNode
     @native def filter(query: String, node: JNode): UastIter
     @native def encode(n: JNode): ByteBuffer
-
     @native def dispose()
-    override def finalize(): Unit = {
-        this.dispose()
-    }
 }
 object Context {
     @native def create(): Long

src/main/scala/org/bblfsh/client/v2/package.scala

@@ -1,11 +1,11 @@
 package org.bblfsh.client
 
 /** Bblfsh client for protocol v2.
- *  See https://github.com/bblfsh/sdk/blob/v3.1.0/protocol/driver.proto
- *
- *  The main class to use is [[org.bblfsh.client.v2.BblfshClient]]
- */
+  * See https://github.com/bblfsh/sdk/blob/v3.1.0/protocol/driver.proto
+  *
+  * The main class to use is [[org.bblfsh.client.v2.BblfshClient]]
+  */
 package object v2 {
-    /** Key, Value representation of [[org.bblfsh.client.v2.JObject]] */
-    type JField = (String, JNode)
+  /** Key, Value representation of [[org.bblfsh.client.v2.JObject]] */
+  type JField = (String, JNode)
 }

src/test/scala/org/bblfsh/client/v2/BblfshClientBaseTest.scala

@@ -21,5 +21,7 @@ class BblfshClientBaseTest extends FlatSpec
 
   override def afterAll {
     client.close()
+    System.runFinalization()
+    System.gc()
   }
 }

src/test/scala/org/bblfsh/client/v2/BblfshClientParseTest.scala

@@ -1,6 +1,9 @@
 package org.bblfsh.client.v2
 
 import java.nio.ByteBuffer
+
+import gopkg.in.bblfsh.sdk.v2.protocol.driver.Mode
+
 import scala.io.Source
 
 class BblfshClientParseTest extends BblfshClientBaseTest {
@@ -12,20 +15,54 @@ class BblfshClientParseTest extends BblfshClientBaseTest {
     assert(resp.errors.isEmpty)
   }
 
+  "Filtering UAST" should "work in Native mode" in {
+    val fileContent = Source.fromFile(fileName).getLines.mkString("\n")
+    val resp = client.parse(fileName, fileContent, Mode.NATIVE)
+    val node = resp.get
+
+    val iter = BblfshClient.filter(node, "//SimpleName")
+    iter.toList should have size (10) // number of Identifiers in the file
+    iter.close()
+  }
+
+// TODO(#110) implement value type returns
+//  "Filtering UAST" should "work for Value types" in {
+//    val iter = BblfshClient.filterNumber(resp.get, "count(//*)")
+//    iter.toList should have size (517) // total number of nodes (not the number of results which is 1)
+//  }
+
+  "Filtering UAST" should "work in Annotated mode" in {
+    val fileContent = Source.fromFile(fileName).getLines.mkString("\n")
+    val resp = client.parse(fileName, fileContent, Mode.ANNOTATED)
+    val node = resp.get
+
+    val iter = BblfshClient.filter(node, "//SimpleName[@role='Call']")
+    iter.toList should have size (1) // number of function called in the file
+    iter.close()
+  }
+
+  "Filtering UAST" should "work in Semantic mode" in {
+    val fileContent = Source.fromFile(fileName).getLines.mkString("\n")
+    val resp = client.parse(fileName, fileContent, Mode.SEMANTIC)
+    val node = resp.get
+
+    val iter = BblfshClient.filter(node, "//uast:Identifier[@role='Call']")
+    iter.toList should have size (1) // number of function called in the file
+    iter.close()
+  }
+
   "Decoded UAST after parsing" should "not be NULL" in {
     val uast = resp.uast.decode()
 
     assert(uast != null)
     assert(uast.nativeContext != 0)
 
-    println(uast)
     uast.dispose()
   }
 
   "Decoded UAST RootNode" should "not be NULL" in {
     val uast = resp.uast.decode()
     val rootNode: NodeExt = uast.root()
-    println(rootNode.getClass)
 
     rootNode should not be null
     rootNode.ctx should not be (0)
@@ -37,15 +74,11 @@ class BblfshClientParseTest extends BblfshClientBaseTest {
   "Encoding UAST to the same ContextExt" should "produce the same bytes" in {
     val uastCtx: ContextExt = resp.uast.decode()
     val rootNode: NodeExt = uastCtx.root()
-    println(s"Root node: $rootNode")
 
     val encodedBytes: ByteBuffer = uastCtx.encode(rootNode)
 
     encodedBytes.capacity should be(resp.uast.asReadOnlyByteBuffer.capacity)
     encodedBytes shouldEqual resp.uast.asReadOnlyByteBuffer
-
-    println(resp.uast.asReadOnlyByteBuffer)
-    println(encodedBytes)
   }
 
   "Encoding java UAST to a new Context" should "produce the same bytes" in {
@@ -62,7 +95,6 @@ class BblfshClientParseTest extends BblfshClientBaseTest {
 
 
   "Encoding python UAST to a new Context" should "produce the same bytes" in {
-    val client = BblfshClient("localhost", 9432)
     val fileName = "src/test/resources/python_file.py"
     val fileContent = Source.fromFile(fileName).getLines.mkString("\n")
     val resp = client.parse(fileName, fileContent)

src/test/scala/org/bblfsh/client/v2/FilterManagedTest.scala

@@ -27,6 +27,11 @@ class FilterManagedTest extends FlatSpec
     ctx = Context()
   }
 
+  override def afterAll() = {
+    System.runFinalization()
+    System.gc()
+  }
+
   "XPath filter" should "find all positions under context" in {
     val it = ctx.filter("//file", managedRoot)
     it.hasNext() should be(true)