Python client v3 (UASTv2)

MANIFEST.in

@@ -7,5 +7,5 @@ include Makefile
 include github.com/gogo/protobuf/gogoproto/gogo.proto
 include gopkg.in/bblfsh/sdk.v1/protocol/generated.proto
 include gopkg.in/bblfsh/sdk.v1/uast/generated.proto
-include bblfsh/memtracker.h
+include bblfsh/libuast/libuast.hpp
 prune bblfsh/libuast

README.md

@@ -23,11 +23,10 @@ python setup.py install
 
 ### Dependencies
 
-You need to install `libxml2` and its header files. You also will need a `curl` cli tool to dowload `libuast`, and a `g++` for building [libtuast Python bindings](https://github.com/bblfsh/client-python/blob/0037d762563ab49b3daac8a7577f7103a5628fc6/setup.py#L17). 
+You also will need a `curl` cli tool to dowload `libuast`, and a `g++` for building [libtuast Python bindings](https://github.com/bblfsh/client-python/blob/0037d762563ab49b3daac8a7577f7103a5628fc6/setup.py#L17).
 The command for Debian and derived distributions would be:
 
 ```bash
-sudo apt install libxml2-dev
 sudo apt install curl
 sudo apt install build-essential
 ```
@@ -49,16 +48,18 @@ Please, read the [getting started](https://doc.bblf.sh/using-babelfish/getting-s
 import bblfsh
 
 client = bblfsh.BblfshClient("0.0.0.0:9432")
-uast = client.parse("/path/to/file.py").uast
-print(uast)
+uast = client.parse("/path/to/file.py")
+print(uast.load())
+
 # "filter' allows you to use XPath queries to filter on result nodes:
-print(bblfsh.filter(uast, "//Import[@roleImport and @roleDeclaration]//alias"))
+it = uast.filter("//Import[@role='Import' and @role='Declaration']//alias")
+for node in it:
+    print(node.load())
 
-# filter\_[bool|string|number] must be used when using XPath functions returning
-# these types:
-print(bblfsh.filter_bool(uast, "boolean(//*[@strtOffset or @endOffset])"))
-print(bblfsh.filter_string(uast, "name(//*[1])"))
-print(bblfsh.filter_number(uast, "count(//*)"))
+# filter must be used when using XPath functions returning these types:
+print(uast.filter("boolean(//*[@strtOffset or @endOffset])"))
+print(uast.filter("name(//*[1])"))
+print(uast.filter("count(//*)"))
 
 # You can also iterate on several tree iteration orders:
 it = bblfsh.iterator(uast, bblfsh.TreeOrder.PRE_ORDER)

bblfsh/__init__.py

@@ -1,5 +1,5 @@
 from bblfsh.client import BblfshClient
-from bblfsh.pyuast import filter, filter_bool, filter_number, filter_string, iterator
+from bblfsh.pyuast import decode, iterator, uast
 from bblfsh.aliases import *
 
 class TreeOrder:

bblfsh/aliases.py

@@ -1,27 +1,26 @@
-__all__ = ["DESCRIPTOR", "Node", "Position", "ParseResponse", "NativeParseResponse",
-           "ParseRequest", "NativeParseRequest", "VersionRequest", "ProtocolServiceStub"]
-
 import importlib
 
-from bblfsh.sdkversion import VERSION
-
 # "in" is a reserved keyword in Python thus can't be used as package name, so
 # we import by string
-uast_module = importlib.import_module(
-    "bblfsh.gopkg.in.bblfsh.sdk.%s.uast.generated_pb2" % VERSION)
-protocol_module = importlib.import_module(
-    "bblfsh.gopkg.in.bblfsh.sdk.%s.protocol.generated_pb2" % VERSION)
-protocol_grpc_module = importlib.import_module(
-    "bblfsh.gopkg.in.bblfsh.sdk.%s.protocol.generated_pb2_grpc" % VERSION)
 
-DESCRIPTOR = uast_module.DESCRIPTOR
-Node = uast_module.Node
-Position = uast_module.Position
-ParseResponse = protocol_module.ParseResponse
-NativeParseResponse = protocol_module.NativeParseResponse
-ParseRequest = protocol_module.ParseRequest
-NativeParseRequest = protocol_module.NativeParseRequest
-VersionRequest = protocol_module.VersionRequest
-SupportedLanguagesRequest = protocol_module.SupportedLanguagesRequest
-SupportedLanguagesResponse = protocol_module.SupportedLanguagesResponse
-ProtocolServiceStub = protocol_grpc_module.ProtocolServiceStub
+uast_v2_module = importlib.import_module("bblfsh.gopkg.in.bblfsh.sdk.v2.uast.generated_pb2")
+protocol_v2_module = importlib.import_module("bblfsh.gopkg.in.bblfsh.sdk.v2.protocol.generated_pb2")
+protocol_grpc_v2_module = importlib.import_module("bblfsh.gopkg.in.bblfsh.sdk.v2.protocol.generated_pb2_grpc")
+protocol_v1_module = importlib.import_module("bblfsh.gopkg.in.bblfsh.sdk.v1.protocol.generated_pb2")
+protocol_grpc_v1_module = importlib.import_module("bblfsh.gopkg.in.bblfsh.sdk.v1.protocol.generated_pb2_grpc")
+
+# Node = importlib.import_module(
+        # "bblfsh.gopkg.in.bblfsh.sdk.v2.uast.generated_pb2").Node
+
+DESCRIPTOR = uast_v2_module.DESCRIPTOR
+ParseRequest = protocol_v2_module.ParseRequest
+ParseResponse = protocol_v2_module.ParseResponse
+ParseError = protocol_v2_module.ParseError
+Mode = protocol_v2_module.Mode
+DriverStub = protocol_grpc_v2_module.DriverStub
+DriverServicer = protocol_grpc_v2_module.DriverServicer
+
+VersionRequest = protocol_v1_module.VersionRequest
+SupportedLanguagesRequest = protocol_v1_module.SupportedLanguagesRequest
+SupportedLanguagesResponse = protocol_v1_module.SupportedLanguagesResponse
+ProtocolServiceStub = protocol_grpc_v1_module.ProtocolServiceStub

bblfsh/client.py

@@ -3,13 +3,13 @@
 
 import grpc
 
-from bblfsh.aliases import (ParseRequest, NativeParseRequest, VersionRequest,
-        ProtocolServiceStub, SupportedLanguagesRequest, SupportedLanguagesResponse)
-from bblfsh.sdkversion import VERSION
+from bblfsh.pyuast import decode as uast_decode
+
+from bblfsh.aliases import ParseRequest, DriverStub, ProtocolServiceStub, VersionRequest, SupportedLanguagesRequest
 
 # The following two insertions fix the broken pb import paths
-sys.path.insert(0, os.path.join(os.path.dirname(__file__),
-                                "gopkg/in/bblfsh/sdk/%s/protocol" % VERSION))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "gopkg/in/bblfsh/sdk/v1/protocol"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "gopkg/in/bblfsh/sdk/v2/protocol"))
 sys.path.insert(0, os.path.dirname(__file__))
 
 
@@ -31,7 +31,8 @@ def __init__(self, endpoint):
         :type endpoint: str
         """
         self._channel = grpc.insecure_channel(endpoint)
-        self._stub = ProtocolServiceStub(self._channel)
+        self._stub_v1 = ProtocolServiceStub(self._channel)
+        self._stub_v2 = DriverStub(self._channel)
 
     @staticmethod
     def _check_utf8(text):
@@ -48,7 +49,7 @@ def _get_contents(contents, filename):
         BblfshClient._check_utf8(contents)
         return contents
 
-    def parse(self, filename, language=None, contents=None, timeout=None):
+    def parse(self, filename, language=None, contents=None, mode=None, raw=False, timeout=None):
         """
         Queries the Babelfish server and receives the UAST response for the specified
         file.
@@ -60,6 +61,8 @@ def parse(self, filename, language=None, contents=None, timeout=None):
                          currently supported languages. None means autodetect.
         :param contents: The contents of the file. IF None, it is read from \
                          filename.
+        :param mode:     UAST transformation mode.
+        :param raw:      Return raw binary UAST without decoding it.
         :param timeout: The request timeout in seconds.
         :type filename: str
         :type language: str
@@ -71,47 +74,30 @@ def parse(self, filename, language=None, contents=None, timeout=None):
         contents = self._get_contents(contents, filename)
         request = ParseRequest(filename=os.path.basename(filename),
                                content=contents,
+                               mode=mode,
                                language=self._scramble_language(language))
-        return self._stub.Parse(request, timeout=timeout)
-
-    def native_parse(self, filename, language=None, contents=None, timeout=None):
+        response = self._stub_v2.Parse(request, timeout=timeout)
         """
-        Queries the Babelfish server and receives the native AST response for the specified
-        file.
-
-        :param filename: The path to the file. Can be arbitrary if contents \
-                         is not None.
-        :param language: The programming language of the file. Refer to \
-                         https://doc.bblf.sh/languages.html for the list of \
-                         currently supported languages. None means autodetect.
-        :param contents: The contents of the file. IF None, it is read from \
-                         filename.
-        :param timeout: The request timeout in seconds.
-        :type filename: str
-        :type language: str
-        :type contents: str
-        :type timeout: float
-        :return: Native AST object.
+        TODO: return detected language
+        TODO: handle syntax errors
         """
-
-        contents = self._get_contents(contents, filename)
-        request = NativeParseRequest(filename=os.path.basename(filename),
-                               content=contents,
-                               language=self._scramble_language(language))
-        return self._stub.NativeParse(request, timeout=timeout)
+        if raw:
+            return response.uast
+        ctx = uast_decode(response.uast, format=0)
+        return ctx
 
     def supported_languages(self):
-        sup_response = self._stub.SupportedLanguages(SupportedLanguagesRequest())
+        sup_response = self._stub_v1.SupportedLanguages(SupportedLanguagesRequest())
         return sup_response.languages
 
     def version(self):
         """
         Queries the Babelfish server for version and runtime information.
 
         :return: A dictionary with the keys "version" for the semantic version and
-                 "build" for the build timestamp.
+                 # "build" for the build timestamp.
         """
-        return self._stub.Version(VersionRequest())
+        return self._stub_v1.Version(VersionRequest())
 
     @staticmethod
     def _scramble_language(lang):