adding baseline tokenization script

Author: urialon

.gitignore

@@ -2,4 +2,6 @@
 *.lst
 .idea/*
 *.iml
-*.xml
+*.xml
+*.pyc
+

baseline_tokenization/input_example.txt

@@ -0,0 +1,10 @@
+requires landscape|boolean (){ return false; }
+get parent key|Object (){ return new ContactsUiKey(); }
+get parent key|Object (){ return new ContactsUiKey(); }
+get layout id|int (){ return R.layout.loose_screen; }
+get parent key|Object (){ return new EditContactKey(contactId); }
+to contact|Contact (){ return new Contact(id, name, email); }
+to string|String (){ return "Welcome!\nClick to continue."; }
+get parent key|Object (){ return new EditContactKey(contactId); }
+tear down services|void (@NonNull Services services){ }
+get layout id|int (){ return R.layout.landscape_screen; }

baseline_tokenization/javalang/__init__.py

@@ -0,0 +1,8 @@
+
+from . import parser
+from . import parse
+from . import tokenizer
+from . import javadoc
+
+
+__version__ = "0.10.1"

baseline_tokenization/javalang/ast.py

@@ -0,0 +1,78 @@
+import pickle
+
+import six
+
+
+class MetaNode(type):
+    def __new__(mcs, name, bases, dict):
+        attrs = list(dict['attrs'])
+        dict['attrs'] = list()
+
+        for base in bases:
+            if hasattr(base, 'attrs'):
+                dict['attrs'].extend(base.attrs)
+
+        dict['attrs'].extend(attrs)
+
+        return type.__new__(mcs, name, bases, dict)
+
+
+@six.add_metaclass(MetaNode)
+class Node(object):
+    attrs = ()
+
+    def __init__(self, **kwargs):
+        values = kwargs.copy()
+
+        for attr_name in self.attrs:
+            value = values.pop(attr_name, None)
+            setattr(self, attr_name, value)
+
+        if values:
+            raise ValueError('Extraneous arguments')
+
+    def __equals__(self, other):
+        if type(other) is not type(self):
+            return False
+
+        for attr in self.attrs:
+            if getattr(other, attr) != getattr(self, attr):
+                return False
+
+        return True
+
+    def __repr__(self):
+        return type(self).__name__
+
+    def __iter__(self):
+        return walk_tree(self)
+
+    def filter(self, pattern):
+        for path, node in self:
+            if ((isinstance(pattern, type) and isinstance(node, pattern)) or
+                (node == pattern)):
+                yield path, node
+
+    @property
+    def children(self):
+        return [getattr(self, attr_name) for attr_name in self.attrs]
+
+def walk_tree(root):
+    children = None
+
+    if isinstance(root, Node):
+        yield (), root
+        children = root.children
+    else:
+        children = root
+
+    for child in children:
+        if isinstance(child, (Node, list, tuple)):
+            for path, node in walk_tree(child):
+                yield (root,) + path, node
+
+def dump(ast, file):
+    pickle.dump(ast, file)
+
+def load(file):
+    return pickle.load(file)

baseline_tokenization/javalang/javadoc.py

@@ -0,0 +1,120 @@
+
+import re
+
+def join(s):
+    return ' '.join(l.strip() for l in s.split('\n'))
+
+class DocBlock(object):
+    def __init__(self):
+        self.description = ''
+        self.return_doc = None
+        self.params = []
+
+        self.authors = []
+        self.deprecated = False
+
+        # @exception and @throw are equivalent
+        self.throws = {}
+        self.exceptions = self.throws
+
+        self.tags = {}
+
+    def add_block(self, name, value):
+        value = value.strip()
+
+        if name == 'param':
+            try:
+                param, description = value.split(None, 1)
+            except ValueError:
+                param, description = value, ''
+            self.params.append((param, join(description)))
+
+        elif name in ('throws', 'exception'):
+            try:
+                ex, description = value.split(None, 1)
+            except ValueError:
+                ex, description = value, ''
+            self.throws[ex] = join(description)
+
+        elif name == 'return':
+            self.return_doc = value
+
+        elif name == 'author':
+            self.authors.append(value)
+
+        elif name == 'deprecated':
+            self.deprecated = True
+
+        self.tags.setdefault(name, []).append(value)
+
+blocks_re = re.compile('(^@)', re.MULTILINE)
+leading_space_re = re.compile(r'^\s*\*', re.MULTILINE)
+blocks_justify_re = re.compile(r'^\s*@', re.MULTILINE)
+
+def _sanitize(s):
+    s = s.strip()
+
+    if not (s[:3] == '/**' and s[-2:] == '*/'):
+        raise ValueError('not a valid Javadoc comment')
+
+    s = s.replace('\t', '    ')
+
+    return s
+
+def _uncomment(s):
+    # Remove /** and */
+    s = s[3:-2].strip()
+
+    return leading_space_re.sub('', s)
+
+def _get_indent_level(s):
+    return len(s) - len(s.lstrip())
+
+def _left_justify(s):
+    lines = s.rstrip().splitlines()
+
+    if not lines:
+        return ''
+
+    indent_levels = []
+    for line in lines:
+        if line.strip():
+            indent_levels.append(_get_indent_level(line))
+    indent_levels.sort()
+
+    common_indent = indent_levels[0]
+    if common_indent == 0:
+        return s
+    else:
+        lines = [line[common_indent:] for line in lines]
+        return '\n'.join(lines)
+
+def _force_blocks_left(s):
+    return blocks_justify_re.sub('@', s)
+
+def parse(raw):
+    sanitized = _sanitize(raw)
+    uncommented = _uncomment(sanitized)
+    justified = _left_justify(uncommented)
+    justified_fixed = _force_blocks_left(justified)
+    prepared = justified_fixed
+
+    blocks = blocks_re.split(prepared)
+
+    doc = DocBlock()
+
+    if blocks[0] != '@':
+        doc.description = blocks[0].strip()
+        blocks = blocks[2::2]
+    else:
+        blocks = blocks[1::2]
+
+    for block in blocks:
+        try:
+            tag, value = block.split(None, 1)
+        except ValueError:
+            tag, value = block, ''
+
+        doc.add_block(tag, value)
+
+    return doc

baseline_tokenization/javalang/parse.py

@@ -0,0 +1,53 @@
+
+from .parser import Parser
+from .tokenizer import tokenize
+
+def parse_expression(exp):
+    if not exp.endswith(';'):
+        exp = exp + ';'
+
+    tokens = tokenize(exp)
+    parser = Parser(tokens)
+
+    return parser.parse_expression()
+
+def parse_member_signature(sig):
+    if not sig.endswith(';'):
+        sig = sig + ';'
+
+    tokens = tokenize(sig)
+    parser = Parser(tokens)
+
+    return parser.parse_member_declaration()
+
+def parse_constructor_signature(sig):
+    # Add an empty body to the signature, replacing a ; if necessary
+    if sig.endswith(';'):
+        sig = sig[:-1]
+    sig = sig + '{ }'
+
+    tokens = tokenize(sig)
+    parser = Parser(tokens)
+
+    return parser.parse_member_declaration()
+
+def parse_type(s):
+    tokens = tokenize(s)
+    parser = Parser(tokens)
+
+    return parser.parse_type()
+
+def parse_type_signature(sig):
+    if sig.endswith(';'):
+        sig = sig[:-1]
+    sig = sig + '{ }'
+
+    tokens = tokenize(sig)
+    parser = Parser(tokens)
+
+    return parser.parse_class_or_interface_declaration()
+
+def parse(s):
+    tokens = tokenize(s)
+    parser = Parser(tokens)
+    return parser.parse()