[clang.py] Implement Token API

llvm-svn: 160111

4 files changed, 128 insertions, 1 deletions

diff --git a/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py
index 979838b21c7..51695e20b0c 100644
--- a/clang/bindings/python/tests/cindex/test_cursor.py
+++ b/clang/bindings/python/tests/cindex/test_cursor.py
@@ -231,3 +231,13 @@ def test_result_type():
     assert foo is not None
     t = foo.result_type
     assert t.kind == TypeKind.INT
+
+def test_get_tokens():
+    """Ensure we can map cursors back to tokens."""
+    tu = get_tu('int foo(int i);')
+    foo = get_cursor(tu, 'foo')
+
+    tokens = list(foo.get_tokens())
+    assert len(tokens) == 7
+    assert tokens[0].spelling == 'int'
+    assert tokens[1].spelling == 'foo'
diff --git a/clang/bindings/python/tests/cindex/test_token_kind.py b/clang/bindings/python/tests/cindex/test_token_kind.py
new file mode 100644
index 00000000000..62ec63e0ad5
--- /dev/null
+++ b/clang/bindings/python/tests/cindex/test_token_kind.py
@@ -0,0 +1,43 @@
+from clang.cindex import TokenKind
+from nose.tools import eq_
+from nose.tools import ok_
+from nose.tools import raises
+
+def test_constructor():
+    """Ensure TokenKind constructor works as expected."""
+
+    t = TokenKind(5, 'foo')
+
+    eq_(t.value, 5)
+    eq_(t.name, 'foo')
+
+@raises(ValueError)
+def test_bad_register():
+    """Ensure a duplicate value is rejected for registration."""
+
+    TokenKind.register(2, 'foo')
+
+@raises(ValueError)
+def test_unknown_value():
+    """Ensure trying to fetch an unknown value raises."""
+
+    TokenKind.from_value(-1)
+
+def test_registration():
+    """Ensure that items registered appear as class attributes."""
+    ok_(hasattr(TokenKind, 'LITERAL'))
+    literal = TokenKind.LITERAL
+
+    ok_(isinstance(literal, TokenKind))
+
+def test_from_value():
+    """Ensure registered values can be obtained from from_value()."""
+    t = TokenKind.from_value(3)
+    ok_(isinstance(t, TokenKind))
+    eq_(t, TokenKind.LITERAL)
+
+def test_repr():
+    """Ensure repr() works."""
+
+    r = repr(TokenKind.LITERAL)
+    eq_(r, 'TokenKind.LITERAL')
diff --git a/clang/bindings/python/tests/cindex/test_tokens.py b/clang/bindings/python/tests/cindex/test_tokens.py
new file mode 100644
index 00000000000..70748429094
--- /dev/null
+++ b/clang/bindings/python/tests/cindex/test_tokens.py
@@ -0,0 +1,52 @@
+from clang.cindex import CursorKind
+from clang.cindex import Index
+from clang.cindex import SourceLocation
+from clang.cindex import SourceRange
+from clang.cindex import TokenKind
+from nose.tools import eq_
+from nose.tools import ok_
+
+from .util import get_tu
+
+def test_token_to_cursor():
+    """Ensure we can obtain a Cursor from a Token instance."""
+    tu = get_tu('int i = 5;')
+    r = tu.get_extent('t.c', (0, 9))
+    tokens = list(tu.get_tokens(extent=r))
+
+    assert len(tokens) == 5
+    assert tokens[1].spelling == 'i'
+    assert tokens[1].kind == TokenKind.IDENTIFIER
+
+    cursor = tokens[1].cursor
+    assert cursor.kind == CursorKind.VAR_DECL
+    assert tokens[1].cursor == tokens[2].cursor
+
+def test_token_location():
+    """Ensure Token.location works."""
+
+    tu = get_tu('int foo = 10;')
+    r = tu.get_extent('t.c', (0, 11))
+
+    tokens = list(tu.get_tokens(extent=r))
+    eq_(len(tokens), 4)
+
+    loc = tokens[1].location
+    ok_(isinstance(loc, SourceLocation))
+    eq_(loc.line, 1)
+    eq_(loc.column, 5)
+    eq_(loc.offset, 4)
+
+def test_token_extent():
+    """Ensure Token.extent works."""
+    tu = get_tu('int foo = 10;')
+    r = tu.get_extent('t.c', (0, 11))
+
+    tokens = list(tu.get_tokens(extent=r))
+    eq_(len(tokens), 4)
+
+    extent = tokens[1].extent
+    ok_(isinstance(extent, SourceRange))
+
+    eq_(extent.start.offset, 4)
+    eq_(extent.end.offset, 7)
diff --git a/clang/bindings/python/tests/cindex/test_translation_unit.py b/clang/bindings/python/tests/cindex/test_translation_unit.py
index 9de12ad462a..c91f126097a 100644
--- a/clang/bindings/python/tests/cindex/test_translation_unit.py
+++ b/clang/bindings/python/tests/cindex/test_translation_unit.py
@@ -1,3 +1,6 @@
+import gc
+import os
+
 from clang.cindex import CursorKind
 from clang.cindex import Cursor
 from clang.cindex import File
@@ -8,7 +11,6 @@ from clang.cindex import TranslationUnitSaveError
 from clang.cindex import TranslationUnit
 from .util import get_cursor
 from .util import get_tu
-import os
 
 kInputsDir = os.path.join(os.path.dirname(__file__), 'INPUTS')
 
@@ -217,3 +219,23 @@ def test_get_source_range():
     assert r.end.offset == 5
     assert r.start.file.name == 't.c'
     assert r.end.file.name == 't.c'
+
+def test_get_tokens_gc():
+    """Ensures get_tokens() works properly with garbage collection."""
+
+    tu = get_tu('int foo();')
+    r = tu.get_extent('t.c', (0, 10))
+    tokens = list(tu.get_tokens(extent=r))
+
+    assert tokens[0].spelling == 'int'
+    gc.collect()
+    assert tokens[0].spelling == 'int'
+
+    del tokens[1]
+    gc.collect()
+    assert tokens[0].spelling == 'int'
+
+    # May trigger segfault if we don't do our job properly.
+    del tokens
+    gc.collect()
+    gc.collect() # Just in case.