diff --git a/.gitignore b/.gitignore index c164478..14a6fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /asttokens-1.1.6.tar.gz /asttokens-1.1.10.tar.gz /asttokens-1.1.13.tar.gz +/asttokens-2.0.0.tar.gz diff --git a/0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch b/0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch deleted file mode 100644 index 5c3cffa..0000000 --- a/0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch +++ /dev/null @@ -1,147 +0,0 @@ -From 21caaaa74105c410b3d84c3d8ff0dc2f612aac9a Mon Sep 17 00:00:00 2001 -From: Dmitry S -Date: Wed, 1 May 2019 11:46:30 -0400 -Subject: [PATCH 1/4] Fixes for Astroid 2.2, which expects typed_ast.ast3 nodes - in TreeRebuilder. - ---- - tests/test_mark_tokens.py | 2 +- - tests/tools.py | 62 ++++++++++++++++++++++++++------------- - 2 files changed, 43 insertions(+), 21 deletions(-) - -diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py -index f24a6cad58..c2e3f5f3ca 100644 ---- a/tests/test_mark_tokens.py -+++ b/tests/test_mark_tokens.py -@@ -125,7 +125,7 @@ b + # line3 - 'astroid/email.py': ( 3, 3, 1, 1, ), - 'astroid/format.py': ( 64, 61, 62, 62, ), - 'astroid/module.py': ( 185, 174, 171, 171, ), -- 'astroid/module2.py': ( 248, 253, 235, 248, ), -+ 'astroid/module2.py': ( 248, 253, 240, 253, ), - 'astroid/noendingnewline.py': ( 57, 59, 57, 63, ), - 'astroid/notall.py': ( 15, 17, 15, 17, ), - 'astroid/recursion.py': ( 6, 6, 4, 4, ), -diff --git a/tests/tools.py b/tests/tools.py -index 546879b568..b2a638b1e3 100644 ---- a/tests/tools.py -+++ b/tests/tools.py -@@ -1,14 +1,13 @@ - from __future__ import unicode_literals, print_function - import ast --import astroid --import asttokens --import copy - import io - import os - import re - import sys --from asttokens import util - -+import astroid -+import asttokens -+from asttokens import util - - def get_fixture_path(*path_parts): - python_dir = 'python%s' % sys.version_info[0] -@@ -18,7 +17,6 @@ def read_fixture(*path_parts): - with io.open(get_fixture_path(*path_parts), "r", newline="\n") as f: - return f.read() - -- - def _parse_stmt(text): - # ast.parse produces a module, but we here want to produce a single statement. - return ast.parse(text, 'exec').body[0] -@@ -35,12 +33,11 @@ def parse_snippet(text, is_expr=False, is_module=False): - indented = re.match(r'^[ \t]+\S', text) - if indented: - return _parse_stmt('def dummy():\n' + text).body[0] -- elif is_expr: -+ if is_expr: - return _parse_stmt('(' + text + ')').value -- elif is_module: -+ if is_module: - return ast.parse(text, 'exec') -- else: -- return _parse_stmt(text) -+ return _parse_stmt(text) - - - def to_source(node): -@@ -52,9 +49,8 @@ def to_source(node): - return node.as_string() - - builder = astroid.rebuilder.TreeRebuilder(astroid.manager.AstroidManager()) -- # We need to make a deep copy of node; not sure why, but node seems affected by the astroid -- # TreeRebuilder. -- node_copy = copy.deepcopy(node) -+ # We need to make a copy of node that astroid can process. -+ node_copy = create_astroid_ast(node) - if isinstance(node, ast.Module): - anode = builder.visit_module(node_copy, '', '', '') - else: -@@ -63,11 +59,38 @@ def to_source(node): - anode = builder.visit(node_copy, amodule) - return anode.as_string() - -+def create_astroid_ast(node): -+ if hasattr(astroid, "_ast"): -+ # A bit of a hack, reaching into astroid, but we need to re-create the tree with the parser -+ # module that astroid understands, to be able to use TreeRebuilder on it. -+ parser_module = astroid._ast._get_parser_module() # pylint: disable=no-member,protected-access -+ else: -+ parser_module = ast -+ return ConvertAST(parser_module).visit(node) -+ -+class ConvertAST(ast.NodeVisitor): -+ """Allows converting from ast nodes to typed_ast.ast27 or typed_ast.ast3 nodes.""" -+ def __init__(self, ast_module): -+ self._ast_module = ast_module -+ -+ def visit(self, node): -+ converted_class = getattr(self._ast_module, node.__class__.__name__) -+ new_node = converted_class() -+ for field, old_value in ast.iter_fields(node): -+ new_value = ([self.maybe_visit(n) for n in old_value] if isinstance(old_value, list) else -+ self.maybe_visit(old_value)) -+ setattr(new_node, field, new_value) -+ for attr in getattr(node, '_attributes', ()): -+ setattr(new_node, attr, getattr(node, attr)) -+ return new_node -+ -+ def maybe_visit(self, node): -+ return self.visit(node) if isinstance(node, ast.AST) else node - - def collect_nodes_preorder(root): - """Returns a list of all nodes using pre-order traversal (i.e. parent before children).""" - nodes = [] -- def append(node, par_value): -+ def append(node, par_value): # pylint: disable=unused-argument - nodes.append(node) - return (None, None) - util.visit_tree(root, append, None) -@@ -117,18 +140,17 @@ class MarkChecker(object): - if not (util.is_stmt(node) or util.is_expr(node) or util.is_module(node)): - continue - -- if isinstance(node, astroid.nodes.Yield): -- # Astroid stringifies Yield nodes differently depending on parent, so these are too -- # annoying to verify. -- continue -- - text = self.atok.get_text(node) - rebuilt_node = parse_snippet(text, is_expr=util.is_expr(node), is_module=util.is_module(node)) - - # Now we need to check if the two nodes are equivalent. -- left = to_source(rebuilt_node) -- right = to_source(node) -+ left = _yield_fix(to_source(rebuilt_node)) -+ right = _yield_fix(to_source(node)) - test_case.assertEqual(left, right) - tested_nodes += 1 - - return tested_nodes -+ -+# Yield nodes are parenthesized depending on context; to ease verifications, parenthesize always. -+def _yield_fix(text): -+ return "(" + text + ")" if text.startswith("yield") else text diff --git a/0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch b/0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch deleted file mode 100644 index 0d0b70b..0000000 --- a/0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch +++ /dev/null @@ -1,142 +0,0 @@ -From 441a88ebac5a192c36cba70919d7eb5e968574c7 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= -Date: Wed, 19 Jun 2019 04:50:58 +0200 -Subject: [PATCH 2/4] tests: use Constant for Num, Str, NameConstant node names - (#30) - -This fixes some test failures under python3.8. All tests still pass on -python3.7. Partially addresses #28, replaces #29. -As suggested in https://github.com/gristlabs/asttokens/pull/29/files#r280123329. ---- - tests/test_mark_tokens.py | 14 +++++++------- - tests/test_util.py | 15 ++++++++------- - tests/tools.py | 6 +++++- - 3 files changed, 20 insertions(+), 15 deletions(-) - -diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py -index c2e3f5f3ca..83d35fa432 100644 ---- a/tests/test_mark_tokens.py -+++ b/tests/test_mark_tokens.py -@@ -181,11 +181,11 @@ b + # line3 - m = self.create_mark_checker(source) - self.assertEqual(len(m.all_nodes), 2104) - self.assertEqual(m.view_node(m.all_nodes[-1]), -- "Str:'F1akOFFiRIgPHTZksKBAgMCLGTdGNIAAQgKfDAcgZbj0odOnUA8GBAA7'") -+ "Constant:'F1akOFFiRIgPHTZksKBAgMCLGTdGNIAAQgKfDAcgZbj0odOnUA8GBAA7'") - self.assertEqual(m.view_node(m.all_nodes[-2]), -- "Str:'Ii0uLDAxLzI0Mh44U0gxMDI5JkM0JjU3NDY6Kjc5Njo7OUE8Ozw+Oz89QTxA'") -+ "Constant:'Ii0uLDAxLzI0Mh44U0gxMDI5JkM0JjU3NDY6Kjc5Njo7OUE8Ozw+Oz89QTxA'") - self.assertEqual(m.view_node(m.all_nodes[1053]), -- "Str:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'") -+ "Constant:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'") - self.assertEqual(m.view_node(m.all_nodes[1052]), - "BinOp:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'\r\n" + - " +'CxsSEhkWDhYYFQ0aJhkaGBweGyccGh8hHiIkIiMmGTEiHhQoPSYoJSkqKDcp'") -@@ -225,7 +225,7 @@ bar = ('x y z' # comment2 - ) - """ - m = self.create_mark_checker(source) -- node_name = 'Const' if self.is_astroid_test else 'Str' -+ node_name = 'Const' if self.is_astroid_test else 'Constant' - self.assertEqual(m.view_nodes_at(2, 6), { - node_name + ":'x y z' \\\n'''a b c''' \"u v w\"" - }) -@@ -336,7 +336,7 @@ bar = ('x y z' # comment2 - name_a = 'AssignName:a' if self.is_astroid_test else 'Name:a' - const_true = ('Const:True' if self.is_astroid_test else - 'Name:True' if six.PY2 else -- 'NameConstant:True') -+ 'Constant:True') - self.assertEqual(m.view_nodes_at(1, 0), - {name_a, "Assign:a = True if True else False", "Module:" + source}) - self.assertEqual(m.view_nodes_at(1, 4), -@@ -388,7 +388,7 @@ bar = ('x y z' # comment2 - if self.is_astroid_test: - self.assertEqual(m.view_nodes_at(1, 5), {'Const:4'}) - else: -- self.assertEqual(m.view_nodes_at(1, 5), {'Num:4'}) -+ self.assertEqual(m.view_nodes_at(1, 5), {'Constant:4'}) - self.assertEqual(m.view_nodes_at(2, 0), {'Delete:del x[4]'}) - self.assertEqual(m.view_nodes_at(2, 4), {'Name:x', 'Subscript:x[4]'}) - -@@ -428,7 +428,7 @@ bar = ('x y z' # comment2 - self.assertEqual(m.view_nodes_at(2, 8), {'Keyword:b=[y]'}) - else: - self.assertEqual(m.view_nodes_at(1, 2), {'keyword:x=1'}) -- self.assertEqual(m.view_nodes_at(1, 4), {'Num:1'}) -+ self.assertEqual(m.view_nodes_at(1, 4), {'Constant:1'}) - self.assertEqual(m.view_nodes_at(2, 2), {'keyword:a=(x)'}) - self.assertEqual(m.view_nodes_at(2, 8), {'keyword:b=[y]'}) - -diff --git a/tests/test_util.py b/tests/test_util.py -index 254116c32f..91a51432ae 100644 ---- a/tests/test_util.py -+++ b/tests/test_util.py -@@ -4,6 +4,7 @@ import ast - import astroid - import unittest - from .context import asttokens -+from .tools import get_node_name - - class TestUtil(unittest.TestCase): - -@@ -37,7 +38,7 @@ class TestUtil(unittest.TestCase): - atok = asttokens.ASTTokens(self.source, parse=True) - - def view(node): -- return "%s:%s" % (node.__class__.__name__, atok.get_text(node)) -+ return "%s:%s" % (get_node_name(node), atok.get_text(node)) - - scan = [view(n) for n in asttokens.util.walk(atok.tree)] - self.assertEqual(scan, [ -@@ -48,20 +49,20 @@ class TestUtil(unittest.TestCase): - 'Call:bar(1 + 2)', - 'Name:bar', - 'BinOp:1 + 2', -- 'Num:1', -- 'Num:2', -+ 'Constant:1', -+ 'Constant:2', - "BinOp:'hello' + ', ' + 'world'", - "BinOp:'hello' + ', '", -- "Str:'hello'", -- "Str:', '", -- "Str:'world'" -+ "Constant:'hello'", -+ "Constant:', '", -+ "Constant:'world'" - ]) - - def test_walk_astroid(self): - atok = asttokens.ASTTokens(self.source, tree=astroid.builder.parse(self.source)) - - def view(node): -- return "%s:%s" % (node.__class__.__name__, atok.get_text(node)) -+ return "%s:%s" % (get_node_name(node), atok.get_text(node)) - - scan = [view(n) for n in asttokens.util.walk(atok.tree)] - self.assertEqual(scan, [ -diff --git a/tests/tools.py b/tests/tools.py -index b2a638b1e3..3f4fea7c0e 100644 ---- a/tests/tools.py -+++ b/tests/tools.py -@@ -96,6 +96,10 @@ def collect_nodes_preorder(root): - util.visit_tree(root, append, None) - return nodes - -+def get_node_name(node): -+ name = node.__class__.__name__ -+ return 'Constant' if name in ('Num', 'Str', 'NameConstant') else name -+ - - class MarkChecker(object): - """ -@@ -112,7 +116,7 @@ class MarkChecker(object): - - def view_node(self, node): - """Returns a representation of a node and its text, such as "Call:foo()". """ -- return "%s:%s" % (node.__class__.__name__, self.atok.get_text(node)) -+ return "%s:%s" % (get_node_name(node), self.atok.get_text(node)) - - def view_nodes_at(self, line, col): - """ diff --git a/0003-Make-tests-pass-under-python3.8-astroid-2.2.patch b/0003-Make-tests-pass-under-python3.8-astroid-2.2.patch deleted file mode 100644 index 5596b48..0000000 --- a/0003-Make-tests-pass-under-python3.8-astroid-2.2.patch +++ /dev/null @@ -1,136 +0,0 @@ -From 3513264ceff0e1cfbd9ad6543bc24b5cc08f1acb Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= -Date: Tue, 10 Sep 2019 11:06:29 +0200 -Subject: [PATCH 4/4] Make tests pass under python3.8 + astroid 2.2 - ---- - asttokens/mark_tokens.py | 27 ++++++++++++++++----------- - tests/test_mark_tokens.py | 32 +++++++++++++++++++++++--------- - 2 files changed, 39 insertions(+), 20 deletions(-) - -diff --git a/asttokens/mark_tokens.py b/asttokens/mark_tokens.py -index 624ac6273b..53e1ed6b16 100644 ---- a/asttokens/mark_tokens.py -+++ b/asttokens/mark_tokens.py -@@ -13,6 +13,7 @@ - # limitations under the License. - - import six -+import sys - import numbers - import token - from . import util -@@ -165,11 +166,14 @@ class MarkTokens(object): - return (first_token, last_token) - - def handle_comp(self, open_brace, node, first_token, last_token): -- # For list/set/dict comprehensions, we only get the token of the first child, so adjust it to -+ # For list/set/dict comprehensions, in the past, we only got the token of the first child, so adjust it to - # include the opening brace (the closing brace will be matched automatically). -- before = self._code.prev_token(first_token) -- util.expect_token(before, token.OP, open_brace) -- return (before, last_token) -+ if util.match_token(first_token, token.OP, open_brace): -+ return (first_token, last_token) -+ else: -+ before = self._code.prev_token(first_token) -+ util.expect_token(before, token.OP, open_brace) -+ return (before, last_token) - - def visit_listcomp(self, node, first_token, last_token): - return self.handle_comp('[', node, first_token, last_token) -@@ -224,13 +228,14 @@ class MarkTokens(object): - return (first_token, self._code.find_token(last_token, token.OP, ']')) - - def visit_tuple(self, node, first_token, last_token): -- # A tuple doesn't include parens; if there is a trailing comma, make it part of the tuple. -- try: -- maybe_comma = self._code.next_token(last_token) -- if util.match_token(maybe_comma, token.OP, ','): -- last_token = maybe_comma -- except IndexError: -- pass -+ if sys.version_info < (3,8): -+ # A tuple doesn't include parens; if there is a trailing comma, make it part of the tuple. -+ try: -+ maybe_comma = self._code.next_token(last_token) -+ if util.match_token(maybe_comma, token.OP, ','): -+ last_token = maybe_comma -+ except IndexError: -+ pass - return (first_token, last_token) - - def visit_str(self, node, first_token, last_token): -diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py -index 83d35fa432..2dbd7e8b3c 100644 ---- a/tests/test_mark_tokens.py -+++ b/tests/test_mark_tokens.py -@@ -98,7 +98,7 @@ b + # line3 - # All other expressions preserve newlines and comments but are parenthesized. - 'b + # line3\n c', - 'b + # line3\n c + # line4\n d', -- 'a, # line2\nb + # line3\n c + # line4\n d', -+ 'a, # line2\nb + # line3\n c + # line4\n d' if sys.version_info < (3,8) else source, - }) - - -@@ -202,8 +202,12 @@ b + # line3 - # Make sure we don't fail on parsing slices of the form `foo[4:]`. - source = "(foo.Area_Code, str(foo.Phone)[:3], str(foo.Phone)[3:], foo[:], bar[::, :])" - m = self.create_mark_checker(source) -- self.assertEqual(m.view_nodes_at(1, 1), -- { "Attribute:foo.Area_Code", "Name:foo", "Tuple:"+source[1:-1] }) -+ -+ at_1_1 = { "Attribute:foo.Area_Code", "Name:foo" } -+ if sys.version_info < (3,8): -+ at_1_1.add( "Tuple:"+source[1:-1] ) -+ self.assertEqual(m.view_nodes_at(1, 1), at_1_1) -+ - self.assertEqual(m.view_nodes_at(1, 16), - { "Subscript:str(foo.Phone)[:3]", "Call:str(foo.Phone)", "Name:str"}) - self.assertEqual(m.view_nodes_at(1, 36), -@@ -226,11 +230,14 @@ bar = ('x y z' # comment2 - """ - m = self.create_mark_checker(source) - node_name = 'Const' if self.is_astroid_test else 'Constant' -+ joined = self.is_astroid_test or sys.version_info < (3,8) - self.assertEqual(m.view_nodes_at(2, 6), { -- node_name + ":'x y z' \\\n'''a b c''' \"u v w\"" -+ node_name + ":'x y z'" + ( -+ " \\\n'''a b c''' \"u v w\"" if joined else "") - }) - self.assertEqual(m.view_nodes_at(4, 7), { -- node_name + ":'x y z' # comment2\n 'a b c' # comment3\n 'u v w'" -+ node_name + ":'x y z'" + ( -+ " # comment2\n 'a b c' # comment3\n 'u v w'" if joined else "") - }) - - -@@ -446,16 +453,23 @@ bar = ('x y z' # comment2 - m.verify_all_nodes(self) - # The `arguments` node has bogus positions here (and whenever there are no arguments). We - # don't let that break our test because it's unclear if it matters to anything anyway. -- self.assertIn('FunctionDef:@deco1\ndef f():\n pass', m.view_nodes_at(2, 0)) -- self.assertEqual(m.view_nodes_at(2, 1), {'Name:deco1'}) -+ -+ nodes_2_0 = m.view_nodes_at(2, 0) -+ nodes_2_1 = m.view_nodes_at(2, 1) -+ nodes_5_0 = m.view_nodes_at(5, 0) -+ nodes_5_1 = m.view_nodes_at(5, 1) -+ -+ self.assertIn('FunctionDef:@deco1\ndef f():\n pass', nodes_2_0 or nodes_2_1) -+ self.assertIn('Name:deco1', nodes_2_1) - if self.is_astroid_test: - self.assertEqual(m.view_nodes_at(5, 0), { - 'FunctionDef:@deco2(a=1)\ndef g(x):\n pass', - 'Decorators:@deco2(a=1)' - }) - else: -- self.assertEqual(m.view_nodes_at(5, 0), {'FunctionDef:@deco2(a=1)\ndef g(x):\n pass'}) -- self.assertEqual(m.view_nodes_at(5, 1), {'Name:deco2', 'Call:deco2(a=1)'}) -+ self.assertIn('FunctionDef:@deco2(a=1)\ndef g(x):\n pass', nodes_5_0 or nodes_5_1) -+ self.assertIn('Name:deco2', nodes_5_1) -+ self.assertIn('Call:deco2(a=1)', nodes_5_1) - - def test_with(self): - source = "with foo: pass" diff --git a/python-asttokens.spec b/python-asttokens.spec index 011f9ee..5a18144 100644 --- a/python-asttokens.spec +++ b/python-asttokens.spec @@ -1,16 +1,12 @@ Name: python-asttokens -Version: 1.1.13 -Release: 3%{?dist} +Version: 2.0.0 +Release: 1%{?dist} Summary: Module to annotate Python abstract syntax trees with source code positions License: ASL 2.0 URL: https://pypi.python.org/pypi/asttokens Source0: https://files.pythonhosted.org/packages/source/a/asttokens/asttokens-%{version}.tar.gz -Patch0001: 0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch -Patch0002: 0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch -Patch0003: 0003-Make-tests-pass-under-python3.8-astroid-2.2.patch - BuildArch: noarch BuildRequires: python3-devel BuildRequires: python3dist(pytest) @@ -51,6 +47,10 @@ pytest-3 tests/ -v --ignore=tests/testdata/ %{python3_sitelib}/* %changelog +* Tue Oct 15 2019 Zbigniew Jędrzejewski-Szmek - 2.0.0-1 +- Update to latest version (#1752074). This should finally fix compatibility + with python3.8. + * Tue Sep 10 2019 Zbigniew Jędrzejewski-Szmek - 1.1.13-3 - Fix build with python3.8 (#1697503) diff --git a/sources b/sources index 47a539f..fed15ef 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (asttokens-1.1.13.tar.gz) = 357ca341ece5dbf57e33a4dbdf6b67209b22920e7b819b63406e2334d08adae5425a3e56bba0def912dddbca12f82a452f61638bc40eb10881387e0b8d7809db +SHA512 (asttokens-2.0.0.tar.gz) = 1f33b04d6ed7cf40b88269400922b8cddc455b05eb820af34f72449b30b584f8d753f9fd4fe15431d773efd8ec9a89b06c1d9d9037089e7787b51e7e2729f187