From 97dd58ab1e92ffad1b3b64ec73ac12b3d69b4c5a Mon Sep 17 00:00:00 2001 From: Zbigniew Jędrzejewski-Szmek Date: Sep 10 2019 10:09:52 +0000 Subject: Add hack patches to build under python3.8 --- diff --git a/0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch b/0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch new file mode 100644 index 0000000..5c3cffa --- /dev/null +++ b/0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch @@ -0,0 +1,147 @@ +From 21caaaa74105c410b3d84c3d8ff0dc2f612aac9a Mon Sep 17 00:00:00 2001 +From: Dmitry S +Date: Wed, 1 May 2019 11:46:30 -0400 +Subject: [PATCH 1/4] Fixes for Astroid 2.2, which expects typed_ast.ast3 nodes + in TreeRebuilder. + +--- + tests/test_mark_tokens.py | 2 +- + tests/tools.py | 62 ++++++++++++++++++++++++++------------- + 2 files changed, 43 insertions(+), 21 deletions(-) + +diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py +index f24a6cad58..c2e3f5f3ca 100644 +--- a/tests/test_mark_tokens.py ++++ b/tests/test_mark_tokens.py +@@ -125,7 +125,7 @@ b + # line3 + 'astroid/email.py': ( 3, 3, 1, 1, ), + 'astroid/format.py': ( 64, 61, 62, 62, ), + 'astroid/module.py': ( 185, 174, 171, 171, ), +- 'astroid/module2.py': ( 248, 253, 235, 248, ), ++ 'astroid/module2.py': ( 248, 253, 240, 253, ), + 'astroid/noendingnewline.py': ( 57, 59, 57, 63, ), + 'astroid/notall.py': ( 15, 17, 15, 17, ), + 'astroid/recursion.py': ( 6, 6, 4, 4, ), +diff --git a/tests/tools.py b/tests/tools.py +index 546879b568..b2a638b1e3 100644 +--- a/tests/tools.py ++++ b/tests/tools.py +@@ -1,14 +1,13 @@ + from __future__ import unicode_literals, print_function + import ast +-import astroid +-import asttokens +-import copy + import io + import os + import re + import sys +-from asttokens import util + ++import astroid ++import asttokens ++from asttokens import util + + def get_fixture_path(*path_parts): + python_dir = 'python%s' % sys.version_info[0] +@@ -18,7 +17,6 @@ def read_fixture(*path_parts): + with io.open(get_fixture_path(*path_parts), "r", newline="\n") as f: + return f.read() + +- + def _parse_stmt(text): + # ast.parse produces a module, but we here want to produce a single statement. + return ast.parse(text, 'exec').body[0] +@@ -35,12 +33,11 @@ def parse_snippet(text, is_expr=False, is_module=False): + indented = re.match(r'^[ \t]+\S', text) + if indented: + return _parse_stmt('def dummy():\n' + text).body[0] +- elif is_expr: ++ if is_expr: + return _parse_stmt('(' + text + ')').value +- elif is_module: ++ if is_module: + return ast.parse(text, 'exec') +- else: +- return _parse_stmt(text) ++ return _parse_stmt(text) + + + def to_source(node): +@@ -52,9 +49,8 @@ def to_source(node): + return node.as_string() + + builder = astroid.rebuilder.TreeRebuilder(astroid.manager.AstroidManager()) +- # We need to make a deep copy of node; not sure why, but node seems affected by the astroid +- # TreeRebuilder. +- node_copy = copy.deepcopy(node) ++ # We need to make a copy of node that astroid can process. ++ node_copy = create_astroid_ast(node) + if isinstance(node, ast.Module): + anode = builder.visit_module(node_copy, '', '', '') + else: +@@ -63,11 +59,38 @@ def to_source(node): + anode = builder.visit(node_copy, amodule) + return anode.as_string() + ++def create_astroid_ast(node): ++ if hasattr(astroid, "_ast"): ++ # A bit of a hack, reaching into astroid, but we need to re-create the tree with the parser ++ # module that astroid understands, to be able to use TreeRebuilder on it. ++ parser_module = astroid._ast._get_parser_module() # pylint: disable=no-member,protected-access ++ else: ++ parser_module = ast ++ return ConvertAST(parser_module).visit(node) ++ ++class ConvertAST(ast.NodeVisitor): ++ """Allows converting from ast nodes to typed_ast.ast27 or typed_ast.ast3 nodes.""" ++ def __init__(self, ast_module): ++ self._ast_module = ast_module ++ ++ def visit(self, node): ++ converted_class = getattr(self._ast_module, node.__class__.__name__) ++ new_node = converted_class() ++ for field, old_value in ast.iter_fields(node): ++ new_value = ([self.maybe_visit(n) for n in old_value] if isinstance(old_value, list) else ++ self.maybe_visit(old_value)) ++ setattr(new_node, field, new_value) ++ for attr in getattr(node, '_attributes', ()): ++ setattr(new_node, attr, getattr(node, attr)) ++ return new_node ++ ++ def maybe_visit(self, node): ++ return self.visit(node) if isinstance(node, ast.AST) else node + + def collect_nodes_preorder(root): + """Returns a list of all nodes using pre-order traversal (i.e. parent before children).""" + nodes = [] +- def append(node, par_value): ++ def append(node, par_value): # pylint: disable=unused-argument + nodes.append(node) + return (None, None) + util.visit_tree(root, append, None) +@@ -117,18 +140,17 @@ class MarkChecker(object): + if not (util.is_stmt(node) or util.is_expr(node) or util.is_module(node)): + continue + +- if isinstance(node, astroid.nodes.Yield): +- # Astroid stringifies Yield nodes differently depending on parent, so these are too +- # annoying to verify. +- continue +- + text = self.atok.get_text(node) + rebuilt_node = parse_snippet(text, is_expr=util.is_expr(node), is_module=util.is_module(node)) + + # Now we need to check if the two nodes are equivalent. +- left = to_source(rebuilt_node) +- right = to_source(node) ++ left = _yield_fix(to_source(rebuilt_node)) ++ right = _yield_fix(to_source(node)) + test_case.assertEqual(left, right) + tested_nodes += 1 + + return tested_nodes ++ ++# Yield nodes are parenthesized depending on context; to ease verifications, parenthesize always. ++def _yield_fix(text): ++ return "(" + text + ")" if text.startswith("yield") else text diff --git a/0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch b/0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch new file mode 100644 index 0000000..0d0b70b --- /dev/null +++ b/0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch @@ -0,0 +1,142 @@ +From 441a88ebac5a192c36cba70919d7eb5e968574c7 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= +Date: Wed, 19 Jun 2019 04:50:58 +0200 +Subject: [PATCH 2/4] tests: use Constant for Num, Str, NameConstant node names + (#30) + +This fixes some test failures under python3.8. All tests still pass on +python3.7. Partially addresses #28, replaces #29. +As suggested in https://github.com/gristlabs/asttokens/pull/29/files#r280123329. +--- + tests/test_mark_tokens.py | 14 +++++++------- + tests/test_util.py | 15 ++++++++------- + tests/tools.py | 6 +++++- + 3 files changed, 20 insertions(+), 15 deletions(-) + +diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py +index c2e3f5f3ca..83d35fa432 100644 +--- a/tests/test_mark_tokens.py ++++ b/tests/test_mark_tokens.py +@@ -181,11 +181,11 @@ b + # line3 + m = self.create_mark_checker(source) + self.assertEqual(len(m.all_nodes), 2104) + self.assertEqual(m.view_node(m.all_nodes[-1]), +- "Str:'F1akOFFiRIgPHTZksKBAgMCLGTdGNIAAQgKfDAcgZbj0odOnUA8GBAA7'") ++ "Constant:'F1akOFFiRIgPHTZksKBAgMCLGTdGNIAAQgKfDAcgZbj0odOnUA8GBAA7'") + self.assertEqual(m.view_node(m.all_nodes[-2]), +- "Str:'Ii0uLDAxLzI0Mh44U0gxMDI5JkM0JjU3NDY6Kjc5Njo7OUE8Ozw+Oz89QTxA'") ++ "Constant:'Ii0uLDAxLzI0Mh44U0gxMDI5JkM0JjU3NDY6Kjc5Njo7OUE8Ozw+Oz89QTxA'") + self.assertEqual(m.view_node(m.all_nodes[1053]), +- "Str:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'") ++ "Constant:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'") + self.assertEqual(m.view_node(m.all_nodes[1052]), + "BinOp:'R0lGODlhigJnAef/AAABAAEEAAkCAAMGAg0GBAYJBQoMCBMODQ4QDRITEBkS'\r\n" + + " +'CxsSEhkWDhYYFQ0aJhkaGBweGyccGh8hHiIkIiMmGTEiHhQoPSYoJSkqKDcp'") +@@ -225,7 +225,7 @@ bar = ('x y z' # comment2 + ) + """ + m = self.create_mark_checker(source) +- node_name = 'Const' if self.is_astroid_test else 'Str' ++ node_name = 'Const' if self.is_astroid_test else 'Constant' + self.assertEqual(m.view_nodes_at(2, 6), { + node_name + ":'x y z' \\\n'''a b c''' \"u v w\"" + }) +@@ -336,7 +336,7 @@ bar = ('x y z' # comment2 + name_a = 'AssignName:a' if self.is_astroid_test else 'Name:a' + const_true = ('Const:True' if self.is_astroid_test else + 'Name:True' if six.PY2 else +- 'NameConstant:True') ++ 'Constant:True') + self.assertEqual(m.view_nodes_at(1, 0), + {name_a, "Assign:a = True if True else False", "Module:" + source}) + self.assertEqual(m.view_nodes_at(1, 4), +@@ -388,7 +388,7 @@ bar = ('x y z' # comment2 + if self.is_astroid_test: + self.assertEqual(m.view_nodes_at(1, 5), {'Const:4'}) + else: +- self.assertEqual(m.view_nodes_at(1, 5), {'Num:4'}) ++ self.assertEqual(m.view_nodes_at(1, 5), {'Constant:4'}) + self.assertEqual(m.view_nodes_at(2, 0), {'Delete:del x[4]'}) + self.assertEqual(m.view_nodes_at(2, 4), {'Name:x', 'Subscript:x[4]'}) + +@@ -428,7 +428,7 @@ bar = ('x y z' # comment2 + self.assertEqual(m.view_nodes_at(2, 8), {'Keyword:b=[y]'}) + else: + self.assertEqual(m.view_nodes_at(1, 2), {'keyword:x=1'}) +- self.assertEqual(m.view_nodes_at(1, 4), {'Num:1'}) ++ self.assertEqual(m.view_nodes_at(1, 4), {'Constant:1'}) + self.assertEqual(m.view_nodes_at(2, 2), {'keyword:a=(x)'}) + self.assertEqual(m.view_nodes_at(2, 8), {'keyword:b=[y]'}) + +diff --git a/tests/test_util.py b/tests/test_util.py +index 254116c32f..91a51432ae 100644 +--- a/tests/test_util.py ++++ b/tests/test_util.py +@@ -4,6 +4,7 @@ import ast + import astroid + import unittest + from .context import asttokens ++from .tools import get_node_name + + class TestUtil(unittest.TestCase): + +@@ -37,7 +38,7 @@ class TestUtil(unittest.TestCase): + atok = asttokens.ASTTokens(self.source, parse=True) + + def view(node): +- return "%s:%s" % (node.__class__.__name__, atok.get_text(node)) ++ return "%s:%s" % (get_node_name(node), atok.get_text(node)) + + scan = [view(n) for n in asttokens.util.walk(atok.tree)] + self.assertEqual(scan, [ +@@ -48,20 +49,20 @@ class TestUtil(unittest.TestCase): + 'Call:bar(1 + 2)', + 'Name:bar', + 'BinOp:1 + 2', +- 'Num:1', +- 'Num:2', ++ 'Constant:1', ++ 'Constant:2', + "BinOp:'hello' + ', ' + 'world'", + "BinOp:'hello' + ', '", +- "Str:'hello'", +- "Str:', '", +- "Str:'world'" ++ "Constant:'hello'", ++ "Constant:', '", ++ "Constant:'world'" + ]) + + def test_walk_astroid(self): + atok = asttokens.ASTTokens(self.source, tree=astroid.builder.parse(self.source)) + + def view(node): +- return "%s:%s" % (node.__class__.__name__, atok.get_text(node)) ++ return "%s:%s" % (get_node_name(node), atok.get_text(node)) + + scan = [view(n) for n in asttokens.util.walk(atok.tree)] + self.assertEqual(scan, [ +diff --git a/tests/tools.py b/tests/tools.py +index b2a638b1e3..3f4fea7c0e 100644 +--- a/tests/tools.py ++++ b/tests/tools.py +@@ -96,6 +96,10 @@ def collect_nodes_preorder(root): + util.visit_tree(root, append, None) + return nodes + ++def get_node_name(node): ++ name = node.__class__.__name__ ++ return 'Constant' if name in ('Num', 'Str', 'NameConstant') else name ++ + + class MarkChecker(object): + """ +@@ -112,7 +116,7 @@ class MarkChecker(object): + + def view_node(self, node): + """Returns a representation of a node and its text, such as "Call:foo()". """ +- return "%s:%s" % (node.__class__.__name__, self.atok.get_text(node)) ++ return "%s:%s" % (get_node_name(node), self.atok.get_text(node)) + + def view_nodes_at(self, line, col): + """ diff --git a/0003-Make-tests-pass-under-python3.8-astroid-2.2.patch b/0003-Make-tests-pass-under-python3.8-astroid-2.2.patch new file mode 100644 index 0000000..5596b48 --- /dev/null +++ b/0003-Make-tests-pass-under-python3.8-astroid-2.2.patch @@ -0,0 +1,136 @@ +From 3513264ceff0e1cfbd9ad6543bc24b5cc08f1acb Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= +Date: Tue, 10 Sep 2019 11:06:29 +0200 +Subject: [PATCH 4/4] Make tests pass under python3.8 + astroid 2.2 + +--- + asttokens/mark_tokens.py | 27 ++++++++++++++++----------- + tests/test_mark_tokens.py | 32 +++++++++++++++++++++++--------- + 2 files changed, 39 insertions(+), 20 deletions(-) + +diff --git a/asttokens/mark_tokens.py b/asttokens/mark_tokens.py +index 624ac6273b..53e1ed6b16 100644 +--- a/asttokens/mark_tokens.py ++++ b/asttokens/mark_tokens.py +@@ -13,6 +13,7 @@ + # limitations under the License. + + import six ++import sys + import numbers + import token + from . import util +@@ -165,11 +166,14 @@ class MarkTokens(object): + return (first_token, last_token) + + def handle_comp(self, open_brace, node, first_token, last_token): +- # For list/set/dict comprehensions, we only get the token of the first child, so adjust it to ++ # For list/set/dict comprehensions, in the past, we only got the token of the first child, so adjust it to + # include the opening brace (the closing brace will be matched automatically). +- before = self._code.prev_token(first_token) +- util.expect_token(before, token.OP, open_brace) +- return (before, last_token) ++ if util.match_token(first_token, token.OP, open_brace): ++ return (first_token, last_token) ++ else: ++ before = self._code.prev_token(first_token) ++ util.expect_token(before, token.OP, open_brace) ++ return (before, last_token) + + def visit_listcomp(self, node, first_token, last_token): + return self.handle_comp('[', node, first_token, last_token) +@@ -224,13 +228,14 @@ class MarkTokens(object): + return (first_token, self._code.find_token(last_token, token.OP, ']')) + + def visit_tuple(self, node, first_token, last_token): +- # A tuple doesn't include parens; if there is a trailing comma, make it part of the tuple. +- try: +- maybe_comma = self._code.next_token(last_token) +- if util.match_token(maybe_comma, token.OP, ','): +- last_token = maybe_comma +- except IndexError: +- pass ++ if sys.version_info < (3,8): ++ # A tuple doesn't include parens; if there is a trailing comma, make it part of the tuple. ++ try: ++ maybe_comma = self._code.next_token(last_token) ++ if util.match_token(maybe_comma, token.OP, ','): ++ last_token = maybe_comma ++ except IndexError: ++ pass + return (first_token, last_token) + + def visit_str(self, node, first_token, last_token): +diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py +index 83d35fa432..2dbd7e8b3c 100644 +--- a/tests/test_mark_tokens.py ++++ b/tests/test_mark_tokens.py +@@ -98,7 +98,7 @@ b + # line3 + # All other expressions preserve newlines and comments but are parenthesized. + 'b + # line3\n c', + 'b + # line3\n c + # line4\n d', +- 'a, # line2\nb + # line3\n c + # line4\n d', ++ 'a, # line2\nb + # line3\n c + # line4\n d' if sys.version_info < (3,8) else source, + }) + + +@@ -202,8 +202,12 @@ b + # line3 + # Make sure we don't fail on parsing slices of the form `foo[4:]`. + source = "(foo.Area_Code, str(foo.Phone)[:3], str(foo.Phone)[3:], foo[:], bar[::, :])" + m = self.create_mark_checker(source) +- self.assertEqual(m.view_nodes_at(1, 1), +- { "Attribute:foo.Area_Code", "Name:foo", "Tuple:"+source[1:-1] }) ++ ++ at_1_1 = { "Attribute:foo.Area_Code", "Name:foo" } ++ if sys.version_info < (3,8): ++ at_1_1.add( "Tuple:"+source[1:-1] ) ++ self.assertEqual(m.view_nodes_at(1, 1), at_1_1) ++ + self.assertEqual(m.view_nodes_at(1, 16), + { "Subscript:str(foo.Phone)[:3]", "Call:str(foo.Phone)", "Name:str"}) + self.assertEqual(m.view_nodes_at(1, 36), +@@ -226,11 +230,14 @@ bar = ('x y z' # comment2 + """ + m = self.create_mark_checker(source) + node_name = 'Const' if self.is_astroid_test else 'Constant' ++ joined = self.is_astroid_test or sys.version_info < (3,8) + self.assertEqual(m.view_nodes_at(2, 6), { +- node_name + ":'x y z' \\\n'''a b c''' \"u v w\"" ++ node_name + ":'x y z'" + ( ++ " \\\n'''a b c''' \"u v w\"" if joined else "") + }) + self.assertEqual(m.view_nodes_at(4, 7), { +- node_name + ":'x y z' # comment2\n 'a b c' # comment3\n 'u v w'" ++ node_name + ":'x y z'" + ( ++ " # comment2\n 'a b c' # comment3\n 'u v w'" if joined else "") + }) + + +@@ -446,16 +453,23 @@ bar = ('x y z' # comment2 + m.verify_all_nodes(self) + # The `arguments` node has bogus positions here (and whenever there are no arguments). We + # don't let that break our test because it's unclear if it matters to anything anyway. +- self.assertIn('FunctionDef:@deco1\ndef f():\n pass', m.view_nodes_at(2, 0)) +- self.assertEqual(m.view_nodes_at(2, 1), {'Name:deco1'}) ++ ++ nodes_2_0 = m.view_nodes_at(2, 0) ++ nodes_2_1 = m.view_nodes_at(2, 1) ++ nodes_5_0 = m.view_nodes_at(5, 0) ++ nodes_5_1 = m.view_nodes_at(5, 1) ++ ++ self.assertIn('FunctionDef:@deco1\ndef f():\n pass', nodes_2_0 or nodes_2_1) ++ self.assertIn('Name:deco1', nodes_2_1) + if self.is_astroid_test: + self.assertEqual(m.view_nodes_at(5, 0), { + 'FunctionDef:@deco2(a=1)\ndef g(x):\n pass', + 'Decorators:@deco2(a=1)' + }) + else: +- self.assertEqual(m.view_nodes_at(5, 0), {'FunctionDef:@deco2(a=1)\ndef g(x):\n pass'}) +- self.assertEqual(m.view_nodes_at(5, 1), {'Name:deco2', 'Call:deco2(a=1)'}) ++ self.assertIn('FunctionDef:@deco2(a=1)\ndef g(x):\n pass', nodes_5_0 or nodes_5_1) ++ self.assertIn('Name:deco2', nodes_5_1) ++ self.assertIn('Call:deco2(a=1)', nodes_5_1) + + def test_with(self): + source = "with foo: pass" diff --git a/21caaaa74105c410b3d84c3d8ff0dc2f612aac9a.patch b/21caaaa74105c410b3d84c3d8ff0dc2f612aac9a.patch deleted file mode 100644 index 7b94dbd..0000000 --- a/21caaaa74105c410b3d84c3d8ff0dc2f612aac9a.patch +++ /dev/null @@ -1,147 +0,0 @@ -From 21caaaa74105c410b3d84c3d8ff0dc2f612aac9a Mon Sep 17 00:00:00 2001 -From: Dmitry S -Date: Wed, 1 May 2019 11:46:30 -0400 -Subject: [PATCH] Fixes for Astroid 2.2, which expects typed_ast.ast3 nodes in - TreeRebuilder. - ---- - tests/test_mark_tokens.py | 2 +- - tests/tools.py | 62 ++++++++++++++++++++++++++------------- - 2 files changed, 43 insertions(+), 21 deletions(-) - -diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py -index f24a6ca..c2e3f5f 100644 ---- a/tests/test_mark_tokens.py -+++ b/tests/test_mark_tokens.py -@@ -125,7 +125,7 @@ def verify_fixture_file(self, path): - 'astroid/email.py': ( 3, 3, 1, 1, ), - 'astroid/format.py': ( 64, 61, 62, 62, ), - 'astroid/module.py': ( 185, 174, 171, 171, ), -- 'astroid/module2.py': ( 248, 253, 235, 248, ), -+ 'astroid/module2.py': ( 248, 253, 240, 253, ), - 'astroid/noendingnewline.py': ( 57, 59, 57, 63, ), - 'astroid/notall.py': ( 15, 17, 15, 17, ), - 'astroid/recursion.py': ( 6, 6, 4, 4, ), -diff --git a/tests/tools.py b/tests/tools.py -index 546879b..b2a638b 100644 ---- a/tests/tools.py -+++ b/tests/tools.py -@@ -1,14 +1,13 @@ - from __future__ import unicode_literals, print_function - import ast --import astroid --import asttokens --import copy - import io - import os - import re - import sys --from asttokens import util - -+import astroid -+import asttokens -+from asttokens import util - - def get_fixture_path(*path_parts): - python_dir = 'python%s' % sys.version_info[0] -@@ -18,7 +17,6 @@ def read_fixture(*path_parts): - with io.open(get_fixture_path(*path_parts), "r", newline="\n") as f: - return f.read() - -- - def _parse_stmt(text): - # ast.parse produces a module, but we here want to produce a single statement. - return ast.parse(text, 'exec').body[0] -@@ -35,12 +33,11 @@ def parse_snippet(text, is_expr=False, is_module=False): - indented = re.match(r'^[ \t]+\S', text) - if indented: - return _parse_stmt('def dummy():\n' + text).body[0] -- elif is_expr: -+ if is_expr: - return _parse_stmt('(' + text + ')').value -- elif is_module: -+ if is_module: - return ast.parse(text, 'exec') -- else: -- return _parse_stmt(text) -+ return _parse_stmt(text) - - - def to_source(node): -@@ -52,9 +49,8 @@ def to_source(node): - return node.as_string() - - builder = astroid.rebuilder.TreeRebuilder(astroid.manager.AstroidManager()) -- # We need to make a deep copy of node; not sure why, but node seems affected by the astroid -- # TreeRebuilder. -- node_copy = copy.deepcopy(node) -+ # We need to make a copy of node that astroid can process. -+ node_copy = create_astroid_ast(node) - if isinstance(node, ast.Module): - anode = builder.visit_module(node_copy, '', '', '') - else: -@@ -63,11 +59,38 @@ def to_source(node): - anode = builder.visit(node_copy, amodule) - return anode.as_string() - -+def create_astroid_ast(node): -+ if hasattr(astroid, "_ast"): -+ # A bit of a hack, reaching into astroid, but we need to re-create the tree with the parser -+ # module that astroid understands, to be able to use TreeRebuilder on it. -+ parser_module = astroid._ast._get_parser_module() # pylint: disable=no-member,protected-access -+ else: -+ parser_module = ast -+ return ConvertAST(parser_module).visit(node) -+ -+class ConvertAST(ast.NodeVisitor): -+ """Allows converting from ast nodes to typed_ast.ast27 or typed_ast.ast3 nodes.""" -+ def __init__(self, ast_module): -+ self._ast_module = ast_module -+ -+ def visit(self, node): -+ converted_class = getattr(self._ast_module, node.__class__.__name__) -+ new_node = converted_class() -+ for field, old_value in ast.iter_fields(node): -+ new_value = ([self.maybe_visit(n) for n in old_value] if isinstance(old_value, list) else -+ self.maybe_visit(old_value)) -+ setattr(new_node, field, new_value) -+ for attr in getattr(node, '_attributes', ()): -+ setattr(new_node, attr, getattr(node, attr)) -+ return new_node -+ -+ def maybe_visit(self, node): -+ return self.visit(node) if isinstance(node, ast.AST) else node - - def collect_nodes_preorder(root): - """Returns a list of all nodes using pre-order traversal (i.e. parent before children).""" - nodes = [] -- def append(node, par_value): -+ def append(node, par_value): # pylint: disable=unused-argument - nodes.append(node) - return (None, None) - util.visit_tree(root, append, None) -@@ -117,18 +140,17 @@ def verify_all_nodes(self, test_case): - if not (util.is_stmt(node) or util.is_expr(node) or util.is_module(node)): - continue - -- if isinstance(node, astroid.nodes.Yield): -- # Astroid stringifies Yield nodes differently depending on parent, so these are too -- # annoying to verify. -- continue -- - text = self.atok.get_text(node) - rebuilt_node = parse_snippet(text, is_expr=util.is_expr(node), is_module=util.is_module(node)) - - # Now we need to check if the two nodes are equivalent. -- left = to_source(rebuilt_node) -- right = to_source(node) -+ left = _yield_fix(to_source(rebuilt_node)) -+ right = _yield_fix(to_source(node)) - test_case.assertEqual(left, right) - tested_nodes += 1 - - return tested_nodes -+ -+# Yield nodes are parenthesized depending on context; to ease verifications, parenthesize always. -+def _yield_fix(text): -+ return "(" + text + ")" if text.startswith("yield") else text diff --git a/python-asttokens.spec b/python-asttokens.spec index ef69022..011f9ee 100644 --- a/python-asttokens.spec +++ b/python-asttokens.spec @@ -6,13 +6,16 @@ Summary: Module to annotate Python abstract syntax trees with source code License: ASL 2.0 URL: https://pypi.python.org/pypi/asttokens Source0: https://files.pythonhosted.org/packages/source/a/asttokens/asttokens-%{version}.tar.gz -Patch0: https://github.com/gristlabs/asttokens/commit/21caaaa74105c410b3d84c3d8ff0dc2f612aac9a.patch + +Patch0001: 0001-Fixes-for-Astroid-2.2-which-expects-typed_ast.ast3-n.patch +Patch0002: 0002-tests-use-Constant-for-Num-Str-NameConstant-node-nam.patch +Patch0003: 0003-Make-tests-pass-under-python3.8-astroid-2.2.patch BuildArch: noarch BuildRequires: python3-devel -BuildRequires: %{py3_dist nose} -BuildRequires: %{py3_dist astroid} -BuildRequires: %{py3_dist six} +BuildRequires: python3dist(pytest) +BuildRequires: python3dist(astroid) +BuildRequires: python3dist(six) %global _description %{expand: The asttokens module annotates Python abstract syntax trees (ASTs) @@ -40,7 +43,7 @@ Requires: %{py3_dist six} %py3_install %check -nosetests-3 tests/ -v +pytest-3 tests/ -v --ignore=tests/testdata/ %files -n python3-asttokens %license LICENSE @@ -48,6 +51,9 @@ nosetests-3 tests/ -v %{python3_sitelib}/* %changelog +* Tue Sep 10 2019 Zbigniew Jędrzejewski-Szmek - 1.1.13-3 +- Fix build with python3.8 (#1697503) + * Mon Aug 19 2019 Miro Hrončok - 1.1.13-3 - Rebuilt for Python 3.8