From 69e8e2ab42353479129f14afc542322d3b02de84 Mon Sep 17 00:00:00 2001
From: Dmitry S <dsagal+git@gmail.com>
Date: Thu, 5 Jul 2018 15:03:08 -0400
Subject: [PATCH] Fixes to make asttokens work with Python3.7 and astroid 2.0.

Fixes detection of non-coding tokens to work with py37.
Skips testing of deep-recursion on astroid 2.0, which no longer supports it.
Adds Python3.7 testing to tox.ini.
---
 asttokens/asttokens.py    |  8 ++++----
 asttokens/util.py         | 14 ++++++++++++++
 tests/test_mark_tokens.py | 20 ++++++++++++++------
 tox.ini                   |  5 +++--
 4 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/asttokens/asttokens.py b/asttokens/asttokens.py
index c0126a6b78..668620d085 100644
--- a/asttokens/asttokens.py
+++ b/asttokens/asttokens.py
@@ -20,7 +20,7 @@ import io
 import six
 from six.moves import xrange      # pylint: disable=redefined-builtin
 from .line_numbers import LineNumbers
-from .util import Token, match_token
+from .util import Token, match_token, is_non_coding_token
 from .mark_tokens import MarkTokens
 
 class ASTTokens(object):
@@ -135,7 +135,7 @@ class ASTTokens(object):
     """
     i = tok.index + 1
     if not include_extra:
-      while self._tokens[i].type >= token.N_TOKENS:
+      while is_non_coding_token(self._tokens[i].type):
         i += 1
     return self._tokens[i]
 
@@ -146,7 +146,7 @@ class ASTTokens(object):
     """
     i = tok.index - 1
     if not include_extra:
-      while self._tokens[i].type >= token.N_TOKENS:
+      while is_non_coding_token(self._tokens[i].type):
         i -= 1
     return self._tokens[i]
 
@@ -168,7 +168,7 @@ class ASTTokens(object):
     include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT.
     """
     for i in xrange(first_token.index, last_token.index + 1):
-      if include_extra or self._tokens[i].type < token.N_TOKENS:
+      if include_extra or not is_non_coding_token(self._tokens[i].type):
         yield self._tokens[i]
 
   def get_tokens(self, node, include_extra=False):
diff --git a/asttokens/util.py b/asttokens/util.py
index 4dd2f27983..3be33abb57 100644
--- a/asttokens/util.py
+++ b/asttokens/util.py
@@ -57,6 +57,20 @@ def expect_token(token, tok_type, tok_str=None):
       token_repr(tok_type, tok_str), str(token),
       token.start[0], token.start[1] + 1))
 
+# These were previously defined in tokenize.py and distinguishable by being greater than
+# token.N_TOKEN. As of python3.7, they are in token.py, and we check for them explicitly.
+if hasattr(token, 'COMMENT'):
+  def is_non_coding_token(token_type):
+    """
+    These are considered non-coding tokens, as they don't affect the syntax tree.
+    """
+    return token_type in (token.NL, token.COMMENT, token.ENCODING)
+else:
+  def is_non_coding_token(token_type):
+    """
+    These are considered non-coding tokens, as they don't affect the syntax tree.
+    """
+    return token_type >= token.N_TOKENS
 
 def iter_children(node):
   """
diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py
index a15934df9c..61c56bab1b 100644
--- a/tests/test_mark_tokens.py
+++ b/tests/test_mark_tokens.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals, print_function
 import astroid
 import six
 import sys
+import token
 import textwrap
 import unittest
 from . import tools
@@ -155,13 +156,19 @@ b +     # line3
     # to_source() on it because it chokes on recursion depth. So we test individual nodes.
     source = tools.read_fixture('astroid/joined_strings.py')
 
-    astroid.MANAGER.optimize_ast = True
-    try:
-      m = self.create_mark_checker(source)
-    finally:
-      astroid.MANAGER.optimize_ast = False
-
     if self.is_astroid_test:
+      if getattr(astroid, '__version__', '1') >= '2':
+        # Astroid 2 no longer supports this; see
+        # https://github.com/PyCQA/astroid/issues/557#issuecomment-396004274
+        self.skipTest('astroid-2.0 does not support this')
+
+      # Astroid < 2 does support this with optimize_ast set to True
+      astroid.MANAGER.optimize_ast = True
+      try:
+        m = self.create_mark_checker(source)
+      finally:
+        astroid.MANAGER.optimize_ast = False
+
       self.assertEqual(len(m.all_nodes), 4)     # This is the result of astroid's optimization
       self.assertEqual(m.view_node_types_at(1, 0), {'Module', 'Assign', 'AssignName'})
       const = next(n for n in m.all_nodes if isinstance(n, astroid.nodes.Const))
@@ -171,6 +178,7 @@ b +     # line3
       # astroid could avoid the need for the optimization by using an explicit stack like we do.
       #self.assertEqual(m.atok.get_text_range(const), (5, len(source) - 1))
     else:
+      m = self.create_mark_checker(source)
       self.assertEqual(len(m.all_nodes), 2104)
       self.assertEqual(m.view_node(m.all_nodes[-1]),
                        "Str:'F1akOFFiRIgPHTZksKBAgMCLGTdGNIAAQgKfDAcgZbj0odOnUA8GBAA7'")