From: Matthias Bussonnier Date: Tue, 3 Jan 2023 11:57:18 +0100 Subject: Fix tests for pygments > 2.14 Pygments 2.14+ have the bash lexer return some tokens as Text.Whitespace instead of Text, this update the test to support this. --- IPython/lib/tests/test_lexers.py | 52 +++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/IPython/lib/tests/test_lexers.py b/IPython/lib/tests/test_lexers.py index efa00d6..000b8fe 100644 --- a/IPython/lib/tests/test_lexers.py +++ b/IPython/lib/tests/test_lexers.py @@ -4,11 +4,14 @@ # Distributed under the terms of the Modified BSD License. from unittest import TestCase +from pygments import __version__ as pygments_version from pygments.token import Token from pygments.lexers import BashLexer from .. import lexers +pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14) + class TestLexers(TestCase): """Collection of lexers tests""" @@ -18,25 +21,26 @@ def setUp(self): def testIPythonLexer(self): fragment = '!echo $HOME\n' - tokens = [ + bash_tokens = [ (Token.Operator, '!'), ] - tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) + ipylex_token = list(self.lexer.get_tokens(fragment)) + assert bash_tokens[:-1] == ipylex_token[:-1] - fragment_2 = '!' + fragment + fragment_2 = "!" + fragment tokens_2 = [ (Token.Operator, '!!'), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = '\t %%!\n' + fragment[1:] tokens_2 = [ (Token.Text, '\t '), (Token.Operator, '%%!'), (Token.Text, '\n'), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = 'x = ' + fragment tokens_2 = [ @@ -44,8 +48,8 @@ def testIPythonLexer(self): (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), - ] + tokens - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x, = ' + fragment tokens_2 = [ @@ -54,8 +58,8 @@ def testIPythonLexer(self): (Token.Text, ' '), (Token.Operator, '='), (Token.Text, ' '), - ] + tokens - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x, = %sx ' + fragment[1:] tokens_2 = [ @@ -67,8 +71,10 @@ def testIPythonLexer(self): (Token.Operator, '%'), (Token.Keyword, 'sx'), (Token.Text, ' '), - ] + tokens[1:] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + ] + bash_tokens[1:] + if tokens_2[7] == (Token.Text, " ") and pyg214: # pygments 2.14+ + tokens_2[7] = (Token.Text.Whitespace, " ") + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'f = %R function () {}\n' tokens_2 = [ @@ -80,7 +86,7 @@ def testIPythonLexer(self): (Token.Keyword, 'R'), (Token.Text, ' function () {}\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = '\t%%xyz\n$foo\n' tokens_2 = [ @@ -89,7 +95,7 @@ def testIPythonLexer(self): (Token.Keyword, 'xyz'), (Token.Text, '\n$foo\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) fragment_2 = '%system?\n' tokens_2 = [ @@ -98,7 +104,7 @@ def testIPythonLexer(self): (Token.Operator, '?'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = 'x != y\n' tokens_2 = [ @@ -109,7 +115,7 @@ def testIPythonLexer(self): (Token.Name, 'y'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment_2 = ' ?math.sin\n' tokens_2 = [ @@ -118,7 +124,7 @@ def testIPythonLexer(self): (Token.Text, 'math.sin'), (Token.Text, '\n'), ] - self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) + assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] fragment = ' *int*?\n' tokens = [ @@ -126,7 +132,7 @@ def testIPythonLexer(self): (Token.Operator, '?'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + assert tokens == list(self.lexer.get_tokens(fragment)) fragment = '%%writefile -a foo.py\nif a == b:\n pass' tokens = [ @@ -145,7 +151,9 @@ def testIPythonLexer(self): (Token.Keyword, 'pass'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + if tokens[10] == (Token.Text, "\n") and pyg214: # pygments 2.14+ + tokens[10] = (Token.Text.Whitespace, "\n") + assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1] fragment = '%%timeit\nmath.sin(0)' tokens = [ @@ -173,4 +181,4 @@ def testIPythonLexer(self): (Token.Punctuation, '>'), (Token.Text, '\n'), ] - self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) + assert tokens == list(self.lexer.get_tokens(fragment))