From 63336e20adafeacf779d8c68d094223f811e508a Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Wed, 24 May 2023 04:43:56 -0400 Subject: [PATCH] gh-104719: IDLE - test existence of all tokenize references. (GH-104767) Class editor.IndentSearcher contains all editor references to tokenize module. Module io tokenize reference cover those other modules. (cherry picked from commit e561c09975bf67ad8bb67c56a81e30a9165bcc84) Co-authored-by: Terry Jan Reedy Co-authored-by: Jelle Zijlstra --- Lib/idlelib/NEWS.txt | 3 ++ Lib/idlelib/editor.py | 19 +++++----- Lib/idlelib/idle_test/test_editor.py | 36 +++++++++++++++++-- Lib/idlelib/idle_test/test_iomenu.py | 6 ++++ ...-05-23-17-19-49.gh-issue-104719.rvYXH-.rst | 2 ++ 5 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 89e86c13b25ea5..efc8bf3ce2d53c 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -4,6 +4,9 @@ Released after 2022-10-24 ========================= +gh-104719: Remove IDLE's modification of tokenize.tabsize and test +other uses of tokenize data and methods. + gh-104499: Fix completions for Tk Aqua 8.7 (currently blank). gh-104486: Make About print both tcl and tk versions if different, diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index df36be8766016f..69b27d0683a104 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1571,7 +1571,7 @@ def reindent_to(self, column): # blocks are found). def guess_indent(self): - opener, indented = IndentSearcher(self.text, self.tabwidth).run() + opener, indented = IndentSearcher(self.text).run() if opener and indented: raw, indentsmall = get_line_indent(opener, self.tabwidth) raw, indentlarge = get_line_indent(indented, self.tabwidth) @@ -1609,15 +1609,10 @@ def get_line_indent(line, tabwidth): class IndentSearcher: + "Manage initial indent guess, returned by run method." - # .run() chews over the Text widget, looking for a block opener - # and the stmt following it. Returns a pair, - # (line containing block opener, line containing stmt) - # Either or both may be None. - - def __init__(self, text, tabwidth): + def __init__(self, text): self.text = text - self.tabwidth = tabwidth self.i = self.finished = 0 self.blkopenline = self.indentedline = None @@ -1633,7 +1628,8 @@ def readline(self): def tokeneater(self, type, token, start, end, line, INDENT=tokenize.INDENT, NAME=tokenize.NAME, - OPENERS=('class', 'def', 'for', 'if', 'try', 'while')): + OPENERS=('class', 'def', 'for', 'if', 'match', 'try', + 'while', 'with')): if self.finished: pass elif type == NAME and token in OPENERS: @@ -1643,6 +1639,10 @@ def tokeneater(self, type, token, start, end, line, self.finished = 1 def run(self): + """Return 2 lines containing block opener and and indent. + + Either the indent line or both may be None. + """ try: tokens = tokenize.generate_tokens(self.readline) for token in tokens: @@ -1654,6 +1654,7 @@ def run(self): ### end autoindent code ### + def prepstr(s): """Extract the underscore from a string. diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index fdb47abf43fb77..9296a6d235fbbe 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -1,10 +1,10 @@ -"Test editor, coverage 35%." +"Test editor, coverage 53%." from idlelib import editor import unittest from collections import namedtuple from test.support import requires -from tkinter import Tk +from tkinter import Tk, Text Editor = editor.EditorWindow @@ -31,7 +31,7 @@ def test_init(self): e._close() -class TestGetLineIndent(unittest.TestCase): +class GetLineIndentTest(unittest.TestCase): def test_empty_lines(self): for tabwidth in [1, 2, 4, 6, 8]: for line in ['', '\n']: @@ -181,6 +181,36 @@ def test_indent_and_newline_event(self): eq(get('1.0', 'end'), ' def f1(self, a,\n \n return a + b\n') +class IndentSearcherTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def test_searcher(self): + text = self.text + searcher = (self.text) + test_info = (# text, (block, indent)) + ("", (None, None)), + ("[1,", (None, None)), # TokenError + ("if 1:\n", ('if 1:\n', None)), + ("if 1:\n 2\n 3\n", ('if 1:\n', ' 2\n')), + ) + for code, expected_pair in test_info: + with self.subTest(code=code): + insert(text, code) + actual_pair = editor.IndentSearcher(text).run() + self.assertEqual(actual_pair, expected_pair) + + class RMenuTest(unittest.TestCase): @classmethod diff --git a/Lib/idlelib/idle_test/test_iomenu.py b/Lib/idlelib/idle_test/test_iomenu.py index 2fb836dba21672..e0642cf0cabef0 100644 --- a/Lib/idlelib/idle_test/test_iomenu.py +++ b/Lib/idlelib/idle_test/test_iomenu.py @@ -8,6 +8,12 @@ from idlelib import util from idlelib.idle_test.mock_idle import Func +# Fail if either tokenize.open and t.detect_encoding does not exist. +# These are used in loadfile and encode. +# Also used in pyshell.MI.execfile and runscript.tabnanny. +from tokenize import open, detect_encoding +# Remove when we have proper tests that use both. + class IOBindingTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst b/Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst new file mode 100644 index 00000000000000..3fbe04ba4f6844 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-05-23-17-19-49.gh-issue-104719.rvYXH-.rst @@ -0,0 +1,2 @@ +Remove IDLE's modification of tokenize.tabsize and test other uses of +tokenize data and methods.