Add t-string support to syntax highlighting

Author: ambv

Lib/_pyrepl/_module_completer.py

@@ -2,6 +2,7 @@
 
 import pkgutil
 import sys
+import token
 import tokenize
 from io import StringIO
 from contextlib import contextmanager
@@ -180,8 +181,8 @@ class ImportParser:
     when parsing multiple statements.
     """
     _ignored_tokens = {
-        tokenize.INDENT, tokenize.DEDENT, tokenize.COMMENT,
-        tokenize.NL, tokenize.NEWLINE, tokenize.ENDMARKER
+        token.INDENT, token.DEDENT, token.COMMENT,
+        token.NL, token.NEWLINE, token.ENDMARKER
     }
     _keywords = {'import', 'from', 'as'}
 
@@ -350,11 +351,11 @@ def peek(self) -> TokenInfo | None:
     def peek_name(self) -> bool:
         if not (tok := self.peek()):
             return False
-        return tok.type == tokenize.NAME
+        return tok.type == token.NAME
 
     def pop_name(self) -> str:
         tok = self.pop()
-        if tok.type != tokenize.NAME:
+        if tok.type != token.NAME:
             raise ParseError('pop_name')
         return tok.string
 

Lib/_pyrepl/mypy.ini

@@ -23,7 +23,3 @@ check_untyped_defs = False
 # Various internal modules that typeshed deliberately doesn't have stubs for:
 [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
 ignore_missing_imports = True
-
-# Other untyped parts of the stdlib
-[mypy-idlelib.*]
-ignore_missing_imports = True

Lib/_pyrepl/utils.py

@@ -113,8 +113,10 @@ def recover_unterminated_string(
     if (
         msg.startswith("unterminated string literal")
         or msg.startswith("unterminated f-string literal")
+        or msg.startswith("unterminated t-string literal")
         or msg.startswith("EOF in multi-line string")
         or msg.startswith("unterminated triple-quoted f-string literal")
+        or msg.startswith("unterminated triple-quoted t-string literal")
     ):
         start = line_lengths[loc[0] - 1] + loc[1] - 1
         end = line_lengths[-1] - 1
@@ -149,7 +151,11 @@ def gen_colors_from_token_stream(
             continue
 
         match token.type:
-            case T.STRING | T.FSTRING_START | T.FSTRING_MIDDLE | T.FSTRING_END:
+            case (
+                T.STRING
+                | T.FSTRING_START | T.FSTRING_MIDDLE | T.FSTRING_END
+                | T.TSTRING_START | T.TSTRING_MIDDLE | T.TSTRING_END
+            ):
                 span = Span.from_token(token, line_lengths)
                 yield ColorSpan(span, "STRING")
             case T.COMMENT:
@@ -202,7 +208,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool:
         case (
             None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"),
             TI(string="match"),
-            TI(T.NUMBER | T.STRING | T.FSTRING_START)
+            TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START)
             | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{" | "~" | "...")
         ):
             return True
@@ -217,7 +223,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool:
         case (
             None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"),
             TI(string="case"),
-            TI(T.NUMBER | T.STRING | T.FSTRING_START)
+            TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START)
             | TI(T.OP, string="(" | "*" | "-" | "[" | "{")
         ):
             return True

Lib/token.py

@@ -0,0 +1 @@
+../../Lib/token.py

Misc/mypy/token.py

@@ -2,4 +2,5 @@
 
 _colorize.py
 _pyrepl
+token.py
 tomllib

Misc/mypy/typed-stdlib.txt

@@ -278,13 +278,13 @@ def make_rst(infile, outfile='Doc/library/token-list.inc',
 %s
 }
 
-def ISTERMINAL(x):
+def ISTERMINAL(x: int) -> bool:
     return x < NT_OFFSET
 
-def ISNONTERMINAL(x):
+def ISNONTERMINAL(x: int) -> bool:
     return x >= NT_OFFSET
 
-def ISEOF(x):
+def ISEOF(x: int) -> bool:
     return x == ENDMARKER
 '''