diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 615ba587c6a400..b2e3d8ebacc6df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,7 @@ repos: name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ + exclude: ^Lib/test/test_dstring.py - id: ruff-check name: Run Ruff (lint) on Platforms/WASI/ args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e74f3262ed540c..c8980e83809b2a 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -862,8 +862,8 @@ A literal pattern corresponds to most The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are -supported. Raw strings and byte strings are supported. :ref:`f-strings` -and :ref:`t-strings` are not supported. +supported. Raw strings, byte strings, and :ref:`d-strings` are supported. +:ref:`f-strings` and :ref:`t-strings` are not supported. The forms ``signed_number '+' NUMBER`` and ``signed_number '-' NUMBER`` are for expressing :ref:`complex numbers `; they require a real number diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index af313f42f9bff6..ff3846f0e6b66e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -298,7 +298,7 @@ To concatenate string expressions at run time, the '+' operator may be used:: Hello Blaise Literal concatenation can freely mix raw strings, triple-quoted strings, -and formatted string literals. +dedented strings, and formatted string literals. For example:: >>> "Hello" r', ' f"{name}!" diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index f3ed1539493b6a..de4852bd5cd646 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -718,13 +718,17 @@ The allowed prefixes are: * ``r``: :ref:`Raw string ` * ``f``: :ref:`Formatted string literal ` ("f-string") * ``t``: :ref:`Template string literal ` ("t-string") +* ``d``: :ref:`Dedented multiline string literal ` ("d-string") * ``u``: No effect (allowed for backwards compatibility) See the linked sections for details on each type. Prefixes are case-insensitive (for example, '``B``' works the same as '``b``'). -The '``r``' prefix can be combined with '``f``', '``t``' or '``b``', so '``fr``', -'``rf``', '``tr``', '``rt``', '``br``', and '``rb``' are also valid prefixes. +The '``r``' prefix can be combined with '``f``', '``t``', '``b``', or +'``d``'. The '``d``' prefix can also be combined with '``f``', '``t``', or +'``b``', so the following prefix families are valid (in any order): +``d``, ``dr``, ``db``, ``drb``, ``df``, ``drf``, ``dt``, and ``drt``. +The ``u`` prefix cannot be combined with ``d``. .. versionadded:: 3.3 The ``'rb'`` prefix of raw bytes literals has been added as a synonym @@ -749,7 +753,10 @@ to indicate that an ending quote ends the literal. :group: python-grammar STRING: [`stringprefix`] (`stringcontent`) - stringprefix: <("r" | "u" | "b" | "br" | "rb"), case-insensitive> + stringprefix: <("r" | "u" | "b" | "br" | "rb" | + "d" | "dr" | "rd" | "db" | "bd" | + "drb" | "dbr" | "rdb" | "rbd" | "bdr" | "brd"), + case-insensitive> stringcontent: | "'''" ( !"'''" `longstringitem`)* "'''" | '"""' ( !'"""' `longstringitem`)* '"""' @@ -1023,6 +1030,65 @@ that a single backslash followed by a newline is interpreted as those two characters as part of the literal, *not* as a line continuation. +.. index:: + single: dedented multiline string literal + single: d-string + +.. _d-strings: +.. _dedented-string-literals: + +Dedented string literals +------------------------ + +.. versionadded:: 3.16 + +A :dfn:`dedented multiline string literal` or :dfn:`d-string` is a string +literal prefixed with '``d``' or '``D``'. It must use triple quotes, and the +opening quotes must be immediately followed by a newline. This newline is not +part of the resulting string. + +The longest prefix consisting only of spaces and tabs that is common to every +non-blank line and to the line containing the closing quotes is removed from +each non-blank line. Blank lines are normalized to a single newline and do not +participate in finding the common indentation. The closing-quotes line always +participates in that calculation, even when it contains no other characters. + +Dedentation is performed on the physical source lines before escape sequences +are processed. Consequently, a line-continuation backslash is treated as part +of the source line while its indentation is removed. Spaces and tabs are +treated as different characters. + +For example:: + + >>> value = d"""spam""" + SyntaxError: d-string must start with a newline + >>> value = d""" + ... first + ... second + ... """ + >>> value + 'first\n second\n' + >>> value = d""" + ... spam + ... ham + ... egg + ... """ + >>> value + ' spam\n ham\n egg\n' + + +The ``d`` prefix can be combined with the compatible ``r``, ``b``, ``f``, and +``t`` prefixes. In a ``df`` or ``dt`` string, a physical line participates in +the outer common indentation calculation only if it starts outside a +replacement field. A line that starts inside a replacement field is ignored +even if constant text resumes later on that line. Only constant text outside +replacement fields is dedented. A nested d-string is dedented independently. + +.. seealso:: + + :pep:`822` -- Dedented Multiline String + + .. index:: single: formatted string literal single: interpolated string literal @@ -1253,7 +1319,9 @@ sequences (``f_quote``). FSTRING_START: `fstringprefix` ("'" | '"' | "'''" | '"""') FSTRING_END: `f_quote` - fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + fstringprefix: <("f" | "fr" | "rf" | "df" | "fd" | "drf" | + "dfr" | "rdf" | "rfd" | "fdr" | "frd"), + case-insensitive> f_debug_specifier: '=' f_quote: @@ -1587,4 +1655,3 @@ is also available in the :mod:`!token` module documentation. A sequence of three consecutive periods (``...``) has a special meaning as an :py:data:`Ellipsis` literal. - diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 012f5da2869cd5..119f4637710daa 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -342,6 +342,16 @@ extern PyObject* _PyUnicode_XStrip( ); +/* Search the longest common leading whitespace of all lines in [src, end). + Returns the length of the common leading whitespace and sets `*output` to + point to the beginning of the common leading whitespace if length > 0. + Export for 'test_peg_generator.test_c_parser'. + */ +PyAPI_FUNC(Py_ssize_t) _Py_search_longest_common_leading_whitespace( + const char *src, + const char *end, + const char **output); + /* Dedent a string. Intended to dedent Python source. Unlike `textwrap.dedent`, this only supports spaces and tabs and doesn't normalize empty lines. diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py new file mode 100644 index 00000000000000..02ae58d4819970 --- /dev/null +++ b/Lib/test/test_dstring.py @@ -0,0 +1,530 @@ +import io +import itertools +import tokenize +import unittest +import warnings + +from test.test_string._support import TStringBaseCase + + +def _prefix_variants(prefix): + variants = set() + for permutation in itertools.permutations(prefix): + for letters in itertools.product( + *((c.lower(), c.upper()) for c in permutation) + ): + variants.add("".join(letters)) + return sorted(variants) + + +_dstring_prefixes = [] +for _prefix in "d db df dt dr drb drf drt".split(): + _dstring_prefixes.extend(_prefix_variants(_prefix)) + + +# Helper functions to evaluate d-strings. +# Use these helper functions to evaluate d-strings in cases where +# you want to test for SyntaxError or special indentation. + +def d(s): + """Helper function to evaluate d-strings.""" + if '"""' in s: + return eval(f"d'''{s}'''") + else: + return eval(f'd"""{s}"""') + +def db(s): + """Helper function to evaluate db-strings.""" + if '"""' in s: + return eval(f"db'''{s}'''") + else: + return eval(f'db"""{s}"""') + +def df(s, globals=None): + """Helper function to evaluate df-strings.""" + if '"""' in s: + return eval(f"df'''{s}'''", globals=globals) + else: + return eval(f'df"""{s}"""', globals=globals) + +def dt(s, globals=None): + """Helper function to evaluate dt-strings.""" + if '"""' in s: + return eval(f"dt'''{s}'''", globals=globals) + else: + return eval(f'dt"""{s}"""', globals=globals) + + +class AllRaisesMixin: + def assertAllRaise(self, exception_type, regex, exprs): + """Assert that all strings in exprs raise exception_type with regex.""" + for expr in exprs: + with self.subTest(expr=expr): + with self.assertRaisesRegex(exception_type, regex): + eval(expr) + + +class DStringTestCase(AllRaisesMixin, unittest.TestCase): + + def test_single_quote(self): + exprs = [ + f"{p}'hello, world'" for p in _dstring_prefixes + ] + [ + f'{p}"hello, world"' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must be triple-quoted", exprs) + + def test_empty_dstring(self): + exprs = [ + f"{p}''''''" for p in _dstring_prefixes + ] + [ + f'{p}""""""' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + for prefix in _dstring_prefixes: + for expr in [f"{prefix}'''\n'''", f'{prefix}"""\n"""']: + with self.subTest(expr=expr): + v = eval(expr) + if 't' in prefix.lower(): + self.assertEqual(v.strings, ("",)) + elif 'b' in prefix.lower(): + self.assertEqual(v, b"") + else: + self.assertEqual(v, "") + + def test_missing_newline_in_plain_and_raw_prefixes(self): + exprs = [ + 'd"""x"""', + 'dr"""x"""', + 'db"""x"""', + 'drb"""x"""', + 'd"""x\n"""', + 'd"""\\\n"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_backslash_after_opening_quotes(self): + exprs = [ + f'{p}"""\\\nhello\n"""' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_u_prefix_is_rejected(self): + exprs = [ + f'{p}"""\nhello\n"""' for p in _prefix_variants("du") + ] + self.assertAllRaise(SyntaxError, "'u' and 'd' prefixes are incompatible", exprs) + + def check_dbstring(self, s, expected): + # check both d- and db-strings with expected and expected.encode() + self.assertEqual(d(s), expected) + self.assertEqual(df(s), expected) + self.assertEqual(db(s), expected.encode()) + + def test_dbstring(self): + # Basic dedent - remove common leading whitespace + source = """ + hello + world + """ + self.check_dbstring(source, "hello\nworld\n") + + # Whitespace before the closing quotes is a blank final line, even + # when it is longer than the common indentation. + source = """ + foo + bar + """ + self.check_dbstring(source, "foo\nbar\n") + + # closing quote on same line as last content line + source = """ + hello + world""" + self.check_dbstring(source, "hello\nworld") + + # Dedent with varying indentation + source = """ + .line1 + ...line2 + + line3 + ..""".replace('.', ' ') + self.check_dbstring(source, " line1\n line2\n\nline3\n") + + # Dedent with tabs + source = """ +\thello +\tworld +\t""" + self.check_dbstring(source, "hello\nworld\n") + + # Mixed spaces and tabs (using common leading whitespace) + source = """ +\t\t hello +\t\t world +\t\t """ + self.check_dbstring(source, " hello\n world\n") + + # Empty lines do not affect the calculation of common leading whitespace + source = """ + hello + + world + """ + self.check_dbstring(source, "hello\n\nworld\n") + + # Blank lines are normalized to single newlines, even when they + # are longer than the common indentation. + source = """ +....hello +.. +...... +....world +....""".replace('.', ' ') + self.check_dbstring(source, "hello\n\n\nworld\n") + + # A blank line that does not match the common indentation is not + # an error; it is just normalized to an empty line. + source = """ +....hello +\t +....world +....""".replace('.', ' ') + self.check_dbstring(source, "hello\n\nworld\n") + + # Blank lines are normalized even when there is no common + # indentation. + source = """ +hello +.. +world +""".replace('.', ' ') + self.check_dbstring(source, "hello\n\nworld\n") + + # Line continuation with backslash works as usual. + # But you cannot put a backslash right after the opening quotes. + source = r""" + Hello \ + World!\ + """ + self.check_dbstring(source, "Hello World!") + + def test_raw_dstring(self): + source = r""" + path\\to\\file + keep\\n + """ + self.check_dbstring(source, "path\\to\\file\nkeep\\n\n") + + def test_braces_are_literal_without_f_prefix(self): + s = d(""" + hello { + 1 +} world + """) + self.assertEqual(s, " hello {\n 1\n} world\n") + + def test_dbstring_non_ascii_error(self): + with self.assertRaisesRegex(SyntaxError, "bytes can only contain ASCII literal characters"): + db('\n \u00e9\n ') + + def test_dbstring_non_ascii_error_precedes_invalid_escape_warning(self): + source = "db'''\n \u00e9\\z\n '''" + with warnings.catch_warnings(): + warnings.simplefilter("error", SyntaxWarning) + with self.assertRaisesRegex( + SyntaxError, "bytes can only contain ASCII literal characters" + ): + eval(source) + + def test_concat_bytes_and_nonbytes_error(self): + exprs = [ + 'd"""\n x\n """ db"""\n y\n """', + 'db"""\n x\n """ d"""\n y\n """', + ] + self.assertAllRaise(SyntaxError, "cannot mix bytes and nonbytes literals", exprs) + + +class DFStringTestCase(AllRaisesMixin, unittest.TestCase): + + def test_missing_newline_in_f_variants(self): + exprs = [ + 'df"""x"""', + 'drf"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_dfstring(self): + world = 42 + + s = df""" + Hello + {world} + """ + self.assertEqual(s, "Hello\n 42\n") + + # '{' is taken into account when calculating the common indentation + s = df""" + Hello + {world} + """ + self.assertEqual(s, " Hello\n42\n") + + # spaces after '}' is not taken into account + s = df""" + Hello {world} Lorum + ipsum dolor sit amet, + """ + self.assertEqual(s, "Hello 42 Lorum\nipsum dolor sit amet,\n") + + # A line that starts inside a replacement field does not affect the + # common indentation. + s = df""" + Hello { + world } Lorum + ipsum""" + self.assertEqual(s, "Hello 42 Lorum\nipsum") + + # This is also true when constant text resumes on the same line. + s = df""" + hello { + 1 + } world + """ + self.assertEqual(s, "hello 1 world\n") + + def test_dfstring_conversion_and_format(self): + x = 3.1415 + name = "Alice" + + s = df""" + {x:.2f} {name!r} + """ + self.assertEqual(s, "3.14 'Alice'\n") + + s = df""" + {x=} + """ + self.assertEqual(s, "x=3.1415\n") + + def test_concat_with_fstring(self): + s = d""" + hello + """ f"world" + self.assertEqual(s, "hello\nworld") + + def test_closing_quote_on_content_line(self): + value = "Python" + s = df""" + hello {value} + world""" + self.assertEqual(s, "hello Python\n world") + + def test_blank_line_normalization(self): + # Blank lines are normalized to single newlines, even when their + # whitespace doesn't match the common indentation. + self.assertEqual(df('\n foo\n\t\n bar {1}\n '), "foo\n\nbar 1\n") + self.assertEqual(df('\n foo\n {1}\n '), "foo\n1\n") + + def test_multiline_format_spec(self): + class Spec: + def __format__(self, spec): + return spec + s = Spec() + + # Lines inside a multi-line format spec are not dedented. + self.assertEqual(df''' + {s:>6 + } + ''', ">6\n \n") + + # A line that starts in a format spec does not affect the outer common + # indentation, and its own indentation is retained. + self.assertEqual(df''' + {s:>6 + } + ''', ">6\n \n") + + # Nested replacement fields in the format spec keep working. + self.assertEqual(df''' + {s:{"a"}b + c} + ''', "ab\n c\n") + + def test_multiline_debug_text(self): + # The static text of a debug expression spanning multiple lines is + # not dedented. + self.assertEqual(df''' + {1 + + 1=} + ''', "1 +\n 1=2\n") + + self.assertEqual(df''' + {24 * + 3=} + ''', " 24 *\n 3=72\n") + + def test_nested_string_lines_do_not_affect_indent(self): + # Physical lines inside nested string literals in replacement fields + # do not take part in the outer common indentation calculation. The + # inner string itself is not dedented. + self.assertEqual(df""" + {0 or ''' + foo'''} + bar + """, "\n foo\nbar\n") + + def test_nested_dstring_inside_dfstring(self): + # A nested d-string literal in a replacement field is dedented + # independently when the expression is evaluated. + s = df""" + outer line + {d""" + nested + line + """} + """ + self.assertEqual(s, "outer line\n nested\nline\n\n") + + # the nested d-string content is dedented independently. + s = df""" + {d""" + foo + bar + """} + outer + """ + self.assertEqual(s, "foo\nbar\n\nouter\n") + + # Nested df-strings also work and interpolate normally. + s = df""" + prefix {df""" + value {42} + """} suffix + """ + self.assertEqual(s, "prefix value 42\n suffix\n") + + # Nested raw d-strings keep backslashes. + s = df""" + {dr""" + path\\to\\file + """} + """ + self.assertEqual(s, r"path\\to\\file" + "\n\n") + + # Lines inside the replacement field do not contribute to the outer + # d-string's common indentation. + s = df""" + foo {0 or d""" + bar + baz + """} spam + """ + self.assertEqual(s, "foo bar\nbaz\n spam\n") + + # Nested d-strings must still start with a newline. + expr = 'df"""\n {d"""x"""}\n """' + with self.assertRaisesRegex(SyntaxError, "d-string must start with a newline"): + eval(expr) + + def test_tokenize_untokenize_roundtrip(self): + source = b'''s = df""" + hello { + 1 +} world + goodbye + """ +''' + tokens = list(tokenize.tokenize(io.BytesIO(source).readline)) + token_pairs = [(tok.type, tok.string) for tok in tokens] + roundtripped = tokenize.untokenize(token_pairs) + + roundtripped_pairs = [ + (tok.type, tok.string) + for tok in tokenize.tokenize(io.BytesIO(roundtripped).readline) + ] + self.assertEqual(roundtripped_pairs, token_pairs) + + namespace = {} + exec(source, namespace) + roundtripped_namespace = {} + exec(roundtripped, roundtripped_namespace) + self.assertEqual(namespace["s"], "hello 1 world\ngoodbye\n") + self.assertEqual(roundtripped_namespace["s"], namespace["s"]) + + +class DTStringTestCase(AllRaisesMixin, TStringBaseCase, unittest.TestCase): + + def test_missing_newline_in_t_variants(self): + exprs = [ + 'dt"""x"""', + 'drt"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_dtstring_basic(self): + name = "Python" + t = dt""" + Hello, {name} + """ + self.assertTStringEqual(t, ("Hello, ", "\n"), [(name, "name")]) + + def test_blank_final_line_normalization(self): + t = dt""" + foo + {1} + """ + self.assertTStringEqual(t, ("foo\n", "\n"), [(1, "1")]) + + def test_closing_quote_on_content_line(self): + value = "Python" + t = dt""" + Hello, {value} + goodbye""" + self.assertTStringEqual(t, ("Hello, ", "\n goodbye"), [(value, "value")]) + + def test_drtstring_raw_content(self): + t = drt""" + keep\n + """ + self.assertTStringEqual(t, ("keep\\n\n",), ()) + + def test_multiline_expression_text(self): + # The captured expression text of a multi-line interpolation is not + # dedented. + t = dt("""\n {1 +\n 1}\n """) + self.assertTStringEqual(t, ("", "\n"), [(2, "1 +\n 1")]) + + def test_multiline_expression_does_not_affect_indent(self): + # A line starting inside a replacement field does not take part in the + # common indentation calculation. Its text is not dedented. + t = dt("""\n Hello {1 +\n 2}\n """) + self.assertTStringEqual(t, ("Hello ", "\n"), [(3, "1 +\n 2")]) + + def test_multiline_format_spec(self): + # Lines inside a multi-line format spec are not dedented. + # t = eval('dt"""\n {1:>6\n }\n """') + t = dt""" + {1:>6 + } + """ + self.assertEqual(t.interpolations[0].format_spec, f">6\n{' '*12}") + + def test_concat_with_tstring_is_rejected(self): + exprs = [ + 'd"""\n x\n """ t"hello"', + 't"hello" d"""\n x\n """', + 'db"""\n x\n """ t"hello"', + 't"hello" db"""\n x\n """', + ] + self.assertAllRaise( + SyntaxError, + "cannot mix t-string literals with string or bytes literals", + exprs, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2db09d61f409b..a37debe1b3739a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3581,7 +3581,7 @@ def determine_valid_prefixes(): # some uppercase-only prefix is added. for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase): try: - eval(f'{letter}""') + eval(f'{letter}"""\n"""') # d-string needs multiline single_char_valid_prefixes.add(letter.lower()) except SyntaxError: pass @@ -3605,7 +3605,7 @@ def determine_valid_prefixes(): # because it's a valid expression: not "" continue try: - eval(f'{p}""') + eval(f'{p}"""\n"""') # d-string needs multiline # No syntax error, so p is a valid string # prefix. diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 3545d92c4f5d7f..3ea6c5b986ea78 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -87,7 +87,8 @@ def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr'] + _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'd', 'br', 'fr', 'tr', + 'bd', 'rd', 'fd', 'td', 'brd', 'frd', 'trd'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 47cb7684eab85f..cbc243a2cf9cbb 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13885,8 +13885,8 @@ of all lines in the [src, end). It returns the length of the common leading whitespace and sets `output` to point to the beginning of the common leading whitespace if length > 0. */ -static Py_ssize_t -search_longest_common_leading_whitespace( +Py_ssize_t +_Py_search_longest_common_leading_whitespace( const char *const src, const char *const end, const char **output) @@ -13982,7 +13982,7 @@ _PyUnicode_Dedent(PyObject *unicode) // [whitespace_start, whitespace_start + whitespace_len) // describes the current longest common leading whitespace const char *whitespace_start = NULL; - Py_ssize_t whitespace_len = search_longest_common_leading_whitespace( + Py_ssize_t whitespace_len = _Py_search_longest_common_leading_whitespace( src, end, &whitespace_start); if (whitespace_len == 0) { diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 809f3c0a7b270e..5e62f02a60a674 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1312,24 +1312,146 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq // Fstring stuff +static int +unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, const char *line_end, + int is_raw, Token* token) +{ + if (is_raw || memchr(line_start, '\\', line_end - line_start) == NULL) { + return PyUnicodeWriter_WriteUTF8(w, line_start, line_end - line_start); + } + else { + PyObject *line = _PyPegen_decode_string(p, 0, line_start, line_end - line_start, token); + if (line == NULL || PyUnicodeWriter_WriteStr(w, line) < 0) { + Py_XDECREF(line); + return -1; + } + Py_DECREF(line); + } + return 0; +} + +static PyObject* +_PyPegen_dedent_string_part( + Parser *p, const char *s, size_t len, const char *indent, Py_ssize_t indent_len, + int is_first, int is_last, int is_raw, Token* token) +{ + const char *line_start = s; + const char *end = s + len; + + int prev_call_invalid = p->call_invalid_rules; + if (!prev_call_invalid && !is_raw) { + // _PyPegen_decode_string() and decode_bytes_with_escapes() may call + // warn_invalid_escape_sequence(). It may emit issue or raise SyntaxError + // for invalid escape sequences. + // We need to call it before dedenting since SyntaxError needs exact lineno + // and col_offset of invalid escape sequences. + PyObject *t = _PyPegen_decode_string(p, 0, s, len, token); + if (t == NULL) { + return NULL; + } + Py_DECREF(t); + p->call_invalid_rules = 1; + } + + PyUnicodeWriter *w = PyUnicodeWriter_Create(len); + if (w == NULL) { + p->call_invalid_rules = prev_call_invalid; + return NULL; + } + + if (is_first) { + assert (line_start[0] == '\n'); + line_start++; // skip the first newline + } + else { + // Example: df""" + // first part {param} second part + // next line + // """" + // The first line in a non-first part doesn't start at the + // beginning of a physical line; copy it verbatim. + const char *line_end = memchr(line_start, '\n', end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + goto error; + } + line_start = line_end; + } + + while (line_start < end) { + // A blank line (whitespace-only line with a newline) is normalized + // to a single newline. A whitespace-only final part is also blank: + // its preceding newline has already been written. Other tails may + // continue with a replacement field, so they are kept verbatim. + const char *q = line_start; + while (q < end && (*q == ' ' || *q == '\t')) { + q++; + } + if (q == end && is_last) { + break; + } + if (q < end && *q == '\n') { + if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { + goto error; + } + line_start = q + 1; + continue; + } + + // A non-blank line. The common indent was computed from all + // physical lines of the literal, so it is always a prefix of + // the leading whitespace of this line. + assert(q - line_start >= indent_len); + assert(memcmp(line_start, indent, (size_t)indent_len) == 0); + line_start += indent_len; + const char *line_end = memchr(line_start, '\n', end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + goto error; + } + line_start = line_end; + } + p->call_invalid_rules = prev_call_invalid; + return PyUnicodeWriter_Finish(w); + +error: + p->call_invalid_rules = prev_call_invalid; + PyUnicodeWriter_Discard(w); + return NULL; +} + static expr_ty -_PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* token) { +_PyPegen_decode_fstring_part(Parser* p, int is_first, int is_last, + int is_raw, const char *indent, Py_ssize_t indent_len, + expr_ty constant, Token* token) +{ assert(PyUnicode_CheckExact(constant->v.Constant.value)); const char* bstr = PyUnicode_AsUTF8(constant->v.Constant.value); if (bstr == NULL) { return NULL; } + is_raw = is_raw || strchr(bstr, '\\') == NULL; - size_t len; - if (strcmp(bstr, "{{") == 0 || strcmp(bstr, "}}") == 0) { - len = 1; - } else { - len = strlen(bstr); + PyObject *str = NULL; + if (indent != NULL) { + str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent, indent_len, + is_first, is_last, is_raw, token); + } + else { + str = _PyPegen_decode_string(p, is_raw, bstr, strlen(bstr), token); } - is_raw = is_raw || strchr(bstr, '\\') == NULL; - PyObject *str = _PyPegen_decode_string(p, is_raw, bstr, len, token); if (str == NULL) { _Pypegen_raise_decode_error(p); return NULL; @@ -1360,12 +1482,50 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } int is_raw = strpbrk(quote_str, "rR") != NULL; + int is_dedent = strpbrk(quote_str, "dD") != NULL; asdl_expr_seq *seq = _Py_asdl_expr_seq_new(total_items, p->arena); if (seq == NULL) { return NULL; } + const char *indent_start = NULL; + Py_ssize_t indent_len = 0; + + if (is_dedent) { + if (total_items == 0) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, + "d-string must start with a newline" + ); + return NULL; + } + expr_ty first_item = asdl_seq_GET(raw_expressions, 0); + if (first_item->kind != Constant_kind + || PyUnicode_ReadChar(first_item->v.Constant.value, 0) != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + first_item, + "d-string must start with a newline" + ); + return NULL; + } + + // The longest common leading whitespace is computed by the tokenizer + // over physical lines that start outside replacement fields and + // attached to the FSTRING_END/TSTRING_END token as metadata. + assert(b->metadata != NULL && PyUnicode_CheckExact(b->metadata)); + if (b->metadata != NULL && PyUnicode_CheckExact(b->metadata)) { + indent_start = PyUnicode_AsUTF8AndSize(b->metadata, &indent_len); + if (indent_start == NULL) { + return NULL; + } + } + else { + indent_start = ""; + indent_len = 0; + } + } + Py_ssize_t index = 0; for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(raw_expressions, i); @@ -1397,7 +1557,8 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b } if (item->kind == Constant_kind) { - item = _PyPegen_decode_fstring_part(p, is_raw, item, b); + item = _PyPegen_decode_fstring_part(p, i == 0, i == n_items - 1, + is_raw, indent_start, indent_len, item, b); if (item == NULL) { return NULL; } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 41f03a408cfc75..f547ac245a8114 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -24,6 +24,73 @@ contains_null_bytes(const char* str, size_t size) return memchr(str, 0, size) != NULL; } +/* PEP 822: fold the leading whitespace of the freshly read physical line + [tok->cur, tok->inp) into the longest common leading whitespace of every + active d-string whose replacement field is not active at the start of the + line. Lines starting inside replacement fields, including lines inside + nested string literals, do not affect the outer d-string. Blank lines + (whitespace-only) are ignored. The line that contains the closing quotes is + naturally non-blank because the quote characters are on it. + Return 1 on success, 0 on memory error (tok->done is set). */ +static int +update_dstring_indents(struct tok_state *tok) +{ + int any_dedent = 0; + for (int i = 1; i <= tok->tok_mode_stack_index; i++) { + tokenizer_mode *mode = &tok->tok_mode_stack[i]; + if (mode->dedent && !INSIDE_FSTRING_EXPR(mode)) { + any_dedent = 1; + break; + } + } + if (!any_dedent) { + return 1; + } + + const char *line = tok->cur; + const char *end = tok->inp; + const char *ws_end = line; + while (ws_end < end && (*ws_end == ' ' || *ws_end == '\t')) { + ws_end++; + } + if (ws_end == end || *ws_end == '\n' || *ws_end == '\r') { + // A blank line. It doesn't affect the common leading whitespace. + return 1; + } + Py_ssize_t len = ws_end - line; + + for (int i = 1; i <= tok->tok_mode_stack_index; i++) { + tokenizer_mode *mode = &(tok->tok_mode_stack[i]); + if (!mode->dedent || INSIDE_FSTRING_EXPR(mode)) { + continue; + } + if (!mode->dedent_seen_line) { + mode->dedent_seen_line = 1; + assert(mode->dedent_indent == NULL); + mode->dedent_indent_len = 0; + if (len > 0) { + mode->dedent_indent = PyMem_Malloc(len); + if (mode->dedent_indent == NULL) { + tok->done = E_NOMEM; + tok->cur = tok->inp; + return 0; + } + memcpy(mode->dedent_indent, line, len); + mode->dedent_indent_len = len; + } + } + else { + Py_ssize_t common = 0; + while (common < mode->dedent_indent_len && common < len + && mode->dedent_indent[common] == line[common]) { + common++; + } + mode->dedent_indent_len = common; + } + } + return 1; +} + /* Get next char, updating state; error code goes into tok->done */ int _PyLexer_nextc(struct tok_state *tok) @@ -60,6 +127,10 @@ _PyLexer_nextc(struct tok_state *tok) tok->cur = tok->inp; return EOF; } + + if (tok->tok_mode_stack_index > 0 && !update_dstring_indents(tok)) { + return EOF; + } } Py_UNREACHABLE(); } @@ -400,8 +471,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str /* Identifier (most frequent token!) */ nonascii = 0; if (is_potential_identifier_start(c)) { - /* Process the various legal combinations of b"", r"", u"", and f"". */ - int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0; + /* Process the various legal combinations of b"", r"", u"", f"", and d"". */ + int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0, saw_d = 0; while (1) { if (!saw_b && (c == 'b' || c == 'B')) { saw_b = 1; @@ -421,6 +492,9 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str else if (!saw_t && (c == 't' || c == 'T')) { saw_t = 1; } + else if (!saw_d && (c == 'd' || c == 'D')) { + saw_d = 1; + } else { break; } @@ -428,7 +502,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str if (c == '"' || c == '\'') { // Raise error on incompatible string prefixes: int status = _PyLexer_check_string_prefixes( - tok, saw_b, saw_r, saw_u, saw_f, saw_t); + tok, saw_b, saw_r, saw_u, saw_f, saw_t, saw_d); if (status < 0) { return MAKE_TOKEN(ERRORTOKEN); } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index c6d3b9045c7292..b4bc2a63824d7e 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -47,7 +47,7 @@ TOK_NEXT_MODE(struct tok_state *tok) int _PyLexer_nextc(struct tok_state *); void _PyLexer_backup(struct tok_state *, int); int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char); -int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); +int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); int _PyLexer_scan_string(struct tok_state *, struct token *, int); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 5cf9b4d768c3eb..429ae64405a154 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -81,6 +81,11 @@ free_fstring_expressions(struct tok_state *tok) mode->last_expr_end = -1; mode->in_format_spec = 0; } + if (mode->dedent_indent != NULL) { + PyMem_Free(mode->dedent_indent); + mode->dedent_indent = NULL; + mode->dedent_indent_len = 0; + } } } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 9cd196a114c7cb..12e7546e81fa15 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -30,6 +30,17 @@ struct token { int level; int lineno, col_offset, end_lineno, end_col_offset; const char *start, *end; + /* Extra data (str or NULL) attached to some tokens, used by the parser + actions. The lexer sets it in two (mutually exclusive) cases: + - On the '}', ':' or '!' token terminating a replacement field + expression: the source text of the expression, for f-string debug + expressions (`f"{expr=}"`) and t-string Interpolation.str + (see set_ftstring_expr()). + - On the FSTRING_END/TSTRING_END token of a d-string: the longest + common leading whitespace of the participating physical lines of the + literal, used to dedent the string parts (PEP 822). + The owner is this struct (freed by _PyToken_Free()); pegen's + initialize_token() moves it into the arena-managed Token. */ PyObject *metadata; }; @@ -67,6 +78,15 @@ typedef struct _tokenizer_mode { int in_debug; int in_format_spec; + /* PEP 822 d-string support. While tokenizing a d-string, the longest + common leading whitespace of all physical lines seen so far is + maintained in [dedent_indent, dedent_indent + dedent_indent_len). + It is attached to the FSTRING_END/TSTRING_END token as metadata. */ + int dedent; + int dedent_seen_line; + char *dedent_indent; + Py_ssize_t dedent_indent_len; + enum string_kind_t string_kind; } tokenizer_mode; diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index e546954ba5c0d8..e05253b136c765 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -178,9 +178,9 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) int _PyLexer_check_string_prefixes(struct tok_state *tok, int saw_b, int saw_r, int saw_u, - int saw_f, int saw_t) { - // Supported: rb, rf, rt (in any order) - // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order) + int saw_f, int saw_t, int saw_d) { + // Supported: rb, rf, rt, db, df, dt (in any order) + // Unsupported: ub, ur, uf, ut, ud, bf, bt, ft (in any order) #define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \ do { \ @@ -203,6 +203,9 @@ _PyLexer_check_string_prefixes(struct tok_state *tok, if (saw_u && saw_t) { RETURN_SYNTAX_ERROR("u", "t"); } + if (saw_u && saw_d) { + RETURN_SYNTAX_ERROR("u", "d"); + } if (saw_b && saw_f) { RETURN_SYNTAX_ERROR("b", "f"); @@ -263,6 +266,7 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->quote = quote; the_current_tok->quote_size = quote_size; + the_current_tok->raw = 0; the_current_tok->start = tok->start; the_current_tok->multi_line_start = tok->line_start; the_current_tok->first_line = tok->lineno; @@ -273,27 +277,35 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) the_current_tok->last_expr_end = -1; the_current_tok->in_format_spec = 0; the_current_tok->in_debug = 0; + the_current_tok->dedent = 0; + the_current_tok->dedent_seen_line = 0; + the_current_tok->dedent_indent = NULL; + the_current_tok->dedent_indent_len = 0; enum string_kind_t string_kind = FSTRING; - switch (*tok->start) { - case 'T': - case 't': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - string_kind = TSTRING; - break; - case 'F': - case 'f': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - break; - case 'R': - case 'r': - the_current_tok->raw = 1; - if (Py_TOLOWER(*(tok->start + 1)) == 't') { + for (const char *p = tok->start; *p != c; p++) { + switch (*p) { + case 'f': + case 'F': + break; + case 't': + case 'T': string_kind = TSTRING; - } - break; - default: - Py_UNREACHABLE(); + break; + case 'r': + case 'R': + the_current_tok->raw = 1; + break; + case 'd': + case 'D': + if (quote_size != 3) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "d-string must be triple-quoted")); + } + the_current_tok->dedent = 1; + break; + default: + Py_UNREACHABLE(); + } } the_current_tok->string_kind = string_kind; @@ -467,6 +479,29 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st current_tok->last_expr_end = -1; } + if (current_tok->dedent) { + // Attach the longest common leading whitespace of the whole + // d-string to the FSTRING_END/TSTRING_END token so that + // _PyPegen_joined_str()/_PyPegen_template_str() can dedent + // the string parts. + PyObject *indent = PyUnicode_FromStringAndSize( + current_tok->dedent_indent == NULL ? "" : current_tok->dedent_indent, + current_tok->dedent_indent_len); + if (current_tok->dedent_indent != NULL) { + PyMem_Free(current_tok->dedent_indent); + current_tok->dedent_indent = NULL; + current_tok->dedent_indent_len = 0; + } + if (indent == NULL) { + return MAKE_TOKEN(ERRORTOKEN); + } + // This token can already have expression metadata when malformed + // nested f-strings cause the tokenizer to reuse it as the end token. + // Expression metadata is not meaningful on an FTSTRING_END token; + // replace it with the d-string indentation metadata. + Py_XSETREF(token->metadata, indent); + } + p_start = tok->start; p_end = tok->cur; tok->tok_mode_stack_index--; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index b164dfbc81a933..061449c32440b4 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -247,6 +247,60 @@ _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t) return decode_unicode_with_escapes(p, s, len, t); } +// Dedent d-string and return result as a bytes. +static PyObject* +_PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, + const char *indent, Py_ssize_t indent_len) +{ + PyBytesWriter *w = PyBytesWriter_Create(0); + if (w == NULL) { + return NULL; + } + + const char *end = s + len; + while (s < end) { + // A blank line (whitespace-only line with a newline) is normalized + // to a single newline. Whitespace before the closing quotes is also + // blank, but its preceding newline has already been written. + const char *q = s; + while (q < end && (*q == ' ' || *q == '\t')) { + q++; + } + if (q == end) { + break; + } + if (q < end && *q == '\n') { + if (PyBytesWriter_WriteBytes(w, "\n", 1) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + s = q + 1; + continue; + } + + // A non-blank line. The common indent was computed from all lines + // including the closing quotes line, so it is always a prefix of + // the leading whitespace of this line. + assert(q - s >= indent_len); + assert(memcmp(s, indent, (size_t)indent_len) == 0); + s += indent_len; + const char *line_end = memchr(s, '\n', end - s); + if (line_end == NULL) { + line_end = end; // last line without newline + } + else { + line_end++; // include the newline in the line + } + + if (PyBytesWriter_WriteBytes(w, s, line_end - s) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + s = line_end; + } + return PyBytesWriter_Finish(w); +} + /* s must include the bracketing quote characters, and r, b &/or f prefixes (if any), and embedded escape sequences (if any). (f-strings are handled by the parser) _PyPegen_parse_string parses it, and returns the decoded Python string object. */ @@ -262,9 +316,10 @@ _PyPegen_parse_string(Parser *p, Token *t) int quote = Py_CHARMASK(*s); int bytesmode = 0; int rawmode = 0; + int dedentmode = 0; if (Py_ISALPHA(quote)) { - while (!bytesmode || !rawmode) { + while (!bytesmode || !rawmode || !dedentmode) { if (quote == 'b' || quote == 'B') { quote =(unsigned char)*++s; bytesmode = 1; @@ -276,6 +331,10 @@ _PyPegen_parse_string(Parser *p, Token *t) quote = (unsigned char)*++s; rawmode = 1; } + else if (quote == 'd' || quote == 'D') { + quote =(unsigned char)*++s; + dedentmode = 1; + } else { break; } @@ -315,25 +374,88 @@ _PyPegen_parse_string(Parser *p, Token *t) return NULL; } } + else if (dedentmode) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must be triple-quoted"); + return NULL; + } /* Avoid invoking escape decoding routines if possible. */ rawmode = rawmode || strchr(s, '\\') == NULL; + + if (dedentmode && (len == 0 || s[0] != '\n')) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must start with a newline"); + return NULL; + } if (bytesmode) { - /* Disallow non-ASCII characters. */ - const char *ch; - for (ch = s; *ch; ch++) { + for (const char *ch = s; ch < s + len; ch++) { if (Py_CHARMASK(*ch) >= 0x80) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - t, - "bytes can only contain ASCII " - "literal characters"); + t, "bytes can only contain ASCII literal characters"); return NULL; } } + } + + int _prev_call_invald = p->call_invalid_rules; + PyObject *dedent_bytes = NULL; + if (dedentmode) { + // _PyPegen_decode_string() and decode_bytes_with_escapes() emit + // a warning for invalid escape sequences. + // We need to call it before dedenting since it shifts the positions. + if (!_prev_call_invald && !rawmode) { + PyObject *temp; + if (bytesmode) { + temp = decode_bytes_with_escapes(p, s, len, t); + } + else { + temp = _PyPegen_decode_string(p, 0, s, len, t); + } + if (temp == NULL) { + return NULL; + } + Py_DECREF(temp); + } + + // We find common indent from [s, end+1) because we want to include the last line + // for indent calculation. + const char *end = s + len; + assert(*end == '"' || *end == '\''); // end[0:3] is the trailing quotes + const char *indent = ""; + Py_ssize_t indent_len = _Py_search_longest_common_leading_whitespace(s+1, end+1, &indent); + + s++; len--; // skip the first newline + // Dedent even when indent_len == 0: blank lines must still be + // normalized to single newlines. + dedent_bytes = _PyPegen_dedent_string(p, s, len, indent, indent_len); + if (dedent_bytes == NULL) { + return NULL; + } + char *dedent_str; + Py_ssize_t dedent_len; + if (PyBytes_AsStringAndSize(dedent_bytes, &dedent_str, &dedent_len) < 0) { + Py_DECREF(dedent_bytes); + return NULL; + } + s = dedent_str; + len = dedent_len; + + p->call_invalid_rules = 1; + } + + PyObject *result; + if (bytesmode) { if (rawmode) { - return PyBytes_FromStringAndSize(s, (Py_ssize_t)len); + result = PyBytes_FromStringAndSize(s, (Py_ssize_t)len); } - return decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); + else { + result = decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); + } + } + else { + result = _PyPegen_decode_string(p, rawmode, s, len, t); } - return _PyPegen_decode_string(p, rawmode, s, len, t); + Py_XDECREF(dedent_bytes); + p->call_invalid_rules = _prev_call_invald; + return result; } +