X Tutup
Skip to content

Commit 69e1097

Browse files
pablogsalpablogsal
authored
bpo-46521: Fix codeop to use a new partial-input mode of the parser (GH-31010)
1 parent 25db2b3 commit 69e1097

File tree

7 files changed

+49
-55
lines changed

7 files changed

+49
-55
lines changed

‎Include/cpython/compile.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@
1818
#define PyCF_IGNORE_COOKIE 0x0800
1919
#define PyCF_TYPE_COMMENTS 0x1000
2020
#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
21+
#define PyCF_ALLOW_INCOMPLETE_INPUT 0x4000
2122
#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \
22-
PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT)
23+
PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \
24+
PyCF_ALLOW_INCOMPLETE_INPUT)
2325

2426
typedef struct {
2527
int cf_flags; /* bitmask of CO_xxx flags relevant to future */

‎Include/errcode.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ extern "C" {
2626
#define E_TOODEEP 20 /* Too many indentation levels */
2727
#define E_DEDENT 21 /* No matching outer block for dedent */
2828
#define E_DECODE 22 /* Error in decoding into Unicode */
29+
#define E_EOFS 23 /* EOF in triple-quoted string */
30+
#define E_EOLS 24 /* EOL in single-quoted string */
2931
#define E_LINECONT 25 /* Unexpected characters after a line continuation */
3032
#define E_BADSINGLE 27 /* Ill-formed single statement input */
3133
#define E_INTERACT_STOP 28 /* Interactive mode stopped tokenization */

‎Lib/codeop.py‎

Lines changed: 11 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -10,30 +10,6 @@
1010
syntax error (OverflowError and ValueError can be produced by
1111
malformed literals).
1212

13-
Approach:
14-
15-
First, check if the source consists entirely of blank lines and
16-
comments; if so, replace it with 'pass', because the built-in
17-
parser doesn't always do the right thing for these.
18-
19-
Compile three times: as is, with \n, and with \n\n appended. If it
20-
compiles as is, it's complete. If it compiles with one \n appended,
21-
we expect more. If it doesn't compile either way, we compare the
22-
error we get when compiling with \n or \n\n appended. If the errors
23-
are the same, the code is broken. But if the errors are different, we
24-
expect more. Not intuitive; not even guaranteed to hold in future
25-
releases; but this matches the compiler's behavior from Python 1.4
26-
through 2.2, at least.
27-
28-
Caveat:
29-
30-
It is possible (but not likely) that the parser stops parsing with a
31-
successful outcome before reaching the end of the source; in this
32-
case, trailing symbols may be ignored instead of causing an error.
33-
For example, a backslash followed by two newlines may be followed by
34-
arbitrary garbage. This will be fixed once the API for the parser is
35-
better.
36-
3713
The two interfaces are:
3814

3915
compile_command(source, filename, symbol):
@@ -64,7 +40,11 @@
6440

6541
__all__ = ["compile_command", "Compile", "CommandCompiler"]
6642

67-
PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h.
43+
# The following flags match the values from Include/cpython/compile.h
44+
# Caveat emptor: These flags are undocumented on purpose and depending
45+
# on their effect outside the standard library is **unsupported**.
46+
PyCF_DONT_IMPLY_DEDENT = 0x200
47+
PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
6848

6949
def _maybe_compile(compiler, source, filename, symbol):
7050
# Check for source consisting of only blank lines and comments.
@@ -86,24 +66,12 @@ def _maybe_compile(compiler, source, filename, symbol):
8666
with warnings.catch_warnings():
8767
warnings.simplefilter("error")
8868

89-
code1 = err1 = err2 = None
90-
try:
91-
code1 = compiler(source + "\n", filename, symbol)
92-
except SyntaxError as e:
93-
err1 = e
94-
9569
try:
96-
code2 = compiler(source + "\n\n", filename, symbol)
70+
compiler(source + "\n", filename, symbol)
9771
except SyntaxError as e:
98-
err2 = e
99-
100-
try:
101-
if not code1 and _is_syntax_error(err1, err2):
102-
raise err1
103-
else:
104-
return None
105-
finally:
106-
err1 = err2 = None
72+
if "incomplete input" in str(e):
73+
return None
74+
raise
10775

10876
def _is_syntax_error(err1, err2):
10977
rep1 = repr(err1)
@@ -115,7 +83,7 @@ def _is_syntax_error(err1, err2):
11583
return False
11684

11785
def _compile(source, filename, symbol):
118-
return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
86+
return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT)
11987

12088
def compile_command(source, filename="<input>", symbol="single"):
12189
r"""Compile a command and determine whether it is incomplete.
@@ -144,7 +112,7 @@ class Compile:
144112
statement, it "remembers" and compiles all subsequent program texts
145113
with the statement in force."""
146114
def __init__(self):
147-
self.flags = PyCF_DONT_IMPLY_DEDENT
115+
self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
148116

149117
def __call__(self, source, filename, symbol):
150118
codeob = compile(source, filename, symbol, self.flags, True)

‎Misc/NEWS.d/next/Library/2022-02-01-19-34-28.bpo-46521.IMUIrs.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix a bug in the :mod:`codeop` module that was incorrectly identifying
2+
invalid code involving string quotes as valid code.

‎Parser/pegen.c‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,9 @@ compute_parser_flags(PyCompilerFlags *flags)
726726
if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) {
727727
parser_flags |= PyPARSE_ASYNC_HACKS;
728728
}
729+
if (flags->cf_flags & PyCF_ALLOW_INCOMPLETE_INPUT) {
730+
parser_flags |= PyPARSE_ALLOW_INCOMPLETE_INPUT;
731+
}
729732
return parser_flags;
730733
}
731734

@@ -811,16 +814,26 @@ reset_parser_state_for_error_pass(Parser *p)
811814
p->tok->interactive_underflow = IUNDERFLOW_STOP;
812815
}
813816

817+
static inline int
818+
_is_end_of_source(Parser *p) {
819+
int err = p->tok->done;
820+
return err == E_EOF || err == E_EOFS || err == E_EOLS;
821+
}
822+
814823
void *
815824
_PyPegen_run_parser(Parser *p)
816825
{
817826
void *res = _PyPegen_parse(p);
818827
assert(p->level == 0);
819828
if (res == NULL) {
829+
if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) {
830+
PyErr_Clear();
831+
return RAISE_SYNTAX_ERROR("incomplete input");
832+
}
820833
if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) {
821834
return NULL;
822835
}
823-
// Make a second parser pass. In this pass we activate heavier and slower checks
836+
// Make a second parser pass. In this pass we activate heavier and slower checks
824837
// to produce better error messages and more complete diagnostics. Extra "invalid_*"
825838
// rules will be active during parsing.
826839
Token *last_token = p->tokens[p->fill - 1];

‎Parser/pegen.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#define PyPARSE_BARRY_AS_BDFL 0x0020
2323
#define PyPARSE_TYPE_COMMENTS 0x0040
2424
#define PyPARSE_ASYNC_HACKS 0x0080
25+
#define PyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100
2526

2627
#define CURRENT_POS (-5)
2728

‎Parser/tokenizer.c‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
static struct tok_state *tok_new(void);
4141
static int tok_nextc(struct tok_state *tok);
4242
static void tok_backup(struct tok_state *tok, int c);
43-
43+
static int syntaxerror(struct tok_state *tok, const char *format, ...);
4444

4545
/* Spaces in this constant are treated as "zero or more spaces or tabs" when
4646
tokenizing. */
@@ -1031,8 +1031,9 @@ tok_nextc(struct tok_state *tok)
10311031
if (tok->cur != tok->inp) {
10321032
return Py_CHARMASK(*tok->cur++); /* Fast path */
10331033
}
1034-
if (tok->done != E_OK)
1035-
return EOF;
1034+
if (tok->done != E_OK) {
1035+
return EOF;
1036+
}
10361037
if (tok->fp == NULL) {
10371038
rc = tok_underflow_string(tok);
10381039
}
@@ -1964,16 +1965,21 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
19641965
tok->line_start = tok->multi_line_start;
19651966
int start = tok->lineno;
19661967
tok->lineno = tok->first_lineno;
1967-
19681968
if (quote_size == 3) {
1969-
return syntaxerror(tok,
1970-
"unterminated triple-quoted string literal"
1971-
" (detected at line %d)", start);
1969+
syntaxerror(tok, "unterminated triple-quoted string literal"
1970+
" (detected at line %d)", start);
1971+
if (c != '\n') {
1972+
tok->done = E_EOFS;
1973+
}
1974+
return ERRORTOKEN;
19721975
}
19731976
else {
1974-
return syntaxerror(tok,
1975-
"unterminated string literal (detected at"
1976-
" line %d)", start);
1977+
syntaxerror(tok, "unterminated string literal (detected at"
1978+
" line %d)", start);
1979+
if (c != '\n') {
1980+
tok->done = E_EOLS;
1981+
}
1982+
return ERRORTOKEN;
19771983
}
19781984
}
19791985
if (c == quote) {

0 commit comments

Comments
 (0)
X Tutup