gh-107450: Check for overflow in the tokenizer and fix overflow test (#110832)

Co-authored-by: Filipe Laíns <lains@riseup.net> Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
2023-10-16 16:42:49 +02:00
parent b3c9faf056
commit a1ac5590e0
4 changed files with 40 additions and 22 deletions
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -59,6 +59,10 @@ tok_nextc(struct tok_state *tok)
    int rc;
    for (;;) {
        if (tok->cur != tok->inp) {
+            if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
+                tok->done = E_COLUMNOVERFLOW;
+                return EOF;
+            }
            tok->col_offset++;
            return Py_CHARMASK(*tok->cur++); /* Fast path */
        }