From 571009c0cdd10125a9926498e2d0d38aed6c11c1 Mon Sep 17 00:00:00 2001 From: jnlt3 Date: Mon, 8 Dec 2025 15:01:53 +0300 Subject: [PATCH 1/2] postgres tokenizer: do not silently ignore the character after a question mark if it is not one of the expected characters --- src/tokenizer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 54a158c1f..622acfc4a 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1684,7 +1684,7 @@ impl<'a> Tokenizer<'a> { } } Some('#') => self.consume_and_return(chars, Token::QuestionMarkSharp), - _ => self.consume_and_return(chars, Token::Question), + _ => Ok(Some(Token::Question)), } } '?' => { From 489fb973d5e003fa8674fcf0c81e26c33c813fd8 Mon Sep 17 00:00:00 2001 From: jnlt3 Date: Sun, 14 Dec 2025 15:16:05 +0300 Subject: [PATCH 2/2] Add test case for the tokenizer consuming the character following a question mark --- tests/sqlparser_postgres.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 11512cf80..d5739cf4e 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -26,7 +26,7 @@ use helpers::attached_token::AttachedToken; use sqlparser::ast::{ DataType, DropBehavior, DropOperator, DropOperatorClass, DropOperatorSignature, }; -use sqlparser::tokenizer::Span; +use sqlparser::tokenizer::{Span, Token, Whitespace}; use test_utils::*; use sqlparser::ast::*; @@ -7492,3 +7492,20 @@ fn parse_create_operator_class() { ) .is_err()); } + +#[test] +fn tokenize_question_mark() { + let sql = "SELECT x ? y"; + pg().tokenizes_to( + sql, + vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::make_word("x", None), + Token::Whitespace(Whitespace::Space), + Token::Question, + Token::Whitespace(Whitespace::Space), + Token::make_word("y", None), + ], + ) +}