diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0427f02499..d93b3939cb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -46,7 +46,7 @@ jobs: - name: Setup Rust Toolchain uses: ./.github/actions/setup-builder with: - rust-version: "1.86.0" + rust-version: "1.89.0" - run: cargo clippy --all-targets --all-features -- -D warnings benchmark-lint: @@ -56,7 +56,7 @@ jobs: - name: Setup Rust Toolchain uses: ./.github/actions/setup-builder with: - rust-version: "1.86.0" + rust-version: "1.89.0" - run: cd sqlparser_bench && cargo clippy --all-targets --all-features -- -D warnings compile: diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index d0b87d962a..bf2184b3cf 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -278,6 +278,24 @@ pub trait Dialect: Debug + Any { false } + /// Determine whether backslash escapes in string literals use Snowflake's + /// escape table instead of the MySQL-style default. + /// + /// Snowflake (verified against the live service) decodes octal `\o`..`\ooo` + /// (one to three digits, greedy), hex `\xhh` (exactly two hex digits, else a + /// tokenizer error), unicode `\uXXXX` (exactly four hex digits, else a + /// tokenizer error), and the single-character escapes `\b \f \n \r \t`; + /// `\'`, `\"` and `\\` escape themselves. Any other escaped character keeps + /// the character and drops the backslash (`'\a' = 'a'`, `'\Z' = 'Z'`, + /// `'\.' = '.'`) — notably `\a`/`\Z` do NOT decode to BEL/^Z as they do in + /// the MySQL-style table used when this returns false. + /// + /// Only consulted when [`Self::supports_string_literal_backslash_escape`] + /// returns true. + fn supports_snowflake_string_literal_escapes(&self) -> bool { + false + } + /// Determine whether the dialect strips the backslash when escaping LIKE wildcards (%, _). /// /// [MySQL] has a special case when escaping single quoted strings which leaves these unescaped diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index 82b50d1e19..9e5c81b62f 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -166,6 +166,12 @@ impl Dialect for SnowflakeDialect { true } + // Snowflake decodes octal/hex/unicode escapes and drops the backslash on + // unknown escapes ('\a' = 'a'); see the trait doc for the verified table. + fn supports_snowflake_string_literal_escapes(&self) -> bool { + true + } + fn supports_within_after_array_aggregation(&self) -> bool { true } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 852b73164f..c93f39549b 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2259,6 +2259,8 @@ impl<'a> Tokenizer<'a> { s.push(ch); s.push(*next); chars.next(); // consume next + } else if self.dialect.supports_snowflake_string_literal_escapes() { + self.push_snowflake_string_escape(&mut s, chars)?; } else { let n = match next { '0' => '\0', @@ -2292,6 +2294,99 @@ impl<'a> Tokenizer<'a> { self.tokenizer_error(error_loc, "Unterminated string literal") } + /// Decode one Snowflake-style backslash escape inside a quoted string. + /// + /// The backslash itself has already been consumed; the character after it is + /// peeked but not consumed. Semantics verified against live Snowflake: + /// octal `\o`..`\ooo` (greedy, 1-3 digits), hex `\xhh` and unicode `\uXXXX` + /// (exact digit counts, malformed sequences are errors — matching + /// Snowflake's compilation errors), C-style `\b \f \n \r \t`, and for any + /// other character the backslash is dropped and the character kept. + fn push_snowflake_string_escape( + &self, + s: &mut String, + chars: &mut State, + ) -> Result<(), TokenizerError> { + let Some(&next) = chars.peek() else { + // Lone trailing backslash: nothing to decode; the enclosing loop + // reports the unterminated string literal. + return Ok(()); + }; + match next { + 'b' | 'f' | 'n' | 'r' | 't' => { + chars.next(); + s.push(match next { + 'b' => '\u{8}', + 'f' => '\u{c}', + 'n' => '\n', + 'r' => '\r', + _ => '\t', + }); + } + '0'..='7' => { + // Octal escape: 1-3 octal digits, greedy ('\101' = 'A', + // '\79' = '\u{7}' followed by '9'). + chars.next(); + let mut v = next.to_digit(8).unwrap_or_default(); + for _ in 0..2 { + match chars.peek().and_then(|c| c.to_digit(8)) { + Some(d) => { + v = v * 8 + d; + chars.next(); + } + None => break, + } + } + // Max \777 = 511, always a valid scalar value. + s.push(char::from_u32(v).unwrap_or('\0')); + } + 'x' | 'u' => { + let error_loc = chars.location(); + let (digits, label) = if next == 'x' { + (2, "hex") + } else { + (4, "unicode") + }; + chars.next(); + let mut v = 0u32; + for _ in 0..digits { + match chars.peek().and_then(|c| c.to_digit(16)) { + Some(d) => { + v = v * 16 + d; + chars.next(); + } + None => { + return self.tokenizer_error( + error_loc, + format!( + "Invalid {label} escape sequence '\\{next}'; should be exactly {digits} digits" + ), + ); + } + } + } + match char::from_u32(v) { + Some(c) => s.push(c), + None => { + return self.tokenizer_error( + error_loc, + format!( + "Invalid {label} escape sequence '\\{next}'; not a valid code point" + ), + ); + } + } + } + // `\'`, `\"`, `\\` escape themselves; any other character keeps + // itself and the backslash is dropped ('\a' = 'a', '\Z' = 'Z'). + _ => { + chars.next(); + s.push(next); + } + } + Ok(()) + } + fn tokenize_multiline_comment( &self, chars: &mut State, @@ -3838,10 +3933,13 @@ mod tests { (r#"'%a\'%b'"#, r#"%a\'%b"#, r#"%a'%b"#), (r#"'a\'\'b\'c\'d'"#, r#"a\'\'b\'c\'d"#, r#"a''b'c'd"#), (r#"'\\'"#, r#"\\"#, r#"\"#), + // Snowflake semantics (verified live): \0 is an octal escape; + // \a and \Z are unknown escapes, so the backslash is dropped and + // the character kept (they are NOT the MySQL BEL/^Z escapes). ( r#"'\0\a\b\f\n\r\t\Z'"#, r#"\0\a\b\f\n\r\t\Z"#, - "\0\u{7}\u{8}\u{c}\n\r\t\u{1a}", + "\0a\u{8}\u{c}\n\r\tZ", ), (r#"'\"'"#, r#"\""#, "\""), (r#"'\\a\\b\'c'"#, r#"\\a\\b\'c"#, r#"\a\b'c"#), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 3c32e627c4..2212aaf23a 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11121,11 +11121,17 @@ fn parse_escaped_string_with_unescape() { let escaping_dialects = &all_dialects_where(|dialect| dialect.supports_string_literal_backslash_escape()); let no_wildcard_exception = &all_dialects_where(|dialect| { - dialect.supports_string_literal_backslash_escape() && !dialect.ignores_wildcard_escapes() + dialect.supports_string_literal_backslash_escape() + && !dialect.ignores_wildcard_escapes() + && !dialect.supports_snowflake_string_literal_escapes() }); let with_wildcard_exception = &all_dialects_where(|dialect| { dialect.supports_string_literal_backslash_escape() && dialect.ignores_wildcard_escapes() }); + let snowflake_escape_table = &all_dialects_where(|dialect| { + dialect.supports_string_literal_backslash_escape() + && dialect.supports_snowflake_string_literal_escapes() + }); let sql = r"SELECT 'I\'m fine'"; assert_mysql_query_value(escaping_dialects, sql, "I'm fine"); @@ -11149,6 +11155,15 @@ fn parse_escaped_string_with_unescape() { sql, "Testing: \0 \\ \\% \\_ \u{8} \n \r \t \u{1a} \u{7} h ", ); + + // Snowflake-style dialects treat \0 as an octal escape and drop the + // backslash on unknown escapes (\Z, \a, \h) instead of mapping them to + // control characters. + assert_mysql_query_value( + snowflake_escape_table, + sql, + "Testing: \0 \\ % _ \u{8} \n \r \t Z a h ", + ); } #[test] diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 7bef4ce1f8..12dba6c142 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -1456,6 +1456,88 @@ fn test_array_agg_func() { } } +#[test] +fn snowflake_string_literal_backslash_escapes() { + // Escape semantics verified against live Snowflake: octal (`\o`..`\ooo`), + // hex `\xhh` and unicode `\u` + 4 hex digit escapes decode (malformed ones + // are compilation errors); `\b \f \n \r \t` decode; `\'`, `\"`, `\\` + // escape themselves; any other escaped character keeps the character and + // drops the backslash. + let unescaped = |sql: &str| -> String { + let mut tokens = Tokenizer::new(&SnowflakeDialect {}, sql) + .tokenize() + .unwrap_or_else(|e| panic!("tokenize failed for {sql}: {e}")); + match tokens.remove(0) { + Token::SingleQuotedString(s) => s, + other => panic!("expected a single-quoted string for {sql}, got {other:?}"), + } + }; + + // Octal escapes: 1-3 digits, greedy; digits 8/9 are not octal. + assert_eq!(unescaped(r"'\2'"), "\u{2}"); + assert_eq!(unescaped(r"'\101'"), "A"); + assert_eq!(unescaped(r"'\79'"), "\u{7}9"); + assert_eq!(unescaped(r"'\8'"), "8"); + assert_eq!(unescaped(r"'\0'"), "\0"); + // Hex and unicode escapes. + assert_eq!(unescaped(r"'\x41'"), "A"); + assert_eq!(unescaped(&format!(r"'\{}'", "u0394")), "\u{394}"); + // Uppercase X/U are NOT escape introducers: backslash dropped, chars kept. + assert_eq!(unescaped(r"'\X41'"), "X41"); + assert_eq!(unescaped(r"'\U0041'"), "U0041"); + // Standard single-character escapes. + assert_eq!(unescaped(r"'\b\f\n\r\t'"), "\u{8}\u{c}\n\r\t"); + // Self-escapes and quote escapes. + assert_eq!(unescaped(r"'\\'"), "\\"); + assert_eq!(unescaped(r"'I\'m'"), "I'm"); + assert_eq!(unescaped(r#"'\"'"#), "\""); + // Unknown escapes drop the backslash and keep the character (unlike the + // MySQL-style table where \a/\Z map to BEL/^Z). + assert_eq!(unescaped(r"'\a\Z\.\q\ '"), "aZ.q "); + assert_eq!(unescaped(r"'\%\_'"), "%_"); + // ClickBench Q29 shape: what REGEXP_REPLACE receives after the lexer. + assert_eq!( + unescaped(r"'^https?://(www\.)?([^/]+)/.*$'"), + "^https?://(www.)?([^/]+)/.*$" + ); + // A doubled backslash protects the next character from escape decoding, + // so `\\2` reaches regex functions as a `\2` backreference. + assert_eq!(unescaped(r"'\\2'"), "\\2"); + + // Malformed hex/unicode escapes are tokenizer errors, matching Snowflake's + // "should be exactly N digits" compilation errors. + for (sql, fragment) in [ + (r"SELECT '\xZZ'".to_string(), "Invalid hex escape sequence"), + (r"SELECT '\x4'".to_string(), "Invalid hex escape sequence"), + ( + format!(r"SELECT '\{}'", "uZZZZ"), + "Invalid unicode escape sequence", + ), + ( + format!(r"SELECT '\{}'", "u12"), + "Invalid unicode escape sequence", + ), + ] { + let err = snowflake() + .parse_sql_statements(&sql) + .expect_err(&format!("expected tokenizer error for {sql}")); + assert!( + err.to_string().contains(fragment), + "error for {sql} should contain {fragment:?}; got: {err}" + ); + } + + // unescape=false mode keeps the raw text untouched. + let mut tokens = Tokenizer::new(&SnowflakeDialect {}, r"'\2 \x41 \q'") + .with_unescape(false) + .tokenize() + .expect("tokenize without unescape"); + assert_eq!( + tokens.remove(0), + Token::SingleQuotedString(r"\2 \x41 \q".to_string()) + ); +} + fn snowflake() -> TestedDialects { TestedDialects::new(vec![Box::new(SnowflakeDialect {})]) }