SQLite: Allow dollar signs in placeholder names

Relevant: apache#1402

SQLite version 3.43.2 2023-10-10 13:08:14
Enter ".help" for usage hints.
Connected to a transient in-memory database.
Use ".open FILENAME" to reopen on a persistent database.
sqlite> .mode box
sqlite> .nullvalue NULL
sqlite> SELECT $$, $$ABC$$, $ABC$, $ABC;
┌──────┬─────────┬───────┬──────┐
│  $$  │ $$ABC$$ │ $ABC$ │ $ABC │
├──────┼─────────┼───────┼──────┤
│ NULL │ NULL    │ NULL  │ NULL │
└──────┴─────────┴───────┴──────┘

src/dialect/mod.rs

@@ -636,6 +636,10 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    fn supports_dollar_quoted_string(&self) -> bool {
+        true
+    }
+
     /// Does the dialect support with clause in create index statement?
     /// e.g. `CREATE INDEX idx ON t WITH (key = value, key2)`
     fn supports_create_index_with_clause(&self) -> bool {

src/dialect/sqlite.rs

@@ -81,4 +81,8 @@ impl Dialect for SQLiteDialect {
     fn supports_asc_desc_in_column_definition(&self) -> bool {
         true
     }
+
+    fn supports_dollar_quoted_string(&self) -> bool {
+        false
+    }
 }

src/tokenizer.rs

@@ -1523,7 +1523,9 @@ impl<'a> Tokenizer<'a> {
 
         chars.next();
 
-        if let Some('$') = chars.peek() {
+        // Check if the second character is a dollar sign
+        let next_is_dollar = matches!(chars.peek(), Some('$'));
+        if next_is_dollar && self.dialect.supports_dollar_quoted_string() {
             chars.next();
 
             let mut is_terminated = false;
@@ -1557,10 +1559,13 @@ impl<'a> Tokenizer<'a> {
             };
         } else {
             value.push_str(&peeking_take_while(chars, |ch| {
-                ch.is_alphanumeric() || ch == '_'
+                ch.is_alphanumeric()
+                    || ch == '_'
+                    || matches!(ch, '$' if !self.dialect.supports_dollar_quoted_string())
             }));
 
-            if let Some('$') = chars.peek() {
+            let next_is_dollar = matches!(chars.peek(), Some('$'));
+            if next_is_dollar && self.dialect.supports_dollar_quoted_string() {
                 chars.next();
 
                 'searching_for_end: loop {
@@ -2151,7 +2156,7 @@ fn take_char_from_hex_digits(
 mod tests {
     use super::*;
     use crate::dialect::{
-        BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect,
+        BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect,
     };
     use core::fmt::Debug;
 
@@ -2604,6 +2609,30 @@ mod tests {
         );
     }
 
+    #[test]
+    fn tokenize_dollar_placeholder_sqlite() {
+        let sql = String::from("SELECT $$, $$ABC$$, $ABC$, $ABC");
+        let dialect = SQLiteDialect {};
+        let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
+        assert_eq!(
+            tokens,
+            vec![
+                Token::make_keyword("SELECT"),
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$$".into()),
+                Token::Comma,
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$$ABC$$".into()),
+                Token::Comma,
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$ABC$".into()),
+                Token::Comma,
+                Token::Whitespace(Whitespace::Space),
+                Token::Placeholder("$ABC".into()),
+            ]
+        );
+    }
+
     #[test]
     fn tokenize_dollar_quoted_string_untagged() {
         let sql =

tests/sqlparser_sqlite.rs

@@ -561,6 +561,16 @@ fn test_dollar_identifier_as_placeholder() {
         }
         _ => unreachable!(),
     }
+
+    // $$ is a valid placeholder in SQLite
+    match sqlite().verified_expr("id = $$") {
+        Expr::BinaryOp { op, left, right } => {
+            assert_eq!(op, BinaryOperator::Eq);
+            assert_eq!(left, Box::new(Expr::Identifier(Ident::new("id"))));
+            assert_eq!(right, Box::new(Expr::Value(Placeholder("$$".to_string()))));
+        }
+        _ => unreachable!(),
+    }
 }
 
 fn sqlite() -> TestedDialects {