Add lifetime to token

src/ast/helpers/attached_token.rs

@@ -80,7 +80,7 @@ use sqlparser_derive::{Visit, VisitMut};
 #[derive(Clone)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
-pub struct AttachedToken(pub TokenWithSpan);
+pub struct AttachedToken(pub TokenWithSpan<'static>);
 
 impl AttachedToken {
     /// Return a new Empty AttachedToken
@@ -123,13 +123,13 @@ impl Hash for AttachedToken {
     }
 }
 
-impl From<TokenWithSpan> for AttachedToken {
-    fn from(value: TokenWithSpan) -> Self {
+impl From<TokenWithSpan<'static>> for AttachedToken {
+    fn from(value: TokenWithSpan<'static>) -> Self {
         AttachedToken(value)
     }
 }
 
-impl From<AttachedToken> for TokenWithSpan {
+impl From<AttachedToken> for TokenWithSpan<'static> {
     fn from(value: AttachedToken) -> Self {
         value.0
     }

src/ast/mod.rs

@@ -2790,7 +2790,7 @@ impl fmt::Display for Declare {
 }
 
 /// Sql options of a `CREATE TABLE` statement.
-#[derive(Debug, Default, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Default)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum CreateTableOptions {

src/ast/spans.rs

@@ -2407,7 +2407,7 @@ pub mod tests {
     #[test]
     fn test_join() {
         let dialect = &GenericDialect;
-        let mut test = SpanTest::new(
+        let test = SpanTest::new(
             dialect,
             "SELECT id, name FROM users LEFT JOIN companies ON users.company_id = companies.id",
         );
@@ -2432,7 +2432,7 @@ pub mod tests {
     #[test]
     pub fn test_union() {
         let dialect = &GenericDialect;
-        let mut test = SpanTest::new(
+        let test = SpanTest::new(
             dialect,
             "SELECT a FROM postgres.public.source UNION SELECT a FROM postgres.public.source",
         );
@@ -2449,7 +2449,7 @@ pub mod tests {
     #[test]
     pub fn test_subquery() {
         let dialect = &GenericDialect;
-        let mut test = SpanTest::new(
+        let test = SpanTest::new(
             dialect,
             "SELECT a FROM (SELECT a FROM postgres.public.source) AS b",
         );
@@ -2474,7 +2474,7 @@ pub mod tests {
     #[test]
     pub fn test_cte() {
         let dialect = &GenericDialect;
-        let mut test = SpanTest::new(dialect, "WITH cte_outer AS (SELECT a FROM postgres.public.source), cte_ignored AS (SELECT a FROM cte_outer), cte_inner AS (SELECT a FROM cte_outer) SELECT a FROM cte_inner");
+        let test = SpanTest::new(dialect, "WITH cte_outer AS (SELECT a FROM postgres.public.source), cte_ignored AS (SELECT a FROM cte_outer), cte_inner AS (SELECT a FROM cte_outer) SELECT a FROM cte_inner");
 
         let query = test.0.parse_query().unwrap();
 
@@ -2486,7 +2486,7 @@ pub mod tests {
     #[test]
     pub fn test_snowflake_lateral_flatten() {
         let dialect = &SnowflakeDialect;
-        let mut test = SpanTest::new(dialect, "SELECT FLATTENED.VALUE:field::TEXT AS FIELD FROM SNOWFLAKE.SCHEMA.SOURCE AS S, LATERAL FLATTEN(INPUT => S.JSON_ARRAY) AS FLATTENED");
+        let test = SpanTest::new(dialect, "SELECT FLATTENED.VALUE:field::TEXT AS FIELD FROM SNOWFLAKE.SCHEMA.SOURCE AS S, LATERAL FLATTEN(INPUT => S.JSON_ARRAY) AS FLATTENED");
 
         let query = test.0.parse_select().unwrap();
 
@@ -2498,7 +2498,7 @@ pub mod tests {
     #[test]
     pub fn test_wildcard_from_cte() {
         let dialect = &GenericDialect;
-        let mut test = SpanTest::new(
+        let test = SpanTest::new(
             dialect,
             "WITH cte AS (SELECT a FROM postgres.public.source) SELECT cte.* FROM cte",
         );
@@ -2524,7 +2524,7 @@ pub mod tests {
     #[test]
     fn test_case_expr_span() {
         let dialect = &GenericDialect;
-        let mut test = SpanTest::new(dialect, "CASE 1 WHEN 2 THEN 3 ELSE 4 END");
+        let test = SpanTest::new(dialect, "CASE 1 WHEN 2 THEN 3 ELSE 4 END");
         let expr = test.0.parse_expr().unwrap();
         let expr_span = expr.span();
         assert_eq!(

src/ast/visitor.rs

@@ -17,6 +17,11 @@
 
 //! Recursive visitors for ast Nodes. See [`Visitor`] for more details.
 
+#[cfg(not(feature = "std"))]
+use alloc::borrow::Cow;
+#[cfg(feature = "std")]
+use std::borrow::Cow;
+
 use crate::ast::{Expr, ObjectName, Query, Statement, TableFactor, Value};
 use core::ops::ControlFlow;
 
@@ -118,6 +123,19 @@ visit_noop!(u8, u16, u32, u64, i8, i16, i32, i64, char, bool, String);
 #[cfg(feature = "bigdecimal")]
 visit_noop!(bigdecimal::BigDecimal);
 
+// Implement Visit and VisitMut for Cow<str> to support the lifetime parameter in BorrowedToken
+impl<'a> Visit for Cow<'a, str> {
+    fn visit<V: Visitor>(&self, _visitor: &mut V) -> ControlFlow<V::Break> {
+        ControlFlow::Continue(())
+    }
+}
+
+impl<'a> VisitMut for Cow<'a, str> {
+    fn visit<V: VisitorMut>(&mut self, _visitor: &mut V) -> ControlFlow<V::Break> {
+        ControlFlow::Continue(())
+    }
+}
+
 /// A visitor that can be used to walk an AST tree.
 ///
 /// `pre_visit_` methods are invoked before visiting all children of the
@@ -751,7 +769,7 @@ mod tests {
 
     fn do_visit<V: Visitor<Break = ()>>(sql: &str, visitor: &mut V) -> Statement {
         let dialect = GenericDialect {};
-        let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
+        let tokens = Tokenizer::new(&dialect, sql).tokenized_owned().unwrap();
         let s = Parser::new(&dialect)
             .with_tokens(tokens)
             .parse_statement()
@@ -942,7 +960,9 @@ mod tests {
         let sql = format!("SELECT x where {cond}");
 
         let dialect = GenericDialect {};
-        let tokens = Tokenizer::new(&dialect, sql.as_str()).tokenize().unwrap();
+        let tokens = Tokenizer::new(&dialect, sql.as_str())
+            .tokenized_owned()
+            .unwrap();
         let s = Parser::new(&dialect)
             .with_tokens(tokens)
             .parse_statement()
@@ -983,7 +1003,7 @@ mod visit_mut_tests {
 
     fn do_visit_mut<V: VisitorMut<Break = ()>>(sql: &str, visitor: &mut V) -> Statement {
         let dialect = GenericDialect {};
-        let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
+        let tokens = Tokenizer::new(&dialect, sql).tokenized_owned().unwrap();
         let mut s = Parser::new(&dialect)
             .with_tokens(tokens)
             .parse_statement()

src/dialect/bigquery.rs

@@ -46,7 +46,7 @@ const RESERVED_FOR_COLUMN_ALIAS: &[Keyword] = &[
 pub struct BigQueryDialect;
 
 impl Dialect for BigQueryDialect {
-    fn parse_statement(&self, parser: &mut Parser) -> Option<Result<Statement, ParserError>> {
+    fn parse_statement(&self, parser: &Parser) -> Option<Result<Statement, ParserError>> {
         if parser.parse_keyword(Keyword::BEGIN) {
             if parser.peek_keyword(Keyword::TRANSACTION)
                 || parser.peek_token_ref().token == Token::SemiColon
@@ -145,7 +145,7 @@ impl Dialect for BigQueryDialect {
         true
     }
 
-    fn is_column_alias(&self, kw: &Keyword, _parser: &mut Parser) -> bool {
+    fn is_column_alias(&self, kw: &Keyword, _parser: &Parser) -> bool {
         !RESERVED_FOR_COLUMN_ALIAS.contains(kw)
     }