Adding upstream version 26.25.3.

Signed-off-by: Daniel Baumann <daniel@debian.org>

sqlglotrs/Cargo.lock

@@ -502,7 +502,7 @@ dependencies = [
 
 [[package]]
 name = "sqlglotrs"
-version = "0.5.0"
+version = "0.6.1"
 dependencies = [
  "criterion",
  "pyo3",

sqlglotrs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sqlglotrs"
-version = "0.5.0"
+version = "0.6.1"
 edition = "2021"
 license = "MIT"
 

sqlglotrs/src/tokenizer.rs

@@ -1,7 +1,6 @@
 use crate::settings::TokenType;
 use crate::trie::{Trie, TrieResult};
 use crate::{Token, TokenTypeSettings, TokenizerDialectSettings, TokenizerSettings};
-use pyo3::exceptions::PyException;
 use pyo3::prelude::*;
 use std::cmp::{max, min};
 
@@ -45,7 +44,7 @@ impl Tokenizer {
         &self,
         sql: &str,
         dialect_settings: &TokenizerDialectSettings,
-    ) -> Result<Vec<Token>, PyErr> {
+    ) -> (Vec<Token>, Option<String>) {
         let mut state = TokenizerState::new(
             sql,
             &self.settings,
@@ -53,9 +52,14 @@ impl Tokenizer {
             dialect_settings,
             &self.keyword_trie,
         );
-        state.tokenize().map_err(|e| {
-            PyException::new_err(format!("Error tokenizing '{}': {}", e.context, e.message))
-        })
+        let tokenize_result = state.tokenize();
+        match tokenize_result {
+            Ok(tokens) => (tokens, None),
+            Err(e) => {
+                let msg = format!("Error tokenizing '{}': {}", e.context, e.message);
+                (state.tokens, Some(msg))
+            }
+        }
     }
 }