Merging upstream version 11.1.3.

Signed-off-by: Daniel Baumann <daniel@debian.org>

sqlglot/tokens.py

@@ -138,7 +138,6 @@ class TokenType(AutoName):
     CASCADE = auto()
     CASE = auto()
     CHARACTER_SET = auto()
-    CHECK = auto()
     CLUSTER_BY = auto()
     COLLATE = auto()
     COMMAND = auto()
@@ -164,7 +163,6 @@ class TokenType(AutoName):
     DIV = auto()
     DROP = auto()
     ELSE = auto()
-    ENCODE = auto()
     END = auto()
     ESCAPE = auto()
     EXCEPT = auto()
@@ -182,17 +180,16 @@ class TokenType(AutoName):
     FROM = auto()
     FULL = auto()
     FUNCTION = auto()
-    GENERATED = auto()
     GLOB = auto()
     GLOBAL = auto()
     GROUP_BY = auto()
     GROUPING_SETS = auto()
     HAVING = auto()
     HINT = auto()
-    IDENTITY = auto()
     IF = auto()
     IGNORE_NULLS = auto()
     ILIKE = auto()
+    ILIKE_ANY = auto()
     IN = auto()
     INDEX = auto()
     INNER = auto()
@@ -211,6 +208,7 @@ class TokenType(AutoName):
     LEADING = auto()
     LEFT = auto()
     LIKE = auto()
+    LIKE_ANY = auto()
     LIMIT = auto()
     LOAD_DATA = auto()
     LOCAL = auto()
@@ -253,6 +251,7 @@ class TokenType(AutoName):
     RECURSIVE = auto()
     REPLACE = auto()
     RESPECT_NULLS = auto()
+    RETURNING = auto()
     REFERENCES = auto()
     RIGHT = auto()
     RLIKE = auto()
@@ -260,7 +259,6 @@ class TokenType(AutoName):
     ROLLUP = auto()
     ROW = auto()
     ROWS = auto()
-    SCHEMA_COMMENT = auto()
     SEED = auto()
     SELECT = auto()
     SEMI = auto()
@@ -441,7 +439,7 @@ class Tokenizer(metaclass=_Tokenizer):
     KEYWORDS = {
         **{
             f"{key}{postfix}": TokenType.BLOCK_START
-            for key in ("{{", "{%", "{#")
+            for key in ("{%", "{#")
             for postfix in ("", "+", "-")
         },
         **{
@@ -449,6 +447,8 @@ class Tokenizer(metaclass=_Tokenizer):
             for key in ("%}", "#}")
             for prefix in ("", "+", "-")
         },
+        "{{+": TokenType.BLOCK_START,
+        "{{-": TokenType.BLOCK_START,
         "+}}": TokenType.BLOCK_END,
         "-}}": TokenType.BLOCK_END,
         "/*+": TokenType.HINT,
@@ -486,11 +486,9 @@ class Tokenizer(metaclass=_Tokenizer):
         "CASE": TokenType.CASE,
         "CASCADE": TokenType.CASCADE,
         "CHARACTER SET": TokenType.CHARACTER_SET,
-        "CHECK": TokenType.CHECK,
         "CLUSTER BY": TokenType.CLUSTER_BY,
         "COLLATE": TokenType.COLLATE,
         "COLUMN": TokenType.COLUMN,
-        "COMMENT": TokenType.SCHEMA_COMMENT,
         "COMMIT": TokenType.COMMIT,
         "COMPOUND": TokenType.COMPOUND,
         "CONSTRAINT": TokenType.CONSTRAINT,
@@ -526,12 +524,10 @@ class Tokenizer(metaclass=_Tokenizer):
         "FOREIGN KEY": TokenType.FOREIGN_KEY,
         "FORMAT": TokenType.FORMAT,
         "FROM": TokenType.FROM,
-        "GENERATED": TokenType.GENERATED,
         "GLOB": TokenType.GLOB,
         "GROUP BY": TokenType.GROUP_BY,
         "GROUPING SETS": TokenType.GROUPING_SETS,
         "HAVING": TokenType.HAVING,
-        "IDENTITY": TokenType.IDENTITY,
         "IF": TokenType.IF,
         "ILIKE": TokenType.ILIKE,
         "IGNORE NULLS": TokenType.IGNORE_NULLS,
@@ -747,11 +743,9 @@ class Tokenizer(metaclass=_Tokenizer):
         "_prev_token_line",
         "_prev_token_comments",
         "_prev_token_type",
-        "_replace_backslash",
     )
 
     def __init__(self) -> None:
-        self._replace_backslash = "\\" in self._STRING_ESCAPES
         self.reset()
 
     def reset(self) -> None:
@@ -855,7 +849,7 @@ class Tokenizer(metaclass=_Tokenizer):
     def _scan_keywords(self) -> None:
         size = 0
         word = None
-        chars = self._text
+        chars: t.Optional[str] = self._text
         char = chars
         prev_space = False
         skip = False
@@ -887,7 +881,7 @@ class Tokenizer(metaclass=_Tokenizer):
                 else:
                     skip = True
             else:
-                chars = None  # type: ignore
+                chars = None
 
         if not word:
             if self._char in self.SINGLE_TOKENS:
@@ -1015,7 +1009,6 @@ class Tokenizer(metaclass=_Tokenizer):
         self._advance(len(quote))
         text = self._extract_string(quote_end)
         text = text.encode(self.ENCODE).decode(self.ENCODE) if self.ENCODE else text  # type: ignore
-        text = text.replace("\\\\", "\\") if self._replace_backslash else text
         self._add(TokenType.NATIONAL if quote[0].upper() == "N" else TokenType.STRING, text)
         return True
 
@@ -1091,13 +1084,18 @@ class Tokenizer(metaclass=_Tokenizer):
         delim_size = len(delimiter)
 
         while True:
-            if (
-                self._char in self._STRING_ESCAPES
-                and self._peek
-                and (self._peek == delimiter or self._peek in self._STRING_ESCAPES)
+            if self._char in self._STRING_ESCAPES and (
+                self._peek == delimiter or self._peek in self._STRING_ESCAPES
             ):
-                text += self._peek
-                self._advance(2)
+                if self._peek == delimiter:
+                    text += self._peek  # type: ignore
+                else:
+                    text += self._char + self._peek  # type: ignore
+
+                if self._current + 1 < self.size:
+                    self._advance(2)
+                else:
+                    raise RuntimeError(f"Missing {delimiter} from {self._line}:{self._current}")
             else:
                 if self._chars(delim_size) == delimiter:
                     if delim_size > 1: