Merging upstream version 18.2.0.

Signed-off-by: Daniel Baumann <daniel@debian.org>

sqlglot/tokens.py

@@ -48,6 +48,7 @@ class TokenType(AutoName):
     HASH_ARROW = auto()
     DHASH_ARROW = auto()
     LR_ARROW = auto()
+    DAT = auto()
     LT_AT = auto()
     AT_GT = auto()
     DOLLAR = auto()
@@ -84,6 +85,7 @@ class TokenType(AutoName):
     UTINYINT = auto()
     SMALLINT = auto()
     USMALLINT = auto()
+    MEDIUMINT = auto()
     INT = auto()
     UINT = auto()
     BIGINT = auto()
@@ -140,6 +142,7 @@ class TokenType(AutoName):
     SMALLSERIAL = auto()
     BIGSERIAL = auto()
     XML = auto()
+    YEAR = auto()
     UNIQUEIDENTIFIER = auto()
     USERDEFINED = auto()
     MONEY = auto()
@@ -157,6 +160,7 @@ class TokenType(AutoName):
     FIXEDSTRING = auto()
     LOWCARDINALITY = auto()
     NESTED = auto()
+    UNKNOWN = auto()
 
     # keywords
     ALIAS = auto()
@@ -180,6 +184,7 @@ class TokenType(AutoName):
     COMMAND = auto()
     COMMENT = auto()
     COMMIT = auto()
+    CONNECT_BY = auto()
     CONSTRAINT = auto()
     CREATE = auto()
     CROSS = auto()
@@ -256,6 +261,7 @@ class TokenType(AutoName):
     NEXT = auto()
     NOTNULL = auto()
     NULL = auto()
+    OBJECT_IDENTIFIER = auto()
     OFFSET = auto()
     ON = auto()
     ORDER_BY = auto()
@@ -298,6 +304,7 @@ class TokenType(AutoName):
     SIMILAR_TO = auto()
     SOME = auto()
     SORT_BY = auto()
+    START_WITH = auto()
     STRUCT = auto()
     TABLE_SAMPLE = auto()
     TEMPORARY = auto()
@@ -319,6 +326,8 @@ class TokenType(AutoName):
     WINDOW = auto()
     WITH = auto()
     UNIQUE = auto()
+    VERSION_SNAPSHOT = auto()
+    TIMESTAMP_SNAPSHOT = auto()
 
 
 class Token:
@@ -530,6 +539,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "COLLATE": TokenType.COLLATE,
         "COLUMN": TokenType.COLUMN,
         "COMMIT": TokenType.COMMIT,
+        "CONNECT BY": TokenType.CONNECT_BY,
         "CONSTRAINT": TokenType.CONSTRAINT,
         "CREATE": TokenType.CREATE,
         "CROSS": TokenType.CROSS,
@@ -636,6 +646,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "SIMILAR TO": TokenType.SIMILAR_TO,
         "SOME": TokenType.SOME,
         "SORT BY": TokenType.SORT_BY,
+        "START WITH": TokenType.START_WITH,
         "TABLE": TokenType.TABLE,
         "TABLESAMPLE": TokenType.TABLE_SAMPLE,
         "TEMP": TokenType.TEMPORARY,
@@ -643,6 +654,7 @@ class Tokenizer(metaclass=_Tokenizer):
         "THEN": TokenType.THEN,
         "TRUE": TokenType.TRUE,
         "UNION": TokenType.UNION,
+        "UNKNOWN": TokenType.UNKNOWN,
         "UNNEST": TokenType.UNNEST,
         "UNPIVOT": TokenType.UNPIVOT,
         "UPDATE": TokenType.UPDATE,
@@ -739,6 +751,8 @@ class Tokenizer(metaclass=_Tokenizer):
         "TRUNCATE": TokenType.COMMAND,
         "VACUUM": TokenType.COMMAND,
         "USER-DEFINED": TokenType.USERDEFINED,
+        "FOR VERSION": TokenType.VERSION_SNAPSHOT,
+        "FOR TIMESTAMP": TokenType.TIMESTAMP_SNAPSHOT,
     }
 
     WHITE_SPACE: t.Dict[t.Optional[str], TokenType] = {
@@ -941,8 +955,8 @@ class Tokenizer(metaclass=_Tokenizer):
             if result == TrieResult.EXISTS:
                 word = chars
 
+            end = self._current + size
             size += 1
-            end = self._current - 1 + size
 
             if end < self.size:
                 char = self.sql[end]
@@ -961,21 +975,20 @@ class Tokenizer(metaclass=_Tokenizer):
                 char = ""
                 chars = " "
 
-        if not word:
-            if self._char in self.SINGLE_TOKENS:
-                self._add(self.SINGLE_TOKENS[self._char], text=self._char)
+        if word:
+            if self._scan_string(word):
                 return
-            self._scan_var()
+            if self._scan_comment(word):
+                return
+            if prev_space or single_token or not char:
+                self._advance(size - 1)
+                word = word.upper()
+                self._add(self.KEYWORDS[word], text=word)
+                return
+        if self._char in self.SINGLE_TOKENS:
+            self._add(self.SINGLE_TOKENS[self._char], text=self._char)
             return
-
-        if self._scan_string(word):
-            return
-        if self._scan_comment(word):
-            return
-
-        self._advance(size - 1)
-        word = word.upper()
-        self._add(self.KEYWORDS[word], text=word)
+        self._scan_var()
 
     def _scan_comment(self, comment_start: str) -> bool:
         if comment_start not in self._COMMENTS:
@@ -1053,8 +1066,8 @@ class Tokenizer(metaclass=_Tokenizer):
                 elif self.IDENTIFIERS_CAN_START_WITH_DIGIT:
                     return self._add(TokenType.VAR)
 
-                self._add(TokenType.NUMBER, number_text)
-                return self._advance(-len(literal))
+                self._advance(-len(literal))
+                return self._add(TokenType.NUMBER, number_text)
             else:
                 return self._add(TokenType.NUMBER)