python/sqlglot
1
0
Fork 0
Code Issues Activity

Merging upstream version 15.2.0.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-02-13 15:58:40 +01:00
parent 2e6df1bcfa
commit 3d4adf9c16
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
81 changed files with 40321 additions and 37940 deletions
Download patch file Download diff file Expand all files Collapse all files

45
sqlglot/tokens.py
View file

@ -113,6 +113,18 @@ class TokenType(AutoName):
DATETIME = auto()
DATETIME64 = auto()
DATE = auto()
INT4RANGE = auto()
INT4MULTIRANGE = auto()
INT8RANGE = auto()
INT8MULTIRANGE = auto()
NUMRANGE = auto()
NUMMULTIRANGE = auto()
TSRANGE = auto()
TSMULTIRANGE = auto()
TSTZRANGE = auto()
TSTZMULTIRANGE = auto()
DATERANGE = auto()
DATEMULTIRANGE = auto()
UUID = auto()
GEOGRAPHY = auto()
NULLABLE = auto()
@ -167,6 +179,7 @@ class TokenType(AutoName):
DELETE = auto()
DESC = auto()
DESCRIBE = auto()
DICTIONARY = auto()
DISTINCT = auto()
DIV = auto()
DROP = auto()
@ -480,6 +493,7 @@ class Tokenizer(metaclass=_Tokenizer):
"ANY": TokenType.ANY,
"ASC": TokenType.ASC,
"AS": TokenType.ALIAS,
"ASOF": TokenType.ASOF,
"AUTOINCREMENT": TokenType.AUTO_INCREMENT,
"AUTO_INCREMENT": TokenType.AUTO_INCREMENT,
"BEGIN": TokenType.BEGIN,
@ -669,6 +683,18 @@ class Tokenizer(metaclass=_Tokenizer):
"TIMESTAMPLTZ": TokenType.TIMESTAMPLTZ,
"DATE": TokenType.DATE,
"DATETIME": TokenType.DATETIME,
"INT4RANGE": TokenType.INT4RANGE,
"INT4MULTIRANGE": TokenType.INT4MULTIRANGE,
"INT8RANGE": TokenType.INT8RANGE,
"INT8MULTIRANGE": TokenType.INT8MULTIRANGE,
"NUMRANGE": TokenType.NUMRANGE,
"NUMMULTIRANGE": TokenType.NUMMULTIRANGE,
"TSRANGE": TokenType.TSRANGE,
"TSMULTIRANGE": TokenType.TSMULTIRANGE,
"TSTZRANGE": TokenType.TSTZRANGE,
"TSTZMULTIRANGE": TokenType.TSTZMULTIRANGE,
"DATERANGE": TokenType.DATERANGE,
"DATEMULTIRANGE": TokenType.DATEMULTIRANGE,
"UNIQUE": TokenType.UNIQUE,
"STRUCT": TokenType.STRUCT,
"VARIANT": TokenType.VARIANT,
@ -709,8 +735,6 @@ class Tokenizer(metaclass=_Tokenizer):
COMMENTS = ["--", ("/*", "*/"), ("{#", "#}")]
KEYWORD_TRIE: t.Dict = {} # autofilled
IDENTIFIER_CAN_START_WITH_DIGIT = False
__slots__ = (
"sql",
"size",
@ -724,6 +748,7 @@ class Tokenizer(metaclass=_Tokenizer):
"_end",
"_peek",
"_prev_token_line",
"identifiers_can_start_with_digit",
)
def __init__(self) -> None:
@ -826,6 +851,12 @@ class Tokenizer(metaclass=_Tokenizer):
def _text(self) -> str:
return self.sql[self._start : self._current]
def peek(self, i: int = 0) -> str:
i = self._current + i
if i < self.size:
return self.sql[i]
return ""
def _add(self, token_type: TokenType, text: t.Optional[str] = None) -> None:
self._prev_token_line = self._line
self.tokens.append(
@ -962,8 +993,12 @@ class Tokenizer(metaclass=_Tokenizer):
if self._peek.isdigit():
self._advance()
elif self._peek == "." and not decimal:
decimal = True
self._advance()
after = self.peek(1)
if after.isdigit() or not after.strip():
decimal = True
self._advance()
else:
return self._add(TokenType.VAR)
elif self._peek in ("-", "+") and scientific == 1:
scientific += 1
self._advance()
@ -984,7 +1019,7 @@ class Tokenizer(metaclass=_Tokenizer):
self._add(TokenType.NUMBER, number_text)
self._add(TokenType.DCOLON, "::")
return self._add(token_type, literal)
elif self.IDENTIFIER_CAN_START_WITH_DIGIT:
elif self.identifiers_can_start_with_digit: # type: ignore
return self._add(TokenType.VAR)
self._add(TokenType.NUMBER, number_text)