Merging upstream version 17.2.0.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
06c5965633
commit
ff2afd7448
91 changed files with 42856 additions and 42624 deletions
|
@ -239,6 +239,7 @@ class TokenType(AutoName):
|
|||
LOCK = auto()
|
||||
MAP = auto()
|
||||
MATCH_RECOGNIZE = auto()
|
||||
MEMBER_OF = auto()
|
||||
MERGE = auto()
|
||||
MOD = auto()
|
||||
NATURAL = auto()
|
||||
|
@ -944,8 +945,6 @@ class Tokenizer(metaclass=_Tokenizer):
|
|||
char = ""
|
||||
chars = " "
|
||||
|
||||
word = None if not single_token and chars[-1] not in self.WHITE_SPACE else word
|
||||
|
||||
if not word:
|
||||
if self._char in self.SINGLE_TOKENS:
|
||||
self._add(self.SINGLE_TOKENS[self._char], text=self._char)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue