Edit on GitHub

sqlglot.executor.python

  1import ast
  2import collections
  3import itertools
  4import math
  5
  6from sqlglot import exp, generator, planner, tokens
  7from sqlglot.dialects.dialect import Dialect, inline_array_sql
  8from sqlglot.errors import ExecuteError
  9from sqlglot.executor.context import Context
 10from sqlglot.executor.env import ENV
 11from sqlglot.executor.table import RowReader, Table
 12from sqlglot.helper import csv_reader, subclasses
 13
 14
 15class PythonExecutor:
 16    def __init__(self, env=None, tables=None):
 17        self.generator = Python().generator(identify=True, comments=False)
 18        self.env = {**ENV, **(env or {})}
 19        self.tables = tables or {}
 20
 21    def execute(self, plan):
 22        running = set()
 23        finished = set()
 24        queue = set(plan.leaves)
 25        contexts = {}
 26
 27        while queue:
 28            node = queue.pop()
 29            try:
 30                context = self.context(
 31                    {
 32                        name: table
 33                        for dep in node.dependencies
 34                        for name, table in contexts[dep].tables.items()
 35                    }
 36                )
 37                running.add(node)
 38
 39                if isinstance(node, planner.Scan):
 40                    contexts[node] = self.scan(node, context)
 41                elif isinstance(node, planner.Aggregate):
 42                    contexts[node] = self.aggregate(node, context)
 43                elif isinstance(node, planner.Join):
 44                    contexts[node] = self.join(node, context)
 45                elif isinstance(node, planner.Sort):
 46                    contexts[node] = self.sort(node, context)
 47                elif isinstance(node, planner.SetOperation):
 48                    contexts[node] = self.set_operation(node, context)
 49                else:
 50                    raise NotImplementedError
 51
 52                running.remove(node)
 53                finished.add(node)
 54
 55                for dep in node.dependents:
 56                    if dep not in running and all(d in contexts for d in dep.dependencies):
 57                        queue.add(dep)
 58
 59                for dep in node.dependencies:
 60                    if all(d in finished for d in dep.dependents):
 61                        contexts.pop(dep)
 62            except Exception as e:
 63                raise ExecuteError(f"Step '{node.id}' failed: {e}") from e
 64
 65        root = plan.root
 66        return contexts[root].tables[root.name]
 67
 68    def generate(self, expression):
 69        """Convert a SQL expression into literal Python code and compile it into bytecode."""
 70        if not expression:
 71            return None
 72
 73        sql = self.generator.generate(expression)
 74        return compile(sql, sql, "eval", optimize=2)
 75
 76    def generate_tuple(self, expressions):
 77        """Convert an array of SQL expressions into tuple of Python byte code."""
 78        if not expressions:
 79            return tuple()
 80        return tuple(self.generate(expression) for expression in expressions)
 81
 82    def context(self, tables):
 83        return Context(tables, env=self.env)
 84
 85    def table(self, expressions):
 86        return Table(
 87            expression.alias_or_name if isinstance(expression, exp.Expression) else expression
 88            for expression in expressions
 89        )
 90
 91    def scan(self, step, context):
 92        source = step.source
 93
 94        if source and isinstance(source, exp.Expression):
 95            source = source.name or source.alias
 96
 97        if source is None:
 98            context, table_iter = self.static()
 99        elif source in context:
100            if not step.projections and not step.condition:
101                return self.context({step.name: context.tables[source]})
102            table_iter = context.table_iter(source)
103        elif isinstance(step.source, exp.Table) and isinstance(step.source.this, exp.ReadCSV):
104            table_iter = self.scan_csv(step)
105            context = next(table_iter)
106        else:
107            context, table_iter = self.scan_table(step)
108
109        return self.context({step.name: self._project_and_filter(context, step, table_iter)})
110
111    def _project_and_filter(self, context, step, table_iter):
112        sink = self.table(step.projections if step.projections else context.columns)
113        condition = self.generate(step.condition)
114        projections = self.generate_tuple(step.projections)
115
116        for reader in table_iter:
117            if len(sink) >= step.limit:
118                break
119
120            if condition and not context.eval(condition):
121                continue
122
123            if projections:
124                sink.append(context.eval_tuple(projections))
125            else:
126                sink.append(reader.row)
127
128        return sink
129
130    def static(self):
131        return self.context({}), [RowReader(())]
132
133    def scan_table(self, step):
134        table = self.tables.find(step.source)
135        context = self.context({step.source.alias_or_name: table})
136        return context, iter(table)
137
138    def scan_csv(self, step):
139        alias = step.source.alias
140        source = step.source.this
141
142        with csv_reader(source) as reader:
143            columns = next(reader)
144            table = Table(columns)
145            context = self.context({alias: table})
146            yield context
147            types = []
148
149            for row in reader:
150                if not types:
151                    for v in row:
152                        try:
153                            types.append(type(ast.literal_eval(v)))
154                        except (ValueError, SyntaxError):
155                            types.append(str)
156                context.set_row(tuple(t(v) for t, v in zip(types, row)))
157                yield context.table.reader
158
159    def join(self, step, context):
160        source = step.name
161
162        source_table = context.tables[source]
163        source_context = self.context({source: source_table})
164        column_ranges = {source: range(0, len(source_table.columns))}
165
166        for name, join in step.joins.items():
167            table = context.tables[name]
168            start = max(r.stop for r in column_ranges.values())
169            column_ranges[name] = range(start, len(table.columns) + start)
170            join_context = self.context({name: table})
171
172            if join.get("source_key"):
173                table = self.hash_join(join, source_context, join_context)
174            else:
175                table = self.nested_loop_join(join, source_context, join_context)
176
177            source_context = self.context(
178                {
179                    name: Table(table.columns, table.rows, column_range)
180                    for name, column_range in column_ranges.items()
181                }
182            )
183            condition = self.generate(join["condition"])
184            if condition:
185                source_context.filter(condition)
186
187        if not step.condition and not step.projections:
188            return source_context
189
190        sink = self._project_and_filter(
191            source_context,
192            step,
193            (reader for reader, _ in iter(source_context)),
194        )
195
196        if step.projections:
197            return self.context({step.name: sink})
198        else:
199            return self.context(
200                {
201                    name: Table(table.columns, sink.rows, table.column_range)
202                    for name, table in source_context.tables.items()
203                }
204            )
205
206    def nested_loop_join(self, _join, source_context, join_context):
207        table = Table(source_context.columns + join_context.columns)
208
209        for reader_a, _ in source_context:
210            for reader_b, _ in join_context:
211                table.append(reader_a.row + reader_b.row)
212
213        return table
214
215    def hash_join(self, join, source_context, join_context):
216        source_key = self.generate_tuple(join["source_key"])
217        join_key = self.generate_tuple(join["join_key"])
218        left = join.get("side") == "LEFT"
219        right = join.get("side") == "RIGHT"
220
221        results = collections.defaultdict(lambda: ([], []))
222
223        for reader, ctx in source_context:
224            results[ctx.eval_tuple(source_key)][0].append(reader.row)
225        for reader, ctx in join_context:
226            results[ctx.eval_tuple(join_key)][1].append(reader.row)
227
228        table = Table(source_context.columns + join_context.columns)
229        nulls = [(None,) * len(join_context.columns if left else source_context.columns)]
230
231        for a_group, b_group in results.values():
232            if left:
233                b_group = b_group or nulls
234            elif right:
235                a_group = a_group or nulls
236
237            for a_row, b_row in itertools.product(a_group, b_group):
238                table.append(a_row + b_row)
239
240        return table
241
242    def aggregate(self, step, context):
243        group_by = self.generate_tuple(step.group.values())
244        aggregations = self.generate_tuple(step.aggregations)
245        operands = self.generate_tuple(step.operands)
246
247        if operands:
248            operand_table = Table(self.table(step.operands).columns)
249
250            for reader, ctx in context:
251                operand_table.append(ctx.eval_tuple(operands))
252
253            for i, (a, b) in enumerate(zip(context.table.rows, operand_table.rows)):
254                context.table.rows[i] = a + b
255
256            width = len(context.columns)
257            context.add_columns(*operand_table.columns)
258
259            operand_table = Table(
260                context.columns,
261                context.table.rows,
262                range(width, width + len(operand_table.columns)),
263            )
264
265            context = self.context(
266                {
267                    None: operand_table,
268                    **context.tables,
269                }
270            )
271
272        context.sort(group_by)
273
274        group = None
275        start = 0
276        end = 1
277        length = len(context.table)
278        table = self.table(list(step.group) + step.aggregations)
279
280        def add_row():
281            table.append(group + context.eval_tuple(aggregations))
282
283        if length:
284            for i in range(length):
285                context.set_index(i)
286                key = context.eval_tuple(group_by)
287                group = key if group is None else group
288                end += 1
289                if key != group:
290                    context.set_range(start, end - 2)
291                    add_row()
292                    group = key
293                    start = end - 2
294                if len(table.rows) >= step.limit:
295                    break
296                if i == length - 1:
297                    context.set_range(start, end - 1)
298                    add_row()
299        elif step.limit > 0 and not group_by:
300            context.set_range(0, 0)
301            table.append(context.eval_tuple(aggregations))
302
303        context = self.context({step.name: table, **{name: table for name in context.tables}})
304
305        if step.projections or step.condition:
306            return self.scan(step, context)
307        return context
308
309    def sort(self, step, context):
310        projections = self.generate_tuple(step.projections)
311        projection_columns = [p.alias_or_name for p in step.projections]
312        all_columns = list(context.columns) + projection_columns
313        sink = self.table(all_columns)
314        for reader, ctx in context:
315            sink.append(reader.row + ctx.eval_tuple(projections))
316
317        sort_ctx = self.context(
318            {
319                None: sink,
320                **{table: sink for table in context.tables},
321            }
322        )
323        sort_ctx.sort(self.generate_tuple(step.key))
324
325        if not math.isinf(step.limit):
326            sort_ctx.table.rows = sort_ctx.table.rows[0 : step.limit]
327
328        output = Table(
329            projection_columns,
330            rows=[r[len(context.columns) : len(all_columns)] for r in sort_ctx.table.rows],
331        )
332        return self.context({step.name: output})
333
334    def set_operation(self, step, context):
335        left = context.tables[step.left]
336        right = context.tables[step.right]
337
338        sink = self.table(left.columns)
339
340        if issubclass(step.op, exp.Intersect):
341            sink.rows = list(set(left.rows).intersection(set(right.rows)))
342        elif issubclass(step.op, exp.Except):
343            sink.rows = list(set(left.rows).difference(set(right.rows)))
344        elif issubclass(step.op, exp.Union) and step.distinct:
345            sink.rows = list(set(left.rows).union(set(right.rows)))
346        else:
347            sink.rows = left.rows + right.rows
348
349        return self.context({step.name: sink})
350
351
352def _ordered_py(self, expression):
353    this = self.sql(expression, "this")
354    desc = "True" if expression.args.get("desc") else "False"
355    nulls_first = "True" if expression.args.get("nulls_first") else "False"
356    return f"ORDERED({this}, {desc}, {nulls_first})"
357
358
359def _rename(self, e):
360    try:
361        values = list(e.args.values())
362
363        if len(values) == 1:
364            values = values[0]
365            if not isinstance(values, list):
366                return self.func(e.key, values)
367            return self.func(e.key, *values)
368
369        if isinstance(e, exp.Func) and e.is_var_len_args:
370            *head, tail = values
371            return self.func(e.key, *head, *tail)
372
373        return self.func(e.key, *values)
374    except Exception as ex:
375        raise Exception(f"Could not rename {repr(e)}") from ex
376
377
378def _case_sql(self, expression):
379    this = self.sql(expression, "this")
380    chain = self.sql(expression, "default") or "None"
381
382    for e in reversed(expression.args["ifs"]):
383        true = self.sql(e, "true")
384        condition = self.sql(e, "this")
385        condition = f"{this} = ({condition})" if this else condition
386        chain = f"{true} if {condition} else ({chain})"
387
388    return chain
389
390
391def _lambda_sql(self, e: exp.Lambda) -> str:
392    names = {e.name.lower() for e in e.expressions}
393
394    e = e.transform(
395        lambda n: exp.var(n.name)
396        if isinstance(n, exp.Identifier) and n.name.lower() in names
397        else n
398    )
399
400    return f"lambda {self.expressions(e, flat=True)}: {self.sql(e, 'this')}"
401
402
403class Python(Dialect):
404    class Tokenizer(tokens.Tokenizer):
405        STRING_ESCAPES = ["\\"]
406
407    class Generator(generator.Generator):
408        TRANSFORMS = {
409            **{klass: _rename for klass in subclasses(exp.__name__, exp.Binary)},
410            **{klass: _rename for klass in exp.ALL_FUNCTIONS},
411            exp.Case: _case_sql,
412            exp.Alias: lambda self, e: self.sql(e.this),
413            exp.Array: inline_array_sql,
414            exp.And: lambda self, e: self.binary(e, "and"),
415            exp.Between: _rename,
416            exp.Boolean: lambda self, e: "True" if e.this else "False",
417            exp.Cast: lambda self, e: f"CAST({self.sql(e.this)}, exp.DataType.Type.{e.args['to']})",
418            exp.Column: lambda self, e: f"scope[{self.sql(e, 'table') or None}][{self.sql(e.this)}]",
419            exp.Distinct: lambda self, e: f"set({self.sql(e, 'this')})",
420            exp.Extract: lambda self, e: f"EXTRACT('{e.name.lower()}', {self.sql(e, 'expression')})",
421            exp.In: lambda self, e: f"{self.sql(e, 'this')} in {{{self.expressions(e, flat=True)}}}",
422            exp.Interval: lambda self, e: f"INTERVAL({self.sql(e.this)}, '{self.sql(e.unit)}')",
423            exp.Is: lambda self, e: self.binary(e, "is"),
424            exp.Lambda: _lambda_sql,
425            exp.Not: lambda self, e: f"not {self.sql(e.this)}",
426            exp.Null: lambda *_: "None",
427            exp.Or: lambda self, e: self.binary(e, "or"),
428            exp.Ordered: _ordered_py,
429            exp.Star: lambda *_: "1",
430        }
class PythonExecutor:
 16class PythonExecutor:
 17    def __init__(self, env=None, tables=None):
 18        self.generator = Python().generator(identify=True, comments=False)
 19        self.env = {**ENV, **(env or {})}
 20        self.tables = tables or {}
 21
 22    def execute(self, plan):
 23        running = set()
 24        finished = set()
 25        queue = set(plan.leaves)
 26        contexts = {}
 27
 28        while queue:
 29            node = queue.pop()
 30            try:
 31                context = self.context(
 32                    {
 33                        name: table
 34                        for dep in node.dependencies
 35                        for name, table in contexts[dep].tables.items()
 36                    }
 37                )
 38                running.add(node)
 39
 40                if isinstance(node, planner.Scan):
 41                    contexts[node] = self.scan(node, context)
 42                elif isinstance(node, planner.Aggregate):
 43                    contexts[node] = self.aggregate(node, context)
 44                elif isinstance(node, planner.Join):
 45                    contexts[node] = self.join(node, context)
 46                elif isinstance(node, planner.Sort):
 47                    contexts[node] = self.sort(node, context)
 48                elif isinstance(node, planner.SetOperation):
 49                    contexts[node] = self.set_operation(node, context)
 50                else:
 51                    raise NotImplementedError
 52
 53                running.remove(node)
 54                finished.add(node)
 55
 56                for dep in node.dependents:
 57                    if dep not in running and all(d in contexts for d in dep.dependencies):
 58                        queue.add(dep)
 59
 60                for dep in node.dependencies:
 61                    if all(d in finished for d in dep.dependents):
 62                        contexts.pop(dep)
 63            except Exception as e:
 64                raise ExecuteError(f"Step '{node.id}' failed: {e}") from e
 65
 66        root = plan.root
 67        return contexts[root].tables[root.name]
 68
 69    def generate(self, expression):
 70        """Convert a SQL expression into literal Python code and compile it into bytecode."""
 71        if not expression:
 72            return None
 73
 74        sql = self.generator.generate(expression)
 75        return compile(sql, sql, "eval", optimize=2)
 76
 77    def generate_tuple(self, expressions):
 78        """Convert an array of SQL expressions into tuple of Python byte code."""
 79        if not expressions:
 80            return tuple()
 81        return tuple(self.generate(expression) for expression in expressions)
 82
 83    def context(self, tables):
 84        return Context(tables, env=self.env)
 85
 86    def table(self, expressions):
 87        return Table(
 88            expression.alias_or_name if isinstance(expression, exp.Expression) else expression
 89            for expression in expressions
 90        )
 91
 92    def scan(self, step, context):
 93        source = step.source
 94
 95        if source and isinstance(source, exp.Expression):
 96            source = source.name or source.alias
 97
 98        if source is None:
 99            context, table_iter = self.static()
100        elif source in context:
101            if not step.projections and not step.condition:
102                return self.context({step.name: context.tables[source]})
103            table_iter = context.table_iter(source)
104        elif isinstance(step.source, exp.Table) and isinstance(step.source.this, exp.ReadCSV):
105            table_iter = self.scan_csv(step)
106            context = next(table_iter)
107        else:
108            context, table_iter = self.scan_table(step)
109
110        return self.context({step.name: self._project_and_filter(context, step, table_iter)})
111
112    def _project_and_filter(self, context, step, table_iter):
113        sink = self.table(step.projections if step.projections else context.columns)
114        condition = self.generate(step.condition)
115        projections = self.generate_tuple(step.projections)
116
117        for reader in table_iter:
118            if len(sink) >= step.limit:
119                break
120
121            if condition and not context.eval(condition):
122                continue
123
124            if projections:
125                sink.append(context.eval_tuple(projections))
126            else:
127                sink.append(reader.row)
128
129        return sink
130
131    def static(self):
132        return self.context({}), [RowReader(())]
133
134    def scan_table(self, step):
135        table = self.tables.find(step.source)
136        context = self.context({step.source.alias_or_name: table})
137        return context, iter(table)
138
139    def scan_csv(self, step):
140        alias = step.source.alias
141        source = step.source.this
142
143        with csv_reader(source) as reader:
144            columns = next(reader)
145            table = Table(columns)
146            context = self.context({alias: table})
147            yield context
148            types = []
149
150            for row in reader:
151                if not types:
152                    for v in row:
153                        try:
154                            types.append(type(ast.literal_eval(v)))
155                        except (ValueError, SyntaxError):
156                            types.append(str)
157                context.set_row(tuple(t(v) for t, v in zip(types, row)))
158                yield context.table.reader
159
160    def join(self, step, context):
161        source = step.name
162
163        source_table = context.tables[source]
164        source_context = self.context({source: source_table})
165        column_ranges = {source: range(0, len(source_table.columns))}
166
167        for name, join in step.joins.items():
168            table = context.tables[name]
169            start = max(r.stop for r in column_ranges.values())
170            column_ranges[name] = range(start, len(table.columns) + start)
171            join_context = self.context({name: table})
172
173            if join.get("source_key"):
174                table = self.hash_join(join, source_context, join_context)
175            else:
176                table = self.nested_loop_join(join, source_context, join_context)
177
178            source_context = self.context(
179                {
180                    name: Table(table.columns, table.rows, column_range)
181                    for name, column_range in column_ranges.items()
182                }
183            )
184            condition = self.generate(join["condition"])
185            if condition:
186                source_context.filter(condition)
187
188        if not step.condition and not step.projections:
189            return source_context
190
191        sink = self._project_and_filter(
192            source_context,
193            step,
194            (reader for reader, _ in iter(source_context)),
195        )
196
197        if step.projections:
198            return self.context({step.name: sink})
199        else:
200            return self.context(
201                {
202                    name: Table(table.columns, sink.rows, table.column_range)
203                    for name, table in source_context.tables.items()
204                }
205            )
206
207    def nested_loop_join(self, _join, source_context, join_context):
208        table = Table(source_context.columns + join_context.columns)
209
210        for reader_a, _ in source_context:
211            for reader_b, _ in join_context:
212                table.append(reader_a.row + reader_b.row)
213
214        return table
215
216    def hash_join(self, join, source_context, join_context):
217        source_key = self.generate_tuple(join["source_key"])
218        join_key = self.generate_tuple(join["join_key"])
219        left = join.get("side") == "LEFT"
220        right = join.get("side") == "RIGHT"
221
222        results = collections.defaultdict(lambda: ([], []))
223
224        for reader, ctx in source_context:
225            results[ctx.eval_tuple(source_key)][0].append(reader.row)
226        for reader, ctx in join_context:
227            results[ctx.eval_tuple(join_key)][1].append(reader.row)
228
229        table = Table(source_context.columns + join_context.columns)
230        nulls = [(None,) * len(join_context.columns if left else source_context.columns)]
231
232        for a_group, b_group in results.values():
233            if left:
234                b_group = b_group or nulls
235            elif right:
236                a_group = a_group or nulls
237
238            for a_row, b_row in itertools.product(a_group, b_group):
239                table.append(a_row + b_row)
240
241        return table
242
243    def aggregate(self, step, context):
244        group_by = self.generate_tuple(step.group.values())
245        aggregations = self.generate_tuple(step.aggregations)
246        operands = self.generate_tuple(step.operands)
247
248        if operands:
249            operand_table = Table(self.table(step.operands).columns)
250
251            for reader, ctx in context:
252                operand_table.append(ctx.eval_tuple(operands))
253
254            for i, (a, b) in enumerate(zip(context.table.rows, operand_table.rows)):
255                context.table.rows[i] = a + b
256
257            width = len(context.columns)
258            context.add_columns(*operand_table.columns)
259
260            operand_table = Table(
261                context.columns,
262                context.table.rows,
263                range(width, width + len(operand_table.columns)),
264            )
265
266            context = self.context(
267                {
268                    None: operand_table,
269                    **context.tables,
270                }
271            )
272
273        context.sort(group_by)
274
275        group = None
276        start = 0
277        end = 1
278        length = len(context.table)
279        table = self.table(list(step.group) + step.aggregations)
280
281        def add_row():
282            table.append(group + context.eval_tuple(aggregations))
283
284        if length:
285            for i in range(length):
286                context.set_index(i)
287                key = context.eval_tuple(group_by)
288                group = key if group is None else group
289                end += 1
290                if key != group:
291                    context.set_range(start, end - 2)
292                    add_row()
293                    group = key
294                    start = end - 2
295                if len(table.rows) >= step.limit:
296                    break
297                if i == length - 1:
298                    context.set_range(start, end - 1)
299                    add_row()
300        elif step.limit > 0 and not group_by:
301            context.set_range(0, 0)
302            table.append(context.eval_tuple(aggregations))
303
304        context = self.context({step.name: table, **{name: table for name in context.tables}})
305
306        if step.projections or step.condition:
307            return self.scan(step, context)
308        return context
309
310    def sort(self, step, context):
311        projections = self.generate_tuple(step.projections)
312        projection_columns = [p.alias_or_name for p in step.projections]
313        all_columns = list(context.columns) + projection_columns
314        sink = self.table(all_columns)
315        for reader, ctx in context:
316            sink.append(reader.row + ctx.eval_tuple(projections))
317
318        sort_ctx = self.context(
319            {
320                None: sink,
321                **{table: sink for table in context.tables},
322            }
323        )
324        sort_ctx.sort(self.generate_tuple(step.key))
325
326        if not math.isinf(step.limit):
327            sort_ctx.table.rows = sort_ctx.table.rows[0 : step.limit]
328
329        output = Table(
330            projection_columns,
331            rows=[r[len(context.columns) : len(all_columns)] for r in sort_ctx.table.rows],
332        )
333        return self.context({step.name: output})
334
335    def set_operation(self, step, context):
336        left = context.tables[step.left]
337        right = context.tables[step.right]
338
339        sink = self.table(left.columns)
340
341        if issubclass(step.op, exp.Intersect):
342            sink.rows = list(set(left.rows).intersection(set(right.rows)))
343        elif issubclass(step.op, exp.Except):
344            sink.rows = list(set(left.rows).difference(set(right.rows)))
345        elif issubclass(step.op, exp.Union) and step.distinct:
346            sink.rows = list(set(left.rows).union(set(right.rows)))
347        else:
348            sink.rows = left.rows + right.rows
349
350        return self.context({step.name: sink})
PythonExecutor(env=None, tables=None)
17    def __init__(self, env=None, tables=None):
18        self.generator = Python().generator(identify=True, comments=False)
19        self.env = {**ENV, **(env or {})}
20        self.tables = tables or {}
generator
env
tables
def execute(self, plan):
22    def execute(self, plan):
23        running = set()
24        finished = set()
25        queue = set(plan.leaves)
26        contexts = {}
27
28        while queue:
29            node = queue.pop()
30            try:
31                context = self.context(
32                    {
33                        name: table
34                        for dep in node.dependencies
35                        for name, table in contexts[dep].tables.items()
36                    }
37                )
38                running.add(node)
39
40                if isinstance(node, planner.Scan):
41                    contexts[node] = self.scan(node, context)
42                elif isinstance(node, planner.Aggregate):
43                    contexts[node] = self.aggregate(node, context)
44                elif isinstance(node, planner.Join):
45                    contexts[node] = self.join(node, context)
46                elif isinstance(node, planner.Sort):
47                    contexts[node] = self.sort(node, context)
48                elif isinstance(node, planner.SetOperation):
49                    contexts[node] = self.set_operation(node, context)
50                else:
51                    raise NotImplementedError
52
53                running.remove(node)
54                finished.add(node)
55
56                for dep in node.dependents:
57                    if dep not in running and all(d in contexts for d in dep.dependencies):
58                        queue.add(dep)
59
60                for dep in node.dependencies:
61                    if all(d in finished for d in dep.dependents):
62                        contexts.pop(dep)
63            except Exception as e:
64                raise ExecuteError(f"Step '{node.id}' failed: {e}") from e
65
66        root = plan.root
67        return contexts[root].tables[root.name]
def generate(self, expression):
69    def generate(self, expression):
70        """Convert a SQL expression into literal Python code and compile it into bytecode."""
71        if not expression:
72            return None
73
74        sql = self.generator.generate(expression)
75        return compile(sql, sql, "eval", optimize=2)

Convert a SQL expression into literal Python code and compile it into bytecode.

def generate_tuple(self, expressions):
77    def generate_tuple(self, expressions):
78        """Convert an array of SQL expressions into tuple of Python byte code."""
79        if not expressions:
80            return tuple()
81        return tuple(self.generate(expression) for expression in expressions)

Convert an array of SQL expressions into tuple of Python byte code.

def context(self, tables):
83    def context(self, tables):
84        return Context(tables, env=self.env)
def table(self, expressions):
86    def table(self, expressions):
87        return Table(
88            expression.alias_or_name if isinstance(expression, exp.Expression) else expression
89            for expression in expressions
90        )
def scan(self, step, context):
 92    def scan(self, step, context):
 93        source = step.source
 94
 95        if source and isinstance(source, exp.Expression):
 96            source = source.name or source.alias
 97
 98        if source is None:
 99            context, table_iter = self.static()
100        elif source in context:
101            if not step.projections and not step.condition:
102                return self.context({step.name: context.tables[source]})
103            table_iter = context.table_iter(source)
104        elif isinstance(step.source, exp.Table) and isinstance(step.source.this, exp.ReadCSV):
105            table_iter = self.scan_csv(step)
106            context = next(table_iter)
107        else:
108            context, table_iter = self.scan_table(step)
109
110        return self.context({step.name: self._project_and_filter(context, step, table_iter)})
def static(self):
131    def static(self):
132        return self.context({}), [RowReader(())]
def scan_table(self, step):
134    def scan_table(self, step):
135        table = self.tables.find(step.source)
136        context = self.context({step.source.alias_or_name: table})
137        return context, iter(table)
def scan_csv(self, step):
139    def scan_csv(self, step):
140        alias = step.source.alias
141        source = step.source.this
142
143        with csv_reader(source) as reader:
144            columns = next(reader)
145            table = Table(columns)
146            context = self.context({alias: table})
147            yield context
148            types = []
149
150            for row in reader:
151                if not types:
152                    for v in row:
153                        try:
154                            types.append(type(ast.literal_eval(v)))
155                        except (ValueError, SyntaxError):
156                            types.append(str)
157                context.set_row(tuple(t(v) for t, v in zip(types, row)))
158                yield context.table.reader
def join(self, step, context):
160    def join(self, step, context):
161        source = step.name
162
163        source_table = context.tables[source]
164        source_context = self.context({source: source_table})
165        column_ranges = {source: range(0, len(source_table.columns))}
166
167        for name, join in step.joins.items():
168            table = context.tables[name]
169            start = max(r.stop for r in column_ranges.values())
170            column_ranges[name] = range(start, len(table.columns) + start)
171            join_context = self.context({name: table})
172
173            if join.get("source_key"):
174                table = self.hash_join(join, source_context, join_context)
175            else:
176                table = self.nested_loop_join(join, source_context, join_context)
177
178            source_context = self.context(
179                {
180                    name: Table(table.columns, table.rows, column_range)
181                    for name, column_range in column_ranges.items()
182                }
183            )
184            condition = self.generate(join["condition"])
185            if condition:
186                source_context.filter(condition)
187
188        if not step.condition and not step.projections:
189            return source_context
190
191        sink = self._project_and_filter(
192            source_context,
193            step,
194            (reader for reader, _ in iter(source_context)),
195        )
196
197        if step.projections:
198            return self.context({step.name: sink})
199        else:
200            return self.context(
201                {
202                    name: Table(table.columns, sink.rows, table.column_range)
203                    for name, table in source_context.tables.items()
204                }
205            )
def nested_loop_join(self, _join, source_context, join_context):
207    def nested_loop_join(self, _join, source_context, join_context):
208        table = Table(source_context.columns + join_context.columns)
209
210        for reader_a, _ in source_context:
211            for reader_b, _ in join_context:
212                table.append(reader_a.row + reader_b.row)
213
214        return table
def hash_join(self, join, source_context, join_context):
216    def hash_join(self, join, source_context, join_context):
217        source_key = self.generate_tuple(join["source_key"])
218        join_key = self.generate_tuple(join["join_key"])
219        left = join.get("side") == "LEFT"
220        right = join.get("side") == "RIGHT"
221
222        results = collections.defaultdict(lambda: ([], []))
223
224        for reader, ctx in source_context:
225            results[ctx.eval_tuple(source_key)][0].append(reader.row)
226        for reader, ctx in join_context:
227            results[ctx.eval_tuple(join_key)][1].append(reader.row)
228
229        table = Table(source_context.columns + join_context.columns)
230        nulls = [(None,) * len(join_context.columns if left else source_context.columns)]
231
232        for a_group, b_group in results.values():
233            if left:
234                b_group = b_group or nulls
235            elif right:
236                a_group = a_group or nulls
237
238            for a_row, b_row in itertools.product(a_group, b_group):
239                table.append(a_row + b_row)
240
241        return table
def aggregate(self, step, context):
243    def aggregate(self, step, context):
244        group_by = self.generate_tuple(step.group.values())
245        aggregations = self.generate_tuple(step.aggregations)
246        operands = self.generate_tuple(step.operands)
247
248        if operands:
249            operand_table = Table(self.table(step.operands).columns)
250
251            for reader, ctx in context:
252                operand_table.append(ctx.eval_tuple(operands))
253
254            for i, (a, b) in enumerate(zip(context.table.rows, operand_table.rows)):
255                context.table.rows[i] = a + b
256
257            width = len(context.columns)
258            context.add_columns(*operand_table.columns)
259
260            operand_table = Table(
261                context.columns,
262                context.table.rows,
263                range(width, width + len(operand_table.columns)),
264            )
265
266            context = self.context(
267                {
268                    None: operand_table,
269                    **context.tables,
270                }
271            )
272
273        context.sort(group_by)
274
275        group = None
276        start = 0
277        end = 1
278        length = len(context.table)
279        table = self.table(list(step.group) + step.aggregations)
280
281        def add_row():
282            table.append(group + context.eval_tuple(aggregations))
283
284        if length:
285            for i in range(length):
286                context.set_index(i)
287                key = context.eval_tuple(group_by)
288                group = key if group is None else group
289                end += 1
290                if key != group:
291                    context.set_range(start, end - 2)
292                    add_row()
293                    group = key
294                    start = end - 2
295                if len(table.rows) >= step.limit:
296                    break
297                if i == length - 1:
298                    context.set_range(start, end - 1)
299                    add_row()
300        elif step.limit > 0 and not group_by:
301            context.set_range(0, 0)
302            table.append(context.eval_tuple(aggregations))
303
304        context = self.context({step.name: table, **{name: table for name in context.tables}})
305
306        if step.projections or step.condition:
307            return self.scan(step, context)
308        return context
def sort(self, step, context):
310    def sort(self, step, context):
311        projections = self.generate_tuple(step.projections)
312        projection_columns = [p.alias_or_name for p in step.projections]
313        all_columns = list(context.columns) + projection_columns
314        sink = self.table(all_columns)
315        for reader, ctx in context:
316            sink.append(reader.row + ctx.eval_tuple(projections))
317
318        sort_ctx = self.context(
319            {
320                None: sink,
321                **{table: sink for table in context.tables},
322            }
323        )
324        sort_ctx.sort(self.generate_tuple(step.key))
325
326        if not math.isinf(step.limit):
327            sort_ctx.table.rows = sort_ctx.table.rows[0 : step.limit]
328
329        output = Table(
330            projection_columns,
331            rows=[r[len(context.columns) : len(all_columns)] for r in sort_ctx.table.rows],
332        )
333        return self.context({step.name: output})
def set_operation(self, step, context):
335    def set_operation(self, step, context):
336        left = context.tables[step.left]
337        right = context.tables[step.right]
338
339        sink = self.table(left.columns)
340
341        if issubclass(step.op, exp.Intersect):
342            sink.rows = list(set(left.rows).intersection(set(right.rows)))
343        elif issubclass(step.op, exp.Except):
344            sink.rows = list(set(left.rows).difference(set(right.rows)))
345        elif issubclass(step.op, exp.Union) and step.distinct:
346            sink.rows = list(set(left.rows).union(set(right.rows)))
347        else:
348            sink.rows = left.rows + right.rows
349
350        return self.context({step.name: sink})
class Python(sqlglot.dialects.dialect.Dialect):
404class Python(Dialect):
405    class Tokenizer(tokens.Tokenizer):
406        STRING_ESCAPES = ["\\"]
407
408    class Generator(generator.Generator):
409        TRANSFORMS = {
410            **{klass: _rename for klass in subclasses(exp.__name__, exp.Binary)},
411            **{klass: _rename for klass in exp.ALL_FUNCTIONS},
412            exp.Case: _case_sql,
413            exp.Alias: lambda self, e: self.sql(e.this),
414            exp.Array: inline_array_sql,
415            exp.And: lambda self, e: self.binary(e, "and"),
416            exp.Between: _rename,
417            exp.Boolean: lambda self, e: "True" if e.this else "False",
418            exp.Cast: lambda self, e: f"CAST({self.sql(e.this)}, exp.DataType.Type.{e.args['to']})",
419            exp.Column: lambda self, e: f"scope[{self.sql(e, 'table') or None}][{self.sql(e.this)}]",
420            exp.Distinct: lambda self, e: f"set({self.sql(e, 'this')})",
421            exp.Extract: lambda self, e: f"EXTRACT('{e.name.lower()}', {self.sql(e, 'expression')})",
422            exp.In: lambda self, e: f"{self.sql(e, 'this')} in {{{self.expressions(e, flat=True)}}}",
423            exp.Interval: lambda self, e: f"INTERVAL({self.sql(e.this)}, '{self.sql(e.unit)}')",
424            exp.Is: lambda self, e: self.binary(e, "is"),
425            exp.Lambda: _lambda_sql,
426            exp.Not: lambda self, e: f"not {self.sql(e.this)}",
427            exp.Null: lambda *_: "None",
428            exp.Or: lambda self, e: self.binary(e, "or"),
429            exp.Ordered: _ordered_py,
430            exp.Star: lambda *_: "1",
431        }
tokenizer_class = <class 'sqlglot.executor.python.Python.Tokenizer'>
parser_class = <class 'sqlglot.parser.Parser'>
generator_class = <class 'sqlglot.executor.python.Python.Generator'>
TIME_TRIE: Dict = {}
FORMAT_TRIE: Dict = {}
INVERSE_TIME_MAPPING: Dict[str, str] = {}
INVERSE_TIME_TRIE: Dict = {}
QUOTE_START = "'"
QUOTE_END = "'"
IDENTIFIER_START = '"'
IDENTIFIER_END = '"'
BIT_START = None
BIT_END = None
HEX_START = None
HEX_END = None
BYTE_START = None
BYTE_END = None
RAW_START = None
RAW_END = None
class Python.Tokenizer(sqlglot.tokens.Tokenizer):
405    class Tokenizer(tokens.Tokenizer):
406        STRING_ESCAPES = ["\\"]
STRING_ESCAPES = ['\\']
class Python.Generator(sqlglot.generator.Generator):
408    class Generator(generator.Generator):
409        TRANSFORMS = {
410            **{klass: _rename for klass in subclasses(exp.__name__, exp.Binary)},
411            **{klass: _rename for klass in exp.ALL_FUNCTIONS},
412            exp.Case: _case_sql,
413            exp.Alias: lambda self, e: self.sql(e.this),
414            exp.Array: inline_array_sql,
415            exp.And: lambda self, e: self.binary(e, "and"),
416            exp.Between: _rename,
417            exp.Boolean: lambda self, e: "True" if e.this else "False",
418            exp.Cast: lambda self, e: f"CAST({self.sql(e.this)}, exp.DataType.Type.{e.args['to']})",
419            exp.Column: lambda self, e: f"scope[{self.sql(e, 'table') or None}][{self.sql(e.this)}]",
420            exp.Distinct: lambda self, e: f"set({self.sql(e, 'this')})",
421            exp.Extract: lambda self, e: f"EXTRACT('{e.name.lower()}', {self.sql(e, 'expression')})",
422            exp.In: lambda self, e: f"{self.sql(e, 'this')} in {{{self.expressions(e, flat=True)}}}",
423            exp.Interval: lambda self, e: f"INTERVAL({self.sql(e.this)}, '{self.sql(e.unit)}')",
424            exp.Is: lambda self, e: self.binary(e, "is"),
425            exp.Lambda: _lambda_sql,
426            exp.Not: lambda self, e: f"not {self.sql(e.this)}",
427            exp.Null: lambda *_: "None",
428            exp.Or: lambda self, e: self.binary(e, "or"),
429            exp.Ordered: _ordered_py,
430            exp.Star: lambda *_: "1",
431        }

Generator converts a given syntax tree to the corresponding SQL string.

Arguments:
  • pretty: Whether or not to format the produced SQL string. Default: False.
  • identify: Determines when an identifier should be quoted. Possible values are: False (default): Never quote, except in cases where it's mandatory by the dialect. True or 'always': Always quote. 'safe': Only quote identifiers that are case insensitive.
  • normalize: Whether or not to normalize identifiers to lowercase. Default: False.
  • pad: Determines the pad size in a formatted string. Default: 2.
  • indent: Determines the indentation size in a formatted string. Default: 2.
  • normalize_functions: Whether or not to normalize all function names. Possible values are: "upper" or True (default): Convert names to uppercase. "lower": Convert names to lowercase. False: Disables function name normalization.
  • unsupported_level: Determines the generator's behavior when it encounters unsupported expressions. Default ErrorLevel.WARN.
  • max_unsupported: Maximum number of unsupported messages to include in a raised UnsupportedError. This is only relevant if unsupported_level is ErrorLevel.RAISE. Default: 3
  • leading_comma: Determines whether or not the comma is leading or trailing in select expressions. This is only relevant when generating in pretty mode. Default: False
  • max_text_width: The max number of characters in a segment before creating new lines in pretty mode. The default is on the smaller end because the length only represents a segment and not the true line length. Default: 80
  • comments: Whether or not to preserve comments in the output SQL code. Default: True
TRANSFORMS = {<class 'sqlglot.expressions.Add'>: <function _rename>, <class 'sqlglot.expressions.And'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.ArrayContained'>: <function _rename>, <class 'sqlglot.expressions.ArrayContains'>: <function _rename>, <class 'sqlglot.expressions.ArrayOverlaps'>: <function _rename>, <class 'sqlglot.expressions.Binary'>: <function _rename>, <class 'sqlglot.expressions.BitwiseAnd'>: <function _rename>, <class 'sqlglot.expressions.BitwiseLeftShift'>: <function _rename>, <class 'sqlglot.expressions.BitwiseOr'>: <function _rename>, <class 'sqlglot.expressions.BitwiseRightShift'>: <function _rename>, <class 'sqlglot.expressions.BitwiseXor'>: <function _rename>, <class 'sqlglot.expressions.Collate'>: <function _rename>, <class 'sqlglot.expressions.Connector'>: <function _rename>, <class 'sqlglot.expressions.DPipe'>: <function _rename>, <class 'sqlglot.expressions.Distance'>: <function _rename>, <class 'sqlglot.expressions.Div'>: <function _rename>, <class 'sqlglot.expressions.Dot'>: <function _rename>, <class 'sqlglot.expressions.EQ'>: <function _rename>, <class 'sqlglot.expressions.Escape'>: <function _rename>, <class 'sqlglot.expressions.GT'>: <function _rename>, <class 'sqlglot.expressions.GTE'>: <function _rename>, <class 'sqlglot.expressions.Glob'>: <function _rename>, <class 'sqlglot.expressions.ILike'>: <function _rename>, <class 'sqlglot.expressions.ILikeAny'>: <function _rename>, <class 'sqlglot.expressions.IntDiv'>: <function _rename>, <class 'sqlglot.expressions.Is'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.JSONBContains'>: <function _rename>, <class 'sqlglot.expressions.JSONBExtract'>: <function _rename>, <class 'sqlglot.expressions.JSONBExtractScalar'>: <function _rename>, <class 'sqlglot.expressions.JSONExtract'>: <function _rename>, <class 'sqlglot.expressions.JSONExtractScalar'>: <function _rename>, <class 'sqlglot.expressions.Kwarg'>: <function _rename>, <class 'sqlglot.expressions.LT'>: <function _rename>, <class 'sqlglot.expressions.LTE'>: <function _rename>, <class 'sqlglot.expressions.Like'>: <function _rename>, <class 'sqlglot.expressions.LikeAny'>: <function _rename>, <class 'sqlglot.expressions.Mod'>: <function _rename>, <class 'sqlglot.expressions.Mul'>: <function _rename>, <class 'sqlglot.expressions.NEQ'>: <function _rename>, <class 'sqlglot.expressions.NullSafeEQ'>: <function _rename>, <class 'sqlglot.expressions.NullSafeNEQ'>: <function _rename>, <class 'sqlglot.expressions.Or'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Overlaps'>: <function _rename>, <class 'sqlglot.expressions.Pow'>: <function _rename>, <class 'sqlglot.expressions.SafeDPipe'>: <function _rename>, <class 'sqlglot.expressions.SimilarTo'>: <function _rename>, <class 'sqlglot.expressions.Slice'>: <function _rename>, <class 'sqlglot.expressions.Sub'>: <function _rename>, <class 'sqlglot.expressions.Abs'>: <function _rename>, <class 'sqlglot.expressions.AnyValue'>: <function _rename>, <class 'sqlglot.expressions.ApproxDistinct'>: <function _rename>, <class 'sqlglot.expressions.ApproxQuantile'>: <function _rename>, <class 'sqlglot.expressions.Array'>: <function inline_array_sql>, <class 'sqlglot.expressions.ArrayAgg'>: <function _rename>, <class 'sqlglot.expressions.ArrayAll'>: <function _rename>, <class 'sqlglot.expressions.ArrayAny'>: <function _rename>, <class 'sqlglot.expressions.ArrayConcat'>: <function _rename>, <class 'sqlglot.expressions.ArrayFilter'>: <function _rename>, <class 'sqlglot.expressions.ArrayJoin'>: <function _rename>, <class 'sqlglot.expressions.ArraySize'>: <function _rename>, <class 'sqlglot.expressions.ArraySort'>: <function _rename>, <class 'sqlglot.expressions.ArraySum'>: <function _rename>, <class 'sqlglot.expressions.ArrayUnionAgg'>: <function _rename>, <class 'sqlglot.expressions.Avg'>: <function _rename>, <class 'sqlglot.expressions.Case'>: <function _case_sql>, <class 'sqlglot.expressions.Cast'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.CastToStrType'>: <function _rename>, <class 'sqlglot.expressions.Ceil'>: <function _rename>, <class 'sqlglot.expressions.Coalesce'>: <function _rename>, <class 'sqlglot.expressions.Concat'>: <function _rename>, <class 'sqlglot.expressions.ConcatWs'>: <function _rename>, <class 'sqlglot.expressions.Count'>: <function _rename>, <class 'sqlglot.expressions.CountIf'>: <function _rename>, <class 'sqlglot.expressions.CurrentDate'>: <function _rename>, <class 'sqlglot.expressions.CurrentDatetime'>: <function _rename>, <class 'sqlglot.expressions.CurrentTime'>: <function _rename>, <class 'sqlglot.expressions.CurrentTimestamp'>: <function _rename>, <class 'sqlglot.expressions.CurrentUser'>: <function _rename>, <class 'sqlglot.expressions.Date'>: <function _rename>, <class 'sqlglot.expressions.DateAdd'>: <function _rename>, <class 'sqlglot.expressions.DateDiff'>: <function _rename>, <class 'sqlglot.expressions.DateFromParts'>: <function _rename>, <class 'sqlglot.expressions.DateStrToDate'>: <function _rename>, <class 'sqlglot.expressions.DateSub'>: <function _rename>, <class 'sqlglot.expressions.DateToDateStr'>: <function _rename>, <class 'sqlglot.expressions.DateToDi'>: <function _rename>, <class 'sqlglot.expressions.DateTrunc'>: <function _rename>, <class 'sqlglot.expressions.DatetimeAdd'>: <function _rename>, <class 'sqlglot.expressions.DatetimeDiff'>: <function _rename>, <class 'sqlglot.expressions.DatetimeSub'>: <function _rename>, <class 'sqlglot.expressions.DatetimeTrunc'>: <function _rename>, <class 'sqlglot.expressions.Day'>: <function _rename>, <class 'sqlglot.expressions.DayOfMonth'>: <function _rename>, <class 'sqlglot.expressions.DayOfWeek'>: <function _rename>, <class 'sqlglot.expressions.DayOfYear'>: <function _rename>, <class 'sqlglot.expressions.Decode'>: <function _rename>, <class 'sqlglot.expressions.DiToDate'>: <function _rename>, <class 'sqlglot.expressions.Encode'>: <function _rename>, <class 'sqlglot.expressions.Exp'>: <function _rename>, <class 'sqlglot.expressions.Explode'>: <function _rename>, <class 'sqlglot.expressions.Extract'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Floor'>: <function _rename>, <class 'sqlglot.expressions.FromBase'>: <function _rename>, <class 'sqlglot.expressions.FromBase64'>: <function _rename>, <class 'sqlglot.expressions.GenerateSeries'>: <function _rename>, <class 'sqlglot.expressions.Greatest'>: <function _rename>, <class 'sqlglot.expressions.GroupConcat'>: <function _rename>, <class 'sqlglot.expressions.Hex'>: <function _rename>, <class 'sqlglot.expressions.Hll'>: <function _rename>, <class 'sqlglot.expressions.If'>: <function _rename>, <class 'sqlglot.expressions.Initcap'>: <function _rename>, <class 'sqlglot.expressions.JSONFormat'>: <function _rename>, <class 'sqlglot.expressions.JSONObject'>: <function _rename>, <class 'sqlglot.expressions.LastDateOfMonth'>: <function _rename>, <class 'sqlglot.expressions.Least'>: <function _rename>, <class 'sqlglot.expressions.Left'>: <function _rename>, <class 'sqlglot.expressions.Length'>: <function _rename>, <class 'sqlglot.expressions.Levenshtein'>: <function _rename>, <class 'sqlglot.expressions.Ln'>: <function _rename>, <class 'sqlglot.expressions.Log'>: <function _rename>, <class 'sqlglot.expressions.Log10'>: <function _rename>, <class 'sqlglot.expressions.Log2'>: <function _rename>, <class 'sqlglot.expressions.LogicalAnd'>: <function _rename>, <class 'sqlglot.expressions.LogicalOr'>: <function _rename>, <class 'sqlglot.expressions.Lower'>: <function _rename>, <class 'sqlglot.expressions.MD5'>: <function _rename>, <class 'sqlglot.expressions.Map'>: <function _rename>, <class 'sqlglot.expressions.MapFromEntries'>: <function _rename>, <class 'sqlglot.expressions.MatchAgainst'>: <function _rename>, <class 'sqlglot.expressions.Max'>: <function _rename>, <class 'sqlglot.expressions.Min'>: <function _rename>, <class 'sqlglot.expressions.Month'>: <function _rename>, <class 'sqlglot.expressions.NextValueFor'>: <function _rename>, <class 'sqlglot.expressions.NumberToStr'>: <function _rename>, <class 'sqlglot.expressions.Nvl2'>: <function _rename>, <class 'sqlglot.expressions.OpenJSON'>: <function _rename>, <class 'sqlglot.expressions.ParameterizedAgg'>: <function _rename>, <class 'sqlglot.expressions.PercentileCont'>: <function _rename>, <class 'sqlglot.expressions.PercentileDisc'>: <function _rename>, <class 'sqlglot.expressions.Posexplode'>: <function _rename>, <class 'sqlglot.expressions.Quantile'>: <function _rename>, <class 'sqlglot.expressions.RangeN'>: <function _rename>, <class 'sqlglot.expressions.ReadCSV'>: <function _rename>, <class 'sqlglot.expressions.Reduce'>: <function _rename>, <class 'sqlglot.expressions.RegexpExtract'>: <function _rename>, <class 'sqlglot.expressions.RegexpILike'>: <function _rename>, <class 'sqlglot.expressions.RegexpLike'>: <function _rename>, <class 'sqlglot.expressions.RegexpSplit'>: <function _rename>, <class 'sqlglot.expressions.Repeat'>: <function _rename>, <class 'sqlglot.expressions.Right'>: <function _rename>, <class 'sqlglot.expressions.Round'>: <function _rename>, <class 'sqlglot.expressions.RowNumber'>: <function _rename>, <class 'sqlglot.expressions.SHA'>: <function _rename>, <class 'sqlglot.expressions.SHA2'>: <function _rename>, <class 'sqlglot.expressions.SafeConcat'>: <function _rename>, <class 'sqlglot.expressions.SafeDivide'>: <function _rename>, <class 'sqlglot.expressions.SetAgg'>: <function _rename>, <class 'sqlglot.expressions.SortArray'>: <function _rename>, <class 'sqlglot.expressions.Split'>: <function _rename>, <class 'sqlglot.expressions.Sqrt'>: <function _rename>, <class 'sqlglot.expressions.StandardHash'>: <function _rename>, <class 'sqlglot.expressions.StarMap'>: <function _rename>, <class 'sqlglot.expressions.Stddev'>: <function _rename>, <class 'sqlglot.expressions.StddevPop'>: <function _rename>, <class 'sqlglot.expressions.StddevSamp'>: <function _rename>, <class 'sqlglot.expressions.StrPosition'>: <function _rename>, <class 'sqlglot.expressions.StrToDate'>: <function _rename>, <class 'sqlglot.expressions.StrToTime'>: <function _rename>, <class 'sqlglot.expressions.StrToUnix'>: <function _rename>, <class 'sqlglot.expressions.Struct'>: <function _rename>, <class 'sqlglot.expressions.StructExtract'>: <function _rename>, <class 'sqlglot.expressions.Substring'>: <function _rename>, <class 'sqlglot.expressions.Sum'>: <function _rename>, <class 'sqlglot.expressions.TimeAdd'>: <function _rename>, <class 'sqlglot.expressions.TimeDiff'>: <function _rename>, <class 'sqlglot.expressions.TimeStrToDate'>: <function _rename>, <class 'sqlglot.expressions.TimeStrToTime'>: <function _rename>, <class 'sqlglot.expressions.TimeStrToUnix'>: <function _rename>, <class 'sqlglot.expressions.TimeSub'>: <function _rename>, <class 'sqlglot.expressions.TimeToStr'>: <function _rename>, <class 'sqlglot.expressions.TimeToTimeStr'>: <function _rename>, <class 'sqlglot.expressions.TimeToUnix'>: <function _rename>, <class 'sqlglot.expressions.TimeTrunc'>: <function _rename>, <class 'sqlglot.expressions.TimestampAdd'>: <function _rename>, <class 'sqlglot.expressions.TimestampDiff'>: <function _rename>, <class 'sqlglot.expressions.TimestampSub'>: <function _rename>, <class 'sqlglot.expressions.TimestampTrunc'>: <function _rename>, <class 'sqlglot.expressions.ToBase64'>: <function _rename>, <class 'sqlglot.expressions.ToChar'>: <function _rename>, <class 'sqlglot.expressions.Trim'>: <function _rename>, <class 'sqlglot.expressions.TryCast'>: <function _rename>, <class 'sqlglot.expressions.TsOrDiToDi'>: <function _rename>, <class 'sqlglot.expressions.TsOrDsAdd'>: <function _rename>, <class 'sqlglot.expressions.TsOrDsToDate'>: <function _rename>, <class 'sqlglot.expressions.TsOrDsToDateStr'>: <function _rename>, <class 'sqlglot.expressions.Unhex'>: <function _rename>, <class 'sqlglot.expressions.UnixToStr'>: <function _rename>, <class 'sqlglot.expressions.UnixToTime'>: <function _rename>, <class 'sqlglot.expressions.UnixToTimeStr'>: <function _rename>, <class 'sqlglot.expressions.Upper'>: <function _rename>, <class 'sqlglot.expressions.VarMap'>: <function _rename>, <class 'sqlglot.expressions.Variance'>: <function _rename>, <class 'sqlglot.expressions.VariancePop'>: <function _rename>, <class 'sqlglot.expressions.Week'>: <function _rename>, <class 'sqlglot.expressions.WeekOfYear'>: <function _rename>, <class 'sqlglot.expressions.When'>: <function _rename>, <class 'sqlglot.expressions.XMLTable'>: <function _rename>, <class 'sqlglot.expressions.Year'>: <function _rename>, <class 'sqlglot.expressions.Alias'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Between'>: <function _rename>, <class 'sqlglot.expressions.Boolean'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Column'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Distinct'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.In'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Interval'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Lambda'>: <function _lambda_sql>, <class 'sqlglot.expressions.Not'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Null'>: <function Python.Generator.<lambda>>, <class 'sqlglot.expressions.Ordered'>: <function _ordered_py>, <class 'sqlglot.expressions.Star'>: <function Python.Generator.<lambda>>}
SELECT_KINDS: Tuple[str, ...] = ()
INVERSE_TIME_MAPPING: Dict[str, str] = {}
INVERSE_TIME_TRIE: Dict = {}
@classmethod
def can_identify(text: str, identify: str | bool = 'safe') -> bool:
247    @classmethod
248    def can_identify(cls, text: str, identify: str | bool = "safe") -> bool:
249        """Checks if text can be identified given an identify option.
250
251        Args:
252            text: The text to check.
253            identify:
254                "always" or `True`: Always returns true.
255                "safe": True if the identifier is case-insensitive.
256
257        Returns:
258            Whether or not the given text can be identified.
259        """
260        if identify is True or identify == "always":
261            return True
262
263        if identify == "safe":
264            return not cls.case_sensitive(text)
265
266        return False

Checks if text can be identified given an identify option.

Arguments:
  • text: The text to check.
  • identify: "always" or True: Always returns true. "safe": True if the identifier is case-insensitive.
Returns:

Whether or not the given text can be identified.

QUOTE_START = "'"
QUOTE_END = "'"
IDENTIFIER_START = '"'
IDENTIFIER_END = '"'
STRING_ESCAPE = '\\'
IDENTIFIER_ESCAPE = '"'
BIT_START: Optional[str] = None
BIT_END: Optional[str] = None
HEX_START: Optional[str] = None
HEX_END: Optional[str] = None
BYTE_START: Optional[str] = None
BYTE_END: Optional[str] = None
RAW_START: Optional[str] = None
RAW_END: Optional[str] = None
Inherited Members
sqlglot.generator.Generator
Generator
NULL_ORDERING_SUPPORTED
LOCKING_READS_SUPPORTED
EXPLICIT_UNION
WRAP_DERIVED_VALUES
CREATE_FUNCTION_RETURN_AS
MATCHED_BY_SOURCE
SINGLE_STRING_INTERVAL
INTERVAL_ALLOWS_PLURAL_FORM
TABLESAMPLE_WITH_METHOD
TABLESAMPLE_SIZE_IS_PERCENT
LIMIT_FETCH
RENAME_TABLE_WITH_DB
GROUPINGS_SEP
INDEX_ON
JOIN_HINTS
TABLE_HINTS
IS_BOOL_ALLOWED
TYPE_MAPPING
STAR_MAPPING
TIME_PART_SINGULARS
TOKEN_MAPPING
STRUCT_DELIMITER
PARAMETER_TOKEN
PROPERTIES_LOCATION
RESERVED_KEYWORDS
WITH_SEPARATED_COMMENTS
UNWRAPPED_INTERVAL_VALUES
SENTINEL_LINE_BREAK
INDEX_OFFSET
UNNEST_COLUMN_ONLY
ALIAS_POST_TABLESAMPLE
IDENTIFIERS_CAN_START_WITH_DIGIT
STRICT_STRING_CONCAT
NORMALIZE_FUNCTIONS
NULL_ORDERING
pretty
identify
normalize
pad
unsupported_level
max_unsupported
leading_comma
max_text_width
comments
normalize_functions
unsupported_messages
generate
unsupported
sep
seg
pad_comment
maybe_comment
wrap
no_identify
normalize_func
indent
sql
uncache_sql
cache_sql
characterset_sql
column_sql
columnposition_sql
columndef_sql
columnconstraint_sql
autoincrementcolumnconstraint_sql
compresscolumnconstraint_sql
generatedasidentitycolumnconstraint_sql
notnullcolumnconstraint_sql
primarykeycolumnconstraint_sql
uniquecolumnconstraint_sql
createable_sql
create_sql
clone_sql
describe_sql
prepend_ctes
with_sql
cte_sql
tablealias_sql
bitstring_sql
hexstring_sql
bytestring_sql
rawstring_sql
datatypesize_sql
datatype_sql
directory_sql
delete_sql
drop_sql
except_sql
except_op
fetch_sql
filter_sql
hint_sql
index_sql
identifier_sql
inputoutputformat_sql
national_sql
partition_sql
properties_sql
root_properties
properties
with_properties
locate_properties
property_sql
likeproperty_sql
fallbackproperty_sql
journalproperty_sql
freespaceproperty_sql
checksumproperty_sql
mergeblockratioproperty_sql
datablocksizeproperty_sql
blockcompressionproperty_sql
isolatedloadingproperty_sql
lockingproperty_sql
withdataproperty_sql
insert_sql
intersect_sql
intersect_op
introducer_sql
pseudotype_sql
onconflict_sql
returning_sql
rowformatdelimitedproperty_sql
withtablehint_sql
indextablehint_sql
table_sql
tablesample_sql
pivot_sql
tuple_sql
update_sql
values_sql
var_sql
into_sql
from_sql
group_sql
having_sql
join_sql
lambda_sql
lateral_sql
limit_sql
offset_sql
setitem_sql
set_sql
pragma_sql
lock_sql
literal_sql
escape_str
loaddata_sql
null_sql
boolean_sql
order_sql
cluster_sql
distribute_sql
sort_sql
ordered_sql
matchrecognize_sql
query_modifiers
offset_limit_modifiers
after_having_modifiers
after_limit_modifiers
select_sql
schema_sql
schema_columns_sql
star_sql
parameter_sql
sessionparameter_sql
placeholder_sql
subquery_sql
qualify_sql
union_sql
union_op
unnest_sql
where_sql
window_sql
partition_by_sql
windowspec_sql
withingroup_sql
between_sql
bracket_sql
safebracket_sql
all_sql
any_sql
exists_sql
case_sql
constraint_sql
nextvaluefor_sql
extract_sql
trim_sql
safeconcat_sql
check_sql
foreignkey_sql
primarykey_sql
if_sql
matchagainst_sql
jsonkeyvalue_sql
jsonobject_sql
openjsoncolumndef_sql
openjson_sql
in_sql
in_unnest_op
interval_sql
return_sql
reference_sql
anonymous_sql
paren_sql
neg_sql
not_sql
alias_sql
aliases_sql
attimezone_sql
add_sql
and_sql
connector_sql
bitwiseand_sql
bitwiseleftshift_sql
bitwisenot_sql
bitwiseor_sql
bitwiserightshift_sql
bitwisexor_sql
cast_sql
currentdate_sql
collate_sql
command_sql
comment_sql
mergetreettlaction_sql
mergetreettl_sql
transaction_sql
commit_sql
rollback_sql
altercolumn_sql
renametable_sql
altertable_sql
droppartition_sql
addconstraint_sql
distinct_sql
ignorenulls_sql
respectnulls_sql
intdiv_sql
dpipe_sql
safedpipe_sql
div_sql
overlaps_sql
distance_sql
dot_sql
eq_sql
escape_sql
glob_sql
gt_sql
gte_sql
ilike_sql
ilikeany_sql
is_sql
like_sql
likeany_sql
similarto_sql
lt_sql
lte_sql
mod_sql
mul_sql
neq_sql
nullsafeeq_sql
nullsafeneq_sql
or_sql
slice_sql
sub_sql
trycast_sql
use_sql
binary
function_fallback_sql
func
format_args
text_width
format_time
expressions
op_expressions
naked_property
set_operation
tag_sql
token_sql
userdefinedfunction_sql
joinhint_sql
kwarg_sql
when_sql
merge_sql
tochar_sql
dictproperty_sql
dictrange_sql
dictsubproperty_sql
oncluster_sql
clusteredbyproperty_sql