dns-oarc/packetq
1
0
Fork 0
Code Issues Activity
packetq/src/sql.cpp

2697 lines
78 KiB
C++
Raw Normal View History

Adding upstream version 1.7.3+dfsg. Signed-off-by: Daniel Baumann <daniel@debian.org>
2025-02-10 05:34:23 +01:00
/*
* Copyright (c) 2017-2024 OARC, Inc.
* Copyright (c) 2011-2017, IIS - The Internet Foundation in Sweden
* All rights reserved.
*
* This file is part of PacketQ.
*
* PacketQ is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* PacketQ is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with PacketQ. If not, see <http://www.gnu.org/licenses/>.
*/
#include "config.h"
#include "sql.h"
#include "output.h"
#include "packet_handler.h"
#include "packetq.h"
#include "reader.h"
#include <unordered_map>
#include <utility>
#include <vector>
#ifdef WIN32
#include <windows.h>
#endif
#ifdef HAVE_LIBMAXMINDDB
#include <maxminddb.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
static MMDB_s* __cc_mmdb = 0;
static MMDB_s* __asn_mmdb = 0;
#endif
namespace packetq {
bool verbose = false;
int g_allocs = 0;
Column* Table::add_column(const char* name, const char* type, int id, bool hidden)
{
if (!type)
return add_column(name, Coltype::_text, id, hidden);
else if (strcmp(type, "bool") == 0)
return add_column(name, Coltype::_bool, id, hidden);
else if (strcmp(type, "int") == 0)
return add_column(name, Coltype::_int, id, hidden);
else if (strcmp(type, "float") == 0)
return add_column(name, Coltype::_float, id, hidden);
else
return add_column(name, Coltype::_text, id, hidden);
}
Column* Table::add_column(const char* name, Coltype::Type type, int id, bool hidden)
{
Column* col = new Column(name, type, id, hidden);
col->m_offset = Table::align(m_curpos, col->m_def.m_align);
m_curpos = col->m_offset + col->m_def.m_size;
if (type == Coltype::_text)
m_text_column_offsets.push_back(col->m_offset);
m_cols.push_back(col);
return col;
}
void Table::delete_row(Row* row)
{
row->decref_text_columns(m_text_column_offsets);
m_row_allocator->deallocate(row);
}
Row* Table::create_row()
{
if (!m_row_allocator) {
m_rsize = sizeof(Row) - ROW_DUMMY_SIZE; // exclude the dummy
m_dsize = m_curpos;
m_row_allocator = new Allocator<Row>(m_rsize + m_dsize, 10000);
}
Row* r = m_row_allocator->allocate();
r->zero_text_columns(m_text_column_offsets);
return r;
}
void Table::add_row(Row* row)
{
m_rows.push_back(row);
}
int g_comp = 0;
void Ordering_terms::compile(const std::vector<Table*>& tables, const std::vector<int>& search_order, Query& q)
{
for (auto it = m_terms.begin(); it != m_terms.end(); it++) {
OP* op = it->m_op;
it->m_op = op->compile(tables, search_order, q);
}
}
class Sorter {
public:
Sorter(Ordering_terms& order)
: m_order(order)
{
}
bool operator()(Row* ia, Row* ib)
{
// this works under the assumption that the ordering terms have
// been compiled with only one table so the row index i is 0
Row** ia_rows = &ia;
Row** ib_rows = &ib;
for (auto it = m_order.m_terms.begin(); it != m_order.m_terms.end(); it++) {
g_comp++;
OP* op = it->m_op;
op->evaluate(it->m_asc ? ia_rows : ib_rows, m_a);
op->evaluate(it->m_asc ? ib_rows : ia_rows, m_b);
int res = m_a.cmp(m_b);
if (res < 0)
return true;
if (res > 0)
return false;
}
return false;
}
bool eq(Row* ia, Row* ib)
{
Row** ia_rows = &ia;
Row** ib_rows = &ib;
for (auto it = m_order.m_terms.begin(); it != m_order.m_terms.end(); it++) {
g_comp++;
OP* op = it->m_op;
op->evaluate(it->m_asc ? ia_rows : ib_rows, m_a);
op->evaluate(it->m_asc ? ib_rows : ia_rows, m_b);
int res = m_a.cmp(m_b);
if (res != 0)
return false;
}
return true;
}
int cmp(Row* ia, Row* ib)
{
Row** ia_rows = &ia;
Row** ib_rows = &ib;
for (auto it = m_order.m_terms.begin(); it != m_order.m_terms.end(); it++) {
g_comp++;
OP* op = it->m_op;
op->evaluate(it->m_asc ? ia_rows : ib_rows, m_a);
op->evaluate(it->m_asc ? ib_rows : ia_rows, m_b);
int res = m_a.cmp(m_b);
if (res != 0)
return res;
}
return 0;
}
Ordering_terms& m_order;
Variant m_a, m_b;
};
struct Stki {
int s;
int l;
int b;
};
struct Spkt {
Variant cache;
int row;
};
class Per_sort {
public:
struct Tlink {
Tlink()
: m_next(0)
, row(0)
, m_eq(0)
{
}
Tlink* get_eq() { return m_eq; }
void reset()
{
m_next = 0;
row = 0;
m_eq = 0;
}
void add_eq(Tlink* o)
{
if (!o->m_eq) {
// add single
if (!m_eq) {
// as first
m_eq = o;
o->m_eq_last = o;
return;
}
// to list
m_eq->m_eq_last->m_eq = o;
m_eq->m_eq_last = o;
return;
} else {
// add list
if (!m_eq) {
// as first
m_eq = o;
o->m_eq_last = o->m_eq->m_eq_last;
return;
}
// to list
m_eq->m_eq_last->m_eq = o;
m_eq->m_eq_last = o->m_eq->m_eq_last;
}
}
union {
Tlink* m_next;
Tlink* m_eq_last;
};
Row* row;
Variant cache;
private:
Tlink* m_eq;
};
struct List {
public:
void reset()
{
m_size = 0;
m_fl[0] = 0;
m_fl[1] = 0;
}
int m_size;
Tlink* m_fl[2];
};
Per_sort(Table& table, Ordering_terms& order)
: m_sorter(order)
, m_table(table)
{
m_escalate_sort = order.m_terms.size() > 1;
m_asc = order.m_terms.begin()->m_asc ? 1 : -1;
m_op = order.m_terms.begin()->m_op;
memset(m_groups, 0, sizeof(m_groups));
m_current.reset();
}
inline bool add_to_list(List& list, Tlink* t)
{
if (list.m_fl[0] == 0) {
list.m_fl[0] = list.m_fl[1] = t;
list.m_size = 1;
return true;
}
list.m_fl[1]->m_next = t;
list.m_fl[1] = t;
list.m_size++;
return true;
}
inline bool insert_into_list(List& list, Tlink* t)
{
if (list.m_fl[0] == 0) {
list.m_fl[0] = list.m_fl[1] = t;
list.m_size = 1;
return true;
}
int cmp0 = cmp(t, list.m_fl[0]);
if (cmp0 == 0) {
list.m_fl[0]->add_eq(t);
return true;
}
if (cmp0 < 0) {
t->m_next = list.m_fl[0];
list.m_fl[0] = t;
list.m_size++;
return true;
}
int cmp1 = cmp(t, list.m_fl[1]);
if (cmp1 == 0) {
list.m_fl[1]->add_eq(t);
return true;
}
if (cmp1 > 0) {
list.m_fl[1]->m_next = t;
list.m_fl[1] = t;
list.m_size++;
return true;
}
return false;
}
int cmp(Tlink* a, Tlink* b)
{
int cmp = a->cache.cmp(b->cache) * m_asc;
if (cmp != 0 || !m_escalate_sort)
return cmp;
return m_sorter.cmp(a->row, b->row);
}
inline void add(Tlink* t)
{
if (!insert_into_list(m_current, t)) {
insert_list(m_current);
m_current.reset();
insert_into_list(m_current, t);
}
}
inline void insert_list(List& l)
{
unsigned int size = l.m_size;
int offs = 0;
size >>= 1;
while (size != 0) {
offs++;
size >>= 1;
}
if (m_groups[offs].m_size != 0) {
List m = merge(m_groups[offs], l);
m_groups[offs].reset();
insert_list(m);
} else
m_groups[offs] = l;
}
List merge(List& l1, List& l2)
{
List r;
r.reset();
Tlink* a = l1.m_fl[0];
Tlink* b = l2.m_fl[0];
if (!a)
return l2;
if (!b)
return l1;
int size = l1.m_size + l2.m_size;
while (a && b) {
int c = cmp(a, b);
if (c == 0) {
Tlink* a2 = a;
Tlink* b2 = b;
a = a->m_next;
b = b->m_next;
a2->m_next = 0;
b2->m_next = 0;
a2->add_eq(b2);
add_to_list(r, a2);
size--;
} else if (c < 0) {
Tlink* a2 = a;
a = a->m_next;
a2->m_next = 0;
add_to_list(r, a2);
} else {
Tlink* b2 = b;
b = b->m_next;
b2->m_next = 0;
add_to_list(r, b2);
}
}
if (a) {
r.m_fl[1]->m_next = a;
r.m_fl[1] = l1.m_fl[1];
}
if (b) {
r.m_fl[1]->m_next = b;
r.m_fl[1] = l2.m_fl[1];
}
l1.reset();
l2.reset();
r.m_size = size;
return r;
}
void sort()
{
int table_size = (int)m_table.m_rows.size();
if (table_size <= 1)
return;
auto it = m_table.m_rows.begin();
Tlink* links = new Tlink[table_size];
int i;
for (i = 0; i < table_size; i++) {
Tlink& r = links[i];
r.reset();
r.row = *it;
// &row works under the assumption that m_op has been compiled with
// this table only so row index is 0
m_op->evaluate(&r.row, r.cache);
it++;
add(&r);
}
if (m_current.m_size)
insert_list(m_current);
List result;
result.reset();
for (i = 0; i < sizeof(m_groups) / sizeof(List); i++) {
if (m_groups[i].m_size > 0) {
result = merge(result, m_groups[i]);
m_groups[i].reset();
}
}
Tlink* p = result.m_fl[0];
it = m_table.m_rows.begin();
while (p) {
*it++ = p->row;
Tlink* e = p->get_eq();
while (e) {
*it++ = e->row;
e = e->get_eq();
}
p = p->m_next;
};
delete[] links;
}
OP* m_op;
Sorter m_sorter;
bool m_escalate_sort;
int m_asc;
Table& m_table;
List m_groups[32];
List m_current;
};
void Table::per_sort(Ordering_terms& order)
{
Per_sort sort(*this, order);
sort.sort();
return;
}
void Table::merge_sort(Ordering_terms& order)
{
Sorter sorter(order);
bool escalate_sort = order.m_terms.size() > 1;
int asc = order.m_terms.begin()->m_asc ? 1 : -1;
OP* op = order.m_terms.begin()->m_op;
int table_size = (int)m_rows.size();
if (table_size <= 1)
return;
Row** row_ptrs = new Row*[table_size];
Spkt* spktpool = new Spkt[table_size * 2];
Spkt* rows[2];
rows[0] = spktpool;
rows[1] = &spktpool[table_size];
auto it = m_rows.begin();
int i = 0;
Spkt* r = rows[0];
for (i = 0; i < table_size; i += 2) {
row_ptrs[i] = *it++;
r[i].row = i;
// &row works under the assumption that m_op has been compiled with
// this table only so row index is 0
op->evaluate(&row_ptrs[r[i].row], r[i].cache);
if (i + 1 < table_size) {
row_ptrs[i + 1] = *it++;
r[i + 1].row = i + 1;
op->evaluate(&row_ptrs[r[i + 1].row], r[i + 1].cache);
}
}
int swap = 0;
Stki stack[64];
Stki* sp = stack;
sp->s = 0;
sp->l = 2;
sp->b = 1;
rows[1][0] = rows[0][1];
rows[1][1] = rows[0][0];
sp--;
int npos = 0;
while (true) {
int start, len;
if (sp > stack && sp->l == sp[-1].l) {
// two equal size -> merge
len = sp->l <<= 1;
start = sp[-1].s;
swap = sp[-1].b;
sp--;
sp--;
} else {
start = npos;
npos += 2;
swap = 0;
len = 2;
}
int cnt = start + len > table_size ? table_size - start : len;
Spkt* s = rows[swap];
Spkt* d = rows[1 - swap];
if (cnt > 0) {
int p1 = start;
int p2 = start + (len >> 1);
int l1 = len >> 1;
int l2 = l1;
if (p1 + l1 > table_size)
l1 = table_size - p1;
if (p2 + l2 > table_size)
l2 = table_size - p2;
i = start;
while (cnt-- > 0) {
if (l1 <= 0) {
d[i++] = s[p2++];
} else if (l2 <= 0) {
d[i++] = s[p1++];
} else {
int cmp = s[p1].cache.cmp(s[p2].cache) * asc;
if (cmp < 0 || (cmp == 0 && escalate_sort && sorter(row_ptrs[s[p1].row], row_ptrs[s[p2].row]))) {
l1--;
d[i++] = s[p1++];
} else {
l2--;
d[i++] = s[p2++];
}
}
}
}
++sp;
sp->l = len;
sp->s = start;
sp->b = 1 - swap;
if (len > table_size) {
for (it = m_rows.begin(); it != m_rows.end(); it++) {
*it = row_ptrs[(d++)->row];
}
break;
}
}
delete[] row_ptrs;
delete[] spktpool;
return;
}
void Table::limit(int limit, int offset)
{
int count = 0;
auto e = m_rows.end();
for (auto it = m_rows.begin(); it != m_rows.end(); it++) {
if (e != m_rows.end()) {
delete_row(*e);
m_rows.erase(e);
e = m_rows.end();
}
int l = count++;
if (l < offset || l >= offset + limit) {
e = it;
}
}
if (e != m_rows.end()) {
delete_row(*e);
m_rows.erase(e);
}
}
void printrep(int n, char c)
{
if (n >= 3000)
return;
char buf[3000];
int i;
for (i = 0; i < n; i++)
buf[i] = c;
buf[i] = 0;
printf("%s", buf);
}
void Table::xml()
{
g_output.reset();
int cols = (int)m_cols.size();
g_output.add_string("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
g_output.add_string("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n");
g_output.add_string("<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\n<head>\n <title>");
g_output.add_string(m_name.c_str());
g_output.add_string("</title>\n");
g_output.add_string("<style type=\"text/css\">\n");
g_output.add_string(" th.int { color: #0F0C00; }\n");
g_output.add_string(" th.float { color: #0F0900; }\n");
g_output.add_string(" th.text { color: #0F0600; }\n");
g_output.add_string(" th.bool { color: #0C0900; }\n");
g_output.add_string("</style>\n");
g_output.add_string("</head>\n");
g_output.add_string("<body>\n");
g_output.add_string("<table>\n");
g_output.add_string("<tr>");
for (int i = 0; i < cols; i++) {
if (m_cols[i]->m_hidden)
continue;
const char* t = "";
switch (m_cols[i]->m_type) {
case (Coltype::_float):
t = "float";
break;
case (Coltype::_int):
t = "int";
break;
case (Coltype::_text):
t = "text";
break;
case (Coltype::_bool):
t = "bool";
break;
}
g_output.add_string("<th class=\"");
g_output.add_string(t);
g_output.add_string("\">");
g_output.add_string(m_cols[i]->m_name.c_str());
g_output.add_string("</th>");
}
g_output.add_string("</tr>\n");
for (auto it = m_rows.begin(); it != m_rows.end(); it++) {
g_output.add_string("<tr>");
Row* r = *it;
Variant v;
for (int i = 0; i < cols; i++) {
Column* c = m_cols[i];
if (c->m_hidden)
continue;
int offset = c->m_offset;
static const int bufsize = 100;
char buf[bufsize];
g_output.add_string("<td>");
switch (c->m_type) {
case Coltype::_bool:
g_output.add_string(r->access_column<bool_column>(offset) ? "1" : "0");
break;
case Coltype::_int:
snprintf(buf, bufsize, "%i", r->access_column<int_column>(offset));
g_output.add_string(buf);
break;
case Coltype::_float:
snprintf(buf, bufsize, "%g", r->access_column<float_column>(offset));
g_output.add_string(buf);
break;
case Coltype::_text:
g_output.add_string(r->access_column<text_column>(offset)->data);
break;
}
g_output.add_string("</td> ");
}
g_output.add_string("</tr>\n");
}
g_output.add_string("</table>\n");
g_output.add_string("</body>\n");
g_output.add_string("</html>\n");
g_output.print();
}
void Table::json(bool trailing_comma)
{
g_output.reset();
int cols = (int)m_cols.size();
g_output.add_string(" {\n ");
g_output.add_q_string("table_name");
g_output.add_string(": ");
g_output.add_q_string(m_name.c_str());
g_output.add_string(",\n ");
g_output.add_q_string("query");
g_output.add_string(": ");
g_output.add_q_string(m_qstring.c_str());
g_output.add_string(",\n ");
g_output.add_q_string("head");
g_output.add_string(": [");
bool append_comma = false;
for (int i = 0; i < cols; i++) {
if (m_cols[i]->m_hidden)
continue;
if (append_comma)
g_output.add_string(",\n");
else
g_output.add_string("\n");
append_comma = true;
g_output.add_string(" { ");
g_output.add_q_string("name");
g_output.add_string(": ");
g_output.add_q_string(m_cols[i]->m_name.c_str());
g_output.add_string(",");
g_output.add_q_string("type");
g_output.add_string(": ");
const char* t = "";
switch (m_cols[i]->m_type) {
case (Coltype::_float):
t = "float";
break;
case (Coltype::_int):
t = "int";
break;
case (Coltype::_text):
t = "text";
break;
case (Coltype::_bool):
t = "bool";
break;
}
g_output.add_q_string(t);
g_output.add_string(" }");
}
g_output.add_string("\n ],\n ");
g_output.add_q_string("data");
g_output.add_string(": [");
bool outer_comma = false;
for (auto it = m_rows.begin(); it != m_rows.end(); it++) {
if (outer_comma)
g_output.add_string(",\n [");
else
g_output.add_string("\n [");
outer_comma = true;
bool comma = false;
Row* r = *it;
for (int i = 0; i < cols; i++) {
Column* c = m_cols[i];
if (c->m_hidden)
continue;
if (comma)
g_output.add_string(",");
comma = true;
int offset = c->m_offset;
static const int bufsize = 100;
char buf[bufsize];
switch (c->m_type) {
case Coltype::_bool:
g_output.add_string(r->access_column<bool_column>(offset) ? "1" : "0");
break;
case Coltype::_int:
snprintf(buf, bufsize, "%i", r->access_column<int_column>(offset));
g_output.add_string(buf);
break;
case Coltype::_float:
snprintf(buf, bufsize, "%g", r->access_column<float_column>(offset));
g_output.add_string(buf);
break;
case Coltype::_text:
g_output.add_q_string(r->access_column<text_column>(offset)->data);
break;
}
}
g_output.add_string("]");
}
g_output.add_string("\n ]\n");
if (trailing_comma) {
g_output.add_string(" },\n");
} else {
g_output.add_string(" }\n");
}
g_output.print();
}
std::string csv_qoute_string(const std::string& s)
{
std::string r = "\"";
int len = s.length();
for (int i = 0; i < len; i++) {
if (s[i] == '"') {
r += '"';
}
r += s[i];
}
r += '"';
return r;
}
void Table::csv(bool format)
{
int cols = (int)m_cols.size();
std::vector<int> col_len(cols);
for (int i = 0; i < cols; i++)
col_len[i] = 0;
int max = 0;
char* tmp = 0;
if (format) {
for (auto it = m_rows.begin(); it != m_rows.end(); it++) {
Row* r = *it;
for (int i = 0; i < cols; i++) {
Column* c = m_cols[i];
if (c->m_hidden)
continue;
int len = 0;
int offset = c->m_offset;
static const int bufsize = 100;
char buf[bufsize];
switch (c->m_type) {
case Coltype::_bool:
len = 1;
break;
case Coltype::_int:
snprintf(buf, bufsize, "%i", r->access_column<int_column>(offset));
len = strlen(buf);
break;
case Coltype::_float:
snprintf(buf, bufsize, "%g", r->access_column<float_column>(offset));
len = strlen(buf);
break;
case Coltype::_text:
len = csv_qoute_string(r->access_column<text_column>(offset)->data).length();
break;
}
len++;
if (len > col_len[i])
col_len[i] = len;
if (len > max)
max = len;
}
}
for (int i = 0; i < cols; i++) {
if (m_cols[i]->m_hidden)
continue;
int l = csv_qoute_string(m_cols[i]->m_name).length();
l++;
if (l > col_len[i])
col_len[i] = l;
if (l > max)
max = l;
}
tmp = new char[max + 1];
for (int i = 0; i < max; i++)
tmp[i] = 32;
tmp[max] = 0;
}
for (int i = 0; i < cols; i++) {
if (m_cols[i]->m_hidden)
continue;
printf("%s", csv_qoute_string(m_cols[i]->m_name).c_str());
if (i < cols - 1) {
size_t pos = csv_qoute_string(m_cols[i]->m_name).length() + max - col_len[i] + 1;
if (format && pos < max) {
printf("%s,", &tmp[pos]);
} else {
printf(",");
}
}
}
printf("\n");
for (auto it = m_rows.begin(); it != m_rows.end(); it++) {
Row* r = *it;
for (int i = 0; i < cols; i++) {
Column* c = m_cols[i];
if (c->m_hidden)
continue;
int offset = c->m_offset;
static const int bufsize = 100;
char buf[bufsize];
std::string out;
switch (c->m_type) {
case Coltype::_bool:
out = r->access_column<bool_column>(offset) ? "1" : "0";
break;
case Coltype::_int:
snprintf(buf, bufsize, "%i", r->access_column<int_column>(offset));
out = buf;
break;
case Coltype::_float:
snprintf(buf, bufsize, "%g", r->access_column<float_column>(offset));
out = buf;
break;
case Coltype::_text:
out = csv_qoute_string(r->access_column<text_column>(offset)->data);
break;
}
fputs(out.c_str(), stdout);
if (i < cols - 1) {
if (format) {
printf("%s,", &tmp[out.length() + max - col_len[i] + 1]);
} else {
printf(",");
}
}
}
printf("\n");
}
delete[] tmp;
}
void Table::dump()
{
int cols = (int)m_cols.size();
int width = 25;
char fmti[40];
snprintf(fmti, sizeof(fmti) - 1, "%%%dd |", width);
fmti[39] = 0;
char fmtd[40];
snprintf(fmtd, sizeof(fmtd) - 1, "%%%dg |", width);
fmtd[39] = 0;
char fmts[40];
snprintf(fmts, sizeof(fmts) - 1, "%%%ds |", width);
fmts[39] = 0;
printf("Table::dump() table:%s cols:%d\n", m_name.c_str(), cols);
printrep((width + 2) * cols + 1, '-');
printf("\n");
printf("|");
for (int i = 0; i < cols; i++)
printf(fmti, m_cols[i]->m_type);
printf("\n");
printf("|");
for (int i = 0; i < cols; i++)
printf(fmts, m_cols[i]->m_name.c_str());
printf("\n");
printrep((width + 2) * cols + 1, '*');
printf("\n");
for (auto it = m_rows.begin(); it != m_rows.end(); it++) {
printf("|");
Row* r = *it;
for (int i = 0; i < cols; i++) {
Column* c = m_cols[i];
int offset = c->m_offset;
switch (c->m_type) {
case Coltype::_bool:
printf(fmts, r->access_column<bool_column>(offset) ? "1" : "0");
break;
case Coltype::_int:
printf(fmti, r->access_column<int_column>(offset));
break;
case Coltype::_float:
printf(fmtd, r->access_column<float_column>(offset));
break;
case Coltype::_text:
printf(fmts, r->access_column<text_column>(offset));
break;
}
}
printf("\n");
}
printrep((width + 2) * cols + 1, '-');
printf("\n");
}
class Parser {
private:
Token::Type m_last;
public:
std::list<Token> m_tokens;
typedef std::list<Token>::iterator Lit;
Parser()
{
m_last = Token::_invalid;
}
void push(Token::Type type, const char* string)
{
if (!(type == Token::_semicolon && m_last == Token::_semicolon))
m_tokens.push_back(Token(type, string));
m_last = type;
}
void dump()
{
for (auto it = m_tokens.begin(); it != m_tokens.end(); it++) {
printf("Type %d: %s\n", it->get_type(), it->get_token());
}
}
bool analyze(Query& q)
{
auto it = m_tokens.begin();
bool ok = true;
while (ok) {
ok = false;
if (get_sample_stmt(q, it))
ok = true;
if (get_select_stmt(q, it))
ok = true;
}
return true;
}
bool get_sample_stmt(Query& q, Lit& i_iter)
{
Lit it = i_iter;
if (!is(it, Token::_label, "sample"))
return false;
it++;
if (!is(it, Token::_number))
return false;
int sample = atoi(it->get_token());
it++;
if (!is(it, Token::_semicolon))
return false;
it++;
q.m_sample = sample;
i_iter = it;
return true;
}
bool get_select_stmt(Query& q, Lit& i_iter)
{
Lit it = i_iter;
if (!get_select_core(q, it)) {
return false;
}
get_from(q, it);
get_where(q, it);
get_group_by(q, it);
get_order_by(q, it);
get_limit(q, it);
get_as(q, it);
if (!is(it, Token::_semicolon)) {
throw Error("Expected ';' but found '%s' !", it->get_token());
}
it++;
i_iter = it;
return true;
}
bool is(Lit& it, Token::Type type, const char* str = 0)
{
if (!str)
return (it->get_type() == type);
return (it->get_type() == type && cmpi(it->get_token(), str));
}
OP* get_result_column(std::list<Token>::iterator& it)
{
OP* res = 0;
Lit save = it;
if (is(it, Token::_op, "*")) {
it++;
return new OP(Token(Token::_column, "*"));
}
if (it->get_type() == Token::_label) {
std::string table = it->get_token();
it++;
if (is(it, Token::_op, ".")) {
it++;
if (is(it, Token::_op, "*")) {
it++;
std::string c = table + ".*";
return new OP(Token(Token::_column, c.c_str()));
}
it = save;
return 0;
}
}
it = save;
if ((res = get_expr(it, 0))) {
save = it;
if (is(it, Token::_label, "as")) {
it++;
if (is(it, Token::_label)) {
res->m_name = it->get_token();
it++;
} else
it = save;
}
}
return res;
// check for valid table
}
bool get_select_core(Query& q, Lit& it)
{
Lit save = it;
if (!is(it, Token::_label, "select"))
return false;
it++;
bool again = true;
bool success = true;
while (again) {
OP* op;
if ((op = get_result_column(it))) {
q.m_select.push_back(op);
} else {
success = false;
break;
}
if (is(it, Token::_op, ","))
it++;
else
again = false;
}
if (success)
return true;
it = save;
return false;
}
bool get_ordering_terms(Ordering_terms& ordering, std::list<Token>::iterator& it)
{
OP* op;
while ((op = get_expr(it, 0))) {
bool asc = true;
if (it->get_type() == Token::_label) {
if (cmpi(it->get_token(), "asc")) {
// default
} else if (cmpi(it->get_token(), "desc")) {
asc = false;
} else if (cmpi(it->get_token(), "collate")) {
throw Error("unhandled option:collate");
} else {
ordering.m_terms.push_back(Ordering_terms::OP_dir(op, asc));
break;
}
it++;
}
ordering.m_terms.push_back(Ordering_terms::OP_dir(op, asc));
if (!is(it, Token::_op, ","))
break;
it++;
}
return true;
}
bool get_group_by(Query& q, Lit& it)
{
if (!is(it, Token::_label, "group")) {
return true;
}
it++;
if (!is(it, Token::_label, "by"))
return false;
it++;
bool res = get_ordering_terms(q.m_group_by, it);
get_having(q, it);
return res;
}
bool get_as(Query& q, Lit& it)
{
if (!is(it, Token::_label, "as")) {
return true;
}
it++;
if (!is(it, Token::_label))
return false;
q.m_result->m_name = it->get_token();
it++;
return true;
}
bool get_order_by(Query& q, Lit& it)
{
if (!is(it, Token::_label, "order")) {
return true;
}
it++;
if (!is(it, Token::_label, "by"))
return false;
it++;
return get_ordering_terms(q.m_order_by, it);
}
bool get_limit(Query& q, Lit& it)
{
Lit save = it;
if (!is(it, Token::_label, "limit")) {
return true;
}
it++;
if (!is(it, Token::_number)) {
it = save;
throw Error("non numeric operand to limit");
}
q.m_limit = atoi(it->get_token());
it++;
save = it;
if (!is(it, Token::_label, "offset")) {
return true;
}
it++;
if (!is(it, Token::_number)) {
it = save;
throw Error("non numeric operand to offset");
}
q.m_offset = atoi(it->get_token());
it++;
return true;
}
bool get_having(Query& q, Lit& it)
{
Lit save = it;
if (!is(it, Token::_label, "having"))
return true;
it++;
OP* res = 0;
if ((res = get_expr(it, 0))) {
q.m_having = res;
return true;
}
it = save;
return false;
}
bool get_where(Query& q, Lit& it)
{
Lit save = it;
if (!is(it, Token::_label, "where"))
return true;
it++;
OP* res = 0;
if ((res = get_expr(it, 0))) {
q.m_where = res;
return true;
}
it = save;
return false;
}
bool get_from(Query& q, Lit& it)
{
if (!is(it, Token::_label, "from"))
return false;
it++;
if (it->get_type() == Token::_label) {
const char* name = it->get_token();
if (get_packet_handler(name)) {
q.m_from_name = name;
it++;
return true;
} else
throw Error("Error in from statement, table '%s' not found", name);
}
throw Error("Error in from statement");
}
int get_stack_precedence(std::stack<OP*>& operator_stack)
{
int pre = 0;
if (!operator_stack.empty())
pre = operator_stack.top()->m_precedence;
return pre;
}
// using The shunting yard algorithm
OP* get_expr(Lit& it, int rec)
{
std::stack<OP*> operator_stack;
std::stack<OP*> operand_stack;
bool success = true;
bool expect_expr = true;
while (success) {
success = false;
Lit save = it;
Lit next = it;
next++;
// match literal
if (expect_expr && is_literal(it)) {
OP* op = new OP(*it);
it++;
operand_stack.push(op);
success = true;
expect_expr = false;
continue;
}
// match literal
if (expect_expr && is_unary_op(it)) {
OP* op = new OP(*it);
op->set_type(Token::_uop);
it++;
if (!(op->m_right = get_expr(it, rec + 1)))
throw Error("Got unary '%s' but could not parse following expression", op->get_token());
operand_stack.push(op);
success = true;
expect_expr = false;
continue;
}
// match function-name (
if (expect_expr && is(it, Token::_label) && is(next, Token::_paren, "(")) {
OP* func = new OP(*it);
func->set_type(Token::_function);
it++;
it++;
if (is(it, Token::_op, "*")) {
it->set_type(Token::_number);
it->set_token("1");
}
func->m_param[0] = get_expr(it, rec + 1);
if (!func->m_param[0])
throw Error("Missing operand to function");
operand_stack.push(func);
int n = 1;
while (n < OP::max_param() && is(it, Token::_op, ",")) {
it++;
func->m_param[n++] = get_expr(it, rec + 1);
}
if (!is(it, Token::_paren, ")"))
throw Error("Expected ) after %s", func->get_token());
it++;
expect_expr = false;
success = true;
continue;
}
// match [[databasename .] table-name . ] column name
if (expect_expr && is(it, Token::_label)) {
OP* op = new OP(*it);
it++;
success = true;
op->set_type(Token::_column);
operand_stack.push(op);
expect_expr = false;
continue;
}
// match ( expr )
if (!expect_expr && is(it, Token::_paren, ",")) {
break;
}
if (!expect_expr && is(it, Token::_paren, ")")) {
break;
}
if (expect_expr && is(it, Token::_paren, "(")) {
it++;
OP* op = 0;
if ((op = get_expr(it, rec + 1))) {
if (is(it, Token::_paren, ")")) {
it++;
operand_stack.push(op);
expect_expr = false;
success = true;
continue;
}
throw Error("Error in expression no )");
}
it = save;
throw Error("Error in expression");
}
// bin op
if (!expect_expr && is(it, Token::_op, "is") && is(next, Token::_op, "not")) {
it++;
it->set_token("is not");
}
// bin op
if (!expect_expr && is(it, Token::_op, "not") && is(next, Token::_op, "like")) {
it++;
it->set_token("not like");
}
if (!expect_expr && is(it, Token::_op) && OP::is_binary(it->get_token())) {
OP* bop = new OP(*it);
while (bop->m_precedence <= get_stack_precedence(operator_stack)) {
OP* stack_op = operator_stack.top();
operator_stack.pop();
if (operand_stack.size() >= 2) {
OP* stk1 = operand_stack.top();
stack_op->m_right = stk1;
operand_stack.pop();
OP* stk2 = operand_stack.top();
stack_op->m_left = stk2;
operand_stack.pop();
operand_stack.push(stack_op);
}
}
operator_stack.push(bop);
it++;
success = true;
expect_expr = true;
continue;
}
}
while (!operator_stack.empty()) {
OP* bop = operator_stack.top();
operator_stack.pop();
if (bop) {
if (operand_stack.size() >= 2) {
OP* stk = operand_stack.top();
bop->m_right = stk;
operand_stack.pop();
OP* stk2 = operand_stack.top();
bop->m_left = stk2;
operand_stack.pop();
operand_stack.push(bop);
}
}
}
if (operand_stack.size() == 0)
return 0;
return operand_stack.top();
}
bool is_unary_op(Lit& it)
{
if (is(it, Token::_op, "-"))
return true;
if (is(it, Token::_op, "+"))
return true;
if (is(it, Token::_op, "~"))
return true;
if (is(it, Token::_op, "not"))
return true;
return false;
}
bool is_literal(Lit& it)
{
if (it->get_type() == Token::_number || it->get_type() == Token::_string)
return true;
if (it->get_type() == Token::_label && cmpi(it->get_token(), "null"))
return true;
return false;
}
};
class Lexer {
public:
Lexer(Parser& p)
: m_parser(p)
, num_state(_nan)
{
}
Parser& m_parser;
enum State {
_unknown,
_white,
_label,
_number,
_op,
_paren,
_string
};
bool is_white(const char c) { return (c == ' ' || c == 9 || c == 10 || c == 13); }
bool is_char(const char c) { return ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')); }
bool is_num(const char c) { return ((c >= '0' && c <= '9')); }
bool is_op(const char c) { return (c == '*' || c == ',' || c == '=' || c == '<' || c == '>' || c == '/' || c == '|' || c == '%' || c == '+' || c == '-' || c == '&' || c == '~' || c == '!'); }
bool is_paren(const char c) { return (c == '(' || c == ')'); }
bool is_termination(const char c) { return (c == 0 || c == ';'); }
bool is_quote(const char c) { return (c == '\''); }
bool is_label_start(const char c) { return (is_char(c)); }
bool is_label_part(const char c) { return (is_num(c) || is_char(c) || c == '_'); }
bool lex(const char* i_str)
{
const char* p = i_str;
State state = _white;
const char* strstart = 0;
bool is_escaped = false;
std::string str = "";
while (true) {
switch (state) {
case (_unknown): {
if (is_white(*p))
state = _white;
else if (is_label_start(*p))
state = _label;
else if (is_num(*p) || (*p == '.' && is_num(p[1])))
state = _number;
else if (is_quote(*p))
state = _string;
else if (is_paren(*p))
state = _paren;
else if (is_op(*p))
state = _op;
else if (is_termination(*p)) {
m_parser.push(Token::_semicolon, ";");
if (*p++ == 0) {
m_parser.push(Token::_end, "END");
return true;
}
} else {
m_parser.push(Token::_end, "END");
printf("Unknown char %c (%d) at %d! in statement %s\n", *p, *p, int(p - i_str), i_str);
return false;
}
} break;
case (_white): {
if (is_white(*p))
p++;
else {
state = _unknown;
}
} break;
case (_string): {
if ((!is_escaped) && is_quote(*p)) {
if (!strstart) {
strstart = p;
str = "";
} else {
m_parser.push(Token::_string, str.c_str());
str = "";
strstart = 0;
state = _unknown;
is_escaped = false;
}
p++;
} else {
if (is_escaped) {
if (!is_quote(*p))
str += '\\';
is_escaped = false;
}
if (*p == 0) {
printf("Unexpected end of string in statement %s\n", strstart);
m_parser.push(Token::_end, "END");
return false;
}
if (*p == '\\')
is_escaped = true;
else
str += (*p);
p++;
}
} break;
case (_number): {
if (parse_num(*p)) {
if (!strstart)
strstart = p;
p++;
} else {
if (!strstart)
throw Error("Numeric problem");
std::string label = strstart;
label = label.substr(0, p - strstart);
m_parser.push(Token::_number, label.c_str());
strstart = 0;
state = _unknown;
}
} break;
case (_label): {
if (!strstart) {
strstart = p++;
} else if (is_label_part(*p)) {
p++;
} else {
std::string label = strstart;
label = label.substr(0, p - strstart);
Token::Type type = Token::_label;
if (cmpi(label, "is"))
type = Token::_op;
if (cmpi(label, "not"))
type = Token::_op;
if (cmpi(label, "in"))
type = Token::_op;
if (cmpi(label, "like"))
type = Token::_op;
if (cmpi(label, "glob"))
type = Token::_op;
if (cmpi(label, "match"))
type = Token::_op;
if (cmpi(label, "regexp"))
type = Token::_op;
if (cmpi(label, "and"))
type = Token::_op;
if (cmpi(label, "or"))
type = Token::_op;
m_parser.push(type, label.c_str());
strstart = 0;
state = _unknown;
}
} break;
case (_paren): {
std::string s;
s = *p;
m_parser.push(Token::_paren, s.c_str());
p++;
state = _unknown;
} break;
case (_op): {
std::string s;
s = *p;
char n = p[1];
switch (*p) {
case ('|'):
if (n == '|') {
p++;
s += *p;
}
break;
case ('>'):
if (n == '=') {
p++;
s += *p;
}
if (n == '>') {
p++;
s += *p;
}
break;
case ('<'):
if (n == '<') {
p++;
s += *p;
}
if (n == '=') {
p++;
s += *p;
}
if (n == '>') {
p++;
s += *p;
}
break;
case ('='):
if (n == '=') {
p++;
s += *p;
}
break;
case ('!'):
if (n == '=') {
p++;
s += *p;
}
break;
}
m_parser.push(Token::_op, s.c_str());
p++;
state = _unknown;
} break;
default:
printf("Missing impl char %c at %d! in statement %s\n", *p, int(p - i_str), i_str);
m_parser.push(Token::_end, "END");
return false;
}
}
}
bool parse_num(const char p)
{
switch (num_state) {
case (_nan):
num_state = _int;
if (p == '.')
num_state = _dot;
break;
case (_int):
if (p == '.')
num_state = _dot;
else if (!is_num(p)) {
num_state = _nan;
return false;
}
break;
case (_dot):
if (p == 'E' || p == 'e')
num_state = _e;
else if (!is_num(p)) {
num_state = _nan;
return false;
}
break;
case (_dec):
if (p == 'E' || p == 'e')
num_state = _e;
else if (!is_num(p)) {
num_state = _nan;
return false;
}
break;
case (_e):
if (p == '+' || p == '-')
num_state = _sign;
else if (is_num(p))
num_state = _exp;
else {
throw Error("expected number digit after E");
}
break;
case (_sign):
if (!is_num(p))
throw Error("expected number digit after E");
num_state = _exp;
break;
case (_exp):
if (!is_num(p)) {
num_state = _nan;
return false;
}
break;
}
return true;
}
enum Num_state {
_nan,
_int,
_dot,
_dec,
_e,
_sign,
_exp,
};
Num_state num_state;
};
void Query::parse()
{
Parser p;
Lexer l(p);
l.lex(m_sql.c_str());
// p.dump();
if (!p.analyze(*this))
throw Error("error parsing select statement");
}
// return column and index in tables, or 0 for column if column isn't found
std::pair<Column*, int> lookup_column_in_tables(const std::vector<Table*>& tables,
const std::vector<int>& search_order,
const char* name)
{
if (strcmp(name, "*") == 0)
return std::pair<Column*, int>((Column*)0, 0);
for (auto i = search_order.begin(); i != search_order.end(); ++i) {
Table* table = tables[*i];
int col_index = table->get_col_index(name);
if (col_index >= 0)
return std::pair<Column*, int>(table->m_cols[col_index], *i);
}
return std::pair<Column*, int>((Column*)0, 0);
}
OP* OP::compile(const std::vector<Table*>& tables, const std::vector<int>& search_order, Query& q)
{
OP* ret = 0;
for (int i = 0; i < max_param(); i++) {
if (m_param[i]) {
m_param[i] = m_param[i]->compile(tables, search_order, q);
if (m_param[i]->m_has_aggregate_function)
m_has_aggregate_function = true;
}
}
if (m_left) {
m_left = m_left->compile(tables, search_order, q);
if (m_left->m_has_aggregate_function)
m_has_aggregate_function = true;
}
if (m_right) {
m_right = m_right->compile(tables, search_order, q);
if (m_right->m_has_aggregate_function)
m_has_aggregate_function = true;
}
// default to destination row
m_row_index = tables.size() - 1;
if (get_type() == _column) {
auto lookup = lookup_column_in_tables(tables, search_order, get_token());
Column* column = lookup.first;
m_row_index = lookup.second;
if (!column)
throw Error("Column '%s' not found", get_token());
int offset = column->m_offset;
m_t = column->m_type;
switch (m_t) {
case Coltype::_int:
ret = new Column_access_int(*this, offset);
break;
case Coltype::_bool:
ret = new Column_access_bool(*this, offset);
break;
case Coltype::_float:
ret = new Column_access_float(*this, offset);
break;
case Coltype::_text:
ret = new Column_access_text(*this, offset);
break;
}
} else if (get_type() == _number) {
const char* p = get_token();
bool integer = true;
while (*p != 0) {
if (*p < '0' || *p > '9')
integer = false;
p++;
}
if (integer) {
m_t = Coltype::_int;
ret = new Static_int(*this);
} else {
m_t = Coltype::_float;
ret = new Static_float(*this);
}
} else if (get_type() == _string) {
m_t = Coltype::_text;
ret = new Static_text(*this);
} else if ((get_type() == _function) && m_param[0]) {
Table* dest_table = tables[m_row_index];
m_t = Coltype::_int;
if (cmpi(get_token(), "if") && m_param[1] && m_param[2]) {
m_t = m_param[1]->m_t;
if (m_param[2]->m_t > m_t)
m_t = m_param[2]->m_t;
ret = new If_func(*this);
} else if (cmpi(get_token(), "name") && m_param[1]) {
m_t = Coltype::_text;
ret = new Name_func(*this);
} else if (cmpi(get_token(), "trim")) {
m_t = Coltype::_text;
ret = new Trim_func(*this);
} else if (cmpi(get_token(), "rsplit") && m_param[1]) {
m_t = Coltype::_text;
ret = new Rsplit_func(*this);
} else if (cmpi(get_token(), "netmask")) {
m_t = Coltype::_text;
ret = new Netmask_func(*this);
} else if (cmpi(get_token(), "cc")) {
m_t = Coltype::_text;
ret = new Cc_func(*this);
} else if (cmpi(get_token(), "asn")) {
m_t = Coltype::_int;
ret = new Asn_func(*this);
} else if (cmpi(get_token(), "count")) {
m_t = Coltype::_int;
ret = new Count_func(*this, dest_table);
} else if (m_param[0]->ret_type() == Coltype::_float && cmpi(get_token(), "min")) {
m_t = Coltype::_float;
ret = new Min_func_float(*this, dest_table);
} else if (m_param[0]->ret_type() == Coltype::_float && cmpi(get_token(), "max")) {
m_t = Coltype::_float;
ret = new Max_func_float(*this, dest_table);
} else if (m_param[0]->ret_type() == Coltype::_float && cmpi(get_token(), "sum")) {
m_t = Coltype::_float;
ret = new Sum_func_float(*this, dest_table);
} else if (cmpi(get_token(), "min")) {
m_t = Coltype::_int;
ret = new Min_func_int(*this, dest_table);
} else if (cmpi(get_token(), "max")) {
m_t = Coltype::_int;
ret = new Max_func_int(*this, dest_table);
} else if (cmpi(get_token(), "sum")) {
m_t = Coltype::_int;
ret = new Sum_func_int(*this, dest_table);
} else if (cmpi(get_token(), "lower")) {
m_t = Coltype::_text;
ret = new Lower_func(*this);
} else if (cmpi(get_token(), "len")) {
m_t = Coltype::_int;
ret = new Len_func(*this);
} else if (cmpi(get_token(), "truncate")) {
m_t = Coltype::_int;
ret = new Truncate_func(*this);
} else if (cmpi(get_token(), "stdev")) {
m_t = Coltype::_float;
ret = new Stdev_func(*this, dest_table);
} else if (cmpi(get_token(), "avg")) {
m_t = Coltype::_float;
ret = new Avg_func(*this, dest_table);
}
} else if ((get_type() == _op) && m_left && m_right) {
if (cmpi(get_token(), "||")) {
m_t = Coltype::_text;
ret = new Bin_op_concatenate(*this);
} else if (cmpi(get_token(), "*")) {
if (m_left->ret_type() == Coltype::_float || m_right->ret_type() == Coltype::_float) {
m_t = Coltype::_float;
ret = new Bin_op_mul_float(*this);
} else {
m_t = Coltype::_int;
ret = new Bin_op_mul(*this);
}
} else if (cmpi(get_token(), "/")) {
m_t = Coltype::_float;
ret = new Bin_op_div(*this);
} else if (cmpi(get_token(), "%")) {
m_t = Coltype::_float;
ret = new Bin_op_modulo(*this);
} else if (cmpi(get_token(), "+")) {
if (m_left->ret_type() == Coltype::_float || m_right->ret_type() == Coltype::_float) {
m_t = Coltype::_float;
ret = new Bin_op_add_float(*this);
} else {
m_t = Coltype::_int;
ret = new Bin_op_add(*this);
}
} else if (cmpi(get_token(), "-")) {
if (m_left->ret_type() == Coltype::_float || m_right->ret_type() == Coltype::_float) {
m_t = Coltype::_float;
ret = new Bin_op_sub_float(*this);
} else {
m_t = Coltype::_int;
ret = new Bin_op_sub(*this);
}
} else if (cmpi(get_token(), "<<")) {
m_t = Coltype::_int;
ret = new Bin_op_arithmetic_shift_left(*this);
} else if (cmpi(get_token(), ">>")) {
m_t = Coltype::_int;
ret = new Bin_op_arithmetic_shift_right(*this);
} else if (cmpi(get_token(), "&")) {
m_t = Coltype::_int;
ret = new Bin_op_bitwise_and(*this);
} else if (cmpi(get_token(), "|")) {
m_t = Coltype::_int;
ret = new Bin_op_bitwise_or(*this);
} else if (cmpi(get_token(), "<")) {
m_t = Coltype::_bool;
ret = new Bin_op_lt(*this);
} else if (cmpi(get_token(), "<=")) {
m_t = Coltype::_bool;
ret = new Bin_op_lteq(*this);
} else if (cmpi(get_token(), ">")) {
m_t = Coltype::_bool;
ret = new Bin_op_gt(*this);
} else if (cmpi(get_token(), ">=")) {
m_t = Coltype::_bool;
ret = new Bin_op_gteq(*this);
} else if (cmpi(get_token(), "=")) {
m_t = Coltype::_bool;
ret = new Bin_op_eq(*this);
} else if (cmpi(get_token(), "==")) {
m_t = Coltype::_bool;
ret = new Bin_op_eq(*this);
} else if (cmpi(get_token(), "like")) {
m_t = Coltype::_bool;
ret = new Bin_op_like(*this);
} else if (cmpi(get_token(), "not like")) {
m_t = Coltype::_bool;
ret = new Bin_op_not_like(*this);
} else if (cmpi(get_token(), "!=")) {
m_t = Coltype::_bool;
ret = new Bin_op_not_eq(*this);
} else if (cmpi(get_token(), "<>")) {
m_t = Coltype::_bool;
ret = new Bin_op_not_eq(*this);
} else if (cmpi(get_token(), "is")) {
m_t = Coltype::_bool;
ret = new Bin_op_eq(*this);
} else if (cmpi(get_token(), "is not")) {
m_t = Coltype::_bool;
ret = new Bin_op_not_eq(*this);
} else if (cmpi(get_token(), "and")) {
m_t = Coltype::_bool;
ret = new Bin_op_and(*this);
} else if (cmpi(get_token(), "or")) {
m_t = Coltype::_bool;
ret = new Bin_op_or(*this);
}
} else if ((get_type() == _uop) && m_right) {
if (cmpi(get_token(), "not")) {
m_t = Coltype::_bool;
ret = new Un_op_not(*this);
} else if (cmpi(get_token(), "+")) {
ret = m_right;
} else if (cmpi(get_token(), "-")) {
if (m_right->ret_type() == Coltype::_float) {
m_t = Coltype::_float;
ret = new Un_op_neg_float(*this);
} else {
m_t = Coltype::_int;
ret = new Un_op_neg(*this);
}
} else if (cmpi(get_token(), "~")) {
m_t = Coltype::_int;
ret = new Un_op_ones_complement(*this);
}
}
clear_ptr();
if (!ret)
throw Error("Unknown operator error '%s' !", get_token());
delete this;
return ret;
}
void OP::evaluate_aggregate_operands(Row** rows)
{
if (m_left)
m_left->evaluate_aggregate_operands(rows);
if (m_right)
m_right->evaluate_aggregate_operands(rows);
for (int i = 0; i < max_param(); ++i)
if (m_param[i])
m_param[i]->evaluate_aggregate_operands(rows);
}
void OP::combine_aggregate(Row* base_row, Row* other_row)
{
if (m_left)
m_left->combine_aggregate(base_row, other_row);
if (m_right)
m_right->combine_aggregate(base_row, other_row);
for (int i = 0; i < max_param(); ++i)
if (m_param[i])
m_param[i]->combine_aggregate(base_row, other_row);
}
// return any column access ops found in given list of op trees - they don't
// have to be compiled beforehand; duplicate column tokens are skipped
std::vector<OP*> find_unique_column_ops(std::vector<OP*> ops)
{
std::vector<OP*> res;
while (!ops.empty()) {
OP* op = ops.back();
ops.pop_back();
if (op->m_left)
ops.push_back(op->m_left);
if (op->m_right)
ops.push_back(op->m_right);
for (int i = 0; i < op->max_param(); ++i)
if (op->m_param[i])
ops.push_back(op->m_param[i]);
if (op->get_type() == Token::_column) {
bool found = false;
for (auto i = res.begin(); i != res.end(); ++i) {
if (cmpii((*i)->get_token(), op->get_token())) {
found = true;
break;
}
}
if (!found)
res.push_back(op);
}
}
return res;
}
void Query::replace_star_column_with_all_columns()
{
bool found_star = false;
for (auto i = m_select.begin(); i != m_select.end(); ++i) {
if (strcmp((*i)->get_token(), "*") == 0) {
found_star = true;
break;
}
}
if (found_star) {
for (auto i = m_select.begin(); i != m_select.end(); ++i)
delete *i;
m_select.clear();
if (!m_from_name.empty()) {
Packet_handler* handler = get_packet_handler(m_from_name);
for (auto i = handler->packet_columns.begin(); i != handler->packet_columns.end(); ++i)
m_select.push_back(new OP(Token(Token::_column, i->name)));
}
}
}
void Query::process_from()
{
replace_star_column_with_all_columns();
if (m_from_name.empty())
return;
std::vector<OP*> all_ops;
all_ops.insert(all_ops.end(), m_select.begin(), m_select.end());
if (m_where)
all_ops.push_back(m_where);
// skip m_having, it can't access source columns
for (auto i = m_order_by.m_terms.begin(); i != m_order_by.m_terms.end(); ++i)
all_ops.push_back(i->m_op);
for (auto i = m_group_by.m_terms.begin(); i != m_group_by.m_terms.end(); ++i)
all_ops.push_back(i->m_op);
auto used_columns = find_unique_column_ops(all_ops);
// add from table with used columns
Packet_handler* handler = get_packet_handler(m_from_name);
for (auto j = handler->packet_columns.begin(); j != handler->packet_columns.end(); ++j)
for (auto i = used_columns.begin(); i != used_columns.end(); ++i)
if (cmpii(j->name, (*i)->get_token()))
m_used_from_column_ids.push_back(j->id);
m_from = handler->create_table(m_used_from_column_ids);
}
void Query::process_select(Row** rows, Row* dest, GenericAccessor* dest_accessors)
{
for (unsigned int i = 0, size = m_select.size(); i < size; ++i) {
OP* op = m_select[i];
if (!op)
continue;
if (op->m_has_aggregate_function) {
// defer evaluating aggregate functions, just eval their operands
op->evaluate_aggregate_operands(rows);
} else {
Variant v;
op->evaluate(rows, v);
dest_accessors[i].set(dest, v);
}
}
}
void Query::combine_aggregates_in_select(Row* base_row, Row* other_row)
{
for (unsigned int i = 0; i < m_select.size(); ++i) {
OP* op = m_select[i];
if (op && op->m_has_aggregate_function)
op->combine_aggregate(base_row, other_row);
}
}
void Query::process_aggregates_in_select(Row** rows, Row* dest, GenericAccessor dest_accessors[])
{
for (unsigned int i = 0; i < m_select.size(); ++i) {
OP* op = m_select[i];
if (op && op->m_has_aggregate_function) {
Variant v;
op->evaluate(rows, v);
dest_accessors[i].set(dest, v);
}
}
}
bool Query::process_where(Row** rows)
{
if (!m_where)
return true;
Variant v;
m_where->evaluate(rows, v);
return v.get_bool();
}
bool Query::process_having(Row** rows)
{
if (!m_having)
return true;
Variant v;
m_having->evaluate(rows, v);
return v.get_bool();
}
std::vector<Variant> process_group_by_key(Ordering_terms& group_by, Row** rows)
{
int size = group_by.m_terms.size();
std::vector<Variant> res(size);
for (int i = 0; i < size; ++i)
group_by.m_terms[i].m_op->evaluate(rows, res[i]);
return res;
}
bool Query::has_aggregate_functions()
{
// this assumes the ops have been compiled
for (auto it = m_select.begin(); it != m_select.end(); it++)
if ((*it)->m_has_aggregate_function)
return true;
return false;
}
void Query::execute(Reader& reader)
{
std::vector<Table*> tables;
std::vector<int> search_results_last, search_results_first, search_results_only;
// set up tables
process_from();
if (m_from)
tables.push_back(m_from);
tables.push_back(m_result);
for (int i = 0; i < int(tables.size()); ++i)
search_results_last.push_back(i);
for (int i = int(tables.size()) - 1; i >= 0; --i)
search_results_first.push_back(i);
search_results_only.push_back(tables.size() - 1);
std::vector<Row*> row_ptrs(tables.size());
Row** rows = &row_ptrs[0];
std::vector<GenericAccessor> result_accessors_vector;
// compile
for (auto i = m_select.begin(); i != m_select.end(); ++i) {
*i = (*i)->compile(tables, search_results_last, *this);
Column* col = m_result->add_column((*i)->get_name(), (*i)->ret_type());
GenericAccessor a;
a.m_offset = col->m_offset;
a.m_type = col->m_type;
result_accessors_vector.push_back(a);
}
if (m_where)
m_where = m_where->compile(tables, search_results_last, *this);
if (m_having)
m_having = m_having->compile(tables, search_results_only, *this);
if (m_group_by.exist())
m_group_by.compile(tables, search_results_last, *this);
if (m_order_by.exist()) {
// copy any missing columns to result table as hidden so we can
// order by them
std::vector<OP*> ops;
for (auto i = m_order_by.m_terms.begin(); i != m_order_by.m_terms.end(); ++i)
ops.push_back(i->m_op);
std::vector<OP*> column_ops = find_unique_column_ops(ops);
for (auto i = column_ops.begin(); i != column_ops.end(); ++i) {
const char* name = (*i)->get_token();
auto lookup = lookup_column_in_tables(tables, search_results_first, name);
if (lookup.first and lookup.second < int(tables.size()) - 1) {
// found, but not in result table
OP* copying_op = new OP(**i);
copying_op = copying_op->compile(tables, search_results_last, *this);
m_select.push_back(copying_op);
Column* col = m_result->add_column(copying_op->get_name(), copying_op->ret_type(), -1, Column::HIDDEN);
GenericAccessor a;
a.m_offset = col->m_offset;
a.m_type = col->m_type;
result_accessors_vector.push_back(a);
}
}
// we only provide access to result table for "order by"; in order
// to make the sort thing work correctly the result table currently
// has to be at index 0
std::vector<Table*> tables_result_only = { m_result };
std::vector<int> tables_result_only_search = { 0 };
m_order_by.compile(tables_result_only, tables_result_only_search, *this);
}
// execute
GenericAccessor* result_accessors = &result_accessors_vector[0];
bool aggregate_functions = has_aggregate_functions();
int count = 0;
bool limiter = !m_order_by.exist() && !m_group_by.exist() && !aggregate_functions && m_limit >= 0;
if (m_from) {
bool first_row = true;
Packet_handler* handler = get_packet_handler(m_from_name);
reader.seek_to_start();
const int src_i = 0, dest_i = tables.size() - 1;
rows[src_i] = m_from->create_row();
if (m_group_by.exist() || aggregate_functions) {
std::unordered_map<std::vector<Variant>, Row*> groups;
rows[dest_i] = 0;
while (reader.read_next(handler, m_used_from_column_ids, *rows[src_i], first_row or m_sample == 0 ? 0 : m_sample - 1)) {
// fill in groups
if (rows[dest_i])
rows[dest_i]->reset_text_columns(m_result->m_text_column_offsets);
else
rows[dest_i] = m_result->create_row();
process_select(rows, rows[dest_i], result_accessors);
if (process_where(rows)) {
auto key = process_group_by_key(m_group_by, rows);
Row*& entry = groups[key];
if (entry) {
combine_aggregates_in_select(entry, rows[dest_i]);
} else {
entry = rows[dest_i];
rows[dest_i] = 0;
}
}
first_row = false;
rows[src_i]->reset_text_columns(m_from->m_text_column_offsets);
}
if (rows[dest_i])
m_result->delete_row(rows[dest_i]);
// put groups into result
for (auto i = groups.begin(); i != groups.end(); ++i) {
rows[dest_i] = i->second;
// propagate the aggregate results through the evaluation tree
process_aggregates_in_select(rows, rows[dest_i], result_accessors);
if (process_having(rows))
m_result->add_row(rows[dest_i]);
else
m_result->delete_row(rows[dest_i]);
}
} else {
rows[dest_i] = m_result->create_row();
while (reader.read_next(handler, m_used_from_column_ids, *rows[src_i], first_row or m_sample == 0 ? 0 : m_sample - 1)) {
// fill in result
process_select(rows, rows[dest_i], result_accessors);
if (process_where(rows)) {
bool commit = true;
if (limiter) {
int l = count++;
if (m_offset > 0)
l -= m_offset;
if (m_limit >= 0 && l >= m_limit)
break;
if (l < 0)
commit = false;
}
if (commit) {
m_result->add_row(rows[dest_i]);
rows[dest_i] = m_result->create_row();
}
}
first_row = false;
rows[src_i]->reset_text_columns(m_from->m_text_column_offsets);
}
m_result->delete_row(rows[dest_i]);
}
m_from->delete_row(rows[src_i]);
} else {
const int dest_i = tables.size() - 1;
rows[dest_i] = m_result->create_row();
process_select(rows, rows[dest_i], result_accessors);
if (process_where(rows))
m_result->add_row(rows[dest_i]);
else
m_result->delete_row(rows[dest_i]);
}
if (m_order_by.exist())
m_result->per_sort(m_order_by);
if (m_limit >= 0 && !limiter)
m_result->limit(m_limit, m_offset);
}
DB::DB()
{
Column::init_defs();
}
DB::~DB()
{
}
bool DB::query(const char* q)
{
return false;
}
Table* DB::get_table(const char* i_name)
{
std::string name = lower(i_name);
Table* t = 0;
auto it = m_tables.find(name);
if (it != m_tables.end())
t = it->second;
return t;
}
Table* DB::create_or_use_table(const char* i_name)
{
std::string name = lower(i_name);
Table* t = get_table(name.c_str());
if (!t)
t = create_table(name.c_str());
return t;
}
Table* DB::create_table(const char* i_name)
{
std::string name = lower(i_name);
Table* t = new Table(name.c_str());
m_tables[std::string(name.c_str())] = t;
return t;
}
Column::Column(const char* name, Coltype::Type type, int id, bool hidden)
: m_name(name)
, m_type(type)
, m_def(Column::m_coldefs[type])
, m_id(id)
, m_offset(0)
{
m_hidden = hidden;
}
void Trim_func::evaluate(Row** rows, Variant& v)
{
Variant str;
m_param[0]->evaluate(rows, str);
RefCountStringHandle str_handle(str.get_text());
const char* s = (*str_handle)->data;
const char* t;
RefCountStringHandle trim_handle;
if (m_param[1]) {
Variant trim;
m_param[1]->evaluate(rows, trim);
trim_handle.set(trim.get_text());
t = (*trim_handle)->data;
} else
t = " ";
int l = strlen(t);
if (l <= 0) {
v = *str_handle;
return;
}
int slen = strlen(s);
int start = 0, end = slen;
// left trim
while (end - start >= l && memcmp(s + start, t, l) == 0)
start += l;
// right trim
while (end - start >= l && memcmp(s + end - l, t, l) == 0)
end -= l;
if (start == 0 && end == slen)
v = *str_handle;
else {
RefCountStringHandle res(RefCountString::construct(s, start, end));
v = *res;
}
}
Cc_func::Cc_func(const OP& op)
: OP(op)
{
#ifdef HAVE_LIBMAXMINDDB
if (__cc_mmdb) {
return;
}
std::string db;
char* env = getenv("PACKETQ_MAXMIND_CC_DB");
if (env) {
db = env;
}
if (db.empty()) {
std::list<std::string> paths = {
"/var/lib/GeoIP", "/usr/share/GeoIP", "/usr/local/share/GeoIP"
};
if ((env = getenv("PACKETQ_MAXMIND_PATH"))) {
paths.push_front(std::string(env));
}
auto i = paths.begin();
for (; i != paths.end(); i++) {
db = (*i) + "/GeoLite2-Country.mmdb";
struct stat s;
if (!stat(db.c_str(), &s)) {
break;
}
}
if (i == paths.end()) {
return;
}
}
MMDB_s* mmdb = new (std::nothrow) MMDB_s;
if (!mmdb) {
return;
}
int ret = MMDB_open(db.c_str(), 0, mmdb);
if (ret != MMDB_SUCCESS) {
fprintf(stderr, "Warning: cannot open MaxMind CC database \"%s\": %s\n", db.c_str(), MMDB_strerror(ret));
delete mmdb;
return;
}
__cc_mmdb = mmdb;
#endif
}
void Cc_func::evaluate(Row** rows, Variant& v)
{
#ifdef HAVE_LIBMAXMINDDB
if (!__cc_mmdb) {
RefCountStringHandle res(RefCountString::construct(""));
v = *res;
return;
}
Variant str;
m_param[0]->evaluate(rows, str);
RefCountStringHandle str_handle(str.get_text());
int gai_error, ret;
MMDB_lookup_result_s mmdb_result = MMDB_lookup_string(__cc_mmdb, (*str_handle)->data, &gai_error, &ret);
if (gai_error || ret != MMDB_SUCCESS || !mmdb_result.found_entry) {
RefCountStringHandle res(RefCountString::construct(""));
v = *res;
return;
}
MMDB_entry_data_s entry_data;
ret = MMDB_get_value(&mmdb_result.entry, &entry_data, "country", "iso_code", NULL);
if (ret != MMDB_SUCCESS || !entry_data.has_data || entry_data.type != MMDB_DATA_TYPE_UTF8_STRING) {
RefCountStringHandle res(RefCountString::construct(""));
v = *res;
return;
}
RefCountStringHandle res(RefCountString::construct(entry_data.utf8_string, 0, entry_data.data_size));
v = *res;
#else
RefCountStringHandle res(RefCountString::construct(""));
v = *res;
#endif
}
Asn_func::Asn_func(const OP& op)
: OP(op)
{
#ifdef HAVE_LIBMAXMINDDB
if (__asn_mmdb) {
return;
}
std::string db;
char* env = getenv("PACKETQ_MAXMIND_ASN_DB");
if (env) {
db = env;
}
if (db.empty()) {
std::list<std::string> paths = {
"/var/lib/GeoIP", "/usr/share/GeoIP", "/usr/local/share/GeoIP"
};
if ((env = getenv("PACKETQ_MAXMIND_PATH"))) {
paths.push_front(std::string(env));
}
auto i = paths.begin();
for (; i != paths.end(); i++) {
db = (*i) + "/GeoLite2-ASN.mmdb";
struct stat s;
if (!stat(db.c_str(), &s)) {
break;
}
}
if (i == paths.end()) {
return;
}
}
MMDB_s* mmdb = new (std::nothrow) MMDB_s;
if (!mmdb) {
return;
}
int ret = MMDB_open(db.c_str(), 0, mmdb);
if (ret != MMDB_SUCCESS) {
fprintf(stderr, "Warning: cannot open MaxMind ASN database \"%s\": %s\n", db.c_str(), MMDB_strerror(ret));
delete mmdb;
return;
}
__asn_mmdb = mmdb;
#endif
}
void Asn_func::evaluate(Row** rows, Variant& v)
{
#ifdef HAVE_LIBMAXMINDDB
if (!__asn_mmdb) {
v = -1;
return;
}
Variant str;
m_param[0]->evaluate(rows, str);
RefCountStringHandle str_handle(str.get_text());
int gai_error, ret;
MMDB_lookup_result_s mmdb_result = MMDB_lookup_string(__asn_mmdb, (*str_handle)->data, &gai_error, &ret);
if (gai_error || ret != MMDB_SUCCESS || !mmdb_result.found_entry) {
v = -1;
return;
}
MMDB_entry_data_s entry_data;
ret = MMDB_get_value(&mmdb_result.entry, &entry_data, "autonomous_system_number", NULL);
if (ret != MMDB_SUCCESS || !entry_data.has_data || entry_data.type != MMDB_DATA_TYPE_UINT32) {
v = -1;
return;
}
v = (int_column)entry_data.uint32;
#else
v = -1;
#endif
}
DB g_db;
Coldef Column::m_coldefs[COLTYPE_MAX];
} // namespace packetq
Copy permalink