dns-oarc/oarc-dsc-datatool
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.4.2.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-04-22 13:23:23 +02:00
parent e344d0b8ae
commit 1ea3e103a7
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
77 changed files with 5760 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

24
.copr/Makefile Normal file
View file

@ -0,0 +1,24 @@
top=..
all: srpm
prereq: $(top)/rpmbuild
rpm -q git rpm-build >/dev/null || dnf -y install git rpm-build
update-dist-tools: $(top)/dist-tools
( cd "$(top)/dist-tools" && git pull )
$(top)/dist-tools:
git clone https://github.com/jelu/dist-tools.git "$(top)/dist-tools"
$(top)/rpmbuild:
mkdir -p "$(top)"/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
srpm: prereq update-dist-tools
git config --global safe.directory "*"
test -f .gitmodules && git submodule update --init || true
echo "$(spec)" | grep -q "develop.spec" && auto_build_number=`date --utc +%s` message="Auto build `date --utc --iso-8601=seconds`" "$(top)/dist-tools/spec-new-changelog-entry" || true
overwrite=yes nosign=yes "$(top)/dist-tools/create-source-packages" rpm
cp ../*.orig.tar.gz "$(top)/rpmbuild/SOURCES/"
echo "$(spec)" | grep -q "develop.spec" && rpmbuild -bs --define "%_topdir $(top)/rpmbuild" --undefine=dist rpm/*.spec || rpmbuild -bs --define "%_topdir $(top)/rpmbuild" --undefine=dist "$(spec)"
cp "$(top)"/rpmbuild/SRPMS/*.src.rpm "$(outdir)"

1
.github/FUNDING.yml vendored Normal file
View file

@ -0,0 +1 @@
custom: https://www.dns-oarc.net/donate

41
.github/workflows/codeql.yml vendored Normal file
View file

@ -0,0 +1,41 @@
name: "CodeQL"
on:
push:
branches: [ "develop", "master" ]
pull_request:
branches: [ "develop" ]
schedule:
- cron: "41 0 * * 3"
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ python ]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
queries: +security-and-quality
- name: Autobuild
uses: github/codeql-action/autobuild@v3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{ matrix.language }}"

32
.github/workflows/test.yml vendored Normal file
View file

@ -0,0 +1,32 @@
name: "Test"
on:
push:
branches: [ "develop" ]
pull_request:
branches: [ "develop" ]
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: "3.10"
- name: Install dependencies
run: |
sudo apt-get install python3-maxminddb python3-yaml wget
python -m pip install --upgrade pip
pip install .
- name: Test
run: |
cd tests
rm ipv4-address-space.csv ipv6-unicast-address-assignments.csv
wget https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv
wget https://www.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.csv
./test.sh

16
.gitignore vendored Normal file
View file

@ -0,0 +1,16 @@
/venv/
*.pyc
__pycache__/
.pytest_cache/
.coverage
htmlcov/
dist/
build/
*.egg-info/
tests/test.out
tests/test.*.tmp
coverage.xml

223
CHANGES Normal file
View file

@ -0,0 +1,223 @@
2024-06-18 Jerry Lundström
Release 1.4.2
This release fixes issues with IANA's IPv6 parameters file,
dsc-datatool expected a RIR in the Designation field but IANA recently
added a title for SRv6 reservation which caused an exception.
Other updates are related to packages and GitHub workflows.
7560d82 Tests
8568c84 Fix client subnet authority
a8c58a9 Workflow
fd8915c RPM SUSE
2023-12-06 Jerry Lundström
Release 1.4.1
This release fixes issue with InfluxDB quoting, was missing to quote
the quote character.
Other changes:
- Dependency correction for SLE 15.5
- Tweaks to test layouts
b44b874 Tests
eef3ae0 SLE 15.5
75c7fc1 Influx quoting
2023-06-15 Jerry Lundström
Release 1.4.0
This release adds the option `--encoding` to set an encoding to use
for reading and writing files.
f64c8b6 encoding man-page
09c0ce9 Encoding
2022-11-10 Jerry Lundström
Release 1.3.0
This release adds option `nonstrict` to `client_subnet_authority`
generator for skipping bad data in datasets.
The contrib DSC+Grafana test site dashboards has been moved to its
own repository, feel free to contribute your own creations to it:
https://github.com/DNS-OARC/dsc-datatool-grafana
90b232d Add CodeQL workflow for GitHub code scanning
e4fa3b0 Test site
474f97d client_subnet_authority non-strict mode
2022-06-13 Jerry Lundström
Release 1.2.0
This release fixes handling of base64'ed strings in DSC XML and will
now decode them back into text when reading, the selected output will
then handling any quoting or escaping needed.
Added a new option for Prometheus output to set a prefix for metrics so
that they can be easily separated from other metrics if needed, see
`man dsc-datatool-output prometheus`.
5f9f972 Fix COPR
3d72019 Prometheus metric prefix
bdc992e base64 labels
2022-04-05 Jerry Lundström
Release 1.1.0
This release adds support for Prometheus' node_exporter using it's
Textfile Collector (see `man dsc-datatool-output prometheus`) and
fixes a bug in InfluxDB output when selecting what timestamp to use.
Also updates packages and Grafana test site dashboards.
4381541 RPM
19bc153 Typo/clarification
2a32dd8 Prometheus, InfluxDB, Copyright
dd5323e debhelper
7352c1e Bye Travis
32b3bbe Grafana dashboards
304ab76 Info
2020-10-21 Jerry Lundström
Release 1.0.2
This release fixed a bug in DAT file parsing that was discovered when
adding coverage tests.
45b1aa3 Coverage
7aedc1a Coverage
64957b9 DAT, Coverage
370fb86 Coverage
891cb7c Coverage
9374faa Coverage
2020-08-07 Jerry Lundström
Release 1.0.1
This release adds compatibility with Python v3.5 which allows
packages to be built for Ubuntu Xenial.
bc0be5b python 3.5
2020-08-03 Jerry Lundström
Release 1.0.0
This release brings a complete rewrite of the tool, from Perl to
Python. This rewrite was made possible thanks to funding from EURid,
and will help with maintainability and packaging.
Core design and command line syntax is kept the same but as the
libraries the generators use have been changed additional command line
options must be used.
- client_subnet_authority (generator)
This generator now uses IANA's IP address space registry CSVs to
look up the network authority, therefor it needs either to fetch
the CSV files or be given them on command line.
See `man dsc-datatool-generator client_subnet_authority` for more
information.
- client_subnet_country (generator)
This generator now uses MaxMind databases to look up country based
on subnet.
See `man dsc-datatool generator client_subnet_country` for more
information and setup guide of the MaxMind databases.
589ea8b Badges
c32038b nonstrict
0ea3e32 LGTM
cff2e1c COPR
02c31b0 COPR
e8332fd COPR
6d9f71c Input, YAML
93ba755 EPEL 8 packages
3e2df6f Authority
f5d023f Debian packaging
1a59f09 Documentation
85cb1e1 restructure
decd3f6 man-pages, URLs
f264854 man-pages
d73c319 man-pages
f5ca007 man-pages
7bfaf53 Fedora dependencies
3452b48 RPM dependencies
7a4edbc Test
ed43406 client_subnet_authority
62c7d9d Server, node
e0c6419 RPM package
938f154 Rewrite
5400464 README
968ccb1 COPR, spec
14d987f RPM requires
ee10efb Package
a25870f Funding
Revision history for App::DSC::DataTool
0.05 2019-05-31
Release 0.05
Fixed issue with empty values in InfluxDB output, they are now
quoted as an empty string.
9917c4e InfluxDB quote keys/values
0.04 2019-01-21
Release 0.04
Package dependency fix and update of example Grafana dashboards.
d3babc9 Copyright years
9955c88 Travis Perl versions
134a8b3 Debian dependency
2d2114d Fix #23: Rework Grafana dashboards to hopefully show more
correct numbers and also split them up.
9bca0d3 Prepare SPEC for OSB/COPR
0.03 2016-12-16
Release 0.03
Support processing of 25 of the 37 DAT files that the Extractor
can produce, the others can not be converted into time series data
since they lack timestamps. Processing of XML is the recommended
approach to secure all information.
72e829c Implement processing of DAT directories
45294d0 RPM spec
4e8ff69 Fix 5.24 forbidden keys usage
7589ad2 Use perl 5.24 also
cfac110 Fix #16: Handle directories in --xml and warn that --dat is
not implemented yet
0.02 2016-11-11
Release 0.02
First release of `dsc-datatool` with support for:
- Reading DSC XML files
- Transformer:
- Labler: convert indexes/keys to textual names such as QTYPE
- ReRanger: (re)compile lists/ranges/buckets into new buckets
- NetRemap: (re)compile IP addresses and subets into new subnets
- Generator:
- client_subnet_authority: Create a dataset with IP Authority for subnets
- client_subnet_country: Create a dataset with Countries for subnets
- Output:
- Graphite
- InfluxDB
See `dsc-datatool -h` for options and wiki article:
https://github.com/DNS-OARC/dsc-datatool/wiki/Setting-up-a-test-Grafana

33
LICENSE Normal file
View file

@ -0,0 +1,33 @@
DSC DataTool
Copyright (c) 2016-2024 OARC, Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

3
MANIFEST.in Normal file
View file

@ -0,0 +1,3 @@
graft dsc_datatool
global-exclude *.pyc
include CHANGES

37
README.md Normal file
View file

@ -0,0 +1,37 @@
# DSC DataTool
[![Total alerts](https://img.shields.io/lgtm/alerts/g/DNS-OARC/dsc-datatool.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/DNS-OARC/dsc-datatool/alerts/) [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=dns-oarc%3Adsc-datatool&metric=bugs)](https://sonarcloud.io/dashboard?id=dns-oarc%3Adsc-datatool) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=dns-oarc%3Adsc-datatool&metric=security_rating)](https://sonarcloud.io/dashboard?id=dns-oarc%3Adsc-datatool)
Tool for converting, exporting, merging and transforming DSC data.
Please have a look at the man-page(s) `dsc-datatool` (1) on how to use or
[the wiki article](https://github.com/DNS-OARC/dsc-datatool/wiki/Setting-up-a-test-Grafana)
on how to set this up using Influx DB and Grafana.
More information about DSC may be found here:
- https://www.dns-oarc.net/tools/dsc
- https://www.dns-oarc.net/oarc/data/dsc
Issues should be reported here:
- https://github.com/DNS-OARC/dsc-datatool/issues
General support and discussion:
- Mattermost: https://chat.dns-oarc.net/community/channels/oarc-software
- mailing-list: https://lists.dns-oarc.net/mailman/listinfo/dsc
## Dependencies
`dsc-datatool` requires the following Python libraries:
- PyYAML
- maxminddb
## Python Development Environment
Using Ubuntu/Debian:
```
sudo apt-get install python3-maxminddb python3-yaml python3-venv
python3 -m venv venv --system-site-packages
. venv/bin/activate
pip install -e . --no-deps
```

44
contrib/iana-dns-params-toyaml.py Normal file
View file

@ -0,0 +1,44 @@
import yaml
import csv
from urllib.request import Request, urlopen
from io import StringIO
rcode = {}
qtype = {}
opcode = {}
for row in csv.reader(StringIO(urlopen(Request('http://www.iana.org/assignments/dns-parameters/dns-parameters-6.csv')).read().decode('utf-8'))):
if row[0] == 'RCODE':
continue
rcode[row[0]] = row[1]
for row in csv.reader(StringIO(urlopen(Request('http://www.iana.org/assignments/dns-parameters/dns-parameters-4.csv')).read().decode('utf-8'))):
if row[0] == 'TYPE':
continue
qtype[row[1]] = row[0]
for row in csv.reader(StringIO(urlopen(Request('http://www.iana.org/assignments/dns-parameters/dns-parameters-5.csv')).read().decode('utf-8'))):
if row[0] == 'OpCode':
continue
opcode[row[0]] = row[1]
y = {}
for n in ['rcode', 'client_addr_vs_rcode', 'rcode_vs_replylen']:
y[n] = { 'Rcode': {} }
for k, v in rcode.items():
y[n]['Rcode'][k] = v
for n in ['qtype', 'transport_vs_qtype', 'certain_qnames_vs_qtype', 'qtype_vs_tld', 'qtype_vs_qnamelen', 'chaos_types_and_names', 'dns_ip_version_vs_qtype']:
y[n] = { 'Qtype': {} }
for k, v in qtype.items():
if v == '*':
v = 'wildcard'
y[n]['Qtype'][k] = v
for n in ['opcode']:
y[n] = { 'Opcode': {} }
for k, v in rcode.items():
y[n]['Opcode'][k] = v
print(yaml.dump(y, explicit_start=True, default_flow_style=False))

484
dsc_datatool/__init__.py Normal file
View file

@ -0,0 +1,484 @@
"""dsc_datatool
The main Python module for the command line tool `dsc-datatool`, see
`man dsc-datatool` on how to run it.
On runtime it will load all plugins under the following module path:
- dsc_datatool.input
- dsc_datatool.output
- dsc_datatool.generator
- dsc_datatool.transformer
Each plugin category should base it class on one of the follow superclasses:
- dsc_datatool.Input
- dsc_datatool.Output
- dsc_datatool.Generator
- dsc_datatool.Transformer
Doing so it will be automatically registered as available and indexed in
the following public dicts using the class name:
- inputs
- outputs
- generators
- transformers
Example of an output:
from dsc_datatool import Output
class ExampleOutput(Output):
def process(self, datasets)
...
:copyright: 2024 OARC, Inc.
"""
__version__ = '1.4.2'
import argparse
import logging
import os
import importlib
import pkgutil
import sys
import traceback
import re
args = argparse.Namespace()
inputs = {}
outputs = {}
generators = {}
transformers = {}
process_dataset = {}
encoding = 'utf-8'
class Dataset(object):
"""A representation of a DSC dataset
A DSC dataset is one to two dimensional structure where the last
dimension holds an array of values and counters.
It is based on the XML structure of DSC:
<array name="pcap_stats" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="ifname"/>
<dimension number="2" type="pcap_stat"/>
<data>
<ifname val="eth0">
<pcap_stat val="filter_received" count="5625"/>
<pcap_stat val="pkts_captured" count="4894"/>
<pcap_stat val="kernel_dropped" count="731"/>
</ifname>
</data>
</array>
Attributes:
- name: The name of the dataset
- start_time: The start time of the dataset in seconds
- stop_time: The stop time of the dataset in seconds
- dimensions: An array with `Dimension`, the first dimension
"""
name = None
start_time = None
stop_time = None
dimensions = None
def __init__(self):
self.dimensions = []
def __repr__(self):
return '<Dataset name=%r dimension=%r>' % (self.name, self.dimensions)
class Dimension(object):
"""A representation of a DSC dimension
A DSC dataset dimension which can be the first or second dimension,
see `Dataset` for more information.
Attributes:
- name: The name of the dimension
- value: Is set to the value of the dimension if it's the first dimension
- values: A dict of values with corresponding counters if it's the second dimension
"""
name = None
value = None
values = None
dimensions = None
def __init__(self, name):
self.name = name
self.values = {}
self.dimensions = []
def __repr__(self):
return '<Dimension name=%r value=%r dimension=%r>' % (self.name, self.values or self.value, self.dimensions)
class Input(object):
"""Base class of an input plugin"""
def process(self, file):
"""Input.process(...) -> [ Dataset, ... ]
Called to process a file and return an array of `Dataset`'s found in it.
"""
raise Exception('process() not overloaded')
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the input plugin in `inputs`."""
global inputs
if cls.__name__ in inputs:
raise Exception('Duplicate input module: %s already exists' % cls.__name__)
inputs[cls.__name__] = cls
class Output(object):
"""Base class of an output plugin"""
def process(self, datasets):
"""Output.process([ Dataset, ... ])
Called to output the `Dataset`'s in the given array."""
raise Exception('process() not overloaded')
def __init__(self, opts):
"""instance = Output({ 'opt': value, ... })
Called to create an instance of the output plugin, will get a dict
with options provided on command line."""
pass
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the output plugin in `outputs`."""
global outputs
if cls.__name__ in outputs:
raise Exception('Duplicate output module: %s already exists' % cls.__name__)
outputs[cls.__name__] = cls
class Generator(object):
"""Base class of a generator plugin"""
def process(self, datasets):
"""Generator.process([ Dataset, ... ]) -> [ Dataset, ... ]
Called to generate additional `Dataset`'s based on the given array
of `Dataset`'s."""
raise Exception('process() not overloaded')
def __init__(self, opts):
"""instance = Generator({ 'opt': value, ... })
Called to create an instance of the generator plugin, will get a dict
with options provided on command line."""
pass
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the generator plugin in `generators`."""
global generators
if cls.__name__ in generators:
raise Exception('Duplicate generator module: %s already exists' % cls.__name__)
generators[cls.__name__] = cls
class Transformer(object):
"""Base class of a transformer plugin"""
def process(self, datasets):
"""Transformer.process([ Dataset, ... ])
Called to do transformation of the given `Dataset`'s, as in modifying
them directly."""
raise Exception('process() not overloaded')
def __init__(self, opts):
"""instance = Transformer({ 'opt': value, ... })
Called to create an instance of the transformer plugin, will get a dict
with options provided on command line."""
pass
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the transformer plugin in `transformers`."""
global transformers
if cls.__name__ in transformers:
raise Exception('Duplicate transformer module: %s already exists' % cls.__name__)
transformers[cls.__name__] = cls
def main():
"""Called when running `dsc-datatool`."""
def iter_namespace(ns_pkg):
return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
def split_arg(arg, num=1):
sep = arg[0]
p = arg.split(sep)
p.pop(0)
ret = ()
while num > 0:
ret += (p.pop(0),)
num -= 1
ret += (p,)
return ret
def parse_opts(opts):
ret = {}
for opt in opts:
p = opt.split('=', maxsplit=1)
if len(p) > 1:
if p[0] in ret:
if isinstance(ret[p[0]], list):
ret[p[0]].append(p[1])
else:
ret[p[0]] = [ ret[p[0]], p[1] ]
else:
ret[p[0]] = p[1]
elif len(p) > 0:
ret[p[0]] = True
return ret
def _process(datasets, generators, transformers, outputs):
gen_datasets = []
for generator in generators:
try:
gen_datasets += generator.process(datasets)
except Exception as e:
logging.warning('Generator %s failed: %s' % (generator, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
datasets += gen_datasets
if '*' in transformers:
for transformer in transformers['*']:
try:
transformer.process(datasets)
except Exception as e:
logging.warning('Transformer %s failed: %s' % (transformer, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
for dataset in datasets:
if dataset.name in transformers:
for transformer in transformers[dataset.name]:
try:
transformer.process([dataset])
except Exception as e:
logging.warning('Transformer %s failed: %s' % (transformer, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
for output in outputs:
try:
output.process(datasets)
except Exception as e:
logging.warning('Output %s failed: %s' % (output, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
return 0
global args, inputs, outputs, generators, transformers, process_dataset
parser = argparse.ArgumentParser(prog='dsc-datatool',
description='Export DSC data into various formats and databases.',
epilog='See man-page dsc-datatool(1) and dsc-datatool-[generator|transformer|output] <name>(5) for more information')
parser.add_argument('-c', '--conf', nargs=1,
help='Not implemented')
# help='Specify the YAML configuration file to use (default to ~/.dsc-datatool.conf), any command line option will override the options in the configuration file. See dsc-datatool.conf(5)for more information.')
parser.add_argument('-s', '--server', nargs=1,
help='Specify the server for where the data comes from. (required)')
parser.add_argument('-n', '--node', nargs=1,
help='Specify the node for where the data comes from. (required)')
parser.add_argument('-x', '--xml', action='append',
help='Read DSC data from the given file or directory, can be specified multiple times. If a directory is given then all files ending with .xml will be read.')
parser.add_argument('-d', '--dat', action='append',
help='Read DSC data from the given directory, can be specified multiple times. Note that the DAT format is depended on the filename to know what type of data it is.')
parser.add_argument('--dataset', action='append',
help='Specify that only the list of datasets will be processed, the list is comma separated and the option can be given multiple times.')
parser.add_argument('-o', '--output', action='append',
help='"<sep><output>[<sep>option=value...]>" Output data to <output> and use <separator> as an options separator.')
parser.add_argument('-t', '--transform', action='append',
help='"<sep><name><sep><datasets>[<sep>option=value...]>" Use the transformer <name> to change the list of datasets in <datasets>.')
parser.add_argument('-g', '--generator', action='append',
help='"<name>[,<name>,...]" or "<sep><name>[<sep>option=value...]>" Use the specified generators to generate additional datasets.')
parser.add_argument('--list', action='store_true',
help='List the available generators, transformers and outputs then exit.')
parser.add_argument('--skipped-key', nargs=1, default='-:SKIPPED:-',
help='Set the special DSC skipped key. (default to "-:SKIPPED:-")')
parser.add_argument('--skipped-sum-key', nargs=1, default='-:SKIPPED_SUM:-',
help='Set the special DSC skipped sum key. (default to "-:SKIPPED_SUM:-")')
parser.add_argument('--encoding', nargs=1, default='utf-8',
help='Encoding to use for all files, default utf-8.')
parser.add_argument('-v', '--verbose', action='count', default=0,
help='Increase the verbose level, can be given multiple times.')
parser.add_argument('-V', '--version', action='version', version='%(prog)s v'+__version__,
help='Display version and exit.')
args = parser.parse_args()
log_level = 30 - (args.verbose * 10)
if log_level < 0:
log_level = 0
logging.basicConfig(format='%(asctime)s %(levelname)s %(module)s: %(message)s', level=log_level, stream=sys.stderr)
import dsc_datatool.input
import dsc_datatool.output
import dsc_datatool.generator
import dsc_datatool.transformer
for finder, name, ispkg in iter_namespace(dsc_datatool.input):
importlib.import_module(name)
for finder, name, ispkg in iter_namespace(dsc_datatool.output):
importlib.import_module(name)
for finder, name, ispkg in iter_namespace(dsc_datatool.generator):
importlib.import_module(name)
for finder, name, ispkg in iter_namespace(dsc_datatool.transformer):
importlib.import_module(name)
if args.list:
print('Generators:')
for name in generators:
print('',name)
print('Transformers:')
for name in transformers:
print('',name)
print('Outputs:')
for name in outputs:
print('',name)
return 0
if not args.server or not args.node:
raise Exception('--server and --node must be given')
if isinstance(args.server, list):
args.server = ' '.join(args.server)
elif not isinstance(args.server, str):
raise Exception('Invalid argument for --server: %r' % args.server)
if isinstance(args.node, list):
args.node = ' '.join(args.node)
elif not isinstance(args.node, str):
raise Exception('Invalid argument for --node: %r' % args.node)
gens = []
if args.generator:
for arg in args.generator:
if not re.match(r'^\w', arg):
name, opts = split_arg(arg)
if not name in generators:
logging.critical('Generator %s does not exist' % name)
return 1
gens.append(generators[name](parse_opts(opts)))
continue
for name in arg.split(','):
if not name in generators:
logging.critical('Generator %s does not exist' % name)
return 1
gens.append(generators[name]({}))
trans = {}
if args.transform:
for arg in args.transform:
name, datasets, opts = split_arg(arg, num=2)
if not name in transformers:
logging.critical('Transformer %s does not exist' % name)
return 1
for dataset in datasets.split(','):
if not dataset in trans:
trans[dataset] = []
trans[dataset].append(transformers[name](parse_opts(opts)))
out = []
if args.output:
for arg in args.output:
name, opts = split_arg(arg)
if not name in outputs:
logging.critical('Output %s does not exist' % name)
return 1
out.append(outputs[name](parse_opts(opts)))
if args.dataset:
for dataset in args.dataset:
for p in dataset.split(','):
process_dataset[p] = True
xml = []
if args.xml:
for entry in args.xml:
if os.path.isfile(entry):
xml.append(entry)
elif os.path.isdir(entry):
with os.scandir(entry) as dir:
for file in dir:
if not file.name.startswith('.') and file.is_file() and file.name.lower().endswith('.xml'):
xml.append(file.path)
else:
logging.error('--xml %r is not a file or directory' % entry)
dat = []
if args.dat:
for entry in args.dat:
if os.path.isdir(entry):
dat.append(entry)
else:
logging.error('--dat %r is not a directory' % entry)
if not xml and not dat:
logging.error('No valid --xml or --dat given')
return 1
xml_input = inputs['XML']()
for file in xml:
try:
datasets = xml_input.process(file)
except Exception as e:
logging.critical('Unable to process XML file %s: %s' % (file, e))
return 1
ret = _process(datasets, gens, trans, out)
if ret > 0:
return ret
dat_input = inputs['DAT']()
for dir in dat:
try:
datasets = dat_input.process(dir)
except Exception as e:
logging.critical('Unable to process DAT files in %s: %s' % (dir, e))
return 1
ret = _process(datasets, gens, trans, out)
if ret > 0:
return ret

181
dsc_datatool/generator/client_subnet_authority.py Normal file
View file

@ -0,0 +1,181 @@
"""dsc_datatool.generator.client_subnet_authority
See `man dsc-datatool-generator client_subnet_authority`.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import csv
import ipaddress
import logging
from urllib.request import Request, urlopen
from io import StringIO
from dsc_datatool import Generator, Dataset, Dimension, args, encoding
_whois2rir = {
'whois.apnic.net': 'APNIC',
'whois.arin.net': 'ARIN',
'whois.ripe.net': 'RIPE NCC',
'whois.lacnic.net': 'LACNIC',
'whois.afrinic.net': 'AFRINIC',
}
_desig2rir = {
'apnic': 'APNIC',
'arin': 'ARIN',
'ripe ncc': 'RIPE NCC',
'lacnic': 'LACNIC',
'afrinic': 'AFRINIC',
'iana': 'IANA',
'6to4': 'IANA',
}
class client_subnet_authority(Generator):
auth = None
nonstrict = False
def _read(self, input):
global _whois2rir, _desig2rir
for row in csv.reader(input):
prefix, designation, date, whois, rdap, status, note = row
if prefix == 'Prefix':
continue
rir = designation.replace('Administered by ', '').lower()
whois = whois.lower()
if whois in _whois2rir:
rir = _whois2rir[whois]
else:
if rir in _desig2rir:
rir = _desig2rir[rir]
else:
found = None
for k, v in _desig2rir.items():
if k in rir:
found = v
break
if found:
rir = found
else:
if status == 'RESERVED':
rir = 'IANA'
elif designation == 'Segment Routing (SRv6) SIDs':
# TODO: How to better handle this weird allocation?
rir = 'IANA'
else:
raise Exception('Unknown whois/designation: %r/%r' % (whois, designation))
try:
net = ipaddress.ip_network(prefix)
except Exception:
ip, net = prefix.split('/')
net = ipaddress.ip_network('%s.0.0.0/%s' % (int(ip), net))
if net.version == 4:
idx = ipaddress.ip_network('%s/8' % net.network_address, strict=False)
else:
idx = ipaddress.ip_network('%s/24' % net.network_address, strict=False)
if idx.network_address in self.auth:
self.auth[idx.network_address].append({'net': net, 'auth': rir})
else:
self.auth[idx.network_address] = [{'net': net, 'auth': rir}]
def __init__(self, opts):
Generator.__init__(self, opts)
self.auth = {}
csvs = opts.get('csv', None)
urlv4 = opts.get('urlv4', 'https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv')
urlv6 = opts.get('urlv6', 'https://www.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.csv')
if opts.get('nonstrict', False):
self.nonstrict = True
if csvs:
if not isinstance(csvs, list):
csvs = [ csvs ]
for file in csvs:
with open(file, newline='', encoding=encoding) as csvfile:
self._read(csvfile)
elif opts.get('fetch', 'no').lower() == 'yes':
urls = opts.get('url', [ urlv4, urlv6 ])
if urls and not isinstance(urls, list):
urls = [ urls ]
logging.info('bootstrapping client subnet authority using URLs')
for url in urls:
logging.info('fetching %s' % url)
self._read(StringIO(urlopen(Request(url)).read().decode('utf-8')))
else:
raise Exception('No authorities bootstrapped, please specify csv= or fetch=yes')
def process(self, datasets):
gen_datasets = []
for dataset in datasets:
if dataset.name != 'client_subnet':
continue
subnets = {}
for d1 in dataset.dimensions:
for d2 in d1.dimensions:
for k, v in d2.values.items():
if k == args.skipped_key:
continue
elif k == args.skipped_sum_key:
continue
if k in subnets:
subnets[k] += v
else:
subnets[k] = v
auth = {}
for subnet in subnets:
try:
ip = ipaddress.ip_address(subnet)
except Exception as e:
if not self.nonstrict:
raise e
continue
if ip.version == 4:
idx = ipaddress.ip_network('%s/8' % ip, strict=False)
ip = ipaddress.ip_network('%s/32' % ip)
else:
idx = ipaddress.ip_network('%s/24' % ip, strict=False)
ip = ipaddress.ip_network('%s/128' % ip)
if not idx.network_address in self.auth:
idx = '??'
else:
for entry in self.auth[idx.network_address]:
if entry['net'].overlaps(ip):
idx = entry['auth']
break
if idx in auth:
auth[idx] += subnets[subnet]
else:
auth[idx] = subnets[subnet]
if auth:
authd = Dataset()
authd.name = 'client_subnet_authority'
authd.start_time = dataset.start_time
authd.stop_time = dataset.stop_time
gen_datasets.append(authd)
authd1 = Dimension('ClientAuthority')
authd1.values = auth
authd.dimensions.append(authd1)
return gen_datasets
import sys
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Generator.__init_subclass__(client_subnet_authority)

98
dsc_datatool/generator/client_subnet_country.py Normal file
View file

@ -0,0 +1,98 @@
"""dsc_datatool.generator.client_subnet_country
See `man dsc-datatool-generator client_subnet_country`.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import maxminddb
import os
import logging
from dsc_datatool import Generator, Dataset, Dimension, args
class client_subnet_country(Generator):
reader = None
nonstrict = False
def __init__(self, opts):
Generator.__init__(self, opts)
paths = opts.get('path', ['/var/lib/GeoIP', '/usr/share/GeoIP', '/usr/local/share/GeoIP'])
if not isinstance(paths, list):
paths = [ paths ]
filename = opts.get('filename', 'GeoLite2-Country.mmdb')
db = opts.get('db', None)
if db is None:
for path in paths:
db = '%s/%s' % (path, filename)
if os.path.isfile(db) and os.access(db, os.R_OK):
break
db = None
if db is None:
raise Exception('Please specify valid Maxmind database with path=,filename= or db=')
logging.info('Using %s' % db)
self.reader = maxminddb.open_database(db)
if opts.get('nonstrict', False):
self.nonstrict = True
def process(self, datasets):
gen_datasets = []
for dataset in datasets:
if dataset.name != 'client_subnet':
continue
subnets = {}
for d1 in dataset.dimensions:
for d2 in d1.dimensions:
for k, v in d2.values.items():
if k == args.skipped_key:
continue
elif k == args.skipped_sum_key:
continue
if k in subnets:
subnets[k] += v
else:
subnets[k] = v
cc = {}
for subnet in subnets:
try:
c = self.reader.get(subnet)
except Exception as e:
if not self.nonstrict:
raise e
continue
if c:
iso_code = c.get('country', {}).get('iso_code', '??')
if iso_code in cc:
cc[iso_code] += subnets[subnet]
else:
cc[iso_code] = subnets[subnet]
if cc:
ccd = Dataset()
ccd.name = 'client_subnet_country'
ccd.start_time = dataset.start_time
ccd.stop_time = dataset.stop_time
gen_datasets.append(ccd)
ccd1 = Dimension('ClientCountry')
ccd1.values = cc
ccd.dimensions.append(ccd1)
return gen_datasets
import sys
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Generator.__init_subclass__(client_subnet_country)

177
dsc_datatool/input/dat.py Normal file
View file

@ -0,0 +1,177 @@
"""dsc_datatool.input.dat
Input plugin to generate `Dataset`'s from DSC DAT files.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import re
from dsc_datatool import Input, Dataset, Dimension, process_dataset, encoding
_dataset1d = [
'client_subnet_count',
'ipv6_rsn_abusers_count',
]
_dataset2d = {
'qtype': 'Qtype',
'rcode': 'Rcode',
'do_bit': 'D0',
'rd_bit': 'RD',
'opcode': 'Opcode',
'dnssec_qtype': 'Qtype',
'edns_version': 'EDNSVersion',
'client_subnet2_count': 'Class',
'client_subnet2_trace': 'Class',
'edns_bufsiz': 'EDNSBufSiz',
'idn_qname': 'IDNQname',
'client_port_range': 'PortRange',
'priming_responses': 'ReplyLen',
}
_dataset3d = {
'chaos_types_and_names': [ 'Qtype', 'Qname' ],
'certain_qnames_vs_qtype': [ 'CertainQnames', 'Qtype' ],
'direction_vs_ipproto': [ 'Direction', 'IPProto' ],
'pcap_stats': [ 'pcap_stat', 'ifname' ],
'transport_vs_qtype': [ 'Transport', 'Qtype' ],
'dns_ip_version': [ 'IPVersion', 'Qtype' ],
'priming_queries': [ 'Transport', 'EDNSBufSiz' ],
'qr_aa_bits': [ 'Direction', 'QRAABits' ],
}
class DAT(Input):
def process(self, dir):
global _dataset1d, _dataset2d, _dataset3d
datasets = []
for d in _dataset1d:
if process_dataset and not d in process_dataset:
continue
try:
datasets += self.process1d('%s/%s.dat' % (dir, d), d)
except FileNotFoundError:
pass
for k, v in _dataset2d.items():
if process_dataset and not k in process_dataset:
continue
try:
datasets += self.process2d('%s/%s.dat' % (dir, k), k, v)
except FileNotFoundError:
pass
for k, v in _dataset3d.items():
if process_dataset and not k in process_dataset:
continue
try:
datasets += self.process3d('%s/%s.dat' % (dir, k), k, v[0], v[1])
except FileNotFoundError:
pass
return datasets
def process1d(self, file, name):
datasets = []
with open(file, 'r', encoding=encoding) as f:
for l in f.readlines():
if re.match(r'^#', l):
continue
l = re.sub(r'[\r\n]+$', '', l)
dat = re.split(r'\s+', l)
if len(dat) != 2:
raise Exception('DAT %r dataset %r: invalid number of elements for a 1d dataset' % (file, name))
dataset = Dataset()
dataset.name = name
dataset.start_time = int(dat.pop(0))
dataset.stop_time = dataset.start_time + 60
d1 = Dimension('All')
d1.values = { 'ALL': int(dat[0]) }
dataset.dimensions.append(d1)
datasets.append(dataset)
return datasets
def process2d(self, file, name, field):
datasets = []
with open(file, 'r', encoding=encoding) as f:
for l in f.readlines():
if re.match(r'^#', l):
continue
l = re.sub(r'[\r\n]+$', '', l)
dat = re.split(r'\s+', l)
dataset = Dataset()
dataset.name = name
dataset.start_time = int(dat.pop(0))
dataset.stop_time = dataset.start_time + 60
d1 = Dimension('All')
d1.value = 'ALL'
dataset.dimensions.append(d1)
d2 = Dimension(field)
while dat:
if len(dat) < 2:
raise Exception('DAT %r dataset %r: invalid number of elements for a 2d dataset' % (file, name))
k = dat.pop(0)
v = dat.pop(0)
d2.values[k] = int(v)
d1.dimensions.append(d2)
datasets.append(dataset)
return datasets
def process3d(self, file, name, first, second):
datasets = []
with open(file, 'r', encoding=encoding) as f:
for l in f.readlines():
if re.match(r'^#', l):
continue
l = re.sub(r'[\r\n]+$', '', l)
dat = re.split(r'\s+', l)
dataset = Dataset()
dataset.name = name
dataset.start_time = int(dat.pop(0))
dataset.stop_time = dataset.start_time + 60
while dat:
if len(dat) < 2:
raise Exception('DAT %r dataset %r: invalid number of elements for a 2d dataset' % (file, name))
k = dat.pop(0)
v = dat.pop(0)
d1 = Dimension(first)
d1.value = k
dataset.dimensions.append(d1)
d2 = Dimension(second)
dat2 = v.split(':')
while dat2:
if len(dat2) < 2:
raise Exception('DAT %r dataset %r: invalid number of elements for a 2d dataset' % (file, name))
k2 = dat2.pop(0)
v2 = dat2.pop(0)
d2.values[k2] = int(v2)
d1.dimensions.append(d2)
datasets.append(dataset)
return datasets
import sys
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Input.__init_subclass__(DAT)

71
dsc_datatool/input/xml.py Normal file
View file

@ -0,0 +1,71 @@
"""dsc_datatool.input.xml
Input plugin to generate `Dataset`'s from DSC XML files.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import logging
from xml.dom import minidom
import base64
from dsc_datatool import Input, Dataset, Dimension, process_dataset
class XML(Input):
def process(self, file):
dom = minidom.parse(file)
datasets = []
for array in dom.getElementsByTagName('array'):
if process_dataset and not array.getAttribute('name') in process_dataset:
continue
dataset = Dataset()
dataset.name = array.getAttribute('name')
dataset.start_time = int(array.getAttribute('start_time'))
dataset.stop_time = int(array.getAttribute('stop_time'))
dimensions = [None, None]
for dimension in array.getElementsByTagName('dimension'):
if dimension.getAttribute('number') == '1':
if dimensions[0]:
logging.warning('Overwriting dimension 1 for %s' % dataset.name)
dimensions[0] = dimension.getAttribute('type')
elif dimension.getAttribute('number') == '2':
if dimensions[1]:
logging.warning('Overwriting dimension 2 for %s' % dataset.name)
dimensions[1] = dimension.getAttribute('type')
else:
logging.warning('Invalid dimension number %r for %s' % (dimension.getAttribute('number'), dataset.name))
for node1 in array.getElementsByTagName(dimensions[0]):
d1 = Dimension(dimensions[0])
d1.value = node1.getAttribute('val')
try:
if node1.getAttribute('base64'):
d1.value = base64.b64decode(d1.value).decode('utf-8')
except Exception as e:
pass
dataset.dimensions.append(d1)
d2 = Dimension(dimensions[1])
d1.dimensions.append(d2)
for node2 in node1.getElementsByTagName(dimensions[1]):
val = node2.getAttribute('val')
try:
if node2.getAttribute('base64'):
val = base64.b64decode(val).decode('utf-8')
except Exception as e:
pass
d2.values[val] = int(node2.getAttribute('count'))
datasets.append(dataset)
return datasets
import sys
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Input.__init_subclass__(XML)

103
dsc_datatool/output/influxdb.py Normal file
View file

@ -0,0 +1,103 @@
"""dsc_datatool.output.influxdb
See `man dsc-datatool-output influxdb`.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import re
import sys
import atexit
from dsc_datatool import Output, args, encoding
_re = re.compile(r'([,=\\\s])')
def _key(key):
return re.sub(_re, r'\\\1', key)
def _val(val):
ret = re.sub(_re, r'\\\1', val)
if ret == '':
return '""'
return ret
def _process(tags, timestamp, dimension, fh):
if dimension.dimensions is None:
return
if len(dimension.dimensions) > 0:
if not (dimension.name == 'All' and dimension.value == 'ALL'):
tags += ',%s=%s' % (_key(dimension.name.lower()), _val(dimension.value))
for d2 in dimension.dimensions:
_process(tags, timestamp, d2, fh)
return
if dimension.values is None:
return
if len(dimension.values) > 0:
tags += ',%s=' % _key(dimension.name.lower())
for k, v in dimension.values.items():
print('%s%s value=%s %s' % (tags, _val(k), v, timestamp), file=fh)
class InfluxDB(Output):
start_timestamp = True
fh = None
def __init__(self, opts):
Output.__init__(self, opts)
timestamp = opts.get('timestamp', 'start')
if timestamp == 'start':
pass
elif timestamp == 'stop':
self.start_timestamp = False
else:
raise Exception('timestamp option invalid')
file = opts.get('file', None)
append = opts.get('append', False)
if file:
if append:
self.fh = open(file, 'a', encoding=encoding)
else:
self.fh = open(file, 'w', encoding=encoding)
atexit.register(self.close)
else:
self.fh = sys.stdout
if opts.get('dml', False):
print('# DML', file=self.fh)
database = opts.get('database', None)
if database:
print('# CONTEXT-DATABASE: %s' % database, file=self.fh)
def close(self):
if self.fh:
self.fh.close()
self.fh = None
def process(self, datasets):
for dataset in datasets:
tags = '%s,server=%s,node=%s' % (_key(dataset.name.lower()), args.server, args.node)
if self.start_timestamp:
timestamp = dataset.start_time * 1000000000
else:
timestamp = dataset.end_time * 1000000000
for d in dataset.dimensions:
_process(tags, timestamp, d, self.fh)
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Output.__init_subclass__(InfluxDB)

112
dsc_datatool/output/prometheus.py Normal file
View file

@ -0,0 +1,112 @@
"""dsc_datatool.output.prometheus
See `man dsc-datatool-output prometheus`.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import re
import sys
import atexit
from dsc_datatool import Output, args, encoding
_re = re.compile(r'([\\\n"])')
def _key(key):
return re.sub(_re, r'\\\1', key)
def _val(val):
ret = re.sub(_re, r'\\\1', val)
if ret == '':
return '""'
return '"%s"' % ret
class Prometheus(Output):
show_timestamp = True
start_timestamp = True
fh = None
type_def = ''
type_printed = False
prefix = ''
def __init__(self, opts):
Output.__init__(self, opts)
timestamp = opts.get('timestamp', 'start')
if timestamp == 'hide':
self.show_timestamp = False
elif timestamp == 'start':
pass
elif timestamp == 'stop':
self.start_timestamp = False
else:
raise Exception('timestamp option invalid')
file = opts.get('file', None)
append = opts.get('append', False)
if file:
if append:
self.fh = open(file, 'a', encoding=encoding)
else:
self.fh = open(file, 'w', encoding=encoding)
atexit.register(self.close)
else:
self.fh = sys.stdout
self.prefix = opts.get('prefix', '')
def close(self):
if self.fh:
self.fh.close()
self.fh = None
def _process(self, tags, timestamp, dimension, fh):
if dimension.dimensions is None:
return
if len(dimension.dimensions) > 0:
if not (dimension.name == 'All' and dimension.value == 'ALL'):
tags += ',%s=%s' % (_key(dimension.name.lower()), _val(dimension.value))
for d2 in dimension.dimensions:
self._process(tags, timestamp, d2, fh)
return
if dimension.values is None:
return
if len(dimension.values) > 0:
tags += ',%s=' % _key(dimension.name.lower())
for k, v in dimension.values.items():
if not self.type_printed:
print(self.type_def, file=fh)
self.type_printed = True
if self.show_timestamp:
print('%s%s} %s %s' % (tags, _val(k), v, timestamp), file=fh)
else:
print('%s%s} %s' % (tags, _val(k), v), file=fh)
def process(self, datasets):
for dataset in datasets:
self.type_def = '# TYPE %s gauge' % _key(dataset.name.lower())
self.type_printed = False
tags = '%s%s{server=%s,node=%s' % (self.prefix, _key(dataset.name.lower()), _val(args.server), _val(args.node))
if self.start_timestamp:
timestamp = dataset.start_time * 1000
else:
timestamp = dataset.end_time * 1000
for d in dataset.dimensions:
self._process(tags, timestamp, d, self.fh)
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Output.__init_subclass__(Prometheus)

69
dsc_datatool/transformer/labler.py Normal file
View file

@ -0,0 +1,69 @@
"""dsc_datatool.transformer.labler
See `man dsc-datatool-transformer labler`.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import yaml
from dsc_datatool import Transformer, encoding
def _process(label, d):
l = label.get(d.name, None)
if d.values:
if l is None:
return
values = d.values
d.values = {}
for k, v in values.items():
nk = l.get(k, None)
d.values[nk or k] = v
return
if l:
v = l.get(d.value, None)
if v:
d.value = v
for d2 in d.dimensions:
_process(label, d2)
class Labler(Transformer):
label = None
def __init__(self, opts):
Transformer.__init__(self, opts)
if not 'yaml' in opts:
raise Exception('yaml=file option required')
f = open(opts.get('yaml'), 'r', encoding=encoding)
try:
self.label = yaml.full_load(f)
except AttributeError:
self.label = yaml.load(f)
f.close()
def process(self, datasets):
if self.label is None:
return
for dataset in datasets:
label = self.label.get(dataset.name, None)
if label is None:
continue
for d in dataset.dimensions:
_process(label, d)
import sys
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Transformer.__init_subclass__(Labler)

77
dsc_datatool/transformer/net_remap.py Normal file
View file

@ -0,0 +1,77 @@
"""dsc_datatool.transformer.net_remap
See `man dsc-datatool-transformer netremap`.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import ipaddress
from dsc_datatool import Transformer, args
class NetRemap(Transformer):
v4net = None
v6net = None
nonstrict = False
def __init__(self, opts):
Transformer.__init__(self, opts)
net = opts.get('net', None)
self.v4net = opts.get('v4net', net)
self.v6net = opts.get('v6net', net)
if not self.v4net:
raise Exception('v4net (or net) must be given')
if not self.v6net:
raise Exception('v6net (or net) must be given')
if opts.get('nonstrict', False):
self.nonstrict = True
def _process(self, dimension):
if not dimension.values:
for d2 in dimension.dimensions:
self._process(d2)
return
values = dimension.values
dimension.values = {}
for k, v in values.items():
if k == args.skipped_key:
continue
elif k == args.skipped_sum_key:
dimension.values['0'] = v
continue
try:
ip = ipaddress.ip_address(k)
except Exception as e:
if not self.nonstrict:
raise e
continue
if ip.version == 4:
nkey = str(ipaddress.IPv4Network('%s/%s' % (ip, self.v4net), strict=False).network_address)
else:
nkey = str(ipaddress.IPv6Network('%s/%s' % (ip, self.v6net), strict=False).network_address)
if not nkey in dimension.values:
dimension.values[nkey] = v
else:
dimension.values[nkey] += v
def process(self, datasets):
for dataset in datasets:
for dimension in dataset.dimensions:
self._process(dimension)
import sys
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Transformer.__init_subclass__(NetRemap)

123
dsc_datatool/transformer/re_ranger.py Normal file
View file

@ -0,0 +1,123 @@
"""dsc_datatool.transformer.re_ranger
See `man dsc-datatool-transformer reranger`.
Part of dsc_datatool.
:copyright: 2024 OARC, Inc.
"""
import re
from dsc_datatool import Transformer, args
_key_re = re.compile(r'^(?:(\d+)|(\d+)-(\d+))$')
class ReRanger(Transformer):
key = None
func = None
allow_invalid_keys = None
range = None
split_by = None
def __init__(self, opts):
Transformer.__init__(self, opts)
self.key = opts.get('key', 'mid')
self.func = opts.get('func', 'sum')
self.allow_invalid_keys = opts.get('allow_invalid_keys', False)
self.range = opts.get('range', None)
if self.allow_invalid_keys != False:
self.allow_invalid_keys = True
if self.range is None:
raise Exception('range must be given')
m = re.match(r'^/(\d+)$', self.range)
if m is None:
raise Exception('invalid range')
self.split_by = int(m.group(1))
if self.key != 'low' and self.key != 'mid' and self.key != 'high':
raise Exception('invalid key %r' % self.key)
if self.func != 'sum':
raise Exception('invalid func %r' % self.func)
def _process(self, dimension):
global _key_re
if not dimension.values:
for d2 in dimension.dimensions:
self._process(d2)
return
values = dimension.values
dimension.values = {}
skipped = None
for k, v in values.items():
low = None
high = None
m = _key_re.match(k)
if m:
low, low2, high = m.group(1, 2, 3)
if high is None:
low = int(low)
high = low
else:
low = int(low2)
high = int(high)
elif k == args.skipped_key:
continue
elif k == args.skipped_sum_key:
if skipped is None:
skipped = v
else:
skipped += v
continue
elif self.allow_invalid_keys:
dimension.values[k] = v
continue
else:
raise Exception('invalid key %r' % k)
if self.key == 'low':
nkey = low
elif self.key == 'mid':
nkey = int(low + ( (high - low) / 2 ))
else:
nkey = high
nkey = int(nkey / self.split_by) * self.split_by
low = nkey
high = nkey + self.split_by - 1
if self.func == 'sum':
if low != high:
nkey = '%d-%d' % (low, high)
else:
nkey = str(nkey)
if nkey in dimension.values:
dimension.values[nkey] += v
else:
dimension.values[nkey] = v
if skipped:
dimension.values['skipped'] = skipped
def process(self, datasets):
for dataset in datasets:
for dimension in dataset.dimensions:
self._process(dimension)
import sys
if sys.version_info[0] == 3 and sys.version_info[1] == 5: # pragma: no cover
Transformer.__init_subclass__(ReRanger)

188
man/man1/dsc-datatool.1 Normal file
View file

@ -0,0 +1,188 @@
.TH "dsc-datatool" "1"
.SH NAME
dsc-datatool \- Tool for converting, exporting, merging and transforming DSC data.
.SH SYNOPSIS
.SY dsc-datatool
.OP \-h
.OP \-c CONF
.OP \-s SERVER
.OP \-n NODE
.OP \-x XML
.OP \-d DAT
.OP \-\-dataset DATASET
.OP \-o OUTPUT
.OP \-t TRANSFORM]
.OP \-g GENERATOR
.OP \-\-list
.OP \-\-skipped\-key SKIPPED_KEY
.OP \-\-skipped\-sum\-key SKIPPED_SUM_KEY
.OP \-v
.OP \-V
.YS
.SH DESCRIPTION
Tool for converting, exporting, merging and transforming DSC data.
Please have a look at the wiki article on how to set this up using
Influx DB and Grafana.
https://github.com/DNS-OARC/dsc-datatool/wiki/Setting-up-a-test-Grafana
.SH OPTIONS
.TP
.B -h, --help
show this help message and exit
.TP
.BI "-c " CONF ", --conf " CONF
Not implemented
.TP
.BI "-s " SERVER ", --server " SERVER
Specify the server for where the data comes from. (required)
.TP
.BI "-n " NODE ", --node " NODE
Specify the node for where the data comes from. (required)
.TP
.BI "-x " XML ", --xml " XML
Read DSC data from the given file or directory, can be specified multiple
times.
If a directory is given then all files ending with .xml will be read.
.TP
.BI "-d " DAT ", --dat " DAT
Read DSC data from the given directory, can be specified multiple times.
Note that the DAT format is depended on the filename to know what type of
data it is.
.TP
.BI "--dataset " DATASET
Specify that only the list of datasets will be processed, the list is
comma separated and the option can be given multiple times.
.TP
.BI "-o " OUTPUT ", --output " OUTPUT
.I OUTPUT
has the following format that uses
.I output
to specify the output module and
.I sep
as an options separator.
.EX
<sep><output>[<sep>option=value...]>
.EE
Can be specified multiple times to output to more then one.
Use
.B dsc-datatool --list
to see a list of modules and the man-page of each output for information
about options.
.TP
.BI "-t " TRANSFORM ", --transform " TRANSFORM
.I TRANSFORM
has the following format that uses
.I name
to specify the transformer module and
.I sep
as an options separator.
The
.I datasets
field can specify which dataset to run the transformer on, or use
.I *
to specify all datasets.
.EX
<sep><name><sep><datasets>[<sep>option=value...]>
.EE
Can be specific multiple times to chain transformation, the chain will be
executed in the order on command line with one exception.
All transformations specified for dataset
.I *
will be executed before named dataset transformations.
Use
.B dsc-datatool --list
to see a list of modules and the man-page of each transformer for
information about options.
For a list of datasets see the DSC configuration that creates the XML files
or the documentation for the Presenter that creates the DAT files.
.TP
.BI "-g " GENERATOR ", --generator " GENERATOR
.I GENERATOR
has two formats, one to specify a comma separated list of generators
and one that uses
.I name
to specify the generator module and
.I sep
as an options separator.
.EX
<name>[,<name>,...]
<sep><name>[<sep>option=value...]>
.EE
This option can be given multiple times.
Use
.B dsc-datatool --list
to see a list of modules and the man-page of each generator for
information about options.
.TP
.B --list
List the available generators, transformers and outputs then exit.
.TP
.BI "--skipped-key " SKIPPED_KEY
Set the special DSC skipped key. (default to "-:SKIPPED:-")
.TP
.BI "--skipped-sum-key " SKIPPED_SUM_KEY
Set the special DSC skipped sum key. (default to "-:SKIPPED_SUM:-")
.TP
.BI "--encoding " ENCODING
Set the encoding to use when reading and writing files, default to utf-8.
.TP
.B -v, --verbose
Increase the verbose level, can be given multiple times.
.TP
.B -V, --version
Display version and exit.
.LP
.SH EXAMPLE
.EX
dsc-datatool \\
--server "$SERVER" \\
--node "$NODE" \\
--output ";InfluxDB;file=influx.txt;dml=1;database=dsc" \\
--transform ";Labler;*;yaml=$HOME/labler.yaml" \\
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \\
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \\
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \\
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \\
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \\
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=8" \\
--generator client_subnet_country \\
--generator ";client_subnet_authority;fetch=yes" \\
--xml "$XML"
.EE
.SH "SEE ALSO"
.BR dsc-datatool.conf (5),
.BI dsc-datatool- [ generator | transformer | output ]
.BR <name> (7)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

29
man/man5/dsc-datatool.conf.5 Normal file
View file

@ -0,0 +1,29 @@
.TH "dsc-datatool.conf" "5"
.SH NAME
dsc-datatool.conf \- Configuration file for dsc-datatool.
.SH DESCRIPTION
Not implemented.
.SH "SEE ALSO"
.BR dsc-datatool (1)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

66
man/man7/dsc-datatool-generator-client_subnet_authority.7 Normal file
View file

@ -0,0 +1,66 @@
.TH "dsc-datatool-generator client_subnet_authority" "7"
.SH NAME
client_subnet_authority \- Generates network authority information (RIR) based on client subnet.
.SH SYNOPSIS
.SY dsc-datatool
.B \-\-generator
.I """;client_subnet_authority;<options...>"""
.YS
.SH DESCRIPTION
This generator will lookup the network authority (RIR) based network
information in the
.I client_subnet
dataset and generate a new dataset
.IR client_subnet_authority .
The network authority lookup will be done from imports of CSV files or by
fetching the required data from IANA.
.SH OPTIONS
.TP
.BR csv =<files>
Read the network authority information from the given
.IR file(s) .
This option can be a single file, a list of comma separated files or given
multiple times.
.TP
.BR fetch =yes
Fetch the network authority information from IANA IP address space registry
for IPv4 and IPv6.
.TP
.BR urlv4 =url
URL for IANA IPv4 address space registry, default to
.IR https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv .
.TP
.BR urlv6 =url
URL for IANA IPv6 address space registry, default to
.IR https://www.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.csv .
.TP
.BR nonstrict =<value>
If set (to any value) then a non-strict mode is enabled which will disregard
any bad data in the dataset, skipping it completely.
Default is strict mode.
.LP
.SH "SEE ALSO"
.BR dsc-datatool (1)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

67
man/man7/dsc-datatool-generator-client_subnet_country.7 Normal file
View file

@ -0,0 +1,67 @@
.TH "dsc-datatool-generator client_subnet_country" "7"
.SH NAME
client_subnet_country \- Generates country information based on client subnet using Maxmind database.
.SH SYNOPSIS
.SY dsc-datatool
.B \-\-generator
.I """;client_subnet_country;<options...>"""
.YS
.SH DESCRIPTION
This generator looks up country code in a Maxmind database for the subnets
in the
.I client_subnet
dataset and the dataset
.I client_subnet_country
from that.
.SH OPTIONS
.TP
.BR path =<directory>
Search for the Maxmind database in the specified
.IR directory .
This option can be given multiple times to search in multiple directories.
Default search paths are:
.IR /var/lib/GeoIP ,
.IR /usr/share/GeoIP ,
.IR /usr/local/share/GeoIP .
.TP
.BR filename =filename
The database filename to search for, default to
.IR GeoLite2-Country.mmdb .
.TP
.BR db =/path/to/database.mmdb
Full path to Maxmind database, this option overrides
.I path
and
.IR filename .
.TP
.BR nonstrict =<value>
If set (to any value) then a non-strict mode is enabled which will disregard
any bad data in the dataset, skipping it completely.
Default is strict mode.
.LP
.SH "SEE ALSO"
.BR dsc-datatool (1),
.BR geoipupdate (1)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

64
man/man7/dsc-datatool-output-influxdb.7 Normal file
View file

@ -0,0 +1,64 @@
.TH "dsc-datatool-output influxdb" "7"
.SH NAME
InfluxDB \- InfluxDB output.
.SH SYNOPSIS
.SY dsc-datatool
.B \-\-output
.I """;InfluxDB;<options...>"""
.YS
.SH DESCRIPTION
This output generates InfluxDB importable output to stdout or to a specified
file.
.SH OPTIONS
.TP
.BR timestamp =[start|stop]
Choose which timestamp from the dataset to use in the InfluxDB output to
specify when the metrics took place.
Default to
.I start
timestamp.
.TP
.BR file =<filename>
Specify a file to output to instead of stdout.
.TP
.BR append
If given, the output will be appended to the file specified rather then
overwritten.
.TP
.BR dml
Add a
.I "# DML"
header to the output.
.TP
.BR database =name
If
.I dml
is used, this can be used to specify which database the output should be
imported into.
.LP
.SH "SEE ALSO"
.BR dsc-datatool (1),
.BR influx (1)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

78
man/man7/dsc-datatool-output-prometheus.7 Normal file
View file

@ -0,0 +1,78 @@
.TH "dsc-datatool-output prometheus" "7"
.SH NAME
Prometheus \- Prometheus output.
.SH SYNOPSIS
.SY dsc-datatool
.B \-\-output
.I """;Prometheus;<options...>"""
.YS
.SH DESCRIPTION
This output generates Prometheus importable output to stdout or to a specified
file.
.SS Prometheus' node_exporter
This output can be used together with Prometheus'
.IR node_exporter 's
Textfile Collector to automate statistics gathering but some specific
setup and requirements must be meet.
You must hide the timestamp with option
.B timestamp=hide
because timestamps are not supported by the Textfile Collector.
You must make sure only one XML file from a server+node combination is
processed at a time.
Because otherwise you will get multiple data point for the same metric
and this will generate errors from the Textfile Collector, since it does
not support timestamps and cannot separate the measurements.
You must make sure that only one file (per server+node combo) is generated
for the Textfile Collector to read, and it should be the same between runs.
See Textfile Collectors documentation how to setup that atomically.
.SH OPTIONS
.TP
.BR timestamp =[hide|start|stop]
Choose which timestamp from the dataset to use in the Prometheus output to
specify when the metrics took place.
Default to
.I start
timestamp.
.TP
.BR file =<filename>
Specify a file to output to instead of stdout.
.TP
.BR append
If given, the output will be appended to the file specified rather then
overwritten.
.TP
.BR prefix =<string>
Use the given string as prefix on all metric names.
.LP
.SH "SEE ALSO"
.BR dsc-datatool (1)
.I https://prometheus.io/docs/guides/node-exporter/
.I https://github.com/prometheus/node_exporter#textfile-collector
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

62
man/man7/dsc-datatool-transformer-labler.7 Normal file
View file

@ -0,0 +1,62 @@
.TH "dsc-datatool-transformer labler" "7"
.SH NAME
Labler \- Rewrite numeric labels to textual labels using the provided YAML data as lookup tables.
.SH SYNOPSIS
.SY dsc-datatool
.B \-\-transform
.I """;Labler;<dataset>;yaml=<file>"""
.YS
.SH DESCRIPTION
This transformer rewrites labels in datasets based on information provided
in a YAML file.
The YAML structure is as follows:
.EX
---
dataset_name:
DimentionName:
DimentionValue: RewriteTo
.EE
The
.I dataset_name
is the name of the dataset,
.I DimentionName
is the name of the dimension to rewrite and
.I DimensionValue
is the value to change to
.IR RewriteTo .
.SH OPTIONS
.TP
.B dataset
See
.IR dsc-datatool (1)
on how to specify which dataset(s) to run the transformer on.
.TP
.BR yaml =<file>
The YAML file to load rewrite data from (required).
.LP
.SH "SEE ALSO"
.BR dsc-datatool (1)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

60
man/man7/dsc-datatool-transformer-netremap.7 Normal file
View file

@ -0,0 +1,60 @@
.TH "dsc-datatool-transformer netremap" "7"
.SH NAME
NetRemap \- Remap network addresses to other ranges/subnets.
.SH SYNOPSIS
.SY dsc-datatool
.B \-\-transform
.I """;NetRemap;<dataset>;<options...>"""
.YS
.SH DESCRIPTION
This transformer can remap network ranges on IP addresses and subnets.
.SH OPTIONS
.TP
.B dataset
See
.IR dsc-datatool (1)
on how to specify which dataset(s) to run the transformer on.
.TP
.BR net =<network>
The network to remap both IPv4 and IPv6 addresses and networks, specify
only the number in a short-form network (e.g. "/<network>").
.TP
.BR v4net =<network>
The IPv4 network to use for IPv4 addresses and networks, default to
.I net
if that is specified.
.TP
.BR v6net =<network>
The IPv6 network to use for IPv6 addresses and networks, default to
.I net
if that is specified.
.TP
.BR nonstrict =<value>
If set (to any value) then a non-strict mode is enabled which will disregard
any bad data in the dataset, skipping it completely.
Default is strict mode.
.LP
.SH "SEE ALSO"
.BR dsc-datatool (1)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

75
man/man7/dsc-datatool-transformer-reranger.7 Normal file
View file

@ -0,0 +1,75 @@
.TH "dsc-datatool-transformer reranger" "7"
.SH NAME
ReRanger \- Rewrite ranged or numerical statistics into other ranges.
.SH SYNOPSIS
.SY dsc-datatool
.B \-\-transform
.I """;ReRanger;<dataset>;<options...>"""
.YS
.SH DESCRIPTION
This transformer can re-range values in datasets to new rangers.
It supports both values that are a single numerical value or one that is
already a range itself
.RI ( num\-num ).
.SH OPTIONS
.TP
.B dataset
See
.IR dsc-datatool (1)
on how to specify which dataset(s) to run the transformer on.
.TP
.BR key =[low|mid|high]
Specify what value to use when re-ranging an already ranged value
.RI (low\-high).
Default to
.IR mid ,
which is high minus low then divided by 2.
.TP
.BR func =<func>
The function to use when aggregating multiple values into a new ranger,
default to
.IR sum .
Currently only one function exists,
.IR sum ,
which adds the sum of all values into the new range.
.TP
.B allow_invalid_keys
If given the "invalid" keys/values that is not numerical or a range will be
passed through unmodified.
.TP
.BR range =</num>
The new range given as
.I /number
(required).
For example
.I range=/64
will re-range all values into buckets of 64, 0-63, 64-127 and so on.
.LP
.SH "SEE ALSO"
.BR dsc-datatool (1)
.SH AUTHORS
Jerry Lundström, DNS-OARC
.LP
Maintained by DNS-OARC
.LP
.RS
.I https://www.dns-oarc.net/tools/dsc
.RE
.LP
.SH BUGS
For issues and feature requests please use:
.LP
.RS
\fIhttps://github.com/DNS-OARC/dsc-datatool/issues\fP
.RE
.LP
For question and help please use:
.LP
.RS
\fIhttps://lists.dns-oarc.net/mailman/listinfo/dsc\fP
.RE
.LP

259
rpm/dsc-datatool.spec Normal file
View file

@ -0,0 +1,259 @@
Name: dsc-datatool
Version: 1.4.2
Release: 1%{?dist}
Summary: Export DSC data to other formats and/or databases
Group: Productivity/Networking/DNS/Utilities
License: BSD-3-Clause
URL: https://www.dns-oarc.net/oarc/data/dsc
# Source needs to be generated by dist-tools/create-source-packages, see
# https://github.com/jelu/dist-tools
Source0: https://github.com/DNS-OARC/dsc-datatool/archive/v%{version}.tar.gz?/%{name}_%{version}.orig.tar.gz
BuildArch: noarch
BuildRequires: python3-devel
BuildRequires: python3-setuptools
BuildRequires: python-rpm-macros
%if 0%{?el7}
BuildRequires: python36-maxminddb
BuildRequires: python36-PyYAML
%else
%if (0%{?sle_version} == 150500 && !0%{?is_opensuse}) || 0%{?sle_version} >= 150600
BuildRequires: python311-maxminddb
%else
BuildRequires: python3-maxminddb
%endif
BuildRequires: python3-PyYAML
%endif
%if 0%{?el7}
Requires: python36-maxminddb
Requires: python36-PyYAML
%else
%if (0%{?sle_version} == 150500 && !0%{?is_opensuse}) || 0%{?sle_version} >= 150600
Requires: python311-maxminddb
%else
Requires: python3-maxminddb
%endif
Requires: python3-PyYAML
%endif
%package doc
Summary: Documentation files for %{name}
Group: Documentation
%description
Tool for converting, exporting, merging and transforming DSC data.
%description doc
Tool for converting, exporting, merging and transforming DSC data.
This package contains the documentation for dsc-datatool.
%prep
%setup -q -n %{name}_%{version}
%build
python3 setup.py build
%install
python3 setup.py install --prefix=%{_prefix} --root=%{buildroot}
mkdir -p %{buildroot}%{_mandir}/man1/
install -m644 man/man1/dsc-datatool.1 %{buildroot}%{_mandir}/man1/
mkdir -p %{buildroot}%{_mandir}/man5/
install -m644 man/man5/dsc-datatool.conf.5 %{buildroot}%{_mandir}/man5/
mkdir -p %{buildroot}%{_mandir}/man7/
install -m644 man/man7/dsc-datatool-transformer-reranger.7 %{buildroot}%{_mandir}/man7/
install -m644 man/man7/dsc-datatool-generator-client_subnet_country.7 %{buildroot}%{_mandir}/man7/
install -m644 man/man7/dsc-datatool-generator-client_subnet_authority.7 %{buildroot}%{_mandir}/man7/
install -m644 man/man7/dsc-datatool-output-influxdb.7 %{buildroot}%{_mandir}/man7/
install -m644 man/man7/dsc-datatool-output-prometheus.7 %{buildroot}%{_mandir}/man7/
install -m644 man/man7/dsc-datatool-transformer-labler.7 %{buildroot}%{_mandir}/man7/
install -m644 man/man7/dsc-datatool-transformer-netremap.7 %{buildroot}%{_mandir}/man7/
%files
%license LICENSE
%{_bindir}/dsc-datatool
%{_mandir}/man1/dsc-datatool.1*
%{_mandir}/man5/dsc-datatool.conf.5*
%{_mandir}/man7/dsc-datatool*.7*
%{python3_sitelib}/dsc_datatool*
%files doc
%doc CHANGES README.md
%license LICENSE
%changelog
* Tue Jun 18 2024 Jerry Lundström <lundstrom.jerry@gmail.com> 1.4.2-1
- Release 1.4.2
* This release fixes issues with IANA's IPv6 parameters file,
dsc-datatool expected a RIR in the Designation field but IANA recently
added a title for SRv6 reservation which caused an exception.
* Other updates are related to packages and GitHub workflows.
* Commits:
7560d82 Tests
8568c84 Fix client subnet authority
a8c58a9 Workflow
fd8915c RPM SUSE
* Wed Dec 06 2023 Jerry Lundström <lundstrom.jerry@gmail.com> 1.4.1-1
- Release 1.4.1
* This release fixes issue with InfluxDB quoting, was missing to quote
the quote character.
* Other changes:
- Dependency correction for SLE 15.5
- Tweaks to test layouts
* Commits:
b44b874 Tests
eef3ae0 SLE 15.5
75c7fc1 Influx quoting
* Thu Jun 15 2023 Jerry Lundström <lundstrom.jerry@gmail.com> 1.4.0-1
- Release 1.4.0
* This release adds the option `--encoding` to set an encoding to use
for reading and writing files.
* Commits:
f64c8b6 encoding man-page
09c0ce9 Encoding
* Thu Nov 10 2022 Jerry Lundström <lundstrom.jerry@gmail.com> 1.3.0-1
- Release 1.3.0
* This release adds option `nonstrict` to `client_subnet_authority`
generator for skipping bad data in datasets.
* The contrib DSC+Grafana test site dashboards has been moved to its
own repository, feel free to contribute your own creations to it:
https://github.com/DNS-OARC/dsc-datatool-grafana
* Commits:
90b232d Add CodeQL workflow for GitHub code scanning
e4fa3b0 Test site
474f97d client_subnet_authority non-strict mode
* Mon Jun 13 2022 Jerry Lundström <lundstrom.jerry@gmail.com> 1.2.0-1
- Release 1.2.0
* This release fixes handling of base64'ed strings in DSC XML and will
now decode them back into text when reading, the selected output will
then handling any quoting or escaping needed.
* Added a new option for Prometheus output to set a prefix for metrics so
that they can be easily separated from other metrics if needed, see
`man dsc-datatool-output prometheus`.
* Commits:
5f9f972 Fix COPR
3d72019 Prometheus metric prefix
bdc992e base64 labels
* Tue Apr 05 2022 Jerry Lundström <lundstrom.jerry@gmail.com> 1.1.0-1
- Release 1.1.0
* This releases adds support for Prometheus' node_exporter using it's
Textfile Collector (see `man dsc-datatool-output prometheus`) and
fixes a bug in InfluxDB output when selecting what timestamp to use.
Also updates packages and Grafana test site dashboards.
* Commits:
4381541 RPM
19bc153 Typo/clarification
2a32dd8 Prometheus, InfluxDB, Copyright
dd5323e debhelper
7352c1e Bye Travis
32b3bbe Grafana dashboards
304ab76 Info
* Wed Oct 21 2020 Jerry Lundström <lundstrom.jerry@gmail.com> 1.0.2-1
- Release 1.0.2
* This release fixed a bug in DAT file parsing that was discovered when
adding coverage tests.
* Commits:
45b1aa3 Coverage
7aedc1a Coverage
64957b9 DAT, Coverage
370fb86 Coverage
891cb7c Coverage
9374faa Coverage
* Fri Aug 07 2020 Jerry Lundström <lundstrom.jerry@gmail.com> 1.0.1-1
- Release 1.0.1
* This release adds compatibility with Python v3.5 which allows
packages to be built for Ubuntu Xenial.
* Commits:
bc0be5b python 3.5
* Mon Aug 03 2020 Jerry Lundström <lundstrom.jerry@gmail.com> 1.0.0-2
- Release 1.0.0
* This release brings a complete rewrite of the tool, from Perl to
Python. This rewrite was made possible thanks to funding from EURid,
and will help with maintainability and packaging.
* Core design and command line syntax is kept the same but as the
libraries the generators use have been changed additional command line
options must be used.
- client_subnet_authority (generator)
This generator now uses IANA's IP address space registry CSVs to
look up the network authority, therefor it needs either to fetch
the CSV files or be given them on command line.
See `man dsc-datatool-generator client_subnet_authority` for more
information.
- client_subnet_country (generator)
This generator now uses MaxMind databases to look up country based
on subnet.
See `man dsc-datatool generator client_subnet_country` for more
information and setup guide of the MaxMind databases.
* Commits:
589ea8b Badges
c32038b nonstrict
0ea3e32 LGTM
cff2e1c COPR
02c31b0 COPR
e8332fd COPR
6d9f71c Input, YAML
93ba755 EPEL 8 packages
3e2df6f Authority
f5d023f Debian packaging
1a59f09 Documentation
85cb1e1 restructure
decd3f6 man-pages, URLs
f264854 man-pages
d73c319 man-pages
f5ca007 man-pages
7bfaf53 Fedora dependencies
3452b48 RPM dependencies
7a4edbc Test
ed43406 client_subnet_authority
62c7d9d Server, node
e0c6419 RPM package
938f154 Rewrite
5400464 README
968ccb1 COPR, spec
14d987f RPM requires
ee10efb Package
a25870f Funding
* Wed Apr 15 2020 Jerry Lundström <lundstrom.jerry@gmail.com> 1.0.0-1
- Prepare for v1.0.0
* Fri May 31 2019 Jerry Lundström <lundstrom.jerry@gmail.com> 0.05-1
- Release 0.05
* Fixed issue with empty values in InfluxDB output, they are now
quoted as an empty string.
* Commits:
9917c4e InfluxDB quote keys/values
* Mon Jan 21 2019 Jerry Lundström <lundstrom.jerry@gmail.com> 0.04-1
- Release 0.04
* Package dependency fix and update of example Grafana dashboards.
* Commits:
d3babc9 Copyright years
9955c88 Travis Perl versions
134a8b3 Debian dependency
2d2114d Fix #23: Rework Grafana dashboards to hopefully show more
correct numbers and also split them up.
9bca0d3 Prepare SPEC for OSB/COPR
* Fri Dec 16 2016 Jerry Lundström <lundstrom.jerry@gmail.com> 0.03-1
- Release 0.03
* Support processing of 25 of the 37 DAT files that the Extractor
can produce, the others can not be converted into time series data
since they lack timestamps. Processing of XML is the recommended
approach to secure all information.
* Commits:
72e829c Implement processing of DAT directories
45294d0 RPM spec
4e8ff69 Fix 5.24 forbidden keys usage
7589ad2 Use perl 5.24 also
cfac110 Fix #16: Handle directories in --xml and warn that --dat is
not implemented yet
* Thu Dec 15 2016 Jerry Lundström <lundstrom.jerry@gmail.com> 0.02-1
- Initial package

7
setup.cfg Normal file
View file

@ -0,0 +1,7 @@
[tool:pytest]
testpaths = tests
[coverage:run]
branch = True
source =
dsc_datatool

28
setup.py Normal file
View file

@ -0,0 +1,28 @@
from setuptools import find_packages, setup
setup(
name='dsc_datatool',
version='1.4.2',
packages=find_packages(),
include_package_data=True,
zip_safe=False,
python_requires='>=3.5.1',
install_requires=[
'maxminddb>=1.2.0',
'PyYAML>=3.11',
],
extras_require={
'dev': [
'pytest>=4',
'coverage',
'watchdog',
],
},
entry_points={
'console_scripts': [
'dsc-datatool = dsc_datatool:main',
],
},
scripts=[
],
)

1
sonar-project.properties.local Normal file
View file

@ -0,0 +1 @@
sonar.sources=dsc_datatool

336
tests/1458044657.xml Normal file
View file

@ -0,0 +1,336 @@
<dscdata>
<array name="pcap_stats" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="ifname"/>
<dimension number="2" type="pcap_stat"/>
<data>
<ifname val="Li8xNDU4MDQ0NjU3LnBjYXAuZGlzdA==" base64="1">
<pcap_stat val="pkts_captured" count="8"/>
</ifname>
</data>
</array>
<array name="label_count" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="LabelCount"/>
<data>
<All val="ALL">
<LabelCount val="3" count="4"/>
<LabelCount val="6" count="4"/>
</All>
</data>
</array>
<array name="third_ld_vs_rcode" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Rcode"/>
<dimension number="2" type="ThirdLD"/>
<data>
<Rcode val="0">
<ThirdLD val="216.in-addr.arpa" count="2"/>
<ThirdLD val="www.google.se" count="1"/>
<ThirdLD val="www.google.com" count="1"/>
</Rcode>
</data>
</array>
<array name="second_ld_vs_rcode" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Rcode"/>
<dimension number="2" type="SecondLD"/>
<data>
<Rcode val="0">
<SecondLD val="in-addr.arpa" count="2"/>
<SecondLD val="google.com" count="1"/>
<SecondLD val="www.google.se" count="1"/>
</Rcode>
</data>
</array>
<array name="server" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="IP"/>
<data>
<All val="ALL">
<IP val="8.8.8.8" count="8"/>
</All>
</data>
</array>
<array name="qr_aa_bits" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Direction"/>
<dimension number="2" type="QRAABits"/>
<data>
<Direction val="sent">
</Direction>
<Direction val="recv">
</Direction>
<Direction val="else">
<QRAABits val="cXI9MCxhYT0w" base64="1" count="4"/>
<QRAABits val="cXI9MSxhYT0w" base64="1" count="4"/>
</Direction>
</data>
</array>
<array name="qname" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Name"/>
<data>
<All val="ALL">
<Name val="100.209.58.216.in-addr.arpa" count="2"/>
<Name val="www.google.se" count="2"/>
<Name val="131.209.58.216.in-addr.arpa" count="2"/>
<Name val="www.google.com" count="2"/>
</All>
</data>
</array>
<array name="qclass" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Class"/>
<data>
<All val="ALL">
<Class val="1" count="8"/>
</All>
</data>
</array>
<array name="dns_ip_version" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Version"/>
<data>
<All val="ALL">
<Version val="IPv4" count="8"/>
</All>
</data>
</array>
<array name="ip_version" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Version"/>
<data>
<All val="ALL">
<Version val="IPv4" count="8"/>
</All>
</data>
</array>
<array name="direction_vs_ipproto" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Direction"/>
<dimension number="2" type="IPProto"/>
<data>
<Direction val="sent">
</Direction>
<Direction val="recv">
</Direction>
<Direction val="else">
<IPProto val="udp" count="8"/>
</Direction>
</data>
</array>
<array name="client_port" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Port"/>
<data>
<All val="ALL">
<Port val="59978" count="4"/>
<Port val="0" count="1"/>
<Port val="53" count="1"/>
<Port val="44275" count="1"/>
<Port val="57483" count="1"/>
</All>
</data>
</array>
<array name="client_port_range" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="PortRange"/>
<data>
<All val="ALL">
<PortRange val="43008-44031" count="1"/>
<PortRange val="56320-57343" count="1"/>
<PortRange val="57344-58367" count="1"/>
<PortRange val="58368-59391" count="1"/>
</All>
</data>
</array>
<array name="transport_vs_qtype" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Transport"/>
<dimension number="2" type="Qtype"/>
<data>
<Transport val="udp">
<Qtype val="1" count="2"/>
<Qtype val="12" count="2"/>
</Transport>
<Transport val="tcp">
</Transport>
</data>
</array>
<array name="ipv6_rsn_abusers" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="ClientAddr"/>
<data>
</data>
</array>
<array name="idn_vs_tld" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="TLD"/>
<data>
</data>
</array>
<array name="tc_bit" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="TC"/>
<data>
<All val="ALL">
<TC val="clr" count="8"/>
</All>
</data>
</array>
<array name="rd_bit" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="RD"/>
<data>
<All val="ALL">
<RD val="set" count="4"/>
</All>
</data>
</array>
<array name="do_bit" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="D0"/>
<data>
<All val="ALL">
<D0 val="clr" count="4"/>
</All>
</data>
</array>
<array name="edns_bufsiz" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="EDNSBufSiz"/>
<data>
<All val="ALL">
<EDNSBufSiz val="None" count="4"/>
</All>
</data>
</array>
<array name="edns_version" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="EDNSVersion"/>
<data>
<All val="ALL">
<EDNSVersion val="none" count="4"/>
</All>
</data>
</array>
<array name="idn_qname" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="IDNQname"/>
<data>
<All val="ALL">
<IDNQname val="normal" count="4"/>
</All>
</data>
</array>
<array name="chaos_types_and_names" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Qtype"/>
<dimension number="2" type="Qname"/>
<data>
</data>
</array>
<array name="client_addr_vs_rcode" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Rcode"/>
<dimension number="2" type="ClientAddr"/>
<data>
<Rcode val="0">
<ClientAddr val="172.17.0.16" count="4"/>
</Rcode>
</data>
</array>
<array name="client_subnet2" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Class"/>
<dimension number="2" type="ClientSubnet"/>
<data>
<Class val="ok">
<ClientSubnet val="172.17.0.0" count="3"/>
</Class>
<Class val="non-auth-tld">
<ClientSubnet val="172.17.0.0" count="1"/>
</Class>
</data>
</array>
<array name="certain_qnames_vs_qtype" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="CertainQnames"/>
<dimension number="2" type="Qtype"/>
<data>
<CertainQnames val="localhost">
</CertainQnames>
<CertainQnames val="X.root-servers.net">
</CertainQnames>
<CertainQnames val="else">
<Qtype val="1" count="2"/>
<Qtype val="12" count="2"/>
</CertainQnames>
</data>
</array>
<array name="qtype_vs_tld" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Qtype"/>
<dimension number="2" type="TLD"/>
<data>
<Qtype val="1">
<TLD val="com" count="1"/>
<TLD val="google.se" count="1"/>
</Qtype>
<Qtype val="12">
<TLD val="arpa" count="2"/>
</Qtype>
</data>
</array>
<array name="qtype_vs_qnamelen" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Qtype"/>
<dimension number="2" type="QnameLen"/>
<data>
<Qtype val="1">
<QnameLen val="13" count="1"/>
<QnameLen val="14" count="1"/>
</Qtype>
<Qtype val="12">
<QnameLen val="27" count="2"/>
</Qtype>
</data>
</array>
<array name="client_subnet" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="ClientSubnet"/>
<data>
<All val="ALL">
<ClientSubnet val="172.17.0.0" count="4"/>
</All>
</data>
</array>
<array name="rcode_vs_replylen" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="Rcode"/>
<dimension number="2" type="ReplyLen"/>
<data>
<Rcode val="0">
<ReplyLen val="142" count="2"/>
<ReplyLen val="47" count="1"/>
<ReplyLen val="48" count="1"/>
</Rcode>
</data>
</array>
<array name="opcode" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Opcode"/>
<data>
<All val="ALL">
<Opcode val="0" count="4"/>
</All>
</data>
</array>
<array name="rcode" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Rcode"/>
<data>
<All val="ALL">
<Rcode val="0" count="4"/>
</All>
</data>
</array>
<array name="qtype" dimensions="2" start_time="1458044655" stop_time="1458044657">
<dimension number="1" type="All"/>
<dimension number="2" type="Qtype"/>
<data>
<All val="ALL">
<Qtype val="1" count="2"/>
<Qtype val="12" count="2"/>
</All>
</data>
</array>
</dscdata>

507
tests/1563520620.dscdata.xml Normal file
View file

@ -0,0 +1,507 @@
<dscdata>
<array name="pcap_stats" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="ifname"/>
<dimension number="2" type="pcap_stat"/>
<data>
<ifname val="eth0">
<pcap_stat val="filter_received" count="5625"/>
<pcap_stat val="pkts_captured" count="4894"/>
<pcap_stat val="kernel_dropped" count="731"/>
</ifname>
</data>
</array>
<array name="direction_vs_ipproto" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Direction"/>
<dimension number="2" type="IPProto"/>
<data>
<Direction val="sent">
<IPProto val="udp" count="289"/>
</Direction>
<Direction val="recv">
<IPProto val="udp" count="219"/>
</Direction>
<Direction val="else">
<IPProto val="udp" count="267"/>
</Direction>
</data>
</array>
<array name="client_port_range" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="PortRange"/>
<data>
<All val="ALL">
<PortRange val="33792-34815" count="12"/>
<PortRange val="35840-36863" count="12"/>
<PortRange val="24576-25599" count="11"/>
<PortRange val="56320-57343" count="11"/>
<PortRange val="55296-56319" count="10"/>
<PortRange val="41984-43007" count="9"/>
<PortRange val="19456-20479" count="8"/>
<PortRange val="58368-59391" count="8"/>
<PortRange val="61440-62463" count="8"/>
<PortRange val="63488-64511" count="8"/>
<PortRange val="3072-4095" count="7"/>
<PortRange val="15360-16383" count="7"/>
<PortRange val="34816-35839" count="7"/>
<PortRange val="37888-38911" count="7"/>
<PortRange val="40960-41983" count="7"/>
<PortRange val="45056-46079" count="7"/>
<PortRange val="46080-47103" count="7"/>
<PortRange val="59392-60415" count="7"/>
<PortRange val="13312-14335" count="6"/>
<PortRange val="14336-15359" count="6"/>
<PortRange val="32768-33791" count="6"/>
<PortRange val="36864-37887" count="6"/>
<PortRange val="43008-44031" count="6"/>
<PortRange val="49152-50175" count="6"/>
<PortRange val="52224-53247" count="6"/>
<PortRange val="2048-3071" count="5"/>
<PortRange val="12288-13311" count="5"/>
<PortRange val="16384-17407" count="5"/>
<PortRange val="28672-29695" count="5"/>
<PortRange val="31744-32767" count="5"/>
<PortRange val="39936-40959" count="5"/>
<PortRange val="47104-48127" count="5"/>
<PortRange val="54272-55295" count="5"/>
<PortRange val="5120-6143" count="4"/>
<PortRange val="7168-8191" count="4"/>
<PortRange val="10240-11263" count="4"/>
<PortRange val="11264-12287" count="4"/>
<PortRange val="17408-18431" count="4"/>
<PortRange val="18432-19455" count="4"/>
<PortRange val="22528-23551" count="4"/>
<PortRange val="23552-24575" count="4"/>
<PortRange val="25600-26623" count="4"/>
<PortRange val="26624-27647" count="4"/>
<PortRange val="29696-30719" count="4"/>
<PortRange val="30720-31743" count="4"/>
<PortRange val="38912-39935" count="4"/>
<PortRange val="48128-49151" count="4"/>
<PortRange val="50176-51199" count="4"/>
<PortRange val="62464-63487" count="4"/>
<PortRange val="1024-2047" count="3"/>
<PortRange val="4096-5119" count="3"/>
<PortRange val="8192-9215" count="3"/>
<PortRange val="9216-10239" count="3"/>
<PortRange val="20480-21503" count="3"/>
<PortRange val="27648-28671" count="3"/>
<PortRange val="44032-45055" count="3"/>
<PortRange val="53248-54271" count="3"/>
<PortRange val="57344-58367" count="3"/>
<PortRange val="6144-7167" count="2"/>
<PortRange val="21504-22527" count="2"/>
<PortRange val="51200-52223" count="2"/>
<PortRange val="60416-61439" count="1"/>
</All>
</data>
</array>
<array name="transport_vs_qtype" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Transport"/>
<dimension number="2" type="Qtype"/>
<data>
<Transport val="udp">
<Qtype val="1" count="128"/>
<Qtype val="12" count="78"/>
<Qtype val="28" count="71"/>
<Qtype val="48" count="25"/>
<Qtype val="16" count="18"/>
<Qtype val="15" count="9"/>
<Qtype val="2" count="7"/>
<Qtype val="43" count="5"/>
<Qtype val="6" count="4"/>
</Transport>
<Transport val="tcp">
</Transport>
</data>
</array>
<array name="ipv6_rsn_abusers" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="ClientAddr"/>
<data>
</data>
</array>
<array name="idn_vs_tld" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="TLD"/>
<data>
</data>
</array>
<array name="rd_bit" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="RD"/>
<data>
<All val="ALL">
<RD val="set" count="186"/>
<RD val="clr" count="159"/>
</All>
</data>
</array>
<array name="do_bit" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="D0"/>
<data>
<All val="ALL">
<D0 val="clr" count="211"/>
<D0 val="set" count="134"/>
</All>
</data>
</array>
<array name="edns_bufsiz" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="EDNSBufSiz"/>
<data>
<All val="ALL">
<EDNSBufSiz val="None" count="204"/>
<EDNSBufSiz val="4096-4607" count="73"/>
<EDNSBufSiz val="512-1023" count="26"/>
<EDNSBufSiz val="1024-1535" count="22"/>
<EDNSBufSiz val="3584-4095" count="12"/>
<EDNSBufSiz val="1536-2047" count="6"/>
<EDNSBufSiz val="2048-2559" count="1"/>
<EDNSBufSiz val="8192-8703" count="1"/>
</All>
</data>
</array>
<array name="edns_version" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="EDNSVersion"/>
<data>
<All val="ALL">
<EDNSVersion val="none" count="204"/>
<EDNSVersion val="0" count="141"/>
</All>
</data>
</array>
<array name="idn_qname" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="IDNQname"/>
<data>
<All val="ALL">
<IDNQname val="normal" count="345"/>
</All>
</data>
</array>
<array name="chaos_types_and_names" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Qtype"/>
<dimension number="2" type="Qname"/>
<data>
</data>
</array>
<array name="client_addr_vs_rcode" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Rcode"/>
<dimension number="2" type="ClientAddr"/>
<data>
<Rcode val="0">
<ClientAddr val="64.191.0.198" count="78"/>
<ClientAddr val="2620:ff:c000::198" count="50"/>
<ClientAddr val="-:SKIPPED:-" count="131"/>
<ClientAddr val="-:SKIPPED_SUM:-" count="131"/>
</Rcode>
<Rcode val="3">
<ClientAddr val="64.191.0.198" count="3"/>
<ClientAddr val="2620:ff:c000::198" count="2"/>
</Rcode>
<Rcode val="2">
<ClientAddr val="64.191.0.198" count="6"/>
</Rcode>
</data>
</array>
<array name="client_subnet2" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Class"/>
<dimension number="2" type="ClientSubnet"/>
<data>
<Class val="ok">
<ClientSubnet val="64.191.0.0" count="180"/>
<ClientSubnet val="2620:ff:c000::198" count="5"/>
</Class>
<Class val="non-auth-tld">
<ClientSubnet val="64.191.0.0" count="18"/>
</Class>
<Class val="root-servers.net">
</Class>
<Class val="localhost">
</Class>
<Class val="a-for-a">
</Class>
<Class val="a-for-root">
</Class>
<Class val="rfc1918-ptr">
<ClientSubnet val="2620:ff:c000::198" count="1"/>
</Class>
<Class val="funny-qclass">
</Class>
</data>
</array>
<array name="certain_qnames_vs_qtype" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="CertainQnames"/>
<dimension number="2" type="Qtype"/>
<data>
<CertainQnames val="localhost">
</CertainQnames>
<CertainQnames val="X.root-servers.net">
</CertainQnames>
<CertainQnames val="else">
<Qtype val="1" count="128"/>
<Qtype val="12" count="78"/>
<Qtype val="28" count="71"/>
<Qtype val="48" count="25"/>
<Qtype val="16" count="18"/>
<Qtype val="15" count="9"/>
<Qtype val="2" count="7"/>
<Qtype val="43" count="5"/>
<Qtype val="6" count="4"/>
</CertainQnames>
</data>
</array>
<array name="qtype_vs_tld" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Qtype"/>
<dimension number="2" type="TLD"/>
<data>
<Qtype val="1">
<TLD val="net" count="128"/>
<TLD val="..." count="128"/>
</Qtype>
<Qtype val="43">
</Qtype>
<Qtype val="12">
<TLD val="arpa" count="78"/>
</Qtype>
<Qtype val="16">
</Qtype>
<Qtype val="48">
</Qtype>
<Qtype val="28">
<TLD val="net" count="71"/>
</Qtype>
<Qtype val="15">
<TLD val="asia" count="8"/>
<TLD val="net" count="1"/>
</Qtype>
<Qtype val="2">
<TLD val="arpa" count="3"/>
<TLD val="net" count="2"/>
<TLD val="com" count="1"/>
<TLD val="org" count="1"/>
</Qtype>
<Qtype val="6">
<TLD val="net" count="2"/>
<TLD val="arpa" count="2"/>
</Qtype>
</data>
</array>
<array name="qtype_vs_qnamelen" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Qtype"/>
<dimension number="2" type="QnameLen"/>
<data>
<Qtype val="1">
<QnameLen val="20" count="84"/>
<QnameLen val="15" count="28"/>
<QnameLen val="16" count="8"/>
<QnameLen val="24" count="2"/>
<QnameLen val="12" count="1"/>
<QnameLen val="18" count="1"/>
<QnameLen val="22" count="1"/>
<QnameLen val="23" count="1"/>
<QnameLen val="28" count="1"/>
<QnameLen val="46" count="1"/>
</Qtype>
<Qtype val="43">
<QnameLen val="15" count="2"/>
<QnameLen val="26" count="2"/>
<QnameLen val="17" count="1"/>
</Qtype>
<Qtype val="12">
<QnameLen val="23" count="35"/>
<QnameLen val="25" count="21"/>
<QnameLen val="26" count="16"/>
<QnameLen val="24" count="2"/>
<QnameLen val="72" count="2"/>
<QnameLen val="27" count="1"/>
<QnameLen val="28" count="1"/>
</Qtype>
<Qtype val="16">
<QnameLen val="74" count="9"/>
<QnameLen val="21" count="8"/>
<QnameLen val="15" count="1"/>
</Qtype>
<Qtype val="48">
<QnameLen val="23" count="11"/>
<QnameLen val="13" count="10"/>
<QnameLen val="12" count="2"/>
<QnameLen val="17" count="2"/>
</Qtype>
<Qtype val="28">
<QnameLen val="20" count="45"/>
<QnameLen val="15" count="12"/>
<QnameLen val="16" count="7"/>
<QnameLen val="24" count="2"/>
<QnameLen val="18" count="1"/>
<QnameLen val="23" count="1"/>
<QnameLen val="26" count="1"/>
<QnameLen val="35" count="1"/>
<QnameLen val="59" count="1"/>
</Qtype>
<Qtype val="15">
<QnameLen val="13" count="8"/>
<QnameLen val="26" count="1"/>
</Qtype>
<Qtype val="2">
<QnameLen val="12" count="3"/>
<QnameLen val="21" count="2"/>
<QnameLen val="9" count="1"/>
<QnameLen val="23" count="1"/>
</Qtype>
<Qtype val="6">
<QnameLen val="21" count="2"/>
<QnameLen val="9" count="1"/>
<QnameLen val="17" count="1"/>
</Qtype>
</data>
</array>
<array name="client_subnet" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="ClientSubnet"/>
<data>
<All val="ALL">
<ClientSubnet val="64.191.0.0" count="198"/>
<ClientSubnet val="2620:ff:c000::" count="5"/>
</All>
</data>
</array>
<array name="rcode_vs_replylen" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="Rcode"/>
<dimension number="2" type="ReplyLen"/>
<data>
<Rcode val="0">
<ReplyLen val="54" count="84"/>
<ReplyLen val="66" count="45"/>
<ReplyLen val="75" count="40"/>
<ReplyLen val="232" count="31"/>
<ReplyLen val="244" count="17"/>
<ReplyLen val="272" count="11"/>
<ReplyLen val="1118" count="10"/>
<ReplyLen val="271" count="9"/>
<ReplyLen val="296" count="8"/>
<ReplyLen val="49" count="7"/>
<ReplyLen val="233" count="7"/>
<ReplyLen val="368" count="7"/>
<ReplyLen val="268" count="6"/>
<ReplyLen val="563" count="6"/>
<ReplyLen val="1472" count="6"/>
<ReplyLen val="82" count="5"/>
<ReplyLen val="309" count="5"/>
<ReplyLen val="42" count="4"/>
<ReplyLen val="88" count="4"/>
<ReplyLen val="104" count="4"/>
<ReplyLen val="241" count="4"/>
<ReplyLen val="379" count="4"/>
<ReplyLen val="722" count="4"/>
<ReplyLen val="62" count="3"/>
<ReplyLen val="245" count="3"/>
<ReplyLen val="293" count="3"/>
<ReplyLen val="298" count="3"/>
<ReplyLen val="478" count="3"/>
<ReplyLen val="587" count="3"/>
<ReplyLen val="41" count="2"/>
<ReplyLen val="50" count="2"/>
<ReplyLen val="61" count="2"/>
<ReplyLen val="84" count="2"/>
<ReplyLen val="87" count="2"/>
<ReplyLen val="96" count="2"/>
<ReplyLen val="99" count="2"/>
<ReplyLen val="102" count="2"/>
<ReplyLen val="110" count="2"/>
<ReplyLen val="123" count="2"/>
<ReplyLen val="131" count="2"/>
<ReplyLen val="132" count="2"/>
<ReplyLen val="239" count="2"/>
<ReplyLen val="253" count="2"/>
<ReplyLen val="261" count="2"/>
<ReplyLen val="396" count="2"/>
<ReplyLen val="574" count="2"/>
<ReplyLen val="1100" count="2"/>
<ReplyLen val="57" count="1"/>
<ReplyLen val="64" count="1"/>
<ReplyLen val="69" count="1"/>
<ReplyLen val="77" count="1"/>
<ReplyLen val="80" count="1"/>
<ReplyLen val="85" count="1"/>
<ReplyLen val="89" count="1"/>
<ReplyLen val="94" count="1"/>
<ReplyLen val="98" count="1"/>
<ReplyLen val="111" count="1"/>
<ReplyLen val="112" count="1"/>
<ReplyLen val="125" count="1"/>
<ReplyLen val="127" count="1"/>
<ReplyLen val="151" count="1"/>
<ReplyLen val="155" count="1"/>
<ReplyLen val="174" count="1"/>
<ReplyLen val="202" count="1"/>
<ReplyLen val="229" count="1"/>
<ReplyLen val="235" count="1"/>
<ReplyLen val="237" count="1"/>
<ReplyLen val="259" count="1"/>
<ReplyLen val="294" count="1"/>
<ReplyLen val="299" count="1"/>
<ReplyLen val="302" count="1"/>
<ReplyLen val="324" count="1"/>
<ReplyLen val="326" count="1"/>
<ReplyLen val="328" count="1"/>
<ReplyLen val="345" count="1"/>
<ReplyLen val="362" count="1"/>
<ReplyLen val="389" count="1"/>
<ReplyLen val="412" count="1"/>
<ReplyLen val="591" count="1"/>
<ReplyLen val="1141" count="1"/>
<ReplyLen val="1146" count="1"/>
</Rcode>
<Rcode val="3">
<ReplyLen val="92" count="2"/>
<ReplyLen val="101" count="2"/>
<ReplyLen val="694" count="2"/>
<ReplyLen val="130" count="1"/>
<ReplyLen val="795" count="1"/>
</Rcode>
<Rcode val="2">
<ReplyLen val="31" count="6"/>
</Rcode>
</data>
</array>
<array name="opcode" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="Opcode"/>
<data>
<All val="ALL">
<Opcode val="0" count="345"/>
</All>
</data>
</array>
<array name="rcode" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="Rcode"/>
<data>
<All val="ALL">
<Rcode val="0" count="416"/>
<Rcode val="3" count="8"/>
<Rcode val="2" count="6"/>
</All>
</data>
</array>
<array name="qtype" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="All"/>
<dimension number="2" type="Qtype"/>
<data>
<All val="ALL">
<Qtype val="1" count="128"/>
<Qtype val="12" count="78"/>
<Qtype val="28" count="71"/>
<Qtype val="48" count="25"/>
<Qtype val="16" count="18"/>
<Qtype val="15" count="9"/>
<Qtype val="2" count="7"/>
<Qtype val="43" count="5"/>
<Qtype val="6" count="4"/>
</All>
</data>
</array>
</dscdata>

2
tests/20190719/certain_qnames_vs_qtype.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 else else:48:1:128:28:71:15:9:12:78:2:7:6:4
#MD5 4bc33435cfa929c19592fa5bbfd4bb69

2
tests/20190719/chaos_types_and_names.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560
#MD5 d0a25752e8594d8c013f127947cbc5f9

8
tests/20190719/client_addr_vs_rcode_accum.dat Normal file
View file

@ -0,0 +1,8 @@
-:SKIPPED:- 0 131
2620:ff:c000::198 0 50
2620:ff:c000::198 3 2
-:SKIPPED_SUM:- 0 131
64.191.0.198 2 6
64.191.0.198 0 78
64.191.0.198 3 3
#MD5 69367713c72c2e8901daffc14632c2d9

2
tests/20190719/client_port_range.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 58368-59391 8 1024-2047 3 45056-46079 7 31744-32767 5 15360-16383 7 30720-31743 4 9216-10239 3 5120-6143 4 25600-26623 4 41984-43007 9 43008-44031 6 62464-63487 4 22528-23551 4 10240-11263 4 39936-40959 5 34816-35839 7 16384-17407 5 32768-33791 6 37888-38911 7 53248-54271 3 3072-4095 7 47104-48127 5 21504-22527 2 49152-50175 6 6144-7167 2 33792-34815 12 61440-62463 8 51200-52223 2 27648-28671 3 4096-5119 3 46080-47103 7 56320-57343 11 63488-64511 8 50176-51199 4 2048-3071 5 8192-9215 3 57344-58367 3 14336-15359 6 52224-53247 6 55296-56319 10 12288-13311 5 24576-25599 11 60416-61439 1 26624-27647 4 17408-18431 4 7168-8191 4 48128-49151 4 38912-39935 4 23552-24575 4 28672-29695 5 20480-21503 3 40960-41983 7 54272-55295 5 13312-14335 6 11264-12287 4 19456-20479 8 36864-37887 6 29696-30719 4 35840-36863 12 18432-19455 4 44032-45055 3 59392-60415 7
#MD5 21f1a5b2da887ab9b0c608f34992d7de

5
tests/20190719/client_subnet2_accum.dat Normal file
View file

@ -0,0 +1,5 @@
2620:ff:c000::198 rfc1918-ptr 1
2620:ff:c000::198 ok 5
64.191.0.0 ok 180
64.191.0.0 non-auth-tld 18
#MD5 dddb87e8a8963e86cb8a503630d4357f

2
tests/20190719/client_subnet2_count.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 rfc1918-ptr 1 ok 2 non-auth-tld 1
#MD5 e883a01b1b022817133db1995aa2c10b

2
tests/20190719/client_subnet2_trace.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 non-auth-tld 18 rfc1918-ptr 1 ok 185
#MD5 b4a4ddc379bfe93124277fc6e2c330cb

3
tests/20190719/client_subnet_accum.dat Normal file
View file

@ -0,0 +1,3 @@
2620:ff:c000:: 5
64.191.0.0 198
#MD5 7b9f4f6c1fad290e8d9f0f6ef2905b0f

2
tests/20190719/client_subnet_count.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 2
#MD5 387ea7cf05e936539c3ec40a63d29dbf

2
tests/20190719/direction_vs_ipproto.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 recv udp:219 sent udp:289 else udp:267
#MD5 425e11a1b3d2f17c5e87e54ff5ecbeb3

2
tests/20190719/dnssec_qtype.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 else 315 48 25 43 5
#MD5 2c7018cb7254ee62b7097fcb6239a57d

2
tests/20190719/do_bit.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 set 134 clr 211
#MD5 7bb564b9b0ebd370f10339a349e82e42

2
tests/20190719/edns_bufsiz.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 3584-4095 12 4096-4607 73 2048-2559 1 512-1023 26 None 204 8192-8703 1 1536-2047 6 1024-1535 22
#MD5 e70ac75f31b3de1176f1975ca1ab03d7

2
tests/20190719/edns_version.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 0 141 none 204
#MD5 4c2f5b7124dea738c503bb5dccb232e2

2
tests/20190719/idn_qname.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 normal 345
#MD5 df6cf377b07e083c57953eda1e9100ae

0
tests/20190719/idn_vs_tld.dat Normal file
View file

0
tests/20190719/ipv6_rsn_abusers_accum.dat Normal file
View file

2
tests/20190719/ipv6_rsn_abusers_count.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 0
#MD5 9b8bcef0308351ed7d24cbed894c2217

2
tests/20190719/opcode.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 0 345
#MD5 3a960c4f0c7ba6d9062f78477f11f1e8

2
tests/20190719/pcap_stats.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 filter_received eth0:5625 kernel_dropped eth0:731 pkts_captured eth0:4894
#MD5 6dbbefe12290b8e7f589f86ec9c30556

2
tests/20190719/qtype.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 12 78 2 7 6 4 15 9 else 48 1 128 28 71
#MD5 afc958bfa6de0678f87f62fe9c9c134f

46
tests/20190719/qtype_vs_qnamelen.dat Normal file
View file

@ -0,0 +1,46 @@
2 23 1
2 9 1
2 21 2
2 12 3
43 26 2
43 17 1
43 15 2
12 28 1
12 72 2
12 23 35
12 26 16
12 25 21
12 27 1
12 24 2
6 9 1
6 21 2
6 17 1
1 18 1
1 46 1
1 12 1
1 20 84
1 23 1
1 16 8
1 24 2
1 15 28
1 22 1
1 28 1
28 16 7
28 24 2
28 15 12
28 18 1
28 35 1
28 20 45
28 26 1
28 23 1
28 59 1
48 12 2
48 13 10
48 23 11
48 17 2
16 74 9
16 15 1
16 21 8
15 26 1
15 13 8
#MD5 1cf280838630abd4d8cc5fe322bbd8ff

13
tests/20190719/qtype_vs_tld.dat Normal file
View file

@ -0,0 +1,13 @@
arpa 2 3
arpa 12 78
arpa 6 2
... 1 128
org 2 1
asia 15 8
net 28 71
net 1 128
net 15 1
net 6 2
net 2 2
com 2 1
#MD5 bebe5fd28997da768999fb36bd299c05

2
tests/20190719/rcode.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 0 416 3 8 2 6
#MD5 7ae0768474205df1d88a4fae8d03b56d

88
tests/20190719/rcode_vs_replylen.dat Normal file
View file

@ -0,0 +1,88 @@
2 31 6
0 293 3
0 112 1
0 232 31
0 396 2
0 89 1
0 302 1
0 94 1
0 298 3
0 1118 10
0 85 1
0 563 6
0 132 2
0 245 3
0 233 7
0 389 1
0 174 1
0 88 4
0 241 4
0 253 2
0 412 1
0 42 4
0 104 4
0 75 40
0 66 45
0 54 84
0 155 1
0 131 2
0 151 1
0 229 1
0 328 1
0 362 1
0 1100 2
0 98 1
0 722 4
0 368 7
0 111 1
0 237 1
0 235 1
0 1141 1
0 62 3
0 99 2
0 84 2
0 102 2
0 345 1
0 61 2
0 239 2
0 261 2
0 259 1
0 309 5
0 244 17
0 80 1
0 69 1
0 1472 6
0 587 3
0 77 1
0 125 1
0 127 1
0 110 2
0 324 1
0 591 1
0 96 2
0 271 9
0 50 2
0 296 8
0 202 1
0 299 1
0 268 6
0 1146 1
0 272 11
0 41 2
0 57 1
0 123 2
0 49 7
0 294 1
0 478 3
0 326 1
0 82 5
0 379 4
0 64 1
0 87 2
0 574 2
3 130 1
3 101 2
3 92 2
3 795 1
3 694 2
#MD5 f7092a6510cfbece8835bc3b41f8e20d

2
tests/20190719/rd_bit.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 clr 159 set 186
#MD5 2dd58158fe3ebb4176faabd9199d06fe

2
tests/20190719/transport_vs_qtype.dat Normal file
View file

@ -0,0 +1,2 @@
1563520560 udp else:48:28:71:1:128:15:9:12:78:2:7:6:4
#MD5 9712453c813073917ee807bfb91dc07a

3
tests/broken.xml Normal file
View file

@ -0,0 +1,3 @@
<dscdata>
<array name="pcap_stats" dimensions="2" start_time="1563520560" stop_time="1563520620">
<dimension number="1" type="ifname"

66
tests/coverage.sh Executable file
View file

@ -0,0 +1,66 @@
#!/bin/sh
set -xe
base=`dirname $0`
export PATH="$base:$PATH"
"$base/test.sh"
dsc-datatool \
-vvv \
-s test-server \
-n test-node \
--output ";InfluxDB;dml=1;database=dsc" \
--transform ";Labler;*;yaml=$base/labler.yaml" \
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
--generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
--generator ";client_subnet_country;path=$HOME/GeoIP" \
--xml "$base/1563520620.dscdata.xml" >/dev/null
! dsc-datatool \
-vvv \
-s test-server \
-n test-node \
--output ";InfluxDB;dml=1;database=dsc" \
--transform ";Labler;*;yaml=$base/labler.yaml" \
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
--generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
--generator ";client_subnet_country;path=$HOME/GeoIP" \
--xml "$base" >/dev/null
dsc-datatool \
-vvv \
-s test-server \
-n test-node \
--output ";InfluxDB;dml=1;database=dsc" \
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
--generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
--generator ";client_subnet_country;path=$HOME/GeoIP" \
--dat "$base/20190719" >/dev/null
dsc-datatool -vvvvvvv --list >/dev/null
! dsc-datatool -s test -n test --output ";InfluxDB;test=a;test=b;test=c" >/dev/null
! dsc-datatool -s test -n test --generator does_not_exist >/dev/null
! dsc-datatool -s test -n test --generator does_not_exist,really_does_not_exist >/dev/null
! dsc-datatool -s test -n test --transform ";does_not_exist;*" >/dev/null
! dsc-datatool -s test -n test --transform ";ReRanger;a,a,a;range=/8" >/dev/null
! dsc-datatool -s test -n test --output does_not_exists >/dev/null
! dsc-datatool -s test -n test --dataset a --dataset b --dataset c,d,e >/dev/null
! dsc-datatool -s test -n test --dat "$base/coverage.sh" >/dev/null

6
tests/dsc-datatool Executable file
View file

@ -0,0 +1,6 @@
#!/bin/sh -e
base=`dirname $0`
export PYTHONPATH="$base/..:$PYTHONPATH"
exec python3-coverage run -a "$base/dsc-datatool.py" "$@"

6
tests/dsc-datatool.py Normal file
View file

@ -0,0 +1,6 @@
#!/usr/bin/python3
import sys
from dsc_datatool import main
if __name__ == "__main__":
sys.exit(main())

374
tests/ipv4-address-space.csv Normal file
View file

@ -0,0 +1,374 @@
Prefix,Designation,Date,WHOIS,RDAP,Status [1],Note
000/8,IANA - Local Identification,1981-09,,,RESERVED,[2][3]
001/8,APNIC,2010-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
002/8,RIPE NCC,2009-09,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
003/8,Administered by ARIN,1994-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
004/8,Administered by ARIN,1992-12,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
005/8,RIPE NCC,2010-11,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
006/8,Army Information Systems Center,1994-02,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
007/8,Administered by ARIN,1995-04,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
008/8,Administered by ARIN,1992-12,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
009/8,Administered by ARIN,1992-08,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
010/8,IANA - Private Use,1995-06,,,RESERVED,[4]
011/8,DoD Intel Information Systems,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
012/8,AT&T Bell Laboratories,1995-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
013/8,Administered by ARIN,1991-09,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
014/8,APNIC,2010-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,[5]
015/8,Administered by ARIN,1994-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
016/8,Administered by ARIN,1994-11,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
017/8,Apple Computer Inc.,1992-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
018/8,Administered by ARIN,1994-01,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
019/8,Ford Motor Company,1995-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
020/8,Administered by ARIN,1994-10,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
021/8,DDN-RVN,1991-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
022/8,Defense Information Systems Agency,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
023/8,ARIN,2010-11,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
024/8,ARIN,2001-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
025/8,Administered by RIPE NCC,1995-01,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
026/8,Defense Information Systems Agency,1995-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
027/8,APNIC,2010-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
028/8,DSI-North,1992-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
029/8,Defense Information Systems Agency,1991-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
030/8,Defense Information Systems Agency,1991-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
031/8,RIPE NCC,2010-05,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
032/8,Administered by ARIN,1994-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
033/8,DLA Systems Automation Center,1991-01,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
034/8,Administered by ARIN,1993-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
035/8,Administered by ARIN,1994-04,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
036/8,APNIC,2010-10,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
037/8,RIPE NCC,2010-11,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
038/8,"PSINet, Inc.",1994-09,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
039/8,APNIC,2011-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
040/8,Administered by ARIN,1994-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
041/8,AFRINIC,2005-04,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",ALLOCATED,
042/8,APNIC,2010-10,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
043/8,Administered by APNIC,1991-01,whois.apnic.net,https://rdap.apnic.net/,LEGACY,
044/8,Administered by ARIN,1992-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
045/8,Administered by ARIN,1995-01,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
046/8,RIPE NCC,2009-09,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
047/8,Administered by ARIN,1991-01,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
048/8,Administered by ARIN,1995-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
049/8,APNIC,2010-08,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
050/8,ARIN,2010-02,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
051/8,Administered by RIPE NCC,1994-08,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
052/8,Administered by ARIN,1991-12,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
053/8,Daimler AG,1993-10,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
054/8,Administered by ARIN,1992-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
055/8,DoD Network Information Center,1995-04,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
056/8,Administered by ARIN,1994-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
057/8,Administered by RIPE NCC,1995-05,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
058/8,APNIC,2004-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
059/8,APNIC,2004-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
060/8,APNIC,2003-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
061/8,APNIC,1997-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
062/8,RIPE NCC,1997-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
063/8,ARIN,1997-04,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
064/8,ARIN,1999-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
065/8,ARIN,2000-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
066/8,ARIN,2000-07,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
067/8,ARIN,2001-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
068/8,ARIN,2001-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
069/8,ARIN,2002-08,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
070/8,ARIN,2004-01,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
071/8,ARIN,2004-08,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
072/8,ARIN,2004-08,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
073/8,ARIN,2005-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
074/8,ARIN,2005-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
075/8,ARIN,2005-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
076/8,ARIN,2005-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
077/8,RIPE NCC,2006-08,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
078/8,RIPE NCC,2006-08,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
079/8,RIPE NCC,2006-08,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
080/8,RIPE NCC,2001-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
081/8,RIPE NCC,2001-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
082/8,RIPE NCC,2002-11,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
083/8,RIPE NCC,2003-11,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
084/8,RIPE NCC,2003-11,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
085/8,RIPE NCC,2004-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
086/8,RIPE NCC,2004-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
087/8,RIPE NCC,2004-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
088/8,RIPE NCC,2004-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
089/8,RIPE NCC,2005-06,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
090/8,RIPE NCC,2005-06,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
091/8,RIPE NCC,2005-06,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
092/8,RIPE NCC,2007-03,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
093/8,RIPE NCC,2007-03,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
094/8,RIPE NCC,2007-07,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
095/8,RIPE NCC,2007-07,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
096/8,ARIN,2006-10,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
097/8,ARIN,2006-10,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
098/8,ARIN,2006-10,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
099/8,ARIN,2006-10,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
100/8,ARIN,2010-11,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,[6]
101/8,APNIC,2010-08,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
102/8,AFRINIC,2011-02,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",ALLOCATED,
103/8,APNIC,2011-02,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
104/8,ARIN,2011-02,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
105/8,AFRINIC,2010-11,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",ALLOCATED,
106/8,APNIC,2011-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
107/8,ARIN,2010-02,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
108/8,ARIN,2008-12,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
109/8,RIPE NCC,2009-01,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
110/8,APNIC,2008-11,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
111/8,APNIC,2008-11,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
112/8,APNIC,2008-05,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
113/8,APNIC,2008-05,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
114/8,APNIC,2007-10,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
115/8,APNIC,2007-10,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
116/8,APNIC,2007-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
117/8,APNIC,2007-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
118/8,APNIC,2007-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
119/8,APNIC,2007-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
120/8,APNIC,2007-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
121/8,APNIC,2006-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
122/8,APNIC,2006-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
123/8,APNIC,2006-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
124/8,APNIC,2005-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
125/8,APNIC,2005-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
126/8,APNIC,2005-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
127/8,IANA - Loopback,1981-09,,,RESERVED,[7]
128/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
129/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
130/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
131/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
132/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
133/8,Administered by APNIC,1997-03,whois.apnic.net,https://rdap.apnic.net/,LEGACY,
134/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
135/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
136/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
137/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
138/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
139/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
140/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
141/8,Administered by RIPE NCC,1993-05,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
142/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
143/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
144/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
145/8,Administered by RIPE NCC,1993-05,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
146/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
147/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
148/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
149/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
150/8,Administered by APNIC,1993-05,whois.apnic.net,https://rdap.apnic.net/,LEGACY,
151/8,Administered by RIPE NCC,1993-05,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
152/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
153/8,Administered by APNIC,1993-05,whois.apnic.net,https://rdap.apnic.net/,LEGACY,
154/8,Administered by AFRINIC,1993-05,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",LEGACY,
155/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
156/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
157/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
158/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
159/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
160/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
161/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
162/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
163/8,Administered by APNIC,1993-05,whois.apnic.net,https://rdap.apnic.net/,LEGACY,
164/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
165/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
166/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
167/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
168/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
169/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,[8]
170/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
171/8,Administered by APNIC,1993-05,whois.apnic.net,https://rdap.apnic.net/,LEGACY,
172/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,[9]
173/8,ARIN,2008-02,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
174/8,ARIN,2008-02,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
175/8,APNIC,2009-08,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
176/8,RIPE NCC,2010-05,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
177/8,LACNIC,2010-06,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
178/8,RIPE NCC,2009-01,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
179/8,LACNIC,2011-02,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
180/8,APNIC,2009-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
181/8,LACNIC,2010-06,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
182/8,APNIC,2009-08,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
183/8,APNIC,2009-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
184/8,ARIN,2008-12,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
185/8,RIPE NCC,2011-02,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
186/8,LACNIC,2007-09,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
187/8,LACNIC,2007-09,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
188/8,Administered by RIPE NCC,1993-05,whois.ripe.net,https://rdap.db.ripe.net/,LEGACY,
189/8,LACNIC,1995-06,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
190/8,LACNIC,1995-06,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
191/8,Administered by LACNIC,1993-05,whois.lacnic.net,https://rdap.lacnic.net/rdap/,LEGACY,
192/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,[10][11]
193/8,RIPE NCC,1993-05,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
194/8,RIPE NCC,1993-05,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
195/8,RIPE NCC,1993-05,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
196/8,Administered by AFRINIC,1993-05,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",LEGACY,
197/8,AFRINIC,2008-10,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",ALLOCATED,
198/8,Administered by ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,[12]
199/8,ARIN,1993-05,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
200/8,LACNIC,2002-11,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
201/8,LACNIC,2003-04,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
202/8,APNIC,1993-05,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
203/8,APNIC,1993-05,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,[13]
204/8,ARIN,1994-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
205/8,ARIN,1994-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
206/8,ARIN,1995-04,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
207/8,ARIN,1995-11,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
208/8,ARIN,1996-04,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
209/8,ARIN,1996-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
210/8,APNIC,1996-06,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
211/8,APNIC,1996-06,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
212/8,RIPE NCC,1997-10,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
213/8,RIPE NCC,1993-10,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
214/8,US-DOD,1998-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
215/8,US-DOD,1998-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",LEGACY,
216/8,ARIN,1998-04,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
217/8,RIPE NCC,2000-06,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
218/8,APNIC,2000-12,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
219/8,APNIC,2001-09,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
220/8,APNIC,2001-12,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
221/8,APNIC,2002-07,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
222/8,APNIC,2003-02,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
223/8,APNIC,2010-04,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
224/8,Multicast,1981-09,,,RESERVED,[14]
225/8,Multicast,1981-09,,,RESERVED,[14]
226/8,Multicast,1981-09,,,RESERVED,[14]
227/8,Multicast,1981-09,,,RESERVED,[14]
228/8,Multicast,1981-09,,,RESERVED,[14]
229/8,Multicast,1981-09,,,RESERVED,[14]
230/8,Multicast,1981-09,,,RESERVED,[14]
231/8,Multicast,1981-09,,,RESERVED,[14]
232/8,Multicast,1981-09,,,RESERVED,[14]
233/8,Multicast,1981-09,,,RESERVED,[14]
234/8,Multicast,1981-09,,,RESERVED,[14][15]
235/8,Multicast,1981-09,,,RESERVED,[14]
236/8,Multicast,1981-09,,,RESERVED,[14]
237/8,Multicast,1981-09,,,RESERVED,[14]
238/8,Multicast,1981-09,,,RESERVED,[14]
239/8,Multicast,1981-09,,,RESERVED,[14][16]
240/8,Future use,1981-09,,,RESERVED,[17]
241/8,Future use,1981-09,,,RESERVED,[17]
242/8,Future use,1981-09,,,RESERVED,[17]
243/8,Future use,1981-09,,,RESERVED,[17]
244/8,Future use,1981-09,,,RESERVED,[17]
245/8,Future use,1981-09,,,RESERVED,[17]
246/8,Future use,1981-09,,,RESERVED,[17]
247/8,Future use,1981-09,,,RESERVED,[17]
248/8,Future use,1981-09,,,RESERVED,[17]
249/8,Future use,1981-09,,,RESERVED,[17]
250/8,Future use,1981-09,,,RESERVED,[17]
251/8,Future use,1981-09,,,RESERVED,[17]
252/8,Future use,1981-09,,,RESERVED,[17]
253/8,Future use,1981-09,,,RESERVED,[17]
254/8,Future use,1981-09,,,RESERVED,[17]
255/8,Future use,1981-09,,,RESERVED,[17][18]
1 Prefix Designation Date WHOIS RDAP Status [1] Note
2 000/8 IANA - Local Identification 1981-09 RESERVED [2][3]
3 001/8 APNIC 2010-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
4 002/8 RIPE NCC 2009-09 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
5 003/8 Administered by ARIN 1994-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
6 004/8 Administered by ARIN 1992-12 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
7 005/8 RIPE NCC 2010-11 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
8 006/8 Army Information Systems Center 1994-02 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
9 007/8 Administered by ARIN 1995-04 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
10 008/8 Administered by ARIN 1992-12 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
11 009/8 Administered by ARIN 1992-08 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
12 010/8 IANA - Private Use 1995-06 RESERVED [4]
13 011/8 DoD Intel Information Systems 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
14 012/8 AT&T Bell Laboratories 1995-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
15 013/8 Administered by ARIN 1991-09 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
16 014/8 APNIC 2010-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED [5]
17 015/8 Administered by ARIN 1994-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
18 016/8 Administered by ARIN 1994-11 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
19 017/8 Apple Computer Inc. 1992-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
20 018/8 Administered by ARIN 1994-01 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
21 019/8 Ford Motor Company 1995-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
22 020/8 Administered by ARIN 1994-10 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
23 021/8 DDN-RVN 1991-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
24 022/8 Defense Information Systems Agency 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
25 023/8 ARIN 2010-11 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
26 024/8 ARIN 2001-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
27 025/8 Administered by RIPE NCC 1995-01 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
28 026/8 Defense Information Systems Agency 1995-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
29 027/8 APNIC 2010-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
30 028/8 DSI-North 1992-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
31 029/8 Defense Information Systems Agency 1991-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
32 030/8 Defense Information Systems Agency 1991-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
33 031/8 RIPE NCC 2010-05 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
34 032/8 Administered by ARIN 1994-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
35 033/8 DLA Systems Automation Center 1991-01 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
36 034/8 Administered by ARIN 1993-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
37 035/8 Administered by ARIN 1994-04 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
38 036/8 APNIC 2010-10 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
39 037/8 RIPE NCC 2010-11 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
40 038/8 PSINet, Inc. 1994-09 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
41 039/8 APNIC 2011-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
42 040/8 Administered by ARIN 1994-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
43 041/8 AFRINIC 2005-04 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ ALLOCATED
44 042/8 APNIC 2010-10 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
45 043/8 Administered by APNIC 1991-01 whois.apnic.net https://rdap.apnic.net/ LEGACY
46 044/8 Administered by ARIN 1992-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
47 045/8 Administered by ARIN 1995-01 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
48 046/8 RIPE NCC 2009-09 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
49 047/8 Administered by ARIN 1991-01 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
50 048/8 Administered by ARIN 1995-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
51 049/8 APNIC 2010-08 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
52 050/8 ARIN 2010-02 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
53 051/8 Administered by RIPE NCC 1994-08 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
54 052/8 Administered by ARIN 1991-12 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
55 053/8 Daimler AG 1993-10 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
56 054/8 Administered by ARIN 1992-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
57 055/8 DoD Network Information Center 1995-04 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
58 056/8 Administered by ARIN 1994-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
59 057/8 Administered by RIPE NCC 1995-05 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
60 058/8 APNIC 2004-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
61 059/8 APNIC 2004-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
62 060/8 APNIC 2003-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
63 061/8 APNIC 1997-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
64 062/8 RIPE NCC 1997-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
65 063/8 ARIN 1997-04 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
66 064/8 ARIN 1999-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
67 065/8 ARIN 2000-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
68 066/8 ARIN 2000-07 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
69 067/8 ARIN 2001-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
70 068/8 ARIN 2001-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
71 069/8 ARIN 2002-08 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
72 070/8 ARIN 2004-01 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
73 071/8 ARIN 2004-08 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
74 072/8 ARIN 2004-08 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
75 073/8 ARIN 2005-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
76 074/8 ARIN 2005-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
77 075/8 ARIN 2005-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
78 076/8 ARIN 2005-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
79 077/8 RIPE NCC 2006-08 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
80 078/8 RIPE NCC 2006-08 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
81 079/8 RIPE NCC 2006-08 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
82 080/8 RIPE NCC 2001-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
83 081/8 RIPE NCC 2001-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
84 082/8 RIPE NCC 2002-11 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
85 083/8 RIPE NCC 2003-11 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
86 084/8 RIPE NCC 2003-11 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
87 085/8 RIPE NCC 2004-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
88 086/8 RIPE NCC 2004-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
89 087/8 RIPE NCC 2004-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
90 088/8 RIPE NCC 2004-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
91 089/8 RIPE NCC 2005-06 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
92 090/8 RIPE NCC 2005-06 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
93 091/8 RIPE NCC 2005-06 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
94 092/8 RIPE NCC 2007-03 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
95 093/8 RIPE NCC 2007-03 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
96 094/8 RIPE NCC 2007-07 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
97 095/8 RIPE NCC 2007-07 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
98 096/8 ARIN 2006-10 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
99 097/8 ARIN 2006-10 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
100 098/8 ARIN 2006-10 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
101 099/8 ARIN 2006-10 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
102 100/8 ARIN 2010-11 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED [6]
103 101/8 APNIC 2010-08 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
104 102/8 AFRINIC 2011-02 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ ALLOCATED
105 103/8 APNIC 2011-02 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
106 104/8 ARIN 2011-02 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
107 105/8 AFRINIC 2010-11 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ ALLOCATED
108 106/8 APNIC 2011-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
109 107/8 ARIN 2010-02 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
110 108/8 ARIN 2008-12 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
111 109/8 RIPE NCC 2009-01 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
112 110/8 APNIC 2008-11 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
113 111/8 APNIC 2008-11 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
114 112/8 APNIC 2008-05 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
115 113/8 APNIC 2008-05 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
116 114/8 APNIC 2007-10 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
117 115/8 APNIC 2007-10 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
118 116/8 APNIC 2007-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
119 117/8 APNIC 2007-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
120 118/8 APNIC 2007-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
121 119/8 APNIC 2007-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
122 120/8 APNIC 2007-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
123 121/8 APNIC 2006-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
124 122/8 APNIC 2006-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
125 123/8 APNIC 2006-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
126 124/8 APNIC 2005-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
127 125/8 APNIC 2005-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
128 126/8 APNIC 2005-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
129 127/8 IANA - Loopback 1981-09 RESERVED [7]
130 128/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
131 129/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
132 130/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
133 131/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
134 132/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
135 133/8 Administered by APNIC 1997-03 whois.apnic.net https://rdap.apnic.net/ LEGACY
136 134/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
137 135/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
138 136/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
139 137/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
140 138/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
141 139/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
142 140/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
143 141/8 Administered by RIPE NCC 1993-05 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
144 142/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
145 143/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
146 144/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
147 145/8 Administered by RIPE NCC 1993-05 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
148 146/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
149 147/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
150 148/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
151 149/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
152 150/8 Administered by APNIC 1993-05 whois.apnic.net https://rdap.apnic.net/ LEGACY
153 151/8 Administered by RIPE NCC 1993-05 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
154 152/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
155 153/8 Administered by APNIC 1993-05 whois.apnic.net https://rdap.apnic.net/ LEGACY
156 154/8 Administered by AFRINIC 1993-05 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ LEGACY
157 155/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
158 156/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
159 157/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
160 158/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
161 159/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
162 160/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
163 161/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
164 162/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
165 163/8 Administered by APNIC 1993-05 whois.apnic.net https://rdap.apnic.net/ LEGACY
166 164/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
167 165/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
168 166/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
169 167/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
170 168/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
171 169/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY [8]
172 170/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
173 171/8 Administered by APNIC 1993-05 whois.apnic.net https://rdap.apnic.net/ LEGACY
174 172/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY [9]
175 173/8 ARIN 2008-02 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
176 174/8 ARIN 2008-02 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
177 175/8 APNIC 2009-08 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
178 176/8 RIPE NCC 2010-05 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
179 177/8 LACNIC 2010-06 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
180 178/8 RIPE NCC 2009-01 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
181 179/8 LACNIC 2011-02 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
182 180/8 APNIC 2009-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
183 181/8 LACNIC 2010-06 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
184 182/8 APNIC 2009-08 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
185 183/8 APNIC 2009-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
186 184/8 ARIN 2008-12 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
187 185/8 RIPE NCC 2011-02 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
188 186/8 LACNIC 2007-09 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
189 187/8 LACNIC 2007-09 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
190 188/8 Administered by RIPE NCC 1993-05 whois.ripe.net https://rdap.db.ripe.net/ LEGACY
191 189/8 LACNIC 1995-06 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
192 190/8 LACNIC 1995-06 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
193 191/8 Administered by LACNIC 1993-05 whois.lacnic.net https://rdap.lacnic.net/rdap/ LEGACY
194 192/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY [10][11]
195 193/8 RIPE NCC 1993-05 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
196 194/8 RIPE NCC 1993-05 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
197 195/8 RIPE NCC 1993-05 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
198 196/8 Administered by AFRINIC 1993-05 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ LEGACY
199 197/8 AFRINIC 2008-10 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ ALLOCATED
200 198/8 Administered by ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY [12]
201 199/8 ARIN 1993-05 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
202 200/8 LACNIC 2002-11 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
203 201/8 LACNIC 2003-04 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
204 202/8 APNIC 1993-05 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
205 203/8 APNIC 1993-05 whois.apnic.net https://rdap.apnic.net/ ALLOCATED [13]
206 204/8 ARIN 1994-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
207 205/8 ARIN 1994-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
208 206/8 ARIN 1995-04 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
209 207/8 ARIN 1995-11 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
210 208/8 ARIN 1996-04 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
211 209/8 ARIN 1996-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
212 210/8 APNIC 1996-06 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
213 211/8 APNIC 1996-06 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
214 212/8 RIPE NCC 1997-10 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
215 213/8 RIPE NCC 1993-10 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
216 214/8 US-DOD 1998-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
217 215/8 US-DOD 1998-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry LEGACY
218 216/8 ARIN 1998-04 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
219 217/8 RIPE NCC 2000-06 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
220 218/8 APNIC 2000-12 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
221 219/8 APNIC 2001-09 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
222 220/8 APNIC 2001-12 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
223 221/8 APNIC 2002-07 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
224 222/8 APNIC 2003-02 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
225 223/8 APNIC 2010-04 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
226 224/8 Multicast 1981-09 RESERVED [14]
227 225/8 Multicast 1981-09 RESERVED [14]
228 226/8 Multicast 1981-09 RESERVED [14]
229 227/8 Multicast 1981-09 RESERVED [14]
230 228/8 Multicast 1981-09 RESERVED [14]
231 229/8 Multicast 1981-09 RESERVED [14]
232 230/8 Multicast 1981-09 RESERVED [14]
233 231/8 Multicast 1981-09 RESERVED [14]
234 232/8 Multicast 1981-09 RESERVED [14]
235 233/8 Multicast 1981-09 RESERVED [14]
236 234/8 Multicast 1981-09 RESERVED [14][15]
237 235/8 Multicast 1981-09 RESERVED [14]
238 236/8 Multicast 1981-09 RESERVED [14]
239 237/8 Multicast 1981-09 RESERVED [14]
240 238/8 Multicast 1981-09 RESERVED [14]
241 239/8 Multicast 1981-09 RESERVED [14][16]
242 240/8 Future use 1981-09 RESERVED [17]
243 241/8 Future use 1981-09 RESERVED [17]
244 242/8 Future use 1981-09 RESERVED [17]
245 243/8 Future use 1981-09 RESERVED [17]
246 244/8 Future use 1981-09 RESERVED [17]
247 245/8 Future use 1981-09 RESERVED [17]
248 246/8 Future use 1981-09 RESERVED [17]
249 247/8 Future use 1981-09 RESERVED [17]
250 248/8 Future use 1981-09 RESERVED [17]
251 249/8 Future use 1981-09 RESERVED [17]
252 250/8 Future use 1981-09 RESERVED [17]
253 251/8 Future use 1981-09 RESERVED [17]
254 252/8 Future use 1981-09 RESERVED [17]
255 253/8 Future use 1981-09 RESERVED [17]
256 254/8 Future use 1981-09 RESERVED [17]
257 255/8 Future use 1981-09 RESERVED [17][18]

80
tests/ipv6-unicast-address-assignments.csv Normal file
View file

@ -0,0 +1,80 @@
Prefix,Designation,Date,WHOIS,RDAP,Status,Note
2001:0000::/23,IANA,1999-07-01,whois.iana.org,,ALLOCATED,"2001:0000::/23 is reserved for IETF Protocol Assignments [RFC2928].
2001:0000::/32 is reserved for TEREDO [RFC4380].
2001:1::1/128 is reserved for Port Control Protocol Anycast [RFC7723].
2001:1::2/128 is reserved for Traversal Using Relays around NAT Anycast [RFC8155].
2001:1::3/128 is reserved for DNS-SD Service Registration Protocol Anycast [RFC-ietf-dnssd-srp-25].
2001:2::/48 is reserved for Benchmarking [RFC5180][RFC Errata 1752].
2001:3::/32 is reserved for AMT [RFC7450].
2001:4:112::/48 is reserved for AS112-v6 [RFC7535].
2001:10::/28 is deprecated (previously ORCHID) [RFC4843].
2001:20::/28 is reserved for ORCHIDv2 [RFC7343].
2001:30::/28 is reserved for Drone Remote ID Protocol Entity Tags (DETs) [RFC9374].
For complete registration details, see [IANA registry iana-ipv6-special-registry]."
2001:0200::/23,APNIC,1999-07-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
2001:0400::/23,ARIN,1999-07-01,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
2001:0600::/23,RIPE NCC,1999-07-01,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:0800::/22,RIPE NCC,2002-11-02,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,2001:0800::/23 was allocated on 2002-05-02. The more recent allocation (2002-11-02) incorporates the previous allocation.
2001:0c00::/23,APNIC,2002-05-02,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,"2001:db8::/32 reserved for Documentation [RFC3849].
For complete registration details, see [IANA registry iana-ipv6-special-registry]."
2001:0e00::/23,APNIC,2003-01-01,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
2001:1200::/23,LACNIC,2002-11-01,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,
2001:1400::/22,RIPE NCC,2003-07-01,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,2001:1400::/23 was allocated on 2003-02-01. The more recent allocation (2003-07-01) incorporates the previous allocation.
2001:1800::/23,ARIN,2003-04-01,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
2001:1a00::/23,RIPE NCC,2004-01-01,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:1c00::/22,RIPE NCC,2004-05-04,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:2000::/19,RIPE NCC,2019-03-12,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,"2001:2000::/20, 2001:3000::/21, and 2001:3800::/22 were allocated on 2004-05-04. The more recent allocation (2019-03-12) incorporates all these previous allocations."
2001:4000::/23,RIPE NCC,2004-06-11,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:4200::/23,AFRINIC,2004-06-01,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",ALLOCATED,
2001:4400::/23,APNIC,2004-06-11,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
2001:4600::/23,RIPE NCC,2004-08-17,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:4800::/23,ARIN,2004-08-24,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
2001:4a00::/23,RIPE NCC,2004-10-15,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:4c00::/23,RIPE NCC,2004-12-17,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:5000::/20,RIPE NCC,2004-09-10,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2001:8000::/19,APNIC,2004-11-30,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
2001:a000::/20,APNIC,2004-11-30,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
2001:b000::/20,APNIC,2006-03-08,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,
2002:0000::/16,6to4,2001-02-01,,,ALLOCATED,"2002::/16 is reserved for 6to4 [RFC3056].
For complete registration details, see [IANA registry iana-ipv6-special-registry]."
2003:0000::/18,RIPE NCC,2005-01-12,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2400:0000::/12,APNIC,2006-10-03,whois.apnic.net,https://rdap.apnic.net/,ALLOCATED,"2400:0000::/19 was allocated on 2005-05-20. 2400:2000::/19 was allocated on 2005-07-08. 2400:4000::/21 was
allocated on 2005-08-08. 2404:0000::/23 was allocated on 2006-01-19. The more recent allocation (2006-10-03)
incorporates all these previous allocations."
2600:0000::/12,ARIN,2006-10-03,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,"2600:0000::/22, 2604:0000::/22, 2608:0000::/22 and 260c:0000::/22 were allocated on 2005-04-19. The more
recent allocation (2006-10-03) incorporates all these previous allocations."
2610:0000::/23,ARIN,2005-11-17,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
2620:0000::/23,ARIN,2006-09-12,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
2630:0000::/12,ARIN,2019-11-06,whois.arin.net,"https://rdap.arin.net/registry
http://rdap.arin.net/registry",ALLOCATED,
2800:0000::/12,LACNIC,2006-10-03,whois.lacnic.net,https://rdap.lacnic.net/rdap/,ALLOCATED,"2800:0000::/23 was allocated on 2005-11-17. The more recent allocation (2006-10-03) incorporates the
previous allocation."
2a00:0000::/12,RIPE NCC,2006-10-03,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,"2a00:0000::/21 was originally allocated on 2005-04-19. 2a01:0000::/23 was allocated on 2005-07-14.
2a01:0000::/16 (incorporating the 2a01:0000::/23) was allocated on 2005-12-15. The more recent allocation
(2006-10-03) incorporates these previous allocations."
2a10:0000::/12,RIPE NCC,2019-06-05,whois.ripe.net,https://rdap.db.ripe.net/,ALLOCATED,
2c00:0000::/12,AFRINIC,2006-10-03,whois.afrinic.net,"https://rdap.afrinic.net/rdap/
http://rdap.afrinic.net/rdap/",ALLOCATED,
2d00:0000::/8,IANA,1999-07-01,,,RESERVED,
2e00:0000::/7,IANA,1999-07-01,,,RESERVED,
3000:0000::/4,IANA,1999-07-01,,,RESERVED,
3ffe::/16,IANA,2008-04,,,RESERVED,"3ffe:831f::/32 was used for Teredo in some old but widely distributed networking stacks. This usage is
deprecated in favor of 2001::/32, which was allocated for the purpose in [RFC4380].
3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156]"
5f00::/16,Segment Routing (SRv6) SIDs,2024-04-23,,,ALLOCATED,"5f00::/16 is reserved for Segment Routing (SRv6) SIDs [RFC-ietf-6man-sids-06].
For complete registration details, see [IANA registry iana-ipv6-special-registry]."
5f01::/16,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
5f02::/15,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
5f04::/14,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
5f08::/13,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
5f10::/12,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
5f20::/11,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
5f40::/10,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
5f80::/9,IANA,2008-04,,,RESERVED,3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
1 Prefix Designation Date WHOIS RDAP Status Note
2 2001:0000::/23 IANA 1999-07-01 whois.iana.org ALLOCATED 2001:0000::/23 is reserved for IETF Protocol Assignments [RFC2928]. 2001:0000::/32 is reserved for TEREDO [RFC4380]. 2001:1::1/128 is reserved for Port Control Protocol Anycast [RFC7723]. 2001:1::2/128 is reserved for Traversal Using Relays around NAT Anycast [RFC8155]. 2001:1::3/128 is reserved for DNS-SD Service Registration Protocol Anycast [RFC-ietf-dnssd-srp-25]. 2001:2::/48 is reserved for Benchmarking [RFC5180][RFC Errata 1752]. 2001:3::/32 is reserved for AMT [RFC7450]. 2001:4:112::/48 is reserved for AS112-v6 [RFC7535]. 2001:10::/28 is deprecated (previously ORCHID) [RFC4843]. 2001:20::/28 is reserved for ORCHIDv2 [RFC7343]. 2001:30::/28 is reserved for Drone Remote ID Protocol Entity Tags (DETs) [RFC9374]. For complete registration details, see [IANA registry iana-ipv6-special-registry].
3 2001:0200::/23 APNIC 1999-07-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
4 2001:0400::/23 ARIN 1999-07-01 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
5 2001:0600::/23 RIPE NCC 1999-07-01 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
6 2001:0800::/22 RIPE NCC 2002-11-02 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED 2001:0800::/23 was allocated on 2002-05-02. The more recent allocation (2002-11-02) incorporates the previous allocation.
7 2001:0c00::/23 APNIC 2002-05-02 whois.apnic.net https://rdap.apnic.net/ ALLOCATED 2001:db8::/32 reserved for Documentation [RFC3849]. For complete registration details, see [IANA registry iana-ipv6-special-registry].
8 2001:0e00::/23 APNIC 2003-01-01 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
9 2001:1200::/23 LACNIC 2002-11-01 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED
10 2001:1400::/22 RIPE NCC 2003-07-01 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED 2001:1400::/23 was allocated on 2003-02-01. The more recent allocation (2003-07-01) incorporates the previous allocation.
11 2001:1800::/23 ARIN 2003-04-01 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
12 2001:1a00::/23 RIPE NCC 2004-01-01 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
13 2001:1c00::/22 RIPE NCC 2004-05-04 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
14 2001:2000::/19 RIPE NCC 2019-03-12 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED 2001:2000::/20, 2001:3000::/21, and 2001:3800::/22 were allocated on 2004-05-04. The more recent allocation (2019-03-12) incorporates all these previous allocations.
15 2001:4000::/23 RIPE NCC 2004-06-11 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
16 2001:4200::/23 AFRINIC 2004-06-01 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ ALLOCATED
17 2001:4400::/23 APNIC 2004-06-11 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
18 2001:4600::/23 RIPE NCC 2004-08-17 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
19 2001:4800::/23 ARIN 2004-08-24 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
20 2001:4a00::/23 RIPE NCC 2004-10-15 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
21 2001:4c00::/23 RIPE NCC 2004-12-17 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
22 2001:5000::/20 RIPE NCC 2004-09-10 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
23 2001:8000::/19 APNIC 2004-11-30 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
24 2001:a000::/20 APNIC 2004-11-30 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
25 2001:b000::/20 APNIC 2006-03-08 whois.apnic.net https://rdap.apnic.net/ ALLOCATED
26 2002:0000::/16 6to4 2001-02-01 ALLOCATED 2002::/16 is reserved for 6to4 [RFC3056]. For complete registration details, see [IANA registry iana-ipv6-special-registry].
27 2003:0000::/18 RIPE NCC 2005-01-12 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
28 2400:0000::/12 APNIC 2006-10-03 whois.apnic.net https://rdap.apnic.net/ ALLOCATED 2400:0000::/19 was allocated on 2005-05-20. 2400:2000::/19 was allocated on 2005-07-08. 2400:4000::/21 was allocated on 2005-08-08. 2404:0000::/23 was allocated on 2006-01-19. The more recent allocation (2006-10-03) incorporates all these previous allocations.
29 2600:0000::/12 ARIN 2006-10-03 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED 2600:0000::/22, 2604:0000::/22, 2608:0000::/22 and 260c:0000::/22 were allocated on 2005-04-19. The more recent allocation (2006-10-03) incorporates all these previous allocations.
30 2610:0000::/23 ARIN 2005-11-17 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
31 2620:0000::/23 ARIN 2006-09-12 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
32 2630:0000::/12 ARIN 2019-11-06 whois.arin.net https://rdap.arin.net/registry http://rdap.arin.net/registry ALLOCATED
33 2800:0000::/12 LACNIC 2006-10-03 whois.lacnic.net https://rdap.lacnic.net/rdap/ ALLOCATED 2800:0000::/23 was allocated on 2005-11-17. The more recent allocation (2006-10-03) incorporates the previous allocation.
34 2a00:0000::/12 RIPE NCC 2006-10-03 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED 2a00:0000::/21 was originally allocated on 2005-04-19. 2a01:0000::/23 was allocated on 2005-07-14. 2a01:0000::/16 (incorporating the 2a01:0000::/23) was allocated on 2005-12-15. The more recent allocation (2006-10-03) incorporates these previous allocations.
35 2a10:0000::/12 RIPE NCC 2019-06-05 whois.ripe.net https://rdap.db.ripe.net/ ALLOCATED
36 2c00:0000::/12 AFRINIC 2006-10-03 whois.afrinic.net https://rdap.afrinic.net/rdap/ http://rdap.afrinic.net/rdap/ ALLOCATED
37 2d00:0000::/8 IANA 1999-07-01 RESERVED
38 2e00:0000::/7 IANA 1999-07-01 RESERVED
39 3000:0000::/4 IANA 1999-07-01 RESERVED
40 3ffe::/16 IANA 2008-04 RESERVED 3ffe:831f::/32 was used for Teredo in some old but widely distributed networking stacks. This usage is deprecated in favor of 2001::/32, which was allocated for the purpose in [RFC4380]. 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156]
41 5f00::/16 Segment Routing (SRv6) SIDs 2024-04-23 ALLOCATED 5f00::/16 is reserved for Segment Routing (SRv6) SIDs [RFC-ietf-6man-sids-06]. For complete registration details, see [IANA registry iana-ipv6-special-registry].
42 5f01::/16 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
43 5f02::/15 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
44 5f04::/14 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
45 5f08::/13 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
46 5f10::/12 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
47 5f20::/11 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
48 5f40::/10 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].
49 5f80::/9 IANA 2008-04 RESERVED 3ffe::/16 and 5f00::/8 were used for the 6bone but were returned. [RFC5156].

782
tests/labler.yaml Normal file
View file

@ -0,0 +1,782 @@
---
certain_qnames_vs_qtype:
Qtype:
'1': A
'10': 'NULL'
'100': UINFO
'101': UID
'102': GID
'103': UNSPEC
'104': NID
'105': L32
'106': L64
'107': LP
'108': EUI48
'109': EUI64
'11': WKS
110-248: Unassigned
'12': PTR
'13': HINFO
'14': MINFO
'15': MX
'16': TXT
'17': RP
'18': AFSDB
'19': X25
'2': NS
'20': ISDN
'21': RT
'22': NSAP
'23': NSAP-PTR
'24': SIG
'249': TKEY
'25': KEY
'250': TSIG
'251': IXFR
'252': AXFR
'253': MAILB
'254': MAILA
'255': wildcard
'256': URI
'257': CAA
'258': AVC
'259': DOA
'26': PX
'260': AMTRELAY
261-32767: Unassigned
'27': GPOS
'28': AAAA
'29': LOC
'3': MD
'30': NXT
'31': EID
'32': NIMLOC
'32768': TA
'32769': DLV
32770-65279: Unassigned
'33': SRV
'34': ATMA
'35': NAPTR
'36': KX
'37': CERT
'38': A6
'39': DNAME
'4': MF
'40': SINK
'41': OPT
'42': APL
'43': DS
'44': SSHFP
'45': IPSECKEY
'46': RRSIG
'47': NSEC
'48': DNSKEY
'49': DHCID
'5': CNAME
'50': NSEC3
'51': NSEC3PARAM
'52': TLSA
'53': SMIMEA
'54': Unassigned
'55': HIP
'56': NINFO
'57': RKEY
'58': TALINK
'59': CDS
'6': SOA
'60': CDNSKEY
'61': OPENPGPKEY
'62': CSYNC
'63': ZONEMD
64-98: Unassigned
65280-65534: Private use
'65535': Reserved
'7': MB
'8': MG
'9': MR
'99': SPF
chaos_types_and_names:
Qtype:
'1': A
'10': 'NULL'
'100': UINFO
'101': UID
'102': GID
'103': UNSPEC
'104': NID
'105': L32
'106': L64
'107': LP
'108': EUI48
'109': EUI64
'11': WKS
110-248: Unassigned
'12': PTR
'13': HINFO
'14': MINFO
'15': MX
'16': TXT
'17': RP
'18': AFSDB
'19': X25
'2': NS
'20': ISDN
'21': RT
'22': NSAP
'23': NSAP-PTR
'24': SIG
'249': TKEY
'25': KEY
'250': TSIG
'251': IXFR
'252': AXFR
'253': MAILB
'254': MAILA
'255': wildcard
'256': URI
'257': CAA
'258': AVC
'259': DOA
'26': PX
'260': AMTRELAY
261-32767: Unassigned
'27': GPOS
'28': AAAA
'29': LOC
'3': MD
'30': NXT
'31': EID
'32': NIMLOC
'32768': TA
'32769': DLV
32770-65279: Unassigned
'33': SRV
'34': ATMA
'35': NAPTR
'36': KX
'37': CERT
'38': A6
'39': DNAME
'4': MF
'40': SINK
'41': OPT
'42': APL
'43': DS
'44': SSHFP
'45': IPSECKEY
'46': RRSIG
'47': NSEC
'48': DNSKEY
'49': DHCID
'5': CNAME
'50': NSEC3
'51': NSEC3PARAM
'52': TLSA
'53': SMIMEA
'54': Unassigned
'55': HIP
'56': NINFO
'57': RKEY
'58': TALINK
'59': CDS
'6': SOA
'60': CDNSKEY
'61': OPENPGPKEY
'62': CSYNC
'63': ZONEMD
64-98: Unassigned
65280-65534: Private use
'65535': Reserved
'7': MB
'8': MG
'9': MR
'99': SPF
client_addr_vs_rcode:
Rcode:
'0': NoError
'1': FormErr
'10': NotZone
'11': DSOTYPENI
12-15: Unassigned
'16': BADSIG
'17': BADKEY
'18': BADTIME
'19': BADMODE
'2': ServFail
'20': BADNAME
'21': BADALG
'22': BADTRUNC
'23': BADCOOKIE
24-3840: Unassigned
'3': NXDomain
3841-4095: Reserved for Private Use
'4': NotImp
4096-65534: Unassigned
'5': Refused
'6': YXDomain
'65535': Reserved, can be allocated by Standards Action
'7': YXRRSet
'8': NXRRSet
'9': NotAuth
dns_ip_version_vs_qtype:
Qtype:
'1': A
'10': 'NULL'
'100': UINFO
'101': UID
'102': GID
'103': UNSPEC
'104': NID
'105': L32
'106': L64
'107': LP
'108': EUI48
'109': EUI64
'11': WKS
110-248: Unassigned
'12': PTR
'13': HINFO
'14': MINFO
'15': MX
'16': TXT
'17': RP
'18': AFSDB
'19': X25
'2': NS
'20': ISDN
'21': RT
'22': NSAP
'23': NSAP-PTR
'24': SIG
'249': TKEY
'25': KEY
'250': TSIG
'251': IXFR
'252': AXFR
'253': MAILB
'254': MAILA
'255': wildcard
'256': URI
'257': CAA
'258': AVC
'259': DOA
'26': PX
'260': AMTRELAY
261-32767: Unassigned
'27': GPOS
'28': AAAA
'29': LOC
'3': MD
'30': NXT
'31': EID
'32': NIMLOC
'32768': TA
'32769': DLV
32770-65279: Unassigned
'33': SRV
'34': ATMA
'35': NAPTR
'36': KX
'37': CERT
'38': A6
'39': DNAME
'4': MF
'40': SINK
'41': OPT
'42': APL
'43': DS
'44': SSHFP
'45': IPSECKEY
'46': RRSIG
'47': NSEC
'48': DNSKEY
'49': DHCID
'5': CNAME
'50': NSEC3
'51': NSEC3PARAM
'52': TLSA
'53': SMIMEA
'54': Unassigned
'55': HIP
'56': NINFO
'57': RKEY
'58': TALINK
'59': CDS
'6': SOA
'60': CDNSKEY
'61': OPENPGPKEY
'62': CSYNC
'63': ZONEMD
64-98: Unassigned
65280-65534: Private use
'65535': Reserved
'7': MB
'8': MG
'9': MR
'99': SPF
opcode:
Opcode:
'0': NoError
'1': FormErr
'10': NotZone
'11': DSOTYPENI
12-15: Unassigned
'16': BADSIG
'17': BADKEY
'18': BADTIME
'19': BADMODE
'2': ServFail
'20': BADNAME
'21': BADALG
'22': BADTRUNC
'23': BADCOOKIE
24-3840: Unassigned
'3': NXDomain
3841-4095: Reserved for Private Use
'4': NotImp
4096-65534: Unassigned
'5': Refused
'6': YXDomain
'65535': Reserved, can be allocated by Standards Action
'7': YXRRSet
'8': NXRRSet
'9': NotAuth
qtype:
Qtype:
'1': A
'10': 'NULL'
'100': UINFO
'101': UID
'102': GID
'103': UNSPEC
'104': NID
'105': L32
'106': L64
'107': LP
'108': EUI48
'109': EUI64
'11': WKS
110-248: Unassigned
'12': PTR
'13': HINFO
'14': MINFO
'15': MX
'16': TXT
'17': RP
'18': AFSDB
'19': X25
'2': NS
'20': ISDN
'21': RT
'22': NSAP
'23': NSAP-PTR
'24': SIG
'249': TKEY
'25': KEY
'250': TSIG
'251': IXFR
'252': AXFR
'253': MAILB
'254': MAILA
'255': wildcard
'256': URI
'257': CAA
'258': AVC
'259': DOA
'26': PX
'260': AMTRELAY
261-32767: Unassigned
'27': GPOS
'28': AAAA
'29': LOC
'3': MD
'30': NXT
'31': EID
'32': NIMLOC
'32768': TA
'32769': DLV
32770-65279: Unassigned
'33': SRV
'34': ATMA
'35': NAPTR
'36': KX
'37': CERT
'38': A6
'39': DNAME
'4': MF
'40': SINK
'41': OPT
'42': APL
'43': DS
'44': SSHFP
'45': IPSECKEY
'46': RRSIG
'47': NSEC
'48': DNSKEY
'49': DHCID
'5': CNAME
'50': NSEC3
'51': NSEC3PARAM
'52': TLSA
'53': SMIMEA
'54': Unassigned
'55': HIP
'56': NINFO
'57': RKEY
'58': TALINK
'59': CDS
'6': SOA
'60': CDNSKEY
'61': OPENPGPKEY
'62': CSYNC
'63': ZONEMD
64-98: Unassigned
65280-65534: Private use
'65535': Reserved
'7': MB
'8': MG
'9': MR
'99': SPF
qtype_vs_qnamelen:
Qtype:
'1': A
'10': 'NULL'
'100': UINFO
'101': UID
'102': GID
'103': UNSPEC
'104': NID
'105': L32
'106': L64
'107': LP
'108': EUI48
'109': EUI64
'11': WKS
110-248: Unassigned
'12': PTR
'13': HINFO
'14': MINFO
'15': MX
'16': TXT
'17': RP
'18': AFSDB
'19': X25
'2': NS
'20': ISDN
'21': RT
'22': NSAP
'23': NSAP-PTR
'24': SIG
'249': TKEY
'25': KEY
'250': TSIG
'251': IXFR
'252': AXFR
'253': MAILB
'254': MAILA
'255': wildcard
'256': URI
'257': CAA
'258': AVC
'259': DOA
'26': PX
'260': AMTRELAY
261-32767: Unassigned
'27': GPOS
'28': AAAA
'29': LOC
'3': MD
'30': NXT
'31': EID
'32': NIMLOC
'32768': TA
'32769': DLV
32770-65279: Unassigned
'33': SRV
'34': ATMA
'35': NAPTR
'36': KX
'37': CERT
'38': A6
'39': DNAME
'4': MF
'40': SINK
'41': OPT
'42': APL
'43': DS
'44': SSHFP
'45': IPSECKEY
'46': RRSIG
'47': NSEC
'48': DNSKEY
'49': DHCID
'5': CNAME
'50': NSEC3
'51': NSEC3PARAM
'52': TLSA
'53': SMIMEA
'54': Unassigned
'55': HIP
'56': NINFO
'57': RKEY
'58': TALINK
'59': CDS
'6': SOA
'60': CDNSKEY
'61': OPENPGPKEY
'62': CSYNC
'63': ZONEMD
64-98: Unassigned
65280-65534: Private use
'65535': Reserved
'7': MB
'8': MG
'9': MR
'99': SPF
qtype_vs_tld:
Qtype:
'1': A
'10': 'NULL'
'100': UINFO
'101': UID
'102': GID
'103': UNSPEC
'104': NID
'105': L32
'106': L64
'107': LP
'108': EUI48
'109': EUI64
'11': WKS
110-248: Unassigned
'12': PTR
'13': HINFO
'14': MINFO
'15': MX
'16': TXT
'17': RP
'18': AFSDB
'19': X25
'2': NS
'20': ISDN
'21': RT
'22': NSAP
'23': NSAP-PTR
'24': SIG
'249': TKEY
'25': KEY
'250': TSIG
'251': IXFR
'252': AXFR
'253': MAILB
'254': MAILA
'255': wildcard
'256': URI
'257': CAA
'258': AVC
'259': DOA
'26': PX
'260': AMTRELAY
261-32767: Unassigned
'27': GPOS
'28': AAAA
'29': LOC
'3': MD
'30': NXT
'31': EID
'32': NIMLOC
'32768': TA
'32769': DLV
32770-65279: Unassigned
'33': SRV
'34': ATMA
'35': NAPTR
'36': KX
'37': CERT
'38': A6
'39': DNAME
'4': MF
'40': SINK
'41': OPT
'42': APL
'43': DS
'44': SSHFP
'45': IPSECKEY
'46': RRSIG
'47': NSEC
'48': DNSKEY
'49': DHCID
'5': CNAME
'50': NSEC3
'51': NSEC3PARAM
'52': TLSA
'53': SMIMEA
'54': Unassigned
'55': HIP
'56': NINFO
'57': RKEY
'58': TALINK
'59': CDS
'6': SOA
'60': CDNSKEY
'61': OPENPGPKEY
'62': CSYNC
'63': ZONEMD
64-98: Unassigned
65280-65534: Private use
'65535': Reserved
'7': MB
'8': MG
'9': MR
'99': SPF
rcode:
Rcode:
'0': NoError
'1': FormErr
'10': NotZone
'11': DSOTYPENI
12-15: Unassigned
'16': BADSIG
'17': BADKEY
'18': BADTIME
'19': BADMODE
'2': ServFail
'20': BADNAME
'21': BADALG
'22': BADTRUNC
'23': BADCOOKIE
24-3840: Unassigned
'3': NXDomain
3841-4095: Reserved for Private Use
'4': NotImp
4096-65534: Unassigned
'5': Refused
'6': YXDomain
'65535': Reserved, can be allocated by Standards Action
'7': YXRRSet
'8': NXRRSet
'9': NotAuth
rcode_vs_replylen:
Rcode:
'0': NoError
'1': FormErr
'10': NotZone
'11': DSOTYPENI
12-15: Unassigned
'16': BADSIG
'17': BADKEY
'18': BADTIME
'19': BADMODE
'2': ServFail
'20': BADNAME
'21': BADALG
'22': BADTRUNC
'23': BADCOOKIE
24-3840: Unassigned
'3': NXDomain
3841-4095: Reserved for Private Use
'4': NotImp
4096-65534: Unassigned
'5': Refused
'6': YXDomain
'65535': Reserved, can be allocated by Standards Action
'7': YXRRSet
'8': NXRRSet
'9': NotAuth
transport_vs_qtype:
Qtype:
'1': A
'10': 'NULL'
'100': UINFO
'101': UID
'102': GID
'103': UNSPEC
'104': NID
'105': L32
'106': L64
'107': LP
'108': EUI48
'109': EUI64
'11': WKS
110-248: Unassigned
'12': PTR
'13': HINFO
'14': MINFO
'15': MX
'16': TXT
'17': RP
'18': AFSDB
'19': X25
'2': NS
'20': ISDN
'21': RT
'22': NSAP
'23': NSAP-PTR
'24': SIG
'249': TKEY
'25': KEY
'250': TSIG
'251': IXFR
'252': AXFR
'253': MAILB
'254': MAILA
'255': wildcard
'256': URI
'257': CAA
'258': AVC
'259': DOA
'26': PX
'260': AMTRELAY
261-32767: Unassigned
'27': GPOS
'28': AAAA
'29': LOC
'3': MD
'30': NXT
'31': EID
'32': NIMLOC
'32768': TA
'32769': DLV
32770-65279: Unassigned
'33': SRV
'34': ATMA
'35': NAPTR
'36': KX
'37': CERT
'38': A6
'39': DNAME
'4': MF
'40': SINK
'41': OPT
'42': APL
'43': DS
'44': SSHFP
'45': IPSECKEY
'46': RRSIG
'47': NSEC
'48': DNSKEY
'49': DHCID
'5': CNAME
'50': NSEC3
'51': NSEC3PARAM
'52': TLSA
'53': SMIMEA
'54': Unassigned
'55': HIP
'56': NINFO
'57': RKEY
'58': TALINK
'59': CDS
'6': SOA
'60': CDNSKEY
'61': OPENPGPKEY
'62': CSYNC
'63': ZONEMD
64-98: Unassigned
65280-65534: Private use
'65535': Reserved
'7': MB
'8': MG
'9': MR
'99': SPF

151
tests/test.gold Normal file
View file

@ -0,0 +1,151 @@
# DML
# CONTEXT-DATABASE: dsc
pcap_stats,server=test-server,node=test-node,ifname=eth0,pcap_stat=filter_received value=5625 1563520560000000000
pcap_stats,server=test-server,node=test-node,ifname=eth0,pcap_stat=pkts_captured value=4894 1563520560000000000
pcap_stats,server=test-server,node=test-node,ifname=eth0,pcap_stat=kernel_dropped value=731 1563520560000000000
direction_vs_ipproto,server=test-server,node=test-node,direction=sent,ipproto=udp value=289 1563520560000000000
direction_vs_ipproto,server=test-server,node=test-node,direction=recv,ipproto=udp value=219 1563520560000000000
direction_vs_ipproto,server=test-server,node=test-node,direction=else,ipproto=udp value=267 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=32768-34815 value=18 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=34816-36863 value=19 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=24576-26623 value=15 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=55296-57343 value=21 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=40960-43007 value=16 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=18432-20479 value=12 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=57344-59391 value=11 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=61440-63487 value=12 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=63488-65535 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=2048-4095 value=12 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=14336-16383 value=13 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=36864-38911 value=13 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=45056-47103 value=14 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=59392-61439 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=12288-14335 value=11 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=43008-45055 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=49152-51199 value=10 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=51200-53247 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=16384-18431 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=28672-30719 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=30720-32767 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=38912-40959 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=47104-49151 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=53248-55295 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=4096-6143 value=7 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=6144-8191 value=6 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=10240-12287 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=22528-24575 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=26624-28671 value=7 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=0-2047 value=3 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=8192-10239 value=6 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=20480-22527 value=5 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=A value=128 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=PTR value=78 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=AAAA value=71 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=DNSKEY value=25 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=TXT value=18 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=MX value=9 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=NS value=7 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=DS value=5 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=SOA value=4 1563520560000000000
rd_bit,server=test-server,node=test-node,rd=set value=186 1563520560000000000
rd_bit,server=test-server,node=test-node,rd=clr value=159 1563520560000000000
do_bit,server=test-server,node=test-node,d0=clr value=211 1563520560000000000
do_bit,server=test-server,node=test-node,d0=set value=134 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=None value=204 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=4096-4607 value=73 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=512-1023 value=26 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=1024-1535 value=22 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=3584-4095 value=12 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=1536-2047 value=6 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=2048-2559 value=1 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=8192-8703 value=1 1563520560000000000
edns_version,server=test-server,node=test-node,ednsversion=none value=204 1563520560000000000
edns_version,server=test-server,node=test-node,ednsversion=0 value=141 1563520560000000000
idn_qname,server=test-server,node=test-node,idnqname=normal value=345 1563520560000000000
client_addr_vs_rcode,server=test-server,node=test-node,rcode=NoError,clientaddr=64.191.0.0 value=78 1563520560000000000
client_addr_vs_rcode,server=test-server,node=test-node,rcode=NoError,clientaddr=2620:: value=50 1563520560000000000
client_addr_vs_rcode,server=test-server,node=test-node,rcode=NoError,clientaddr=0 value=131 1563520560000000000
client_addr_vs_rcode,server=test-server,node=test-node,rcode=NXDomain,clientaddr=64.191.0.0 value=3 1563520560000000000
client_addr_vs_rcode,server=test-server,node=test-node,rcode=NXDomain,clientaddr=2620:: value=2 1563520560000000000
client_addr_vs_rcode,server=test-server,node=test-node,rcode=ServFail,clientaddr=64.191.0.0 value=6 1563520560000000000
client_subnet2,server=test-server,node=test-node,class=ok,clientsubnet=64.191.0.0 value=180 1563520560000000000
client_subnet2,server=test-server,node=test-node,class=ok,clientsubnet=2620:: value=5 1563520560000000000
client_subnet2,server=test-server,node=test-node,class=non-auth-tld,clientsubnet=64.191.0.0 value=18 1563520560000000000
client_subnet2,server=test-server,node=test-node,class=rfc1918-ptr,clientsubnet=2620:: value=1 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=A value=128 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=PTR value=78 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=AAAA value=71 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=DNSKEY value=25 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=TXT value=18 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=MX value=9 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=NS value=7 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=DS value=5 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=SOA value=4 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=A,tld=net value=128 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=A,tld=... value=128 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=PTR,tld=arpa value=78 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=AAAA,tld=net value=71 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=MX,tld=asia value=8 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=MX,tld=net value=1 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=NS,tld=arpa value=3 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=NS,tld=net value=2 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=NS,tld=com value=1 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=NS,tld=org value=1 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=SOA,tld=net value=2 1563520560000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=SOA,tld=arpa value=2 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=A,qnamelen=16-31 value=98 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=A,qnamelen=0-15 value=29 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=A,qnamelen=32-47 value=1 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=DS,qnamelen=0-15 value=2 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=DS,qnamelen=16-31 value=3 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=PTR,qnamelen=16-31 value=76 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=PTR,qnamelen=64-79 value=2 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=TXT,qnamelen=64-79 value=9 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=TXT,qnamelen=16-31 value=8 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=TXT,qnamelen=0-15 value=1 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=DNSKEY,qnamelen=16-31 value=13 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=DNSKEY,qnamelen=0-15 value=12 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=AAAA,qnamelen=16-31 value=57 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=AAAA,qnamelen=0-15 value=12 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=AAAA,qnamelen=32-47 value=1 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=AAAA,qnamelen=48-63 value=1 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=MX,qnamelen=0-15 value=8 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=MX,qnamelen=16-31 value=1 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=NS,qnamelen=0-15 value=4 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=NS,qnamelen=16-31 value=3 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=SOA,qnamelen=16-31 value=3 1563520560000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=SOA,qnamelen=0-15 value=1 1563520560000000000
client_subnet,server=test-server,node=test-node,clientsubnet=64.191.0.0 value=198 1563520560000000000
client_subnet,server=test-server,node=test-node,clientsubnet=2620:: value=5 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=0-63 value=105 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=64-127 value=124 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=192-255 value=70 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=256-319 value=51 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=1088-1151 value=14 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=320-383 value=16 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=512-575 value=8 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=1472-1535 value=6 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=704-767 value=4 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=448-511 value=3 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=576-639 value=4 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=128-191 value=7 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=384-447 value=4 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NXDomain,replylen=64-127 value=4 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NXDomain,replylen=640-703 value=2 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NXDomain,replylen=128-191 value=1 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NXDomain,replylen=768-831 value=1 1563520560000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=ServFail,replylen=0-63 value=6 1563520560000000000
opcode,server=test-server,node=test-node,opcode=NoError value=345 1563520560000000000
rcode,server=test-server,node=test-node,rcode=NoError value=416 1563520560000000000
rcode,server=test-server,node=test-node,rcode=NXDomain value=8 1563520560000000000
rcode,server=test-server,node=test-node,rcode=ServFail value=6 1563520560000000000
qtype,server=test-server,node=test-node,qtype=A value=128 1563520560000000000
qtype,server=test-server,node=test-node,qtype=PTR value=78 1563520560000000000
qtype,server=test-server,node=test-node,qtype=AAAA value=71 1563520560000000000
qtype,server=test-server,node=test-node,qtype=DNSKEY value=25 1563520560000000000
qtype,server=test-server,node=test-node,qtype=TXT value=18 1563520560000000000
qtype,server=test-server,node=test-node,qtype=MX value=9 1563520560000000000
qtype,server=test-server,node=test-node,qtype=NS value=7 1563520560000000000
qtype,server=test-server,node=test-node,qtype=DS value=5 1563520560000000000
qtype,server=test-server,node=test-node,qtype=SOA value=4 1563520560000000000
client_subnet_authority,server=test-server,node=test-node,clientauthority=ARIN value=203 1563520560000000000

91
tests/test.gold2 Normal file
View file

@ -0,0 +1,91 @@
# DML
# CONTEXT-DATABASE: dsc
client_subnet_count,server=test-server,node=test-node,all=ALL value=2 1563520560000000000
ipv6_rsn_abusers_count,server=test-server,node=test-node,all=ALL value=0 1563520560000000000
qtype,server=test-server,node=test-node,qtype=PTR value=78 1563520560000000000
qtype,server=test-server,node=test-node,qtype=NS value=7 1563520560000000000
qtype,server=test-server,node=test-node,qtype=SOA value=4 1563520560000000000
qtype,server=test-server,node=test-node,qtype=MX value=9 1563520560000000000
qtype,server=test-server,node=test-node,qtype=else value=48 1563520560000000000
qtype,server=test-server,node=test-node,qtype=A value=128 1563520560000000000
qtype,server=test-server,node=test-node,qtype=AAAA value=71 1563520560000000000
rcode,server=test-server,node=test-node,rcode=NoError value=416 1563520560000000000
rcode,server=test-server,node=test-node,rcode=NXDomain value=8 1563520560000000000
rcode,server=test-server,node=test-node,rcode=ServFail value=6 1563520560000000000
do_bit,server=test-server,node=test-node,d0=set value=134 1563520560000000000
do_bit,server=test-server,node=test-node,d0=clr value=211 1563520560000000000
rd_bit,server=test-server,node=test-node,rd=clr value=159 1563520560000000000
rd_bit,server=test-server,node=test-node,rd=set value=186 1563520560000000000
opcode,server=test-server,node=test-node,opcode=NoError value=345 1563520560000000000
dnssec_qtype,server=test-server,node=test-node,qtype=else value=315 1563520560000000000
dnssec_qtype,server=test-server,node=test-node,qtype=48 value=25 1563520560000000000
dnssec_qtype,server=test-server,node=test-node,qtype=43 value=5 1563520560000000000
edns_version,server=test-server,node=test-node,ednsversion=0 value=141 1563520560000000000
edns_version,server=test-server,node=test-node,ednsversion=none value=204 1563520560000000000
client_subnet2_count,server=test-server,node=test-node,class=rfc1918-ptr value=1 1563520560000000000
client_subnet2_count,server=test-server,node=test-node,class=ok value=2 1563520560000000000
client_subnet2_count,server=test-server,node=test-node,class=non-auth-tld value=1 1563520560000000000
client_subnet2_trace,server=test-server,node=test-node,class=non-auth-tld value=18 1563520560000000000
client_subnet2_trace,server=test-server,node=test-node,class=rfc1918-ptr value=1 1563520560000000000
client_subnet2_trace,server=test-server,node=test-node,class=ok value=185 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=3584-4095 value=12 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=4096-4607 value=73 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=2048-2559 value=1 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=512-1023 value=26 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=None value=204 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=8192-8703 value=1 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=1536-2047 value=6 1563520560000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=1024-1535 value=22 1563520560000000000
idn_qname,server=test-server,node=test-node,idnqname=normal value=345 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=57344-59391 value=11 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=0-2047 value=3 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=45056-47103 value=14 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=30720-32767 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=14336-16383 value=13 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=8192-10239 value=6 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=4096-6143 value=7 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=24576-26623 value=15 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=40960-43007 value=16 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=43008-45055 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=61440-63487 value=12 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=22528-24575 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=10240-12287 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=38912-40959 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=34816-36863 value=19 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=16384-18431 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=32768-34815 value=18 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=36864-38911 value=13 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=53248-55295 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=2048-4095 value=12 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=47104-49151 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=20480-22527 value=5 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=49152-51199 value=10 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=6144-8191 value=6 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=51200-53247 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=26624-28671 value=7 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=55296-57343 value=21 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=63488-65535 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=12288-14335 value=11 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=59392-61439 value=8 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=28672-30719 value=9 1563520560000000000
client_port_range,server=test-server,node=test-node,portrange=18432-20479 value=12 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=else value=48 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=A value=128 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=AAAA value=71 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=MX value=9 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=PTR value=78 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=NS value=7 1563520560000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=SOA value=4 1563520560000000000
direction_vs_ipproto,server=test-server,node=test-node,direction=recv,ipproto=udp value=219 1563520560000000000
direction_vs_ipproto,server=test-server,node=test-node,direction=sent,ipproto=udp value=289 1563520560000000000
direction_vs_ipproto,server=test-server,node=test-node,direction=else,ipproto=udp value=267 1563520560000000000
pcap_stats,server=test-server,node=test-node,pcap_stat=filter_received,ifname=eth0 value=5625 1563520560000000000
pcap_stats,server=test-server,node=test-node,pcap_stat=kernel_dropped,ifname=eth0 value=731 1563520560000000000
pcap_stats,server=test-server,node=test-node,pcap_stat=pkts_captured,ifname=eth0 value=4894 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=else value=48 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=AAAA value=71 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=A value=128 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=MX value=9 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=PTR value=78 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=NS value=7 1563520560000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=SOA value=4 1563520560000000000

56
tests/test.gold3 Normal file
View file

@ -0,0 +1,56 @@
# DML
# CONTEXT-DATABASE: dsc
pcap_stats,server=test-server,node=test-node,ifname=./1458044657.pcap.dist,pcap_stat=pkts_captured value=8 1458044655000000000
label_count,server=test-server,node=test-node,labelcount=3 value=4 1458044655000000000
label_count,server=test-server,node=test-node,labelcount=6 value=4 1458044655000000000
third_ld_vs_rcode,server=test-server,node=test-node,rcode=0,thirdld=216.in-addr.arpa value=2 1458044655000000000
third_ld_vs_rcode,server=test-server,node=test-node,rcode=0,thirdld=www.google.se value=1 1458044655000000000
third_ld_vs_rcode,server=test-server,node=test-node,rcode=0,thirdld=www.google.com value=1 1458044655000000000
second_ld_vs_rcode,server=test-server,node=test-node,rcode=0,secondld=in-addr.arpa value=2 1458044655000000000
second_ld_vs_rcode,server=test-server,node=test-node,rcode=0,secondld=google.com value=1 1458044655000000000
second_ld_vs_rcode,server=test-server,node=test-node,rcode=0,secondld=www.google.se value=1 1458044655000000000
server,server=test-server,node=test-node,ip=8.8.8.8 value=8 1458044655000000000
qr_aa_bits,server=test-server,node=test-node,direction=else,qraabits=qr\=0\,aa\=0 value=4 1458044655000000000
qr_aa_bits,server=test-server,node=test-node,direction=else,qraabits=qr\=1\,aa\=0 value=4 1458044655000000000
qname,server=test-server,node=test-node,name=100.209.58.216.in-addr.arpa value=2 1458044655000000000
qname,server=test-server,node=test-node,name=www.google.se value=2 1458044655000000000
qname,server=test-server,node=test-node,name=131.209.58.216.in-addr.arpa value=2 1458044655000000000
qname,server=test-server,node=test-node,name=www.google.com value=2 1458044655000000000
qclass,server=test-server,node=test-node,class=1 value=8 1458044655000000000
dns_ip_version,server=test-server,node=test-node,version=IPv4 value=8 1458044655000000000
ip_version,server=test-server,node=test-node,version=IPv4 value=8 1458044655000000000
direction_vs_ipproto,server=test-server,node=test-node,direction=else,ipproto=udp value=8 1458044655000000000
client_port,server=test-server,node=test-node,port=59978 value=4 1458044655000000000
client_port,server=test-server,node=test-node,port=0 value=1 1458044655000000000
client_port,server=test-server,node=test-node,port=53 value=1 1458044655000000000
client_port,server=test-server,node=test-node,port=44275 value=1 1458044655000000000
client_port,server=test-server,node=test-node,port=57483 value=1 1458044655000000000
client_port_range,server=test-server,node=test-node,portrange=43008-45055 value=1 1458044655000000000
client_port_range,server=test-server,node=test-node,portrange=55296-57343 value=1 1458044655000000000
client_port_range,server=test-server,node=test-node,portrange=57344-59391 value=2 1458044655000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=A value=2 1458044655000000000
transport_vs_qtype,server=test-server,node=test-node,transport=udp,qtype=PTR value=2 1458044655000000000
tc_bit,server=test-server,node=test-node,tc=clr value=8 1458044655000000000
rd_bit,server=test-server,node=test-node,rd=set value=4 1458044655000000000
do_bit,server=test-server,node=test-node,d0=clr value=4 1458044655000000000
edns_bufsiz,server=test-server,node=test-node,ednsbufsiz=None value=4 1458044655000000000
edns_version,server=test-server,node=test-node,ednsversion=none value=4 1458044655000000000
idn_qname,server=test-server,node=test-node,idnqname=normal value=4 1458044655000000000
client_addr_vs_rcode,server=test-server,node=test-node,rcode=NoError,clientaddr=172.17.0.0 value=4 1458044655000000000
client_subnet2,server=test-server,node=test-node,class=ok,clientsubnet=172.17.0.0 value=3 1458044655000000000
client_subnet2,server=test-server,node=test-node,class=non-auth-tld,clientsubnet=172.17.0.0 value=1 1458044655000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=A value=2 1458044655000000000
certain_qnames_vs_qtype,server=test-server,node=test-node,certainqnames=else,qtype=PTR value=2 1458044655000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=A,tld=com value=1 1458044655000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=A,tld=google.se value=1 1458044655000000000
qtype_vs_tld,server=test-server,node=test-node,qtype=PTR,tld=arpa value=2 1458044655000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=A,qnamelen=0-15 value=2 1458044655000000000
qtype_vs_qnamelen,server=test-server,node=test-node,qtype=PTR,qnamelen=16-31 value=2 1458044655000000000
client_subnet,server=test-server,node=test-node,clientsubnet=172.17.0.0 value=4 1458044655000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=128-191 value=2 1458044655000000000
rcode_vs_replylen,server=test-server,node=test-node,rcode=NoError,replylen=0-63 value=2 1458044655000000000
opcode,server=test-server,node=test-node,opcode=NoError value=4 1458044655000000000
rcode,server=test-server,node=test-node,rcode=NoError value=4 1458044655000000000
qtype,server=test-server,node=test-node,qtype=A value=2 1458044655000000000
qtype,server=test-server,node=test-node,qtype=PTR value=2 1458044655000000000
client_subnet_authority,server=test-server,node=test-node,clientauthority=ARIN value=4 1458044655000000000

5
tests/test.gold4 Normal file
View file

@ -0,0 +1,5 @@
# CONTEXT-DATABASE: dsc
# DML
pcap_stats,server=test-server-åäö,node=test-node,ifname=eth0åäö,pcap_stat=filter_received value=5625 1563520560000000000
pcap_stats,server=test-server-åäö,node=test-node,ifname=eth0åäö,pcap_stat=kernel_dropped value=731 1563520560000000000
pcap_stats,server=test-server-åäö,node=test-node,ifname=eth0åäö,pcap_stat=pkts_captured value=4894 1563520560000000000

75
tests/test.sh Executable file
View file

@ -0,0 +1,75 @@
#!/bin/sh -xe
base=`dirname $0`
dsc-datatool \
-vvv \
-s test-server \
-n test-node \
--output ";InfluxDB;dml=1;database=dsc" \
--transform ";Labler;*;yaml=$base/labler.yaml" \
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
--generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
--xml "$base/1563520620.dscdata.xml" | sort -s > "$base/test.out"
sort -s "$base/test.gold" > "$base/test.gold.tmp"
diff -u "$base/test.gold.tmp" "$base/test.out"
dsc-datatool \
-vvv \
-s test-server \
-n test-node \
--output ";InfluxDB;dml=1;database=dsc" \
--transform ";Labler;*;yaml=$base/labler.yaml" \
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
--generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
--dat "$base/20190719" | sort -s > "$base/test.out"
sort -s "$base/test.gold2" > "$base/test.gold2.tmp"
diff -u "$base/test.gold2.tmp" "$base/test.out"
dsc-datatool \
-vvv \
-s test-server \
-n test-node \
--output ";InfluxDB;dml=1;database=dsc" \
--transform ";Labler;*;yaml=$base/labler.yaml" \
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
--generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
--xml "$base/1458044657.xml" | sort -s > "$base/test.out"
sort -s "$base/test.gold3" > "$base/test.gold3.tmp"
diff -u "$base/test.gold3.tmp" "$base/test.out"
dsc-datatool \
-vvv \
-s test-server-åäö \
-n test-node \
--output ";InfluxDB;dml=1;database=dsc" \
--transform ";Labler;*;yaml=$base/labler.yaml" \
--transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
--transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
--transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
--transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
--transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
--transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
--generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
--xml "$base/utf8.xml" | sort -s > "$base/test.out"
sort -s "$base/test.gold4" > "$base/test.gold4.tmp"
diff -u "$base/test.gold4.tmp" "$base/test.out"

7
tests/test_main.py Normal file
View file

@ -0,0 +1,7 @@
import pytest
import dsc_datatool as app
def test_main():
with pytest.raises(Exception):
app.main()

68
tests/test_objects.py Normal file
View file

@ -0,0 +1,68 @@
import pytest
from dsc_datatool import Dataset, Dimension, Input, Output, Generator, Transformer
def test_dataset():
o = Dataset()
assert '%r' % o == '<Dataset name=None dimension=[]>'
def test_dimension():
o = Dimension('test')
assert '%r' % o == '<Dimension name=\'test\' value=None dimension=[]>'
def test_input():
o = Input()
with pytest.raises(Exception):
o.process("test")
class Input1(Input):
def process(self, file):
pass
with pytest.raises(Exception):
class Input1(Input):
def process(self, file):
pass
def test_output():
o = Output({})
with pytest.raises(Exception):
o.process([])
class Output1(Output):
def process(self, file):
pass
with pytest.raises(Exception):
class Output1(Output):
def process(self, file):
pass
def test_generator():
o = Generator({})
with pytest.raises(Exception):
o.process([])
class Generator1(Generator):
def process(self, file):
pass
with pytest.raises(Exception):
class Generator1(Generator):
def process(self, file):
pass
def test_transformer():
o = Transformer({})
with pytest.raises(Exception):
o.process([])
class Transformer1(Transformer):
def process(self, file):
pass
with pytest.raises(Exception):
class Transformer1(Transformer):
def process(self, file):
pass

13
tests/utf8.xml Normal file
View file

@ -0,0 +1,13 @@
<dscdata>
<array name="pcap_stats" dimensions="2" start_time="1563520560" stop_time="1563520621">
<dimension number="1" type="ifname"/>
<dimension number="2" type="pcap_stat"/>
<data>
<ifname val="eth0åäö">
<pcap_stat val="filter_received" count="5625"/>
<pcap_stat val="pkts_captured" count="4894"/>
<pcap_stat val="kernel_dropped" count="731"/>
</ifname>
</data>
</array>
</dscdata>