diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..1bd4430 --- /dev/null +++ b/.clang-format @@ -0,0 +1,6 @@ +BasedOnStyle: webkit +IndentWidth: 4 +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignOperands: true +SortIncludes: false diff --git a/.copr/Makefile b/.copr/Makefile new file mode 100644 index 0000000..29ed0bc --- /dev/null +++ b/.copr/Makefile @@ -0,0 +1,23 @@ +top=.. + +all: srpm + +prereq: $(top)/rpmbuild + rpm -q git rpm-build >/dev/null || dnf -y install git rpm-build + +update-dist-tools: $(top)/dist-tools + ( cd "$(top)/dist-tools" && git pull ) + +$(top)/dist-tools: + git clone https://github.com/jelu/dist-tools.git "$(top)/dist-tools" + +$(top)/rpmbuild: + mkdir -p "$(top)"/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} + +srpm: prereq update-dist-tools + test -f .gitmodules && git submodule update --init || true + echo "$(spec)" | grep -q "develop.spec" && auto_build_number=`date --utc +%s` message="Auto build `date --utc --iso-8601=seconds`" "$(top)/dist-tools/spec-new-changelog-entry" || true + overwrite=yes nosign=yes "$(top)/dist-tools/create-source-packages" rpm + cp ../*.orig.tar.gz "$(top)/rpmbuild/SOURCES/" + echo "$(spec)" | grep -q "develop.spec" && rpmbuild -bs --define "%_topdir $(top)/rpmbuild" --undefine=dist rpm/*.spec || rpmbuild -bs --define "%_topdir $(top)/rpmbuild" --undefine=dist "$(spec)" + cp "$(top)"/rpmbuild/SRPMS/*.src.rpm "$(outdir)" diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..38cc1c4 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://www.dns-oarc.net/donate diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..26bcab6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +*.o +*.lo +*.la +config.log +config.status +stamp-h1 +ar-lib +config.guess +config.sub +libtool +ltmain.sh +.deps +.libs +Makefile +Makefile.in +src/dnscap +src/dnscap.1 +autom4te.cache +Makefile.old +aclocal.m4 +compile +configure +depcomp +install-sh +missing +test-driver +config.h +config.h.in~ +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 +build/ +config.h.in +dnscap-[0-9]*tar* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..4d2f1bc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/pcap-thread"] + path = src/pcap-thread + url = https://github.com/DNS-OARC/pcap-thread.git diff --git a/.lgtm.yml b/.lgtm.yml new file mode 100644 index 0000000..a1c94c7 --- /dev/null +++ b/.lgtm.yml @@ -0,0 +1,26 @@ +extraction: + cpp: + prepare: + packages: + - build-essential + - automake + - autoconf + - libtool + - pkg-config + - libpcap-dev + - libldns-dev + - libyaml-perl + - zlib1g-dev + - libssl-dev + after_prepare: + - git clone https://github.com/DNS-OARC/cryptopANT.git + - cd cryptopANT + - ./autogen.sh + - ./configure --prefix="$PWD/../root" + - make + - make install + - cd .. + configure: + command: + - ./autogen.sh + - ./configure --with-extra-cflags="-I $PWD/root/include" --with-extra-ldflags="-L$PWD/root/lib" diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9b598f2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,30 @@ +dist: xenial +addons: + apt: + sources: + - sourceline: 'ppa:dns-oarc/dnscap-pr' + update: true + packages: + - libpcap-dev + - libldns-dev + - libyaml-perl + - zlib1g-dev + - libssl-dev + - libcryptopant-dev +language: c +compiler: + - clang + - gcc +install: ./autogen.sh +script: + - ./configure --enable-warn-all + - make dist + - tar zxvf *.tar.gz + - cd dnscap-[0-9]* + - mkdir build + - cd build + - ../configure --enable-warn-all + - make + - make test + - cat src/test/test*.sh.log + - cat plugins/*/test*.sh.log diff --git a/CBOR_DNS_STREAM.md b/CBOR_DNS_STREAM.md new file mode 100644 index 0000000..a54dc63 --- /dev/null +++ b/CBOR_DNS_STREAM.md @@ -0,0 +1,399 @@ +# CBOR DNS Stream Format version 1 (CDSv1) + +This is an experimental format for representing DNS information in CBOR +with the goals to: +- Be able to stream the information +- Support incomplete, broken and/or invalid DNS +- Have close to no data quality and signature degradation +- Support additional non-DNS meta data (such as ICMP/TCP attributes) + +## Overview + +In CBOR you are expected to have one root element, most likely an array or +map. This format does not have a root element, instead you are expected to +read one CBOR array element at a time as a stream of CBOR elements with the +first array element being the stream initiator object. + +``` +[stream_init] +[message] +... +[message] +``` + +Here are some number on the compression rate compared to PCAP: + +Uncompressed | PCAP | CDS | Factor +-------------|------------|-----------|------- +client | 458373 | 133640 | 0,2915 +zonalizer | 51769844 | 9450475 | 0,1825 +large ditl | 1003931674 | 298167709 | 0,2970 +small ditl | 1651252 | 603314 | 0,3653 + +Gzipped | PCAP | CDS | Factor | F/Uncompressed +-------------|------------|-----------|---------|--------------- +client | 108136 | 45944 | 0,4248 | 0,1002 +zonalizer | 12468329 | 2485620 | 0,1993 | 0,0480 +large ditl | 327227203 | 117569598 | 0,3592 | 0,1171 +small ditl | 539323 | 253402 | 0,4698 | 0,1534 + +Xzipped | PCAP | CDS | Factor | F/Uncompressed +-------------|------------|-----------|---------|--------------- +client | 76248 | 36308 | 0,4761 | 0,0792 +zonalizer | 7894356 | 1695920 | 0,2148 | 0,0327 +large ditl | 267031412 | 86747604 | 0,3248 | 0,0864 +small ditl | 442260 | 206596 | 0,4671 | 0,1251 + +- `client` is a couple of hours of DNS from my workstation +- `zonalizer` is half a day from [Zonalizer](https://zonalizer.makeinstall.se) which continuously tests gTLDs +- `large ditl`, `small ditl` are capture from [DITL](https://www.dns-oarc.net/oarc/data/ditl) + +## Types + +- `int`: A CBOR integer (major type 0x00) +- `uint`: A CBOR integer (value >= 0, major type 0x00) +- `nint`: A CBOR negative integer (value < 0, major type 0x00), this type has special meaning see `Negative Integers` +- `simple`: A CBOR simple value (major type 0xe0) +- `bytes`: A CBOR byte string (major type 0x40) +- `string`: A CBOR UTF-8 string (major type 0x60) +- `any`: Any CBOR value +- `bool`: A CBOR boolean +- `rindex`: A CBOR negative integer that is a reverse index, see `Deduplication` + +## Special Keywords + +- `union`: Can be used to merge the given array or map into the current object +- `optional`: The attribute or object reference is optional + +## Negative Integers + +CBOR encodes negative numbers in a special way and this format uses that +for none negative number to tell them apart. + +Because of that, all negative numbers needs special decoding: + +``` +value = -value - 1 +``` + +## Objects + +The object code below uses: +- `[` and `]` to indicate the start and end of an array +- `type name` per object attribute +- `name` per object reference +- `...` to indicate a list of previous definition +- `(`, `|` and `)` to indicate list of various types that the attribute can be + +### stream_init + +The initial object in the stream. + +``` +[ + string version, + union stream_option option, + ... +] +``` + +- `version`: The version of the format +- `option`: A list of stream option objects + +### stream_option + +A stream option that can specify critical information about the stream and +how it should be decoded, see `Stream Options` for more information. + +``` +[ + uint option_type, + optional any option_value +] +``` + +- `option_type`: The type of option represented as a number +- `option_value`: The option value + +### message + +A message object that describes various DNS packets or other information. + +``` +[ + optional bool is_complete, + union timestamp timestamp, + simple message_bits, + union ip_header ip_header, + union ( icmp_message | udp_message | tcp_message | dns_message ) content +] +``` + +- `is_complete`: Will exist and be false if the message is not complete and following attributes may not exists +- `timestamp`: A timestamp object +- `message_bits`: Bitmap indicating message content + - Bit 0: 0=Not DNS 1=DNS + - Bit 1: if DNS: 0=UDP 1=TCP else: 0=ICMP/ICMPv6 1=TCP + - Bit 2: Fragmented (0=no 1=yes) + - Bit 3: Malformed (0=no 1=yes) +- `ip_header`: An IP header object +- `content`: The message content, may be an ICMP, UDP, TCP or DNS message object + +### timestamp + +The timestamp object of a message. + +``` +[ + ( uint seconds | nint diff_from_last ), + optional uint useconds + optional uint nseconds +] +``` + +- `seconds`: The seconds of a UNIX timestamp +- `diff_from_last`: The differentially from last `timestamp.seconds` +- `useconds`: The microseconds of a UNIX timestamp or if `diff_from_last` is used it will be the differentially from last `timestamp.useconds` +- `nseconds`: The nanoseconds of a UNIX timestamp or if `diff_from_last` is used it will be the differentially from last `timestamp.nseconds` + +### ip_header + +The IP header of a message. + +``` +[ + ( uint | nint ) ip_bits, + optional bytes src_addr, + optional bytes dest_addr, + optional ( uint | nint ) src_dest_port +] +``` + +- `ip_bits`: Bitmap indicating IP header content, if the type is `nint` it also indicates that it is a reverse from last, see `Deduplication` for more information + - Bit 0: address family (0=AF_INET, 1=AF_INET6) + - Bit 1: src_addr present + - Bit 2: dest_addr present + - Bit 3: port present +- `src_addr`: The source address with length specifying address family, 4 bytes is IPv4 and 16 is IPv6 +- `dest_addr`: The destination address with length specifying address family, 4 bytes is IPv4 and 16 is IPv6 +- `src_dest_port`: A combined source and destination port, see `Source And Destination Port` + +#### Source And Destination Port + +The source and destination port are combined into one value. If both source +and destination exists then the value is larger then 65535, the destination +will be the high 16 bits and source the low otherwise it will only be the +source. If the value is negative then only the destination exists. + +``` +if value > 0xffff then + src_port = value & 0xffff + dest_port = value >> 16 +else if value < 0 then + dest_port = -value - 1 +else + src_port = value +``` + +### icmp_message + +`if ip_header.ip_bits.1=0 && ip_header.ip_bits.2=0` + +``` +[ + uint type, + uint code +] +``` + +- `type`: TODO +- `code`: TODO + +### udp_message + +`if ip_header.ip_bits.1=1 && ip_header.ip_bits.2=0` + +TODO + +### tcp_message + +`if ip_header.ip_bits.2=1` + +``` +[ + uint seq_nr, + uint ack_nr, + uint tcp_bits, + uint window +] +``` + +- `seq_nr`: TODO +- `ack_nr`: TODO +- `tcp_bits`: TODO + - 0: URG + - 1: ACK + - 2: PSH + - 3: RST + - 4: SYN + - 5: FIN +- `window`: TODO + +### dns_message + +A DNS packet. + +``` +[ + optional bool is_complete, + uint id, + uint raw_dns_header, # TODO + optional nint count_bits, + optional uint qdcount, + optional uint ancount, + optional uint nscount, + optional uint arcount, + optional simple rr_bits, + optional [ + dns_question question, + ... + ], + optional [ + resource_record answer, + ... + ], + optional [ + resource_record authority, + ... + ], + optional [ + resource_record additional, + ... + ], + optional bytes malformed +] +``` + +- `is_complete`: Will exist and be false if the message is not complete and following attributes may not exists +- `id`: DNS identifier +- `raw_dns_header`: TODO +- `count_bits`: Bitmap indicating which counts are present, see `Negative Integers` and `Deduplication` + - Bit 0: qdcount present + - Bit 1: ancount present + - Bit 2: nscount present + - Bit 3: arcount present +- `qdcount`: Number of question records if different from the number of entries in `question` +- `ancount`: Number of answer resource records if different from the number of entries in `answer` +- `nscount`: Number of authority resource records if different from the number of entries in `authority` +- `arcount`: Number of additional resource records if different from the number of entries in `additional` +- `question`: The question records +- `answer`: The answer resource records +- `authority`: The authority resource records +- `additional`: The additional resource records +- `malformed`: Holds the bytes of the message that was not parsed + +### question + +A DNS question record. + +``` +[ + optional bool is_complete, + ( bytes | compressed_name | rindex ) qname, + optional uint qtype, + optional nint qclass +] +``` + +- `is_complete`: Will exist and be false if the message is not complete and following attributes may not exists +- `qname`: The QNAME as byte string, a name compression object or a reverse index, see `Deduplication` +- `qtype`: The QTYPE, see `Deduplication` +- `qclass`: The QCLASS, see `Negative Integers` and `Deduplication` + +### compressed_name + +An compressed name which has references to other labels within the same message. + +``` +[ + ( bytes label | uint label_index | nint offset | simple extension_bits ), + ... +] +``` + +- `label`: A byte string with a label part +- `label_index`: An index to the N byte string label in the message +- `offset`: The offset specified in the DNS message which could not be translated into a label index +- `extension_bits`: The extension bits if not 0b00 or 0b11 # TODO: add the extension bits + +### resource_record + +A DNS resource record. + +``` +[ + optional bool is_complete, + ( bytes | compressed_name | rindex ) name, + optional simple rr_bits, + optional uint type, + optional uint class, + optional uint ttl, + optional uint rdlength, + ( bytes | mixed_rdata ) rdata +] +``` + +- `is_complete`: Will exist and be false if the message is not complete and following attributes may not exists +- `name`: +- `rr_bits`: Bitmap indicating what is present, see `Deduplication` + - Bit 0: type + - Bit 1: class + - Bit 2: ttl + - Bit 3: rdlength # TODO: reverse index for TTL? +- `type`: The resource record type +- `class`: The resource record class +- `ttl`: The resource record ttl +- `rdlength`: The resource record rdata length +- `rdata`: The resource record data + +### mixed_rdata + +An array mixed with resource data and compressed names. + +``` +[ + ( bytes | compressed_name ) rdata_part, + ... +] +``` +- `rdata_part`: The parts of the resource records data + +## Stream Options + +Each option is specified here as OptionName(OptionNumber) and optional +OptionValue type. + +- `RLABELS(0) uint`: Indicates how many labels should be stored in the reverse label index before discarding them +- `RLABEL_MIN_SIZE(1) uint`: The minimum size a label must be to be put in the reverse label index +- `RDATA_RINDEX_SIZE(2) uint`: Indicates how many rdata should be stored in the reverse rdata index before discarding them +- `RDATA_RINDEX_MIN_SIZE(3) uint`: The minimum size a rdata must be to be put in the reverse rdata index +- `USE_RDATA_INDEX(4)`: If present then the stream uses rdata indexing +- `RDATA_INDEX_MIN_SIZE(5) uint`: The minimum size a rdata must be to be put in the rdata index + +## Deduplication + +Deduplication is done in a few different ways, data may be left out to +indicate that it is the same as the previous value, an index may be used to +indicate that it is the same as the N previous value and a reverse index +may be used to indicate that it is the N previous value looking backwards +across the stream. + +In other words, using the index deduplication you will need to build a table +of the values you come across during the decoding of the stream, this table +can grow very large. + +As an smaller alternative a reverse index can indicate often used data from +the N previous value looking back over the stream. This type of index also +reorder itself to try and put the most used data always in the index. + +TODO: details of each attribute and it's deduplication diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..c6a0909 --- /dev/null +++ b/CHANGES @@ -0,0 +1,796 @@ +2021-02-12 Jerry Lundström + + Release 2.0.0 + + This major release contains three backward incompatible changes, two + new command line options and a completely restructured man-page(!), + please read the change notes carefully before upgrading! + + The first backward incompatible change has to do with the removal of + libbind dependency. This library was causing segfaults on OpenBSD due to + shared (and overwritten) symbols with OpenBSD's libc. + It was replaced with LDNS and LDNS renders domain names as Fully + Qualified Domain Names (FQDN, the trailing dot!) so every output of a + domain name has been changed to a FQDN. + This also changes `-X`/`-x`, which will now match against FQDNs. + + The second backward incompatible change is that `-6` has been removed. + This was used to alter the BPF in order to "fix" it, dnscap adds + specific filters to IP and UDP headers which does not work for IPv6 + traffic. + The generated BPF has been changed to allow IPv6 to always pass, making + the option obsolete. IPv6 filtering is then done in dnscap. + + The last backward incompatible change has to do with the output format + of `-g` related to EDNS0 and is now more consistent with the rest of + the parsable output: + - No more spaces in the output + - Fix incorrect `\` and extra empty new-line + - All EDNS0 options are added after `edns0[...]` using comma separation, example: `edns0[],edns0opt[],...` + - Client Subnet format: `edns0opt[ECS,family=nn,source=nn,scope=nn,addr=...]` + - Unknown/unsupported code: `edns0opt[code=nn,codelen=nn]` + - Parsing error messages have changed, they came from libbind, now comes from LDNS + + New options: + - Add `-q` and `-Q` to filter on matched/not matched QTYPE + + Bugfixes: + - Fix memory leak in EDNS0 ECS address parsing + - `network`: Fix sonarcloud issues, potential `memcpy()` of null pointer + + Other changes: + - Fix CBOR output inclusion, LDNS is always available now + - Add macros for Apple and Windows endian functions + - Restructure and correct the man-page + + 557e5f5 man-page + 025529f v6bug, interval + 37b79e9 FQDN + ebcf434 QTYPE match, args, tests + 0cb5562 v6bug + 75f6115 Endian + aaeb213 Sonarcloud + 8685946 CBOR output + 3e26802 Sonarcloud + 30aa366 libbind + 3f94d0b Mattermost + +2020-10-22 Jerry Lundström + + Release 1.12.0 + + This release fixes the handling of `-?` option for dnscap and all plugins, + previously the handling varied between places and depending on `getopt()` + implementation an invalid option could return the wrong exit code. + + Other changes: + - Fix typo in configure help text + - `plugins/anonmask`: Fix typo in help text + - `plugins/rzkeychange`: + - Add `-D`, dry run mode, for testing + - Fix handling of `-a` and error on too many + + KNOWN ISSUES: + + On OpenBSD the system library libc exports the same symbols as libbind + does and this causes runtime warnings. Until now this has not caused any + known problems but is now also causing segfaults if the packet filter used + (BPF) includes IPv6 addresses. + On all other platforms OARC supports, these symbols are macros and in so + should not cause any problem. + + ee478c0 Known issues + 2f9d957 Tests + 3c663a2 Tests + c88efc5 rzkeychange test + f062f33 Tests + +2020-08-20 Jerry Lundström + + Release 1.11.1 + + This release fixes a lot of issues found by code analysis, adds a + explicit memory zeroing function to remove account information (read + when dropping privileges) and adds code coverage reporting. + + The `dnscap_memzero()` will use `explicit_bzero()` on FreeBSD and + OpenBSD, or `memset_s()` (if supported), otherwise it will manually + set the memory to zero. This will hopefully ensure that the memory + is zeroed as compilers can optimize out `memset()`'s that is just + before `free()`. + + The plugins exit code for the help option `-?` has been changed to 0 + to have the same as `dnscap -?`. + + d9747ee memzero + 1cf17c6 Coverage + 19c7120 Coverage + 7435676 Sonarcloud + 928e181 Sonarcloud + ca4afd0 Sonarcloud + 028f5e0 Badges + db0d6a1 LGTM + +2020-06-01 Jerry Lundström + + Release 1.11.0 + + This release includes a new plugin called `eventlog`, contributed + by Byron Darrah (@ByronDarrah), output DNS activity as log events, + including answers to A and AAAA queries. + + Other changes includes compile warning and code analysis fixes. + + 382eac4 COPR + 4c03650 Compile warn + 21d6a67 Slight change -- wording now matches usage() output. + dd19b0b Added the eventlog.so plugin... + 1ebf504 Added new dnscap plugin: evenlog.so... + f3f9aaa Compile warnings + +2020-03-02 Jerry Lundström + + Release 1.10.4 + + Fixed a bug that would not drop privileges when not specifying any + interface (which is equal to capturing on all interfaces). + Added functionality to set the supplemental groups when dropping + privileges and changing user, or clear them if that is not supported. + Other changes includes corrected man-page about '-w' and update to + documentation. + + a0285e4 drop privileges errors, initgroups/setgroups + 96336f3 daemon: Attempt to drop supplemental groups + 467a9a7 Drop privileges + de940a8 man-page -w + 187ec43 README + +2019-10-02 Jerry Lundström + + Release 1.10.3 + + Fixed plugins inclusion in deb packages for Debian and Ubuntu. + + 017ebb2 Deb packages + cf59143 COPR, spec + +2019-08-05 Jerry Lundström + + Release 1.10.2 + + Fixed bug in the handling of defragmentation configuration which lead + to the use of a local scope variable later on and caused unexpected + behavior. + + 91692b8 Frag conf + 6a74376 Package + d0d1a6d Package + +2019-07-08 Jerry Lundström + + Release 1.10.1 + + Fix various issues found by code analysis tools, a few compiler warnings + removed, undefined bit shift behavior fixed, parameter memory leaks + plugged and documentation updates. + + Fixes: + - `dump_dns`: Remove usage of `strcpy()` and use `snprintf()` instead + of `sprintf()` + - `bpft`: + - Use `text_ptr->len` to store length of generated text + - Use `memcpy()` instead of `strcat()` + - Remove unneeded `realloc()` and `strcpy()` + - `plugins/cryptopan`: Fix strict-aliasing warnings + - `network`: Rework part of `dl_pkt()` to remove usage of `strcpy()` + and use `snprintf()` instead of `sprintf()` + - `plugins/anonaes128`: Use `a6` as dest when copying v4 addresses for + readability and code analysis + - `plugins/cryptopan`: Run first pass separate to eliminate a 32bit + shift by 32 (undefined behavior) + - `plugins/cryptopant`: Fix memory leak of `keyfile` if `-k` is + specified more then once + + Documentation: + - Update `README.md` with correction to building from git and note + about PCAP on OpenBSD + - Fix #190: Update link to `libbind` source + + 074923c Funding + 5d2e84c libbind + 8ee9f2a Travis-CI + 6babd09 Fixes + bb2d1c7 README, compile warnings + 0d9cd9c LGTM, Travis-CI + +2018-12-03 Jerry Lundström + + Release 1.10.0 + + This release adds a new plugin type "filter" and 5 new plugins that can + do anonymization, deanonymization and masking of the IP addresses. + + New features: + - Check plugins for `pluginname_type()` which returns `enum plugin_type`, + if missing the plugin is counted as an "output" plugin + - New plugin type "filter" which calls `pluginname_filter()` prior of + outputting any data or calling of "output" plugins, if the new function + returns non-zero then the packet is filtered out (dropped) + - New extension `DNSCAP_EXT_SET_IADDR` that gives access to a function + for setting the from and to IP addresses both in the extracted data + and the wire + + New plugins: + - `anonaes128`: Anonymize IP addresses using AES128 + - `anonmask`: Pseudo-anonymize IP addresses by masking them + - `cryptopan`: Anonymize IP addresses using an extension to Crypto-PAn + (College of Computing, Georgia Tech) made by David Stott (Lucent) + - `cryptopant`: Anonymize IP addresses using cryptopANT, a different + implementation of Crypto-PAn made by the ANT project at USC/ISI + - `ipcrypt`: Anonymize IP addresses using ipcrypt create by + Jean-Philippe Aumasson + + Bugfixes: + - Fix changing `royparse` and `txtout` with other plugins (thanks to + Duane Wessels and Paul Hoffman) + - Free pointers to allocated strings in `text_free()` (thanks to Michał + Kępień) + - Fix IP checksum calculation + + Other changes: + - `-B` and `-E` can be used without `-w` (thanks to Duane Wessels) + - Use `pcap_findalldevs()` instead of `pcap_lookupdev()` (thanks to + Michał Kępień) + - Document and add `-?` option to all plugins + - Fix clang `scan-build` bugs and LGTM alerts + - Use `gmtime_r()` instead of `gmtime()` + - Update `pcap-thread` to v4.0.0 + + 67d8e2c Fix + fb0ed02 Plugin documentation + a2c9a6c cryptopant + 39db1ca Deanonymize, IPv6 test + afc7107 Crypto-PAn, cryptopANT + f1912cc OpenSSL, anonaes128 + f2bab62 ipcrypt, anonmask + 158b1e7 anonmask help + 60ece58 anonmask + 8f1b138 Plugin types, filter plugin, set iaddr extension, anonymization + by masking + b7d7991 IP checksum + 641a23a Free pointers to allocated strings in text_free() + 4d313bf pcap_findalldevs() + 091e0ca Use pcap_findalldevs() instead of pcap_lookupdev() + 6a7b25e Clean up use of feature test macros on Linux + cbba14c Configure, uninitialized + f228c9c Code formatting + 3fd738c man-page + 770168a Test + 714e4f5 Fix -B so that it works when reading offline pcap files. + 8675bea Test + 911fec9 Implementing test9 as a test of -B and -E command line args. + a7cc72d -B and -E can work fine without -w . + 04c4928 Made the same changes to txtout as were in 165a786 + 165a786 Workaround for stdio mystery causing duplicate royparse output. + +2018-02-28 Jerry Lundström + + Release 1.9.0 + + This release adds a new option to change how the Berkeley Packet Filter + is generated to include the host restrictions for all selections, + previously this restriction would only apply to specific parts. + + Additional tweaks to the RSSM plugin has been made to conform to the + RSSAC002v3 specification. One noticeable change is that the plugin now + requires the DNS to be parsed before counted, any error in the parsing + will result in the message being left out of the statistics. + + Changes: + - Fix spacing in BPF filter to look better + - Fix #146: Add `bpf_hosts_apply_all`, apply any host restriction to all + - `plugin/rssm`: + - Remove quoting of `start-period` and correctly handle empty hashes + - Issue #152, Issue #91: Parse DNS before processing RSSM counters + - `plugin/rssm/dnscap-rssm-rssac002`: Use `YAML::Dump()` for output + + 47d892b Issue #152: RSSM YAML output + d4f1466 Issue #152, Issue #91: Parse DNS before processing RSSM counters + 68fc1ff BPF, `bpf_hosts_apply_all` + +2018-02-07 Jerry Lundström + + Release 1.8.0 + + This release updates the TCP stream code in order to be able to look + at more then just the first query, for handling already ongoing TCP + connections without having seen SYN/ACK and for reassembly of the TCP + stream prior of parsing it for DNS with an additional layer of parsing + (see `reassemble_tcp_bfbparsedns`). + + Updates to the Root Server Scaling Measurement (RSSM) plugin have also + been made to bring it up to date with RSSAC002v3 specification, be + able to output the YAML format described and an additional script to + merge YAML files if the interval is less then the RSSAC002v3 24 hour + period. See "Updates to the RSSM plugin" below and + `plugins/rssm/README.md`. + + New extended options: + - `parse_ongoing_tcp`: Start tracking TCP connections even if SYN/ACK + has not been seen + - `allow_reset_tcpstate`: Allow external reset of TCP state + - `reassemble_tcp`: Use to enable TCP stream reassembly + - `reassemble_tcp_faultreset`: Number of faults before reseting TCP + state when reassembly is enabled + - `reassemble_tcp_bfbparsedns`: Enable an experimental additional layer + of reassemble that uses `libbind` to parse the payload before accepting + it. If the DNS is invalid it will move 2 bytes within the payload and + treat it as a new payload, taking the DNS length again and restart + the process. Requires `libbind` and `reassemble_tcp`. + + New extension functions for plugins: + - `DNSCAP_EXT_TCPSTATE_GETCURR`: Function to get a pointer for the + current TCP state + - `DNSCAP_EXT_TCPSTATE_RESET`: Function to reset a TCP state + + New features: + - Parse additional DNS queries in TCP connections + - `-g` and the `txtout` plugin will reset TCP state (if allowed) on + failure to parse DNS + + Bugfixes: + - Fix `-g` output, separate error message with a space + - Fix TCP packets wrongfully flagged as DNS when using layers. + - Fix TCP debug output when using layers, `ia_str()` is not safe to call + twice in the same `printf` because of local buffer. + - Fix exported extension functions, need to be file local + + New tests for: + - Multiple DNS queries in one TCP connection + - Query over TCP without SYN + - Queries over TCP with first query missing length + - Queries over TCP with middle payloads missing + - Add test with TCP stream that missing multiple packets in the middle + + Updates to the RSSM plugin (`plugins/rssm`): + - Add info about saving counts and sources + - Fix memory leak on `fopen()` errors + - Update to RSSAC002v3 specification + - New options: + - `-D` to disable forking on close + - `-Y`: Use RSSAC002v3 YAML format when writing counters, the file + will contain multiple YAML documents, one for each RSSAC002v3 metric + Used with; -S adds custom metric `dnscap-rssm-sources` and -A adds + `dnscap-rssm-aggregated-sources` + - `-n`: Set the service name to use in RSSAC002v3 YAML + - `-S`: Write source IPs into counters file with the prefix `source` + - `-A`: Write aggregated IPv6(/64) sources into counters file with + the prefix `aggregated-source` + - `-a`: Write aggregated IPv6(/64) sources to + `..` + - Add `dnscap-rssm-rssac002` Perl script for merging RSSAC002v3 YAML files + - Add README.md for the plugin man-page for `dnscap-rssm-rssac002` + - Add test for YAML output and merging of YAML files + + c7058c8 Use file local functions for all extensions + 66b352d RSSM RSSAC002v3 YAML Tool + b09efc2 `plugins/rssm` RSSAC002v3 + 709aba6 Fix #89: Add additional reassembly layers that parses the + payload byte for byte for valid DNS + 04fa013 Fix CID 1463944 (again) + b1cf623 RSSM saving data and forking + fb23305 Fix CID 1463944 + 0fca1a8 Issue #89: TCP stream reassemble + bb6428c CID 1463814: Check `ns_initparse()` for errors + a57066f Fix #88: TCP handling + +2017-12-27 Jerry Lundström + + Release 1.7.1 + + The library used for parsing DNS (libbind) is unable to parse DNS + messages when there is padding at the end (the UDP/TCP payload is larger + then the DNS message). This has been fixed by trying to find the actual + DNS message size, walking all labels and RR data, and then retry parsing. + + Other changes and bug-fixes: + - Fix size when there is a VLAN to match output of `use_layers` yes/no + - Add test of VLAN matching + - Fix `hashtbl.c` building in `rssm` + - Add test with padded DNS message + + 49e5400 Fix #127: If `ns_initparse()` returns `EMSGSIZE`, try and get + actual size and reparse + 99bda0b Fix #98: VLAN + +2017-12-19 Jerry Lundström + + Release 1.7.0 + + This release adds IP fragmentation handling by using layers in pcap-thread + which also adds a new flag to output and modules. `DNSCAP_OUTPUT_ISLAYER` + indicates that `pkt_copy` is equal to `payload` since the layers of the + traffic have already been parsed. IP fragments are reassembled with the + `pcap_thread_ext_frag` extension that is included in pcap-thread. + + New extended (`-o`) options: + - `use_layers`: Use pcap-thread layers to handle the traffic + - `defrag_ipv4`: Enabled IPv4 de-fragmentation + - `defrag_ipv6`: Enabled IPv6 de-fragmentation + - `max_ipv4_fragments`: Set maximum fragmented IPv4 packets to track + - `max_ipv4_fragments_per_packet`: Set the maximum IPv4 fragments per + tracked packet + - `max_ipv6_fragments`: Set maximum fragmented IPv6 packets to track + - `max_ipv6_fragments_per_packet`: Set the maximum IPv6 fragments per + tracked packet + + Currently `-w` does not work with `use_layers` and the plugins `pcapdump` + and `royparse` will discard output with the flag `DNSCAP_OUTPUT_ISLAYER` + because they need access to the original packet. + + The `rzkeychange` plugin now encodes certain flag bits in the data that + it reports for RFC8145 key tag signaling. The flags of interest are: + `DO`, `CD`, and `RD`. These are encoded in an bit-mask as a hexadecimal + value before the `_ta` component of the query name. + + Other changes and bug-fixes: + - Fix #115: document `-g` output, see `OUTPUT FORMATS` `diagnostic` in + `dnscap(1)` man-page + - Add test to match output from non-layers runs with those using layers + - Add test with fragmented DNS queries + - Fix #120: CBOR/CDS compiles again, update tinycbor to v0.4.2 + - Fix `ip->ip_len` byte order + - Fix parsing of IP packets with padding or missing parts of payload + + 0347f74 Add AUTHORS section in man-page + ef1b68c Fix CID 1463073 + 8a79f89 Layers + a404d08 Update pcap-thread to v3.1.0, add test for padding fixes + 08402f1 Fix byte order bug. ip->ip_len must be evaluated with ntohs(). + d6d2340 CBOR/CDS and formatting + 85ec2d8 Fix #87: IP fragmentation reassembly + 22bfd4a Documentation + c35f19f Adding flag bits to rzkeychange RFC8145 key tag signaling data. + This may be useful to find "false" key tag signals from sources + that don't actually perform DNSSEC validation. + +2017-12-01 Jerry Lundström + + Release 1.6.0 + + New additions to the plugins: + - `rzkeychange` can now collect RFC8145 key tag signaling. Signals are + saved during the collection interval, and then sent to the specified + `-k `, one at a time, at the end of the interval. Only root zone + signals are collected. Added by Duane Wessels (@wessels). + - `royparse` is a new plugin to splits a PCAP into two streams, queries + in PCAP format and responses in ASCII format. Created by Roy Arends + (@RoyArends). + - `txtout` new option `-s` for short output, only print QTYPE and QNAME + for IN records. Added by Paul Hoffman (@paulehoffman) + - The extension interface has been extended with `DNSCAP_EXT_IA_STR` to + export the `ia_str()` function. + + Bugfixes and other changes: + - Remove duplicated hashtbl code + - `rssm`: fix bug where count in table was taken out as `uint16_t` but + was a `uint64_t` + - Handle return values from hashtbl functions + - `txtout`: removed unused `-f` options + - Change `ia_str()` to use buffers with correct sizes, thanks to + @RoyArends for spotting this! + + Commits: + 3f78a31 Add copy/author text + 1bd914d Fix CID 1462343, 1462344, 1462345 + f9bb955 Fix `fprintf()` format for message size + abedf84 Fix #105: `inet_ntop` buffers + bfdcd0d Addresses the suggestions from Jerry. + dda0996 royparse :) + 4f6520a royparse plugin finished + f1aa4f2 Fix #103: Remove `opt_f` + 32355b7 Rearrange code to keep the change smaller and fix indentation + d6612c1 Added -s to txtout for short output + 9d8d1ef Check return of `snprintf()` + 55f5aba Format code + 9f19ec3 Fixed memory leak in rzkeychange_keytagsignal() + 58b8784 Fix memory leaks and better return value checks in + rzkeychange_submit_counts() + b06659f Add server and node to keytag signal query name + 705a866 Always free response packets in rzkeychange plugin. + e802843 Implement RFC8145 key tag signal collection in rzkeychange plugin + 5fbf6d0 Added extension for ia_str() so it can be used by rzkeychange + plugin. + 3be8b8f Split `dnscap.c` into more files + e431d14 Fix #92: hashtbl + +2017-08-21 Jerry Lundström + + Release 1.5.1 + + Compatibility fixes for FreeBSD 11.1+ which is now packing `struct ip` + and for OpenBSD. + + Commits: + 17e3c92 FreeBSD is packing `struct ip`, need to `memcpy()` + f8add66 Code formatting + 38cd585 Add documentation about libbind + d1dd55b Fix #82: Update dependencies for OpenBSD + +2017-06-06 Jerry Lundström + + Release 1.5.0 + + Added support for writing gzipped PCAP if the `-W` suffix ends with + `.gz` and made `-X` work without `-x`. New inteface for plugins to + tell them what extensions are available and a new plugin `rzkeychange`. + + Plugin extensions: + - Call `plugin_extension(ext, arg)` to tell plugin what extensions exists + - Add extension for checking responder (`is_responder()`) + + The rzkeychange plugin was developed by Duane Wessels 2016 in support + of the root zone ZSK size increase. It is also being used in support of + the 2017 root KSK rollover and collects the following measurements: + - total number of responses sent + - number of responses with TC bit set + - number of responses over TCP + - number of DNSKEY responses + - number of ICMP_UNREACH_NEEDFRAG messages received + - number of ICMP_TIMXCEED_INTRANS messages received + - number of ICMP_TIMXCEED_REASS messages received + + Other fixes (author Duane Wessels): + - 232cbd0: Correct comment description for meaning of IPPROTO_AH + - 181eaa4: Add #include for struct timeval on NetBSD + + Commits: + + 1d894e2 Make -x and -X work correctly together and update man-page + 34bc54c Make the -X option work without requiring a -x option. + f43222e Fix CID 1440488, 1440489, 1440490 + aa54395 Update pcap-thread to v2.1.3 + 81174ce Prepare SPEC for OSB/COPR + 21d7468 New plugin rzkeychange and plugin extensions + 38491a3 Config header is generated by autotools + 419a8ab Small tweaks and fixes for gzip support + 1967abc updated for earlier BSD versions + f135c90 added auto gzip if the -W suffix ends with .gz + + Commits during development of rzkeychange (author Duane Wessels): + - 620828d: Add rzkeychange -z option to specify resolver IP addresses + - 1f77987: Add -p and -t options to rzkeychange plugin to configure an + alternate port and TCP. Useful for ssh tunnels. + - 2a571f1: Split ICMP time exceeded counter into two counters for time + exceeded due to TTL and another due to fragmentation + - e4ee2d3: The rzkeychange data collection plugin uses + `DNSCAP_EXT_IS_RESPONDER` extension to know if an IP address is a + "responder" or not, because when dnscap is instructed to collect ICMP + with -I, it processes all ICMP packets, not just those limited to + responders (or initiators). + - cee16b8: Add ICMP Time Exceeded to counters + - ad8a227: Counting source IPs has performance impacts. #ifdef'd out for + now add ICMP "frag needed" counts + - c25e72b: Implemented DNS queries with ldns. First there will be some + test queries to ensure the zone is reachable and configured to receive + data. Then a query naming the fields, followed by the periodic queries + delivering counts. + - fd23be7: Make report zone, server, node command line argumements mandatory + - 137789b: Adding rzkeychange plugin files + +2017-03-29 Jerry Lundström + + Release 1.4.1 + + Fixed an issue that when compiled with libpcap that had a specific + feature enabled it would result in a runtime error which could not be + worked around. + + Also fixed various compatibility issues and updated dependency + documentation for CentOS. + + Commits: + + 785d4c4 Fix compiler warnings + 2d4df8d Fix #65: Update pcap-thread to v2.1.2 + 26d3fbc Fix #64: Add missing dependency + 55e6741 Update pcap-thread to v2.1.1, fix issue with libpcap timestamp + type + c6fdb7a Fix typo and remove unused variables + +2017-02-27 Jerry Lundström + + Release 1.4.0 + + Until it can be confirmed that the threaded code works as well as the + non-threaded code it has been made optional and requires a configuration + option to enable it during compilation. + + New extended option: + - `-o pcap_buffer_size=` can be used to increase the capture + buffer within pcap-thread/libpcap, this can help mitigate dropped + packets by the kernel during breaks (like when closing dump file). + + Commits: + + 1c6fbb2 Update copyright year + 63ef665 Suppress OpenBSD warnings about symbols + 2c99946 pcap-thread v2.0.0, disable threads, errors handling + 4cade97 Fix #56: Update pcap-thread to v1.2.2 and add test + +2016-12-23 Jerry Lundström + + Release 1.3.0 + + Rare lockup has been fixed that could happen if a signal was received + in the wrong thread at the wrong time due to `pcap_thread_stop()` + canceling and waiting on threads to join again. The handling of signals + have been improved for threaded and non-threaded operations. + + New features: + - Experimental CBOR DNS Stream format output, see `CBOR_DNS_STREAM.md` + - Extended options to specify user and group to use when dropping + privileges, see EXTENDED OPTIONS in man-page + + Commits: + + a5fa14e Signal and threads + 3868104 Use old style C comments + 7946be5 Clarify building + d5463b4 RPM spec and various automake fixes + df206bf Resource data indexing and documentation + 0e2d0fe Fix #22, fix #43: Update README + 5921d73 Add stream option RLABELS and RLABEL_MIN_SIZE + 6dd6ec1 Implement experimental CBOR DNS Stream Format + 4baf695 Fix #37: Extended options to specifty user/group to use when + dropping privileges + 61d830a Fix #35: Use `AC_HEADER_TIME` and fix warning + +2016-10-27 Jerry Lundström + + Release 1.2.0 + + Update `pcap-thread` to v1.2.0 to get the new callback queue mode which + puts that mode into using pthread conditions if all pcaps are offline and + keeps us from losing packets. + + Use `pcap_thread_dropback()` callback to get the notification when a + packet was dropped because the queue was full, indicating that we can't + process all the packets. Added this stats to the `-S` output as total + and per interface as `ptdrop`. Changed the output for each interface + to not cut of information, for example interface name was cut to + 4 characters. + + Other changes: + + - Add extended options `-o