From f220c0449c950923dde67ff7d67a958351bcb411 Mon Sep 17 00:00:00 2001 From: GalaxP Date: Wed, 4 Jun 2025 16:34:17 +0200 Subject: [PATCH 1/8] Added new module - geolite --- geolite/Makefile.am | 18 ++++++ geolite/README.md | 86 +++++++++++++++++++++++++ geolite/geolite.py | 152 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 geolite/Makefile.am create mode 100644 geolite/README.md create mode 100755 geolite/geolite.py diff --git a/geolite/Makefile.am b/geolite/Makefile.am new file mode 100644 index 00000000..b2b1ea71 --- /dev/null +++ b/geolite/Makefile.am @@ -0,0 +1,18 @@ +EXTRA_DIST=geolite.py readme.md +bin_SCRIPTS=geolite.py + +pkgdocdir=${docdir}/geolite +pkgdoc_DATA=readme.md + +pylint: + pylint-3 geolite.py + +flake8: + flake8 geolite.py + +pycodestyle: + pycodestyle-3 geolite.py + +lint: pylint flake8 pycodestyle + +include ../aminclude.am \ No newline at end of file diff --git a/geolite/README.md b/geolite/README.md new file mode 100644 index 00000000..c9c47d33 --- /dev/null +++ b/geolite/README.md @@ -0,0 +1,86 @@ +# Geolite + +## Module description + +This module outputs flow records with geolocation data using a [geolite database](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data/). + + +## Input data + +This module expects flow records in Unirec format. The required fields +are determined by run time parameters. + + +## Output data + +Flows are sent on the output interface, also in Unirec format, they +contain geolocation data. The fields included in the output interface will vary depending on the selected database type. + +Below are the fields that will be set for each database type: +* `country` + * ipaddr `ip` + * string `name` + * string `iso_code` + * uint32 `geoname_id` + * uint32 `is_in_european_union` + +* `city` + * ipaddr `ip` + * string `country_name` + * string `country_iso_code` + * uint32 `country_geoname_id` + * uint32 `is_in_european_union` + * string `city_name` + * float `latitude` + * float `longitude` + * uint32 `accuracy_radius` + +* `asn` + * ipaddr `ip` + * uint32 `asn` + * string `string autonomous_system_organization` + + +## Module parameters + +In addition to the implicit *libtrap* parameters `-i IFC_SPEC`, `-h` +and `-v` (see [Execute a +module](https://github.com/CESNET/Nemea#try-out-nemea-modules)) this +module takes the following parameters: + +* `-d` `--db` path + + * Specify path to the database file. + +* `-f` `--fields` field1,field2,... + + * Specify the name of field(s) from the input interface, which will be used for geolocation and lookup in the database (case sensitive). + If multiple fields are specified, they must be separated by a comma. + +* `-t` `--type` {country, city, asn} + + * Specify the type of GeoLite database. The default value is `country`. + + +## Example +The following command : + +`./geolite.py -i f:/etc/nemea/data/data.dan.trapcap,f:test.trapcap -d '/usr/share/GeoIP/GeoLite2-Country.mmdb' -t country -f "SRC_IP,DST_IP"` + +will be interpreted as follows: + +* `-i f:/etc/nemea/data/data.dan.trapcap,f:test.trapcap` + sets the input and output interfaces to a file. + +* `-d '/usr/share/GeoIP/GeoLite2-Country.mmdb'` sets the path to the database file. + +* `-t country` sets the database type to `country` (can be omitted as it is the default). + +* `-f "SRC_IP,DST_IP"` specifies the names of the fields containing IP addresses to be used for geolocation. + + + + + + + diff --git a/geolite/geolite.py b/geolite/geolite.py new file mode 100755 index 00000000..a0fe6dc0 --- /dev/null +++ b/geolite/geolite.py @@ -0,0 +1,152 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2025 CESNET +# +# LICENSE TERMS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Company nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# ALTERNATIVELY, provided that this notice is retained in full, this +# product may be distributed under the terms of the GNU General Public +# License (GPL) version 2 or later, in which case the provisions +# of the GPL apply INSTEAD OF those given above. +# +# This software is provided ``as is'', and any express or implied +# warranties, including, but not limited to, the implied warranties of +# merchantability and fitness for a particular purpose are disclaimed. +# In no event shall the company or contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential +# damages (including, but not limited to, procurement of substitute +# goods or services; loss of use, data, or profits; or business +# interruption) however caused and on any theory of liability, whether +# in contract, strict liability, or tort (including negligence or +# otherwise) arising in any way out of the use of this software, even +# if advised of the possibility of such damage. + + +import argparse +import pytrap +import geoip2.database + + +CITY_OUTPUTSPEC = "ipaddr ip, string country_name, string country_iso_code, uint32 country_geoname_id, uint32 is_in_european_union, string city_name, float latitude, float longitude, uint32 accuracy_radius" +COUNTRY_OUTPUTSPEC = "ipaddr ip, string name, string iso_code, uint32 geoname_id, uint32 is_in_european_union" +ASN_OUTPUTSPEC = "ipaddr ip, uint32 asn, string autonomous_system_organization" + + +parser = argparse.ArgumentParser(description='Module for geolocation using GeoLite2 database') +parser.add_argument('-i', "--ifcspec", + help="select TRAP IFC specifier") +parser.add_argument('-d', "--db", + help="path to the GeoLite2 database file", + required=True) +parser.add_argument('-f', "--fields", + help="input fields to use for geolocation seperated by comma", + default="SRC_IP") +parser.add_argument('-t', "--type", + help="type of GeoLite database", + choices=['country', 'city', 'asn'], + default="country") + +# parse command line arguments +args = parser.parse_args() +fields = args.fields.split(',') + + +# initialize TRAP context +trap = pytrap.TrapCtx() +trap.init(['-i', args.ifcspec], 1, 1) + +# output interface +fmttype = pytrap.FMT_UNIREC + +# set the correct output specification based on the type of GeoLite database +if args.type == 'asn': + outputspec = ASN_OUTPUTSPEC +elif args.type == 'city': + outputspec = CITY_OUTPUTSPEC +else: + outputspec = COUNTRY_OUTPUTSPEC + +trap.setDataFmt(0, fmttype, outputspec) +output = pytrap.UnirecTemplate(outputspec) +output.createMessage() + +# input interface +fmtspec = "" +trap.setRequiredFmt(0, fmttype, fmtspec) +rec = pytrap.UnirecTemplate(fmtspec) + + +# open GeoLite2 database reader +with geoip2.database.Reader(args.db) as reader: + # main loop + while True: + try: + data = trap.recv() + except pytrap.FormatChanged as e: + fmttype, fmtspec = trap.getDataFmt(0) + rec = pytrap.UnirecTemplate(fmtspec) + data = e.data + except pytrap.Terminated: + print("Terminated trap.") + break + except pytrap.TrapError: + print("Trap error, exiting.") + break + if len(data) <= 1: + break + + else: + rec.setData(data) + for field in fields: + ip = rec.get(data, field) + output.ip = ip + try: + if args.type == 'city': + geolocation = reader.city(str(ip)) + elif args.type == 'country': + geolocation = reader.country(str(ip)) + elif args.type == 'asn': + geolocation = reader.asn(str(ip)) + except: + continue + if geolocation is None: + continue + + # fill output fields based on geolocation type + if args.type == 'country': + output.name = geolocation.country.name or "unkown" + output.iso_code = geolocation.country.iso_code or "unkown" + output.geoname_id = geolocation.country.geoname_id or 0 + output.is_in_european_union = geolocation.country.is_in_european_union or False + elif args.type == 'city': + output.country_name = geolocation.country.name or "unkown" + output.country_iso_code = geolocation.country.iso_code or "unkown" + output.country_geoname_id = geolocation.country.geoname_id or 0 + output.is_in_european_union = geolocation.country.is_in_european_union or False + output.city_name = geolocation.city.name or "unknown" + output.latitude = geolocation.location.latitude or 0.0 + output.longitude = geolocation.location.longitude or 0.0 + output.accuracy_radius = geolocation.location.accuracy_radius or 0 + elif args.type == 'asn': + output.asn = geolocation.autonomous_system_number or 0 + output.autonomous_system_organization = geolocation.autonomous_system_organization or "unknown" + + # send output data + trap.send(output.getData(), 0) + +# send end-of-stream message and exit +trap.sendFlush(0) +trap.finalize() From 3221f8a72431c8923c755363ff363a5781e2f892 Mon Sep 17 00:00:00 2001 From: GalaxP Date: Thu, 5 Jun 2025 00:56:28 +0200 Subject: [PATCH 2/8] added caching to lookup function --- geolite/README.md | 8 ++++++-- geolite/geolite.py | 24 +++++++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/geolite/README.md b/geolite/README.md index c9c47d33..f6c63a2c 100644 --- a/geolite/README.md +++ b/geolite/README.md @@ -31,8 +31,8 @@ Below are the fields that will be set for each database type: * uint32 `country_geoname_id` * uint32 `is_in_european_union` * string `city_name` - * float `latitude` - * float `longitude` + * float `latitude` + * float `longitude` * uint32 `accuracy_radius` * `asn` @@ -61,6 +61,10 @@ module takes the following parameters: * Specify the type of GeoLite database. The default value is `country`. +* `-c` `--cache` number + + * Specify the number of lookup calls that will be cached. Set to `0` to disable caching. The default value is `128`. + ## Example The following command : diff --git a/geolite/geolite.py b/geolite/geolite.py index a0fe6dc0..53d730d9 100755 --- a/geolite/geolite.py +++ b/geolite/geolite.py @@ -36,6 +36,7 @@ import argparse +from functools import lru_cache import pytrap import geoip2.database @@ -58,6 +59,10 @@ help="type of GeoLite database", choices=['country', 'city', 'asn'], default="country") +parser.add_argument('-c', "--cache", + type=int, + help="number of cached lookups. If set to 0, caching is disabled.", + default=128) # parse command line arguments args = parser.parse_args() @@ -88,9 +93,19 @@ trap.setRequiredFmt(0, fmttype, fmtspec) rec = pytrap.UnirecTemplate(fmtspec) - # open GeoLite2 database reader with geoip2.database.Reader(args.db) as reader: + @lru_cache(maxsize=args.cache) + def lookup_in_database(lookup_ip): + #Function to look up the IP address in the GeoLite2 database. + if args.type == 'city': + return reader.city(lookup_ip) + elif args.type == 'country': + return reader.country(lookup_ip) + elif args.type == 'asn': + return reader.asn(lookup_ip) + + # main loop while True: try: @@ -114,12 +129,7 @@ ip = rec.get(data, field) output.ip = ip try: - if args.type == 'city': - geolocation = reader.city(str(ip)) - elif args.type == 'country': - geolocation = reader.country(str(ip)) - elif args.type == 'asn': - geolocation = reader.asn(str(ip)) + geolocation = lookup_in_database(str(ip)) except: continue if geolocation is None: From 00380bf48598a7164dc58ba353ede8f73cbf9702 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Thu, 10 Jul 2025 08:16:59 +0200 Subject: [PATCH 3/8] Biflow aggregator - Update configuration --- biflow_aggregator/configuration.cpp | 21 ++++++++++++ biflow_aggregator/configuration.h | 51 +++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/biflow_aggregator/configuration.cpp b/biflow_aggregator/configuration.cpp index 0df48889..b3a224b7 100644 --- a/biflow_aggregator/configuration.cpp +++ b/biflow_aggregator/configuration.cpp @@ -71,6 +71,27 @@ bool Configuration::get_eof_termination() noexcept return _eof_terminate; } +void Configuration::set_global_flush_configuration(const char *input) +{ + std::size_t mode_start_index; + _global_flush_configuration.interval = std::stoul(input, &mode_start_index); + if (std::strcmp(input + mode_start_index, "a") == 0 || + std::strcmp(input + mode_start_index, "absolute") == 0) { + _global_flush_configuration.type = Global_flush_configuration::Type::ABSOLUTE; + } else if (std::strcmp(input + mode_start_index, "r") == 0 || + std::strcmp(input + mode_start_index, "relative") == 0 || + std::strcmp(input + mode_start_index, "") == 0) { + _global_flush_configuration.type = Global_flush_configuration::Type::RELATIVE; + } else { + throw std::invalid_argument("Invalid flush timeout format. Expected: [a|absolute|r|relative|]."); + } +} + +Configuration::Global_flush_configuration Configuration::get_global_flush_configuration() noexcept +{ + return _global_flush_configuration; +} + void Configuration::print() noexcept { std::cout << "***** Configuration *****" << std::endl; diff --git a/biflow_aggregator/configuration.h b/biflow_aggregator/configuration.h index b2b53a92..7b218d2d 100644 --- a/biflow_aggregator/configuration.h +++ b/biflow_aggregator/configuration.h @@ -23,6 +23,35 @@ * @brief Class thas holds module configuration */ class Configuration { +public: + + /** + * @brief Global flush configuration + * + * Flush interval is used to flush all records in flow cache to output interface once per given amount of seconds. + * If not set, no flush is performed. + */ + struct Global_flush_configuration { + enum class Type { + ABSOLUTE, ///< Flows must be flushed every interval seconds starting from epoch + RELATIVE, ///< Flows must be flushed every interval seconds starting from module start + } type; + time_t interval = 0; ///< Interval in seconds + + /** + * @brief Check if flush interval is set + * + * @return true Flush interval is set + * @return false Flush interval is not set + */ + [[nodiscard]] inline + bool is_set() const noexcept + { + return interval > 0; + } + }; + +private: /** * @brief Configuration of fields from config file. @@ -45,6 +74,14 @@ class Configuration { */ time_t _t_passive; + /** + * @brief Periodic flush configuration + * + * If set, module flush all records in flow cache to output interface once per given amount of seconds. + * If flush interval is set to 0, no flush is performed. + */ + Global_flush_configuration _global_flush_configuration; + /** * @brief active timeout * @@ -155,6 +192,20 @@ class Configuration { */ time_t get_active_timeout() noexcept; + /** + * @brief Set the flush timeout + * + * See _periodic_flush_configuration for more info. + * + * @param input Timeout in text format. + */ + void set_global_flush_configuration(const char *input); + + /** + * @brief Get the flush timeout object + */ + Global_flush_configuration get_global_flush_configuration() noexcept; + /** * @brief Set the passive timeout * From 753e79e84fc35823030aa2c8c5973f6aa710d76a Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Thu, 10 Jul 2025 08:17:58 +0200 Subject: [PATCH 4/8] Biflow aggregator - Update main.cpp --- biflow_aggregator/main.cpp | 39 ++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/biflow_aggregator/main.cpp b/biflow_aggregator/main.cpp index 146d9109..78ff6790 100644 --- a/biflow_aggregator/main.cpp +++ b/biflow_aggregator/main.cpp @@ -79,7 +79,8 @@ UR_FIELDS ( PARAM('e', "eof", "End when receive EOF.", no_argument, "flag") \ PARAM('s', "size", "Max number of elements in flow cache.", required_argument, "number") \ PARAM('a', "active-timeout", "Active timeout.", required_argument, "number") \ - PARAM('p', "passive-timeout", "Passive timeout.", required_argument, "number") + PARAM('p', "passive-timeout", "Passive timeout.", required_argument, "number") \ + PARAM('g', "global-timeout", "Global timeout.", required_argument, "number") trap_module_info_t *module_info = NULL; static volatile int stop = 0; @@ -364,6 +365,20 @@ void update_flow( if (pt != t_data->value.passive_timeout) dll.swap(t_data); } + +static void flush_all(agg::Aggregator& aggregator, + ur_template_t* out_template, void* out_record, Dll& dll) +{ + for (auto flow_data : aggregator.flow_cache) { + proccess_and_send(aggregator, flow_data.first, flow_data.second, out_template, out_record); + agg::Flow_key_allocator::release_ptr(static_cast(flow_data.first.get_key().first)); + agg::Flow_data_context_allocator::release_ptr(flow_data.second.ctx); + } + dll.clear(); + aggregator.flow_cache.clear(); + trap_send_flush(0); +} + static int do_mainloop(Configuration& config) { @@ -383,8 +398,11 @@ do_mainloop(Configuration& config) time_t time_first; time_t time_last = 0; + time_t last_flush_time = 0; time_t t_passive = config.get_passive_timeout() >> 32; time_t t_active = config.get_active_timeout() >> 32; + const Configuration::Global_flush_configuration& flush_configuration + = config.get_global_flush_configuration(); std::size_t flow_cnt = 0; Dll dll; @@ -432,13 +450,7 @@ do_mainloop(Configuration& config) // clear all memory // flush all flows - for (auto flow_data : agg.flow_cache) { - proccess_and_send(agg, flow_data.first, flow_data.second, out_tmplt, out_rec); - agg::Flow_key_allocator::release_ptr(static_cast(flow_data.first.get_key().first)); - agg::Flow_data_context_allocator::release_ptr(flow_data.second.ctx); - } - - trap_send_flush(0); + flush_all(agg, out_tmplt, out_rec, dll); // Free previous record and temlate ur_free_template(out_tmplt); @@ -490,6 +502,14 @@ do_mainloop(Configuration& config) trap_send_flush(0); timeouted = false; } + + if (unlikely(flush_configuration.is_set() && time_last - last_flush_time >= flush_configuration.interval)) { + last_flush_time = time_last; + if (flush_configuration.type == Configuration::Global_flush_configuration::Type::ABSOLUTE) { + last_flush_time = last_flush_time / flush_configuration.interval * flush_configuration.interval; + } + flush_all(agg, out_tmplt, out_rec, dll); + } bool is_key_reversed = key.generate(in_data, in_tmplt, config.is_biflow_key()); @@ -662,6 +682,9 @@ main(int argc, char **argv) case 's': config.set_flow_cache_size(optarg); break; + case 'g': + config.set_global_flush_configuration(optarg); + break; default: std::cerr << "Invalid argument " << opt << ", skipped..." << std::endl; } From d9fde3d6755f20f69585cb70b59f4c1616322266 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sat, 12 Jul 2025 15:39:28 +0200 Subject: [PATCH 5/8] Biflow aggregator - Add tests --- biflow_aggregator/Makefile.am | 2 + biflow_aggregator/tests/config.xml | 85 +++++++++++++++++++ .../tests/inputs/input1_packet_aggregation | 3 + .../tests/inputs/input2_packet_aggregation | 6 ++ .../inputs/input3_generic_flow_key_min_ports | 6 ++ .../inputs/input4_src_dst_ip_all_aggregations | 3 + .../tests/references/reference1_gt0 | 1 + .../tests/references/reference1_gt5a | 2 + .../tests/references/reference1_gt5r | 1 + .../tests/references/reference2_gt0 | 2 + .../tests/references/reference2_gt5a | 5 ++ .../tests/references/reference2_gt5r | 4 + .../tests/references/reference3_gt0 | 2 + .../tests/references/reference3_gt5a | 5 ++ .../tests/references/reference3_gt5r | 4 + .../tests/references/reference4_gt0 | 1 + .../tests/references/reference4_gt5a | 2 + .../tests/references/reference4_gt5r | 1 + biflow_aggregator/tests/test.sh | 82 ++++++++++++++++++ 19 files changed, 217 insertions(+) create mode 100644 biflow_aggregator/tests/config.xml create mode 100644 biflow_aggregator/tests/inputs/input1_packet_aggregation create mode 100644 biflow_aggregator/tests/inputs/input2_packet_aggregation create mode 100644 biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports create mode 100644 biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations create mode 100644 biflow_aggregator/tests/references/reference1_gt0 create mode 100644 biflow_aggregator/tests/references/reference1_gt5a create mode 100644 biflow_aggregator/tests/references/reference1_gt5r create mode 100644 biflow_aggregator/tests/references/reference2_gt0 create mode 100644 biflow_aggregator/tests/references/reference2_gt5a create mode 100644 biflow_aggregator/tests/references/reference2_gt5r create mode 100644 biflow_aggregator/tests/references/reference3_gt0 create mode 100644 biflow_aggregator/tests/references/reference3_gt5a create mode 100644 biflow_aggregator/tests/references/reference3_gt5r create mode 100644 biflow_aggregator/tests/references/reference4_gt0 create mode 100644 biflow_aggregator/tests/references/reference4_gt5a create mode 100644 biflow_aggregator/tests/references/reference4_gt5r create mode 100755 biflow_aggregator/tests/test.sh diff --git a/biflow_aggregator/Makefile.am b/biflow_aggregator/Makefile.am index 5b5f699c..c68445bb 100644 --- a/biflow_aggregator/Makefile.am +++ b/biflow_aggregator/Makefile.am @@ -5,3 +5,5 @@ biflow_aggregator_SOURCES=main.cpp fields.c fields.h configuration.cpp configura rapidxml.hpp biflow_aggregator_LDADD=-lunirec -ltrap include ../aminclude.am + +TESTS = tests/test.sh diff --git a/biflow_aggregator/tests/config.xml b/biflow_aggregator/tests/config.xml new file mode 100644 index 00000000..da1a851b --- /dev/null +++ b/biflow_aggregator/tests/config.xml @@ -0,0 +1,85 @@ + + + + + SUM + PACKETS + + + + + FLOW_ID + KEY + + + + SRC_PORT + MIN + + + DST_PORT + MIN + + + + + SRC_IP + KEY + + + DST_IP + KEY + + + + SUM + SUM + + + MIN + MIN + + + MAX + MAX + + + FIRST_NON_EMPTY + FIRST_NON_EMPTY + + + LAST_NON_EMPTY + LAST_NON_EMPTY + + + FIRST + FIRST + + + LAST + LAST + + + AVG + AVG + + + BITOR + BITOR + + + STR_APPEND + APPEND + : + 10 + + + SORTED_MERGE_VALUE + SORTED_MERGE + : + SORTED_MERGE_KEY + ASCENDING + 10 + + + diff --git a/biflow_aggregator/tests/inputs/input1_packet_aggregation b/biflow_aggregator/tests/inputs/input1_packet_aggregation new file mode 100644 index 00000000..4ae4e83d --- /dev/null +++ b/biflow_aggregator/tests/inputs/input1_packet_aggregation @@ -0,0 +1,3 @@ +ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST +192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0 +192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0 diff --git a/biflow_aggregator/tests/inputs/input2_packet_aggregation b/biflow_aggregator/tests/inputs/input2_packet_aggregation new file mode 100644 index 00000000..34fd6f24 --- /dev/null +++ b/biflow_aggregator/tests/inputs/input2_packet_aggregation @@ -0,0 +1,6 @@ +ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST +192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0 +192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0 +192.168.1.8,192.168.1.9,10,2016-10-28T17:00:13.0,2016-10-28T17:00:17.0 +192.168.1.8,192.168.1.9,17,2016-10-28T17:00:21.0,2016-10-28T17:00:23.0 +192.168.1.8,192.168.1.9,53,2016-10-28T17:00:26.0,2016-10-28T17:00:41.0 diff --git a/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports b/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports new file mode 100644 index 00000000..8abaa7d5 --- /dev/null +++ b/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports @@ -0,0 +1,6 @@ +ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST,uint32 FLOW_ID,uint16 SRC_PORT,uint16 DST_PORT +192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0,1,0,6666 +192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0,1,6666,0 +192.168.1.8,192.168.1.9,10,2016-10-28T17:00:13.0,2016-10-28T17:00:17.0,2,6666,0 +192.168.1.8,192.168.1.9,17,2016-10-28T17:00:21.0,2016-10-28T17:00:23.0,2,3333,3333 +192.168.1.8,192.168.1.9,53,2016-10-28T17:00:26.0,2016-10-28T17:00:41.0,2,0,6666 diff --git a/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations b/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations new file mode 100644 index 00000000..5798ebe4 --- /dev/null +++ b/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations @@ -0,0 +1,3 @@ +ipaddr DST_IP,ipaddr SRC_IP,time TIME_FIRST,time TIME_LAST,uint32 SUM,uint32 MIN,uint32 MAX,string FIRST_NON_EMPTY,string LAST_NON_EMPTY,uint32 FIRST,uint32 LAST,double AVG,string STR_APPEND,uint32 BITOR,uint32* SORTED_MERGE_KEY,uint32* SORTED_MERGE_VALUE +192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0,333,5,5,,222,16,32,7,test1,1,[3|2|1],[1|2|3] +192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:11.0,333,1,4,test,,555,33,9,test2,3,[8|7|6],[6|5|4] diff --git a/biflow_aggregator/tests/references/reference1_gt0 b/biflow_aggregator/tests/references/reference1_gt0 new file mode 100644 index 00000000..a62b2f5e --- /dev/null +++ b/biflow_aggregator/tests/references/reference1_gt0 @@ -0,0 +1 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667 diff --git a/biflow_aggregator/tests/references/reference1_gt5a b/biflow_aggregator/tests/references/reference1_gt5a new file mode 100644 index 00000000..a4ad9a5e --- /dev/null +++ b/biflow_aggregator/tests/references/reference1_gt5a @@ -0,0 +1,2 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1 +2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,666 diff --git a/biflow_aggregator/tests/references/reference1_gt5r b/biflow_aggregator/tests/references/reference1_gt5r new file mode 100644 index 00000000..a62b2f5e --- /dev/null +++ b/biflow_aggregator/tests/references/reference1_gt5r @@ -0,0 +1 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667 diff --git a/biflow_aggregator/tests/references/reference2_gt0 b/biflow_aggregator/tests/references/reference2_gt0 new file mode 100644 index 00000000..1d94778e --- /dev/null +++ b/biflow_aggregator/tests/references/reference2_gt0 @@ -0,0 +1,2 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:23.000000,4,694 +2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,53 diff --git a/biflow_aggregator/tests/references/reference2_gt5a b/biflow_aggregator/tests/references/reference2_gt5a new file mode 100644 index 00000000..eac7e949 --- /dev/null +++ b/biflow_aggregator/tests/references/reference2_gt5a @@ -0,0 +1,5 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1 +2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,666 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,10 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,17 +2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,53 diff --git a/biflow_aggregator/tests/references/reference2_gt5r b/biflow_aggregator/tests/references/reference2_gt5r new file mode 100644 index 00000000..1be870c0 --- /dev/null +++ b/biflow_aggregator/tests/references/reference2_gt5r @@ -0,0 +1,4 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,10 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,17 +2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,53 diff --git a/biflow_aggregator/tests/references/reference3_gt0 b/biflow_aggregator/tests/references/reference3_gt0 new file mode 100644 index 00000000..24210f65 --- /dev/null +++ b/biflow_aggregator/tests/references/reference3_gt0 @@ -0,0 +1,2 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,1,0,0 +2016-10-28T17:00:13.000000,2016-10-28T17:00:41.000000,3,2,0,0 diff --git a/biflow_aggregator/tests/references/reference3_gt5a b/biflow_aggregator/tests/references/reference3_gt5a new file mode 100644 index 00000000..913131ae --- /dev/null +++ b/biflow_aggregator/tests/references/reference3_gt5a @@ -0,0 +1,5 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1,6666,0 +2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,1,0,6666 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,2,0,6666 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,2,3333,3333 +2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,2,6666,0 diff --git a/biflow_aggregator/tests/references/reference3_gt5r b/biflow_aggregator/tests/references/reference3_gt5r new file mode 100644 index 00000000..4d58ba78 --- /dev/null +++ b/biflow_aggregator/tests/references/reference3_gt5r @@ -0,0 +1,4 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,1,0,0 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,2,0,6666 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,2,3333,3333 +2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,2,6666,0 diff --git a/biflow_aggregator/tests/references/reference4_gt0 b/biflow_aggregator/tests/references/reference4_gt0 new file mode 100644 index 00000000..f79fed90 --- /dev/null +++ b/biflow_aggregator/tests/references/reference4_gt0 @@ -0,0 +1 @@ +192.168.1.1,192.168.1.2,8.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,2,16,33,5,1,666,"test","222","test1:",[3|2|1|4|5|6] diff --git a/biflow_aggregator/tests/references/reference4_gt5a b/biflow_aggregator/tests/references/reference4_gt5a new file mode 100644 index 00000000..8cd58c84 --- /dev/null +++ b/biflow_aggregator/tests/references/reference4_gt5a @@ -0,0 +1,2 @@ +192.168.1.1,192.168.1.2,7.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1,16,32,5,5,333,"","222","test1:",[3|2|1] +192.168.1.1,192.168.1.2,9.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,1,555,33,4,1,333,"test","","test2:",[4|5|6] diff --git a/biflow_aggregator/tests/references/reference4_gt5r b/biflow_aggregator/tests/references/reference4_gt5r new file mode 100644 index 00000000..f79fed90 --- /dev/null +++ b/biflow_aggregator/tests/references/reference4_gt5r @@ -0,0 +1 @@ +192.168.1.1,192.168.1.2,8.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,2,16,33,5,1,666,"test","222","test1:",[3|2|1|4|5|6] diff --git a/biflow_aggregator/tests/test.sh b/biflow_aggregator/tests/test.sh new file mode 100755 index 00000000..06c9e3f3 --- /dev/null +++ b/biflow_aggregator/tests/test.sh @@ -0,0 +1,82 @@ +#!/bin/bash +#set -e + +die_if_not_running() { + local pid=$1 + local error_message=$2 + if not kill -0 "$pid" 2>/dev/null; then + echo $error_message + exit 1 + fi +} + +run_test_with_global_timeout() { + local input=$1 + local output=$2 + local reference=$3 + local config=$4 + local global_timeout=$5 + + (../biflow_aggregator -i "u:lr,u:ba" -e -c config.xml -n $config -g $global_timeout & ) || true + local AGGREGATOR_PID=$! + sleep 0.5 + die_if_not_running $AGGREGATOR_PID "Failed to start biflow aggregator" + + ../../logger/logger -w $output -i "u:ba" & + local LOGGER_PID=$! + sleep 0.5 + die_if_not_running $LOGGER_PID "Failed to start logger" + + ../../logreplay/logreplay -f $input -i "u:lr" & + local LOGREPLAY_PID=$! + sleep 0.5 + die_if_not_running $LOGREPLAY_PID "Failed to start logreplay" + + sleep 2 + wait $LOGREPLAY_PID + kill $AGGREGATOR_PID 2>/dev/null + sleep 0.2 + kill $LOGGER_PID 2>/dev/null + wait $LOGGER_PID + + if ! colordiff $output $reference; then + echo $output doesnt match $reference + success="false" + fi +} + +script_path="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +cd $script_path +success="true" +mkdir -p outputs +for input in inputs/input*; do + basename=$(basename "$input") + echo Found $basename + if [[ $basename =~ input([0-9]+)_(.+) ]]; then + index="${BASH_REMATCH[1]}" + config="${BASH_REMATCH[2]}" + output=outputs/output$index + reference=references/reference$index + else + echo $input + echo "Incorrect input name. Must be input_" + exit 1 + fi + + echo "Running test without global timeout..." + run_test_with_global_timeout $input ${output}_gt0 ${reference}_gt0 $config "0" + + echo "Running test with relative global timeout..." + run_test_with_global_timeout $input ${output}_gt5r ${reference}_gt5r $config "5r" + + echo "Running test with absolute global timeout..." + run_test_with_global_timeout $input ${output}_gt5a ${reference}_gt5a $config "5a" + +done + +if [ "$success" = "true" ]; then + echo "All tests passed successfully." +else + echo "Some tests failed." + exit 1 +fi \ No newline at end of file From 5ef7aa49239c6530cfddd8097bf755853241cd19 Mon Sep 17 00:00:00 2001 From: Jaroslav Pesek Date: Tue, 15 Jul 2025 17:43:23 +0200 Subject: [PATCH 6/8] geoip - fixed building procedure --- Makefile.am | 1 + configure.ac | 1 + geolite/Makefile.am | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 2b95a9fd..d51f4b92 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,6 +11,7 @@ email_reporter \ flow_age_stats \ flowcounter \ flow_meter \ +geolite \ ipv6stats \ json_dump \ json_replay \ diff --git a/configure.ac b/configure.ac index 1935d3ef..bf61aaa4 100644 --- a/configure.ac +++ b/configure.ac @@ -241,6 +241,7 @@ AC_CONFIG_FILES([Makefile email_reporter/Makefile flowcounter/Makefile flow_meter/Makefile + geolite/Makefile googletest_example/Makefile ipv6stats/Makefile json_dump/Makefile diff --git a/geolite/Makefile.am b/geolite/Makefile.am index b2b1ea71..f9c25416 100644 --- a/geolite/Makefile.am +++ b/geolite/Makefile.am @@ -1,8 +1,8 @@ -EXTRA_DIST=geolite.py readme.md +EXTRA_DIST=geolite.py README.md bin_SCRIPTS=geolite.py pkgdocdir=${docdir}/geolite -pkgdoc_DATA=readme.md +pkgdoc_DATA=README.md pylint: pylint-3 geolite.py From 33b7706d543e2baf89fc8a63fe11f779173974d5 Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Thu, 10 Jul 2025 08:17:58 +0200 Subject: [PATCH 7/8] Biflow aggregator - Update main.cpp --- biflow_aggregator/main.cpp | 43 +++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/biflow_aggregator/main.cpp b/biflow_aggregator/main.cpp index 146d9109..6b252eba 100644 --- a/biflow_aggregator/main.cpp +++ b/biflow_aggregator/main.cpp @@ -79,7 +79,8 @@ UR_FIELDS ( PARAM('e', "eof", "End when receive EOF.", no_argument, "flag") \ PARAM('s', "size", "Max number of elements in flow cache.", required_argument, "number") \ PARAM('a', "active-timeout", "Active timeout.", required_argument, "number") \ - PARAM('p', "passive-timeout", "Passive timeout.", required_argument, "number") + PARAM('p', "passive-timeout", "Passive timeout.", required_argument, "number") \ + PARAM('g', "global-timeout", "Global timeout.", required_argument, "number") trap_module_info_t *module_info = NULL; static volatile int stop = 0; @@ -364,6 +365,20 @@ void update_flow( if (pt != t_data->value.passive_timeout) dll.swap(t_data); } + +static void flush_all(agg::Aggregator& aggregator, + ur_template_t* out_template, void* out_record, Dll& dll) +{ + for (auto flow_data : aggregator.flow_cache) { + proccess_and_send(aggregator, flow_data.first, flow_data.second, out_template, out_record); + agg::Flow_key_allocator::release_ptr(static_cast(flow_data.first.get_key().first)); + agg::Flow_data_context_allocator::release_ptr(flow_data.second.ctx); + } + dll.clear(); + aggregator.flow_cache.clear(); + trap_send_flush(0); +} + static int do_mainloop(Configuration& config) { @@ -383,8 +398,11 @@ do_mainloop(Configuration& config) time_t time_first; time_t time_last = 0; + time_t last_flush_time = 0; time_t t_passive = config.get_passive_timeout() >> 32; time_t t_active = config.get_active_timeout() >> 32; + const Configuration::Global_flush_configuration& flush_configuration + = config.get_global_flush_configuration(); std::size_t flow_cnt = 0; Dll dll; @@ -401,7 +419,7 @@ do_mainloop(Configuration& config) while (unlikely(stop == false)) { // Check timeouted flows - for (node *t_data = dll.begin(); t_data != NULL; t_data = t_data->next) { + for (node *t_data = dll.begin(); !flush_configuration.is_set() && t_data != NULL; t_data = t_data->next) { if (time_last >= t_data->value.passive_timeout) { // timeouted auto data = agg.flow_cache.find(t_data->value.key); proccess_and_send(agg, data->first, data->second, out_tmplt, out_rec); @@ -432,13 +450,7 @@ do_mainloop(Configuration& config) // clear all memory // flush all flows - for (auto flow_data : agg.flow_cache) { - proccess_and_send(agg, flow_data.first, flow_data.second, out_tmplt, out_rec); - agg::Flow_key_allocator::release_ptr(static_cast(flow_data.first.get_key().first)); - agg::Flow_data_context_allocator::release_ptr(flow_data.second.ctx); - } - - trap_send_flush(0); + flush_all(agg, out_tmplt, out_rec, dll); // Free previous record and temlate ur_free_template(out_tmplt); @@ -474,7 +486,7 @@ do_mainloop(Configuration& config) time_last = ur_time_get_sec(ur_get(in_tmplt, in_data, F_TIME_LAST)); // Check timeouted flows - for (node *t_data = dll.begin(); t_data != NULL; t_data = t_data->next) { + for (node *t_data = dll.begin(); !flush_configuration.is_set() && t_data != NULL; t_data = t_data->next) { if (time_first >= t_data->value.passive_timeout || time_last >= t_data->value.active_timeout) { // timeouted auto data = agg.flow_cache.find(t_data->value.key); proccess_and_send(agg, data->first, data->second, out_tmplt, out_rec); @@ -490,6 +502,14 @@ do_mainloop(Configuration& config) trap_send_flush(0); timeouted = false; } + + if (unlikely(flush_configuration.is_set() && time_last - last_flush_time >= flush_configuration.interval)) { + last_flush_time = time_last; + if (flush_configuration.type == Configuration::Global_flush_configuration::Type::ABSOLUTE) { + last_flush_time = last_flush_time / flush_configuration.interval * flush_configuration.interval; + } + flush_all(agg, out_tmplt, out_rec, dll); + } bool is_key_reversed = key.generate(in_data, in_tmplt, config.is_biflow_key()); @@ -662,6 +682,9 @@ main(int argc, char **argv) case 's': config.set_flow_cache_size(optarg); break; + case 'g': + config.set_global_flush_configuration(optarg); + break; default: std::cerr << "Invalid argument " << opt << ", skipped..." << std::endl; } From 6b89b96fb514485cff90f2a0e0b5138ddf9a8add Mon Sep 17 00:00:00 2001 From: Damir Zainullin Date: Sat, 12 Jul 2025 15:39:28 +0200 Subject: [PATCH 8/8] Biflow aggregator - Add tests --- biflow_aggregator/Makefile.am | 2 + biflow_aggregator/tests/config.xml | 85 +++++++++++++++++++ .../tests/inputs/input1_packet_aggregation | 3 + .../tests/inputs/input2_packet_aggregation | 6 ++ .../inputs/input3_generic_flow_key_min_ports | 6 ++ .../inputs/input4_src_dst_ip_all_aggregations | 3 + .../tests/references/reference1_gt0 | 1 + .../tests/references/reference1_gt5a | 2 + .../tests/references/reference1_gt5r | 1 + .../tests/references/reference2_gt0 | 2 + .../tests/references/reference2_gt5a | 5 ++ .../tests/references/reference2_gt5r | 4 + .../tests/references/reference3_gt0 | 2 + .../tests/references/reference3_gt5a | 5 ++ .../tests/references/reference3_gt5r | 4 + .../tests/references/reference4_gt0 | 1 + .../tests/references/reference4_gt5a | 2 + .../tests/references/reference4_gt5r | 1 + biflow_aggregator/tests/test.sh | 82 ++++++++++++++++++ 19 files changed, 217 insertions(+) create mode 100644 biflow_aggregator/tests/config.xml create mode 100644 biflow_aggregator/tests/inputs/input1_packet_aggregation create mode 100644 biflow_aggregator/tests/inputs/input2_packet_aggregation create mode 100644 biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports create mode 100644 biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations create mode 100644 biflow_aggregator/tests/references/reference1_gt0 create mode 100644 biflow_aggregator/tests/references/reference1_gt5a create mode 100644 biflow_aggregator/tests/references/reference1_gt5r create mode 100644 biflow_aggregator/tests/references/reference2_gt0 create mode 100644 biflow_aggregator/tests/references/reference2_gt5a create mode 100644 biflow_aggregator/tests/references/reference2_gt5r create mode 100644 biflow_aggregator/tests/references/reference3_gt0 create mode 100644 biflow_aggregator/tests/references/reference3_gt5a create mode 100644 biflow_aggregator/tests/references/reference3_gt5r create mode 100644 biflow_aggregator/tests/references/reference4_gt0 create mode 100644 biflow_aggregator/tests/references/reference4_gt5a create mode 100644 biflow_aggregator/tests/references/reference4_gt5r create mode 100755 biflow_aggregator/tests/test.sh diff --git a/biflow_aggregator/Makefile.am b/biflow_aggregator/Makefile.am index 5b5f699c..c68445bb 100644 --- a/biflow_aggregator/Makefile.am +++ b/biflow_aggregator/Makefile.am @@ -5,3 +5,5 @@ biflow_aggregator_SOURCES=main.cpp fields.c fields.h configuration.cpp configura rapidxml.hpp biflow_aggregator_LDADD=-lunirec -ltrap include ../aminclude.am + +TESTS = tests/test.sh diff --git a/biflow_aggregator/tests/config.xml b/biflow_aggregator/tests/config.xml new file mode 100644 index 00000000..da1a851b --- /dev/null +++ b/biflow_aggregator/tests/config.xml @@ -0,0 +1,85 @@ + + + + + SUM + PACKETS + + + + + FLOW_ID + KEY + + + + SRC_PORT + MIN + + + DST_PORT + MIN + + + + + SRC_IP + KEY + + + DST_IP + KEY + + + + SUM + SUM + + + MIN + MIN + + + MAX + MAX + + + FIRST_NON_EMPTY + FIRST_NON_EMPTY + + + LAST_NON_EMPTY + LAST_NON_EMPTY + + + FIRST + FIRST + + + LAST + LAST + + + AVG + AVG + + + BITOR + BITOR + + + STR_APPEND + APPEND + : + 10 + + + SORTED_MERGE_VALUE + SORTED_MERGE + : + SORTED_MERGE_KEY + ASCENDING + 10 + + + diff --git a/biflow_aggregator/tests/inputs/input1_packet_aggregation b/biflow_aggregator/tests/inputs/input1_packet_aggregation new file mode 100644 index 00000000..4ae4e83d --- /dev/null +++ b/biflow_aggregator/tests/inputs/input1_packet_aggregation @@ -0,0 +1,3 @@ +ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST +192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0 +192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0 diff --git a/biflow_aggregator/tests/inputs/input2_packet_aggregation b/biflow_aggregator/tests/inputs/input2_packet_aggregation new file mode 100644 index 00000000..453ba648 --- /dev/null +++ b/biflow_aggregator/tests/inputs/input2_packet_aggregation @@ -0,0 +1,6 @@ +ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST +192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0 +192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0 +192.168.1.8,192.168.1.9,10,2016-10-28T17:00:13.0,2016-10-28T17:00:17.0 +192.168.1.8,192.168.1.9,17,2016-10-28T17:00:21.0,2016-10-28T17:00:23.0 +192.168.1.8,192.168.1.9,53,2016-10-28T17:00:26.0,2016-10-28T17:00:51.0 diff --git a/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports b/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports new file mode 100644 index 00000000..8abaa7d5 --- /dev/null +++ b/biflow_aggregator/tests/inputs/input3_generic_flow_key_min_ports @@ -0,0 +1,6 @@ +ipaddr DST_IP,ipaddr SRC_IP,uint32 PACKETS,time TIME_FIRST,time TIME_LAST,uint32 FLOW_ID,uint16 SRC_PORT,uint16 DST_PORT +192.168.1.1,192.168.1.2,1,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0,1,0,6666 +192.168.1.5,192.168.1.6,666,2016-10-28T17:00:3.0,2016-10-28T17:00:11.0,1,6666,0 +192.168.1.8,192.168.1.9,10,2016-10-28T17:00:13.0,2016-10-28T17:00:17.0,2,6666,0 +192.168.1.8,192.168.1.9,17,2016-10-28T17:00:21.0,2016-10-28T17:00:23.0,2,3333,3333 +192.168.1.8,192.168.1.9,53,2016-10-28T17:00:26.0,2016-10-28T17:00:41.0,2,0,6666 diff --git a/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations b/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations new file mode 100644 index 00000000..5798ebe4 --- /dev/null +++ b/biflow_aggregator/tests/inputs/input4_src_dst_ip_all_aggregations @@ -0,0 +1,3 @@ +ipaddr DST_IP,ipaddr SRC_IP,time TIME_FIRST,time TIME_LAST,uint32 SUM,uint32 MIN,uint32 MAX,string FIRST_NON_EMPTY,string LAST_NON_EMPTY,uint32 FIRST,uint32 LAST,double AVG,string STR_APPEND,uint32 BITOR,uint32* SORTED_MERGE_KEY,uint32* SORTED_MERGE_VALUE +192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:7.0,333,5,5,,222,16,32,7,test1,1,[3|2|1],[1|2|3] +192.168.1.1,192.168.1.2,2016-10-28T17:00:1.0,2016-10-28T17:00:11.0,333,1,4,test,,555,33,9,test2,3,[8|7|6],[6|5|4] diff --git a/biflow_aggregator/tests/references/reference1_gt0 b/biflow_aggregator/tests/references/reference1_gt0 new file mode 100644 index 00000000..a62b2f5e --- /dev/null +++ b/biflow_aggregator/tests/references/reference1_gt0 @@ -0,0 +1 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667 diff --git a/biflow_aggregator/tests/references/reference1_gt5a b/biflow_aggregator/tests/references/reference1_gt5a new file mode 100644 index 00000000..a4ad9a5e --- /dev/null +++ b/biflow_aggregator/tests/references/reference1_gt5a @@ -0,0 +1,2 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1 +2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,666 diff --git a/biflow_aggregator/tests/references/reference1_gt5r b/biflow_aggregator/tests/references/reference1_gt5r new file mode 100644 index 00000000..a62b2f5e --- /dev/null +++ b/biflow_aggregator/tests/references/reference1_gt5r @@ -0,0 +1 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667 diff --git a/biflow_aggregator/tests/references/reference2_gt0 b/biflow_aggregator/tests/references/reference2_gt0 new file mode 100644 index 00000000..1c168c6b --- /dev/null +++ b/biflow_aggregator/tests/references/reference2_gt0 @@ -0,0 +1,2 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:23.000000,4,694 +2016-10-28T17:00:26.000000,2016-10-28T17:00:51.000000,1,53 diff --git a/biflow_aggregator/tests/references/reference2_gt5a b/biflow_aggregator/tests/references/reference2_gt5a new file mode 100644 index 00000000..bfcda95f --- /dev/null +++ b/biflow_aggregator/tests/references/reference2_gt5a @@ -0,0 +1,5 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1 +2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,666 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,10 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,17 +2016-10-28T17:00:26.000000,2016-10-28T17:00:51.000000,1,53 diff --git a/biflow_aggregator/tests/references/reference2_gt5r b/biflow_aggregator/tests/references/reference2_gt5r new file mode 100644 index 00000000..74d4a06d --- /dev/null +++ b/biflow_aggregator/tests/references/reference2_gt5r @@ -0,0 +1,4 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,667 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,10 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,17 +2016-10-28T17:00:26.000000,2016-10-28T17:00:51.000000,1,53 diff --git a/biflow_aggregator/tests/references/reference3_gt0 b/biflow_aggregator/tests/references/reference3_gt0 new file mode 100644 index 00000000..24210f65 --- /dev/null +++ b/biflow_aggregator/tests/references/reference3_gt0 @@ -0,0 +1,2 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,1,0,0 +2016-10-28T17:00:13.000000,2016-10-28T17:00:41.000000,3,2,0,0 diff --git a/biflow_aggregator/tests/references/reference3_gt5a b/biflow_aggregator/tests/references/reference3_gt5a new file mode 100644 index 00000000..913131ae --- /dev/null +++ b/biflow_aggregator/tests/references/reference3_gt5a @@ -0,0 +1,5 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1,6666,0 +2016-10-28T17:00:03.000000,2016-10-28T17:00:11.000000,1,1,0,6666 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,2,0,6666 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,2,3333,3333 +2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,2,6666,0 diff --git a/biflow_aggregator/tests/references/reference3_gt5r b/biflow_aggregator/tests/references/reference3_gt5r new file mode 100644 index 00000000..4d58ba78 --- /dev/null +++ b/biflow_aggregator/tests/references/reference3_gt5r @@ -0,0 +1,4 @@ +2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,2,1,0,0 +2016-10-28T17:00:13.000000,2016-10-28T17:00:17.000000,1,2,0,6666 +2016-10-28T17:00:21.000000,2016-10-28T17:00:23.000000,1,2,3333,3333 +2016-10-28T17:00:26.000000,2016-10-28T17:00:41.000000,1,2,6666,0 diff --git a/biflow_aggregator/tests/references/reference4_gt0 b/biflow_aggregator/tests/references/reference4_gt0 new file mode 100644 index 00000000..f79fed90 --- /dev/null +++ b/biflow_aggregator/tests/references/reference4_gt0 @@ -0,0 +1 @@ +192.168.1.1,192.168.1.2,8.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,2,16,33,5,1,666,"test","222","test1:",[3|2|1|4|5|6] diff --git a/biflow_aggregator/tests/references/reference4_gt5a b/biflow_aggregator/tests/references/reference4_gt5a new file mode 100644 index 00000000..8cd58c84 --- /dev/null +++ b/biflow_aggregator/tests/references/reference4_gt5a @@ -0,0 +1,2 @@ +192.168.1.1,192.168.1.2,7.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:07.000000,1,1,16,32,5,5,333,"","222","test1:",[3|2|1] +192.168.1.1,192.168.1.2,9.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,1,555,33,4,1,333,"test","","test2:",[4|5|6] diff --git a/biflow_aggregator/tests/references/reference4_gt5r b/biflow_aggregator/tests/references/reference4_gt5r new file mode 100644 index 00000000..f79fed90 --- /dev/null +++ b/biflow_aggregator/tests/references/reference4_gt5r @@ -0,0 +1 @@ +192.168.1.1,192.168.1.2,8.000000,2016-10-28T17:00:01.000000,2016-10-28T17:00:11.000000,3,2,16,33,5,1,666,"test","222","test1:",[3|2|1|4|5|6] diff --git a/biflow_aggregator/tests/test.sh b/biflow_aggregator/tests/test.sh new file mode 100755 index 00000000..06c9e3f3 --- /dev/null +++ b/biflow_aggregator/tests/test.sh @@ -0,0 +1,82 @@ +#!/bin/bash +#set -e + +die_if_not_running() { + local pid=$1 + local error_message=$2 + if not kill -0 "$pid" 2>/dev/null; then + echo $error_message + exit 1 + fi +} + +run_test_with_global_timeout() { + local input=$1 + local output=$2 + local reference=$3 + local config=$4 + local global_timeout=$5 + + (../biflow_aggregator -i "u:lr,u:ba" -e -c config.xml -n $config -g $global_timeout & ) || true + local AGGREGATOR_PID=$! + sleep 0.5 + die_if_not_running $AGGREGATOR_PID "Failed to start biflow aggregator" + + ../../logger/logger -w $output -i "u:ba" & + local LOGGER_PID=$! + sleep 0.5 + die_if_not_running $LOGGER_PID "Failed to start logger" + + ../../logreplay/logreplay -f $input -i "u:lr" & + local LOGREPLAY_PID=$! + sleep 0.5 + die_if_not_running $LOGREPLAY_PID "Failed to start logreplay" + + sleep 2 + wait $LOGREPLAY_PID + kill $AGGREGATOR_PID 2>/dev/null + sleep 0.2 + kill $LOGGER_PID 2>/dev/null + wait $LOGGER_PID + + if ! colordiff $output $reference; then + echo $output doesnt match $reference + success="false" + fi +} + +script_path="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +cd $script_path +success="true" +mkdir -p outputs +for input in inputs/input*; do + basename=$(basename "$input") + echo Found $basename + if [[ $basename =~ input([0-9]+)_(.+) ]]; then + index="${BASH_REMATCH[1]}" + config="${BASH_REMATCH[2]}" + output=outputs/output$index + reference=references/reference$index + else + echo $input + echo "Incorrect input name. Must be input_" + exit 1 + fi + + echo "Running test without global timeout..." + run_test_with_global_timeout $input ${output}_gt0 ${reference}_gt0 $config "0" + + echo "Running test with relative global timeout..." + run_test_with_global_timeout $input ${output}_gt5r ${reference}_gt5r $config "5r" + + echo "Running test with absolute global timeout..." + run_test_with_global_timeout $input ${output}_gt5a ${reference}_gt5a $config "5a" + +done + +if [ "$success" = "true" ]; then + echo "All tests passed successfully." +else + echo "Some tests failed." + exit 1 +fi \ No newline at end of file