diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml new file mode 100644 index 00000000..57c1021d --- /dev/null +++ b/.github/workflows/c-cpp.yml @@ -0,0 +1,33 @@ +name: C/C++ CI + +on: + push: + branches: '*' + pull_request: + branches: '*' + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get -y install git wget build-essential autoconf libtool libpcap-dev pkg-config libxml2-dev gawk libcurl4-openssl-dev libxml2-dev libyaml-dev make python3 python3-dev python3-pip python3-setuptools libxml2-dev libxml2 libpcap0.8-dev libpcap0.8 libidn2-0-dev + ( cd /tmp; wget -qO - http://libnf.net/packages/libnf-1.25.tar.gz | tar -xzf - && cd libnf-1.25 && ./configure -q && sudo make -j3 install; ) + - name: init repository + run: ( git submodule init; git submodule update --recursive; cd modules; git submodule init; git submodule update --recursive; ) + - name: autoreconf + run: autoreconf -i + - name: configure + run: ./configure --enable-repobuild --prefix=/usr --bindir=/usr/bin/nemea/ + - name: make + run: make -j10; sudo make install; sudo ldconfig + - name: Install python components + run: ( cd nemea-framework/pytrap; sudo python3 setup.py install; cd ../pycommon; sudo python3 setup.py install; ) + - name: make check + run: make check || find -name '*.sh.log' -exec cat {} + + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bdf802a8..00000000 --- a/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: c -compiler: - - gcc -os: - - linux - -branches: - only: - - master - -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-c++ - - autoconf - - pkg-config - - make - - automake - - libxml2-devel - - libxml2 - - libpcap-devel - - libpcap - -script: - - git submodule init; git submodule update - - ./bootstrap.sh && ./configure && make clean && make -j5 - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..f7aae787 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +## How to contribute + +#### **Did you find a bug?** + +* **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/CESNET/NEMEA/issues). + +* If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/CESNET/NEMEA/issues/new). Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring. + +#### **Did you write a patch that fixes a bug?** + +* For patches in the C language, check if the patch meets the [coding style](https://github.com/CESNET/Nemea-Framework/blob/master/libtrap/CodingStyle.md). + +* Open a new GitHub pull request with the patch. + +* Ensure the PR description clearly describes the problem and solution. Include the relevant issue number if applicable. + +#### **Do you intend to add a new feature or change an existing one?** + +* Suggest your change in the [NEMEA mailing list](https://random.cesnet.cz/mailman/listinfo/nemea) and start writing code. + +* Do not open an issue on GitHub until you have collected positive feedback about the change. + +#### **Do you have questions about the source code?** + +* Ask any question about how to use NEMEA or its parts in the [NEMEA mailing list](https://random.cesnet.cz/mailman/listinfo/nemea). + +Thanks for any patch or feedback! + +NEMEA Team diff --git a/COPYING b/COPYING new file mode 100644 index 00000000..36bbc711 --- /dev/null +++ b/COPYING @@ -0,0 +1,31 @@ +COPYRIGHT AND PERMISSION NOTICE + +Copyright (C) 2012-2016 CESNET, z.s.p.o. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + 3. Neither the name of the Company nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +ALTERNATIVELY, provided that this notice is retained in full, this product +may be distributed under the terms of the GNU General Public License (GPL) +version 2 or later, in which case the provisions of the GPL apply INSTEAD OF +those given above. + +This software is provided "as is", and any express or implied warranties, +including, but not limited to, the implied warranties of merchantability +and fitness for a particular purpose are disclaimed. In no event shall the +company or contributors be liable for any direct, indirect, incidental, +special, exemplary, or consequential damages (including, but not limited to, +procurement of substitute goods or services; loss of use, data, or profits; +or business interruption) however caused and on any theory of liability, +whether in contract, strict liability, or tort (including negligence or +otherwise) arising in any way out of the use of this software, even if +advised of the possibility of such damage. diff --git a/Makefile.am b/Makefile.am index 4181bc80..ad4c3c9e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1 +1,18 @@ -SUBDIRS=nemea-framework nemea-supervisor modules detectors +SUBDIRS=nemea-framework + +if ENABLE_SUPERVISOR +SUBDIRS+=nemea-supervisor +endif + +if ENABLE_MODULES +SUBDIRS+=modules +endif + +if ENABLE_DETECTORS +SUBDIRS+=detectors +endif + +# because of supervisor, disable systemd checking and installation during +# distcheck +DISTCHECK_CONFIGURE_FLAGS="--without-systemd" + diff --git a/README-on_biflow.md b/README-on_biflow.md new file mode 100644 index 00000000..5c766c5c --- /dev/null +++ b/README-on_biflow.md @@ -0,0 +1,32 @@ +This documents briefly explains "biflow" representation and how it is used in +[ipfixprobe](https://github.com/CESNET/ipfixprobe) and how it should be used in +NEMEA. Biflow is an abbreviation of bidirectional flow (record) which +represents both directions of one connection between two devices in form of one +"message"/flow record. + +Interpretation of UniRec record (how to work with UniRec template and UniRec +message): + +1. If the UniRec template contains both fields: `BYTES_REV` and `PACKETS_REV`, + it is considered as biflow template. Otherwise, it is not (i.e., it is + probably unidirectional flow record). Note: ipfixprobe ALWAYS export both + fields for biflow. +2. If the UniRec template is not biflow (due to missing fields in 1.), the + UniRec messages should be paired/aggregated "manually" to get biflow. +3. If a biflow message contains `PACKETS_REV == 0`, it is still a biflow + record, however, it contains only one direction of the communication. This + means one of the cases: + 1. the communication was really unidirectional (e.g., broadcast messages, UDP streams), + 2. the communication was bidirectional, but the flow exporter observed only + one direction of it, + 3. the communication was bidirectional, but the flow exporter was not able + to pair/aggregated both directions. + +3.2 can happen in case of assymetric routing, where packets of different +directions might be routed via different links. +3.3 can happen in case of splitting flow records due to timeouts or limited +flow cache. + +In any case, it is highly recommended to do pairing/aggregation of UniRec +message before processing. + diff --git a/README.md b/README.md index 0e7557bc..ff7293c7 100644 --- a/README.md +++ b/README.md @@ -1,203 +1,626 @@ -NEMEA System -============ - +## README outline +- [Project status](#project-status) +- [NEMEA System](#nemea-system) + - [Parts of the system](#parts-of-the-system) + - [Repositories](#repositories) +- [Dependencies](#dependencies) +- [Installation](#installation) + - [Vagrant](#vagrant) + - [Binary packages](#binary-packages) + - [Source codes](#source-codes) + - Packer +- [Quick start and how to](#quick-start-and-how-to) + - [Try out NEMEA modules](#try-out-nemea-modules) + - [Deploy NEMEA](#deploy-nemea) + - [Create your own module in C](#create-your-own-module-in-c) + - [Add new module to running configuration](#add-new-module-to-running-configuration) +- [Further Information](#further-information) +- [NEMEA Related publications](#nemea-related-publications) + + +## Project status Travis CI build: [![Build Status](https://travis-ci.org/CESNET/Nemea.svg?branch=master)](https://travis-ci.org/CESNET/Nemea) -This file describes the Nemea system in detail. To see more general information, -please have a look at https://www.liberouter.org/nemea. -Installation -============ +# NEMEA System + +NEMEA (Network Measurements Analysis) system is a **stream-wise**, **flow-based** and **modular** detection system for network traffic analysis. It consists of many independent modules which are interconnected via communication interfaces and each of the modules has its own task. Communication between modules is done by message passing where the messages contain flow records, alerts, some statistics or preprocessed data. + +## Parts of the system + +The following picture shows all important parts of the system. + +![NEMEA parts](doc/NEMEA-parts.png) + +1. Modules - basic building blocks; separate system processes; receive stream of data on their input interfaces, process it and send another stream of data to their output interfaces; all modules are simply divided into two groups according to their task: + * **Detectors** (*red*) - detect some malicious traffic, e.g. *DNS tunnel*, *DoS*, *scanning* + * **Modules** (*yellow*) - export&storage of flow data, preprocess or postprocess the data (filter, aggregate, merge etc.) +2. NEMEA Framework - set of libraries implementing features common for all modules + * **TRAP** (Traffic Analysis Platform) (*blue*) - implements communication interfaces and functions for sending/receiving the messages between interfaces + * **UniRec** (Unified Record) (*orange*) - implements efficient data format of the sent/received messages + * **Common** library (*purple*) - implements common algorithms and data structures used in modules +3. **Supervisor** (*green*) - central management and monitoring tool of the NEMEA system. It takes care of running modules according to a specified configuration. + +## Repositories + +The project is divided into four repositories added as submodules: + +* [NEMEA framework](https://github.com/CESNET/Nemea-Framework) +* [NEMEA modules](https://github.com/CESNET/Nemea-Modules) +* [NEMEA detectors](https://github.com/CESNET/Nemea-Detectors) +* [NEMEA Supervisor](https://github.com/CESNET/Nemea-Supervisor) + + +## Dependencies + +### Building environment + +* autoconf +* automake +* gcc +* gcc-c++ +* libtool +* libxml2-devel +* libxml2-utils (contains xmllint on Debian) +* make +* pkg-config + +### Optional dependencies of modules and detectors + +* rpm-build (build of RPM packages) +* libpcap ([flow_meter](https://github.com/CESNET/Nemea-Modules/tree/master/flow_meter)) +* [libnf](https://github.com/VUTBR/libnf) ([nfreader](https://github.com/CESNET/Nemea-Modules/tree/master/nfreader)) +* libidn ([blacklistfilter](https://github.com/CESNET/Nemea-Detectors/tree/master/blacklistfilter)) +* bison and flex ([unirecfilter](https://github.com/CESNET/Nemea-Modules/tree/master/unirecfilter)) + +### How to install dependencies: + +Debian/Ubuntu: +``` +apt-get install -y gawk bc autoconf automake gcc g++ libtool libxml2-dev make pkg-config libpcap-dev libidn11-dev bison flex +``` + +RHEL/CentOS/Fedora: +``` +yum install -y bc autoconf automake gcc gcc-c++ libtool libxml2-devel make pkg-config libpcap-devel libidn-devel bison flex +``` + +Note: Latest systems (e.g. Fedora) use `dnf` instead of `yum`. + +# Installation -Vagrant -------- +There are three different ways of installation of the NEMEA system covered +in this document: **vagrant**, **binary packages** and **source codes**. + + +## Vagrant To try the system "out-of-box", you can use [Vagrant](https://www.vagrantup.com/). For more information see [./vagrant/](./vagrant/). -Binary packages ---------------- -The Nemea system can be also installed using binary packages. Information will -be supplied soon. +## Binary packages + +Latest RPM packages can be found in COPR: https://copr.fedorainfracloud.org/groups/g/CESNET/coprs/ +The NEMEA repository is at https://copr.fedorainfracloud.org/coprs/g/CESNET/NEMEA/ + +The repository can be added trough `dnf copr enable` or by using the following commands (for CentOS/Fedora): + +``` +wget -O /etc/yum.repos.d/cesnet-nemea.repo https://copr.fedorainfracloud.org/coprs/g/CESNET/NEMEA/repo/epel-7/group_CESNET-NEMEA-epel-7.repo +rpm --import https://copr-be.cloud.fedoraproject.org/results/@CESNET/NEMEA/pubkey.gpg +``` + +After installation of the COPR repository, NEMEA can be installed as any other package (run as root/sudo): + +``` +yum install nemea +``` + +Note: Latest systems (e.g. Fedora) use `dnf` instead of `yum`. -Source Codes installation -------------------------- +For development purposes, there is `nemea-framework-devel` package that installs +all needed development files and docs. To install this package, also add the `NEMEA-testing` repository. -The Nemea system consists of the [Nemea framework](cejkato2/Nemea-Framework), Nemea modules (basic and detection) and Nemea Supervisor. The whole system -is based on GNU/Autotools build system that makes dependency checking and +Currently, we do not have .deb packages (for Debian/Ubuntu/...) but we are working on it. Please follow installation from [source codes](#source-codes) + +## Source codes + +The whole system is based on GNU/Autotools build system that makes dependency checking and building process much more easier. -To clone the read-only repositories, use: +To clone the NEMEA repositories, use: ``` git clone --recursive https://github.com/CESNET/nemea ``` -After successful clone, use: +After successful clone and [dependencies](#dependencies) installation (**!**), use: + ``` ./bootstrap.sh ``` + that will create `configure` scripts and other needed files. The `configure` script supplies various possibilities of -configuration and uses some environmental variables that influence the build +configuration and it uses some environmental variables that influence the build and compilation process. For more information see: + ``` ./configure --help ``` -Build process can be started by: +We recommend to set paths according to the used operating system, e.g.: + +``` +./configure --enable-repobuild --prefix=/usr --bindir=/usr/bin/nemea --sysconfdir=/etc/nemea --libdir=/usr/lib64 +``` + +After finishing `./configure`, build process can be started by: ``` make ``` -The make(1) tool has various parameters, to build the Nemea package faster on +The make(1) tool has various parameters, to build the NEMEA package faster on multicore systems, we recommend to use parameter -j with the number of jobs that should be run in parallel. When the compilation process ends without any error, the package can be installed into paths that were set by `configure`. It is recommended NOT to change -target paths by passing variables to make(1). +target paths by passing variables directly to make(1). The installation can be done by (usually it requires root / sudo): ``` make install ``` -Congratulations, the whole Nemea system should be installed right now... :-) +Congratulations, the whole NEMEA system should be installed right now... :-) + + +# Quick start and how to -Quick Start Guide -================= -TODO: add logreplay and logger, since nfreader requires additional -dependency. +## Try out NEMEA modules + +### Execute a module + +NEMEA modules using have two implicit arguments. `module -h` for help +(optional) and `module -i IFC_SPEC` for communication interface (IFC) +specification. The `-i` parameter is mandatory for all NEMEA modules. + + +**Module help `-h`** + +The example below shows part of help output of [logger](https://github.com/CESNET/Nemea-Modules/tree/master/logger). +It contains module's name, description, number of input and output IFC, modules +parameters and TRAP library parameters (common for all modules). + + +``` +TRAP module, libtrap version: 0.7.6 b258bb4 +=========================================== +Name: Logger +Inputs: variable +Outputs: 0 +Description: + This module logs all incoming UniRec records to standard output or into a specified file. Each record + is written as one line containing values of its fields in human-readable format separated by chosen + delimiters (CSV format). If you use more than one input interface you have to specify output format by + parameter "-o". + +Usage: logger [COMMON]... [OPTIONS]... + +Parameters of module [OPTIONS]: +------------------------------- + -w --write Write output to FILE instead of stdout (rewrite the file). + + -a --append Write output to FILE instead of stdout (append to the end). + + -t --title Write names of fields on the first line. + + -c --cut Quit after N records are received, 0 can be useful in combination + with -t to print UniRec. + +Common TRAP parameters [COMMON]: +-------------------------------- + -h [trap,1] If no argument, print this message. If "trap" or 1 is given, print + TRAP help. + + -i IFC_SPEC Specification of interface types and their parameters, see "-h trap" + (mandatory parameter). + + -v Be verbose. + +Environment variables that affects output: +------------------------------------------ + LIBTRAP_OUTPUT_FORMAT If set to "json", information about module is printed in JSON format. + + PAGER Show the help output in the set PAGER. +``` + + +**Interface specifier `-i`** + +The `-i` parameter with the interface specifier *IFC_SPEC* (`module -i IFC_SPEC`) +specifies modules interfaces - their types and parameters. The interface +specifier has the following format: + +`,,...,` + +where `` looks like + +`:::...:`. + +`` can be one of the following: `t` - TCP socket (for remote +communication), `u` - UNIX socket (for local communication), `b` - blackhole to +drop all messages during sending, `f` - File IFC. + +Interfaces are separated by `,` and their parameters are separated by `:`. +Input IFCs must be specified at first, output IFCs follow. Examples below show + +Example: -This section shows how to manually start Nemea modules. Module description and -specific parameters could be shown by running module with parameter "-h": ``` -nfdump_reader -h +module1 -i t:address:port1,t:port2 ``` -There are two possible data inputs for Nemea modules: - a) from nfdump file (static data) - b) from [IPFIXcol](https://github.com/CESNET/ipfixcol/) with UniRec plugin - (real-time data). +*module1* uses TCP interfaces (for machine to machine communication). Let's +assume it has one input and one output interface (number of input and output +IFCs is given by programmer of the module). Therefore, input IFC will connect +to `address:port1` and output IFC will listen on `port2`. + +TCP type of IFC expects mandatory parameter `port` and optionally, additional +parameter `address` (as it is used in example). Default value of `address` is +`localhost`. + + +Example: -a) Data from nfdump files could be read and sent to Nemea by "nfdump_reader" -module. Following command will send records from file "nfcapd.201406001" to TCP -interface on port 9988 in UniRec format: ``` -nfdump_reader -i t:9988 /data/nfcapd.201406001 +module2 -i u:sock1,u:sock2 ``` -Another module then could capture and process this data, e.g. DNS amplification -detector: +UNIX type of IFC expects unique identifier of the socket. For compatibility +with TCP IFC, `address` can be specified but **it has no effect!** + + +**Important findings:** + +* TCP interface for machine to machine communication, UNIX-SOCKET for communication on the same machine +* input interface behaves as a client, output interfaces behaves as a server + +Detailed information and another examples of *IFC_SPEC* can be found in [libtrap/README.ifcspec.md](https://github.com/CESNET/Nemea-Framework/blob/master/libtrap/README.ifcspec.md). + + +### Interconnect two modules + +Let´s try to interconnect +[logreplay](https://github.com/CESNET/Nemea-Modules/tree/master/logreplay) and +[logger](https://github.com/CESNET/Nemea-Modules/tree/master/logger) modules to +see them communicate. +Logreplay module has one output IFC. It reads CSV file created by logger +module and sends it in UniRec format. Logger has one input interface and it +logs all incoming UniRec records to standard output or into specified file in +CSV format. These two modules can be interconnected using one input IFC and +one output IFC. + +[use-cases/logger-repeater.sh](https://github.com/CESNET/Nemea/blob/master/use-cases/logger-repeater.sh) +can be used for the demonstration. With no parameter, it prints help with +description. With *generate* parameter, the script creates a CSV file with header and 3 +flow records (see +[use-cases/logger-repeater.sh#L53](https://github.com/CESNET/Nemea/blob/master/use-cases/logger-repeater.sh#L53)). +Thereafter it executes logreplay and logger modules + +``` +logreplay -i "u:my_socket"` -f CSV_file +``` +and +``` +logger -i "u:my_socket" -t +``` + +Logreplay has one UNIX output IFC listening on *my_socket* and logger has one UNIX input IFC which connects to *my_socket*. + +To see the effect, go to `use-cases/` and execute the script: ``` -dns_amplification -i t:localhost:9988,u:DNS_amp +./logger-repeater.sh generate ``` - -After executing both commands, DNS amplification detection on data from given -file will be done. Possible attacks will be reported on Unix socket "DNS_amp". -b) Real-time data from IPFIXcol could be provided to Nemea also. First run -IPFIXcol, e.g.: +It should print exactly the same output as generated CSV tmp input (header and +3 records). In +[use-cases](https://github.com/CESNET/Nemea/tree/master/use-cases) there are +more examples with basic modules. +`logreplay` is one of possible ways of getting data into the NEMEA system. + +Other data sources are discussed later in [Get flows to your system](https://github.com/CESNET/Nemea#4-get-flows-to-your-system). + + +## Deploy NEMEA + +This section shows how to deploy NEMEA in four steps. + +It only covers the basics needed to run the system in its default +configuration. Keep in mind that NEMEA was designed as a very flexible +framework, so every user can (and should) adjust the configuration of NEMEA +modules to their own purposes. + + +### 1. Installation + +First of all, the whole system (NEMEA Framework, Modules, Detectors and +Supervisor) has to be installed. Follow [installation instructions](#installation) to +install the system from RPM or from source codes. + + +### 2. Prepare configurations + +To avoid manual control of the system, there is [NEMEA Supervisor](https://github.com/CESNET/Nemea-Supervisor). +It is a central management and monitoring tool of the system and it takes care of running +modules **according to a specified XML configuration**. + +We need to prepare XML configuration file for Supervisor. Fortunately, almost +everything is already done. + +After installation (from RPM or from source codes with recommended `configure` +parameters), there are 2 important paths with configurations: +* `/ush/share/nemea-supervisor/` - contains default prepared XML configuraions of all NEMEA modules (like [nemea-supervisor/configs/](https://github.com/CESNET/Nemea-Supervisor/tree/master/configs)) +* `/etc/nemea/` - contains XML configuration file for Supervisor and directories with used modules configurations (they are empty after installation) + +Note: these two paths depend on *datarootdir* and *sysconfdir* parameters of the `configure` script during the installation. + +The only thing we have to do is this (probably with sudo / root): ``` -ipfixcol -d -c /data/configs/startup.xml +cp -r /usr/share/nemea-supervisor/*/ /etc/nemea ``` -File `startup.xml` should contains configuration for IPFIXcol plugin, among -others there will be settings for output UniRec format and output interface. -Example of such configuration file for UniRec plugin is at the end of this -section. Command for running Nemea module with input from IPFIXcol (e.g. on TCP -port 9966) remains same as with input from `nfdump_reader`: +After this command, supervisor will use default configurations of the modules. +It is shown in [nemea-supervisor/configs/supervisor_config_template.xml.in#L8](https://github.com/CESNET/Nemea-Supervisor/blob/master/configs/supervisor_config_template.xml.in#L8) +that the paths from `sysconfdir` (`/etc/nemea/` in our case) are included in the +configuration file. For detailed information about supervisor configuration +see [README](https://github.com/CESNET/Nemea-Supervisor#configuration) of Supervisor. + + +### 3. Start and control modules + +Once the configurations are prepared, modules can be managed by Supervisor. It can be easily started as a systemd service with + +`service nemea-supervisor start` (recommended, probably with root / sudo) + +or manually + +`/usr/bin/nemea/supervisor --daemon -T /etc/nemea/supervisor_config_template.xml -L /var/log/nemea-supervisor` +Note: manual approach does not change UID that supervisor runs with. +Contrary, using `service`, NEMEA runs as `nemead` UID and `nemead` GID. + +See all service commands in +[README](https://github.com/CESNET/Nemea-Supervisor#program-modes) and all +program parameters with `/usr/bin/nemea/supervisor -h`. You can also check +whether the process is running or not with `ps -ef | grep supervisor`. +If Supervisor has not started successfully, it should print error info directly +to system log (in case of service), which can be browsed with `journalctl -xe`, +or to stdout (in case of manual start). Runtime errors and events can be +found in `supervisor_log` file located in the -L directory +(`/var/log/nemea-supervisor` by default). + +Now we can connect to running supervisor with supervisor client simply with +command `supcli`. The menu with options is described in detail +in [README](https://github.com/CESNET/Nemea-Supervisor#supervisor-functions). +After pressing number *4* and *enter*, it prints current status of the system. +By default, all *detectors* and *loggers* (except flow_meter logger) should be +enabled and running. + +The modules are running, but they don't receive any data yet. We need to send +some flow data to the system... + + +### 4. Get flows to your system + +**IPFIXcol** + +*(recommended)* Use IPFIXcol to collect NetFlow/IPFIX data from routers/probes +and an [IPFIXcol](https://github.com/CESNET/ipfixcol) [unirec plugin](https://github.com/CESNET/ipfixcol/tree/master/plugins/storage/unirec) +to re-send the data to NEMEA. + * needed to install IPFIXcol and the plugin and to set up the routers/probes + * default and recommended solution for production + + +**FlowMeter** + +Use NEMEA internal flow exporter (*flow_meter* module). + * it reads data directly from network interface (via libpcap), measures flows and export it to other NEMEA modules + * simple, but not very performing solution (flow_meter was not designed for performance), suitable only for testing or very small networks + * *TODO*: measure how much traffic can flow_meter handle and make recommendation what "very small network" means? + + +**NfReader** + +[NfReader](https://github.com/CESNET/Nemea-Modules/tree/master/nfreader) reads +**nfdump** files and sends flow records in UniRec format on its output TRAP +interface. + + +**LogReplay** + +[LogReplay](https://github.com/CESNET/Nemea-Modules/tree/master/logreplay) +converts CSV format of data, from logger module to UniRec format and sends it +to the output interface. + + +## Create your own module in C + +**Important**: Nemea-Framework has to be installed in advance. Follow +[installation instructions](#installation) + + +#### Use Example module as a template + +Let `~/mighty-module/` be the directory we want to develop our module in (replace path `~/mighty-module/` in all commands with another directory if needed) and *mighty_module* the name of our module. We will use example module as a template - copy the directory [nemea-framework/examples/c/module/](https://github.com/CESNET/Nemea-Framework/tree/master/examples/c/module) to `~/mighty-module/`. + +In `~/mighty-module/configure.ac` update the following lines +``` +AC_INIT([example_module], [1.0.0], [traffic-analysis@cesnet.cz]) +AC_CONFIG_SRCDIR([example_module.c]) ``` -dns_amplification -i t:localhost:9966,u:RT_DNS_amp +with +``` +AC_INIT([migty_module], [1.0.0], [YOUR EMAIL ADDRESS]) +AC_CONFIG_SRCDIR([mighty_module.c]) ``` - -Another examples of starting Nemea modules follows: -ex1) This example starts two `nfdump_readers` on two different files. Records -from both files are then merged into one stream and sent on TCP interface, -port 9920: +In `~/mighty-module/Makefile.am` update the following lines +``` +bin_PROGRAMS=example_module +example_module_SOURCES=example_module.c fields.c fields.h +example_module_LDADD=-lunirec -ltrap ``` -nfdump_reader -i "t:9911" /data/link1/nfcapd.201406001 -nfdump_reader -i "t:9912" /data/link2/nfcapd.201406001 -merger -i t:localhost:9911,t:localhost:9912,t:9920 -n 2 +with +``` +bin_PROGRAMS=mighty_module +mighty_module_SOURCES=mighty_module.c fields.c fields.h +mighty_module_LDADD=-lunirec -ltrap ``` -ex2) This more complex example start one "nfdump_reader" on multiple files. All -files are read sequentially and sent on output interface. Data from -"nfdump_reader" are then anonymized and sent to "hoststatsnemea" detector and -to "flowcounter". Reports from detector are then stored to CSV file via -"logger": +Finally, execute ``` -nfdump_reader -i "u:HS_src" /data/0601/nfcapd.0000 /data/0601/nfcapd.0005 - /data/0601/nfcapd.0010 /data/0601/nfcapd.0015 - anonymizer -i u:localhost:HS_src,u:HS_an -k 0AnonymizationKeyWithLengthof32B - flowcounter -i u:localhost:HS_an -p 100000 - hoststatsnemea -i u:HS_an,u:HS_report -F - logger -i "u:HS_report" +mv ~/mighty-module/example_module.c ~/mighty-module/mighty_module.c ``` +to rename the source file. + + +#### Build the module -note: In hoststatsnemea configuration file should be "port-flowdir = 1". +Execute the following commands in `~/might-module/`: -Example of configuration file for UniRec plugin for IPFIXcol: +1) Let Autotools process the configuration files. + +``` +autoreconf -i +``` -TODO: update if needed +2) Configure the module directory. ``` - - +./configure +``` + +3) Build the module. - - UDP collector - - Listening port 4740 - 4740 - - - UniRec output - +``` +make +``` - - UniRec output - - Make unirec from the flow data - - unirec - - - t - 9966,16 - 0 - 10000000 - 1 - DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST, - PACKETS,?DST_PORT,?SRC_PORT,DIR_BIT_FIELD,PROTOCOL,?TCP_FLAGS - - - - - +4) (**Optional**) Install the module. The command should be performed as root (e.g. using sudo). ``` +make install +``` + + +### Code explanation + +The example module already links **TRAP** (libtrap) and **UniRec** libraries. +It is a simple module with one input and one output interface which receives on +input inteface a message in UniRec format with two numbers and sends them +together with their sum to output interface. + +The code contains comments but here is the list of important operations: + + +#### Libtrap + +Generated doxygen doc for module developers: https://rawgit.com/CESNET/Nemea-Framework/master/libtrap/doc/doxygen/html/index.html + +Generated doxygen doc for libtrap developers: https://rawgit.com/CESNET/Nemea-Framework/master/libtrap/doc/devel/html/index.html + +1. [Basic module information](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L71) - specify name, description and number of input / output interfaces of the module +2. [Module parameters](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L84) - define parameters the module accepts as program arguments +3. [Module info structure initialization](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L114) - initialize a structure with information from the two previous points +4. [TRAP initialization](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L120) - initialize module interfaces +5. [GETOPT macro](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L131) - parse program arguments +6. Main loop: + * [Receive a message](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L176) - receive a message in UniRec format from input interface + * [Handle receive error](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L179) - check whether an error has occurred during receive + * [Send a message](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L204) - send a message in UniRec format via output interface + * [Handle send error](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L207) - check whether an error has occurred during send +7. [TRAP and module info clean-up](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L214) - free everything, libtrap finalization + -Manage Nemea modules efficiently -================================ +#### UniRec -The Nemea system can be managed and monitored by a special module called -[Supervisor](https://github.com/CESNET/Nemea-Supervisor). -For examples and more information see its README. +Generated doxygen doc: https://rawgit.com/CESNET/Nemea-Framework/master/unirec/doc/html/index.html -Nemea Related Publications +1. [UniRec fields definition](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L59) - define data types and names of the fields which will be used in UniRec messages (both received and sent messages), e.g. *uint32 PACKETS* +2. [Templates creation](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L144) - create UniRec templates separately for every interface (a template defines set of fields in the message) note: two input interfaces receiving same messages can use one template +3. [Output record allocation](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L157) - allocate a memory for message sent via output interface +4. Main loop (*fields manipulation*): + * [get field](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L195) - get a value of specified field from received message according to UniRec template + * [set field](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L200) - set a value of specified field in message which will be sent according to UniRec template + * [copy fields](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L199) - copy values of fields in received message to fields in message which will be sent according to UniRec templates of both interfaces (only fields that are common for both interfaces are copied) +5. [UniRec cleanup](https://github.com/CESNET/Nemea-Framework/blob/master/examples/c/module/example_module.c#L219) - free everything, UniRec finalization + + +### Execute the module + +#### Module help + +After executing `~/mighty-module/mighty_module -h`, program prints help which contains information from module info structure: + +* module basic information - name, description, number of input / output interfaces +* module parameters - short opt, long opt, description, argument data type +* TRAP library parameters - parameters common for all modules using libtrap + + +### Develop the module + +Now just modify the algorithm in the main loop and the job is done :-) + + +## Add new module to running configuration + +This section is for those who has already deployed the system ([Deploy NEMEA](#deploy-nemea) +section) and wants to add their module to the running configuration. It can be done in 3 steps: + +1. Create a *.sup* config file for your module. You can use [this](https://github.com/CESNET/Nemea-Supervisor/blob/master/configs/template.sup#L10) empty template and fill it according to [this](https://github.com/CESNET/Nemea-Supervisor/blob/master/configs/detectors/dnstunnel_detection.sup) example ([example with comments](https://github.com/CESNET/Nemea-Supervisor/blob/master/configs/config_example.xml#L19)). +2. Add the new *.sup* file to directory included in the Supervisor configuration file. If you have used recommended parameters of the `configure` script during the installation, both the configuration file and the directories should be located in `/etc/nemea`, otherwise check the paths in the configuration file the Supervisor is running with. Than copy the file to one of the directories you want e.g. `cp ./your_module.sup /etc/nemea/others`. +3. Connect to Supervisor using `supcli` command and select option 6 *reload configuration*. New module should be added and if the enabled flag is set to *true*, it should be also running. + +For detailed information about Supervisor configuration see its [README](https://github.com/CESNET/Nemea-Supervisor#configuration). + +Further Information +=================== + +* Public mailing list: nemea@cesnet.cz, subscribe at: [https://random.cesnet.cz/mailman/listinfo/nemea](https://random.cesnet.cz/mailman/listinfo/nemea) +* The archive of mailing list: [https://random.cesnet.cz/pipermail/nemea/](https://random.cesnet.cz/pipermail/nemea/) +* Twitter: [@NEMEA_System](https://twitter.com/NEMEA_System) + +Attribution +=========== + +We kindly ask anyone using NEMEA for research and write an academic paper about it to cite NEMEA as follows: +``` +@inproceedings{nemea16, + author = {Tomas Cejka, Vaclav Bartos, Marek Svepes, Zdenek Rosa, Hana Kubatova}, + title = {NEMEA: A Framework for Network Traffic Analysis}, + booktitle = {12th International Conference on Network and Service Management (CNSM 2016)}, + year = 2016 +} +``` + +NEMEA Related Publications ========================== -* Tomáš Čejka, Radoslav Bodó, Hana Kubátová: Nemea: Searching for Botnet Footprints. In: Proceedings of the 3rd Prague Embedded Systems Workshop, Prague, Czech Republic, 2015. +* Tomas Cejka, Václav Bartoš, Marek Svepes, Zdenek Rosa, Hana Kubatova. [NEMEA: A Framework for Network Traffic Analysis](https://www.liberouter.org/wp-content/uploads/2016/11/cnsm2016-nemea.pdf). In 12th International Conference on Network and Service Management (CNSM 2016), Montreal, Canada, 2016. + +* Zdenek Rosa, Tomas Cejka, Martin Zadnik, Viktor Puš. [Building a Feedback Loop to Capture Evidence of Network Incidents](https://www.liberouter.org/wp-content/uploads/2016/11/cnsm2016-timemachine.pdf). In 12th International Conference on Network and Service Management (CNSM 2016), Montreal, Canada, 2016. + +* Tomas Cejka, Marek Svepes. [Analysis of Vertical Scans Discovered by Naive Detection](http://dx.doi.org/10.1007/978-3-319-39814-3_19). Management and Security in the Age of Hyperconnectivity: 10th IFIP WG 6.6 International Conference on Autonomous Infrastructure, Management, and Security, AIMS 2016. + +* Tomáš Čejka, Radoslav Bodó, Hana Kubátová: Nemea: Searching for Botnet Footprints. In: Proceedings of the 3rd Prague Embedded Systems Workshop (PESW), Prague, CZ, 2015. * Tomáš Čejka, Václav Bartoš, Lukáš Truxa, Hana Kubátová: [Using Application-Aware Flow Monitoring for SIP Fraud Detection](http://link.springer.com/chapter/10.1007/978-3-319-20034-7_10). In: Proc. of 9th International Conference on Autonomous Infrastructure, Management and Security (AIMS15), 2015. -* Tomáš Čejka, Zdeněk Rosa and Hana Kubátová: [Stream-wise Detection of Surreptitious Traffic over DNS](http://ieeexplore.ieee.org/xpl/articleDetails.jsp?reload=true&arnumber=7033254). In: 2014 IEEE 19th International Workshop on Computer Aided Modeling and Design of Communication Links and Networks (CAMAD) (CAMAD 2014). Athens, 2014 +* Tomáš Čejka, Zdeněk Rosa and Hana Kubátová: [Stream-wise Detection of Surreptitious Traffic over DNS](http://ieeexplore.ieee.org/xpl/articleDetails.jsp?reload=true&arnumber=7033254). In: Proc. of 19th IEEE International Workshop on Computer Aided Modeling and Design of Communication Links and Networks (CAMAD 2014). Athens, 2014. -* V. Bartos, M. Zadnik, T. Cejka: [Nemea: Framework for stream-wise analysis of network traffic](http://www.cesnet.cz/wp-content/uploads/2014/02/trapnemea.pdf), CESNET technical report 6/2013. +* Václav Bartoš, Martin Žádník, Tomáš Čejka: [Nemea: Framework for stream-wise analysis of network traffic](http://www.cesnet.cz/wp-content/uploads/2014/02/trapnemea.pdf), CESNET technical report 6/2013. diff --git a/config-examples/Makefile.am b/config-examples/Makefile.am deleted file mode 100644 index 9c9c2827..00000000 --- a/config-examples/Makefile.am +++ /dev/null @@ -1,12 +0,0 @@ -EXTRA_DIST=amplification_detection.mkdir \ - amplification_detection.sup \ - bruteforce-detector.mkdir \ - bruteforce-detector.sup \ - hoststats.mkdir \ - hoststats.sup \ - ntp-amplification_detection.sup \ - prepare_default_config.sh \ - README.md \ - supervisor.mkdir \ - traffic_repeater.sup - diff --git a/config-examples/README.md b/config-examples/README.md deleted file mode 100644 index 5edeb151..00000000 --- a/config-examples/README.md +++ /dev/null @@ -1,31 +0,0 @@ -How to use this directory? -========================== - -This directory can be used for automatic generation of default -configuration for deployment of the Nemea system with supervisor. - -To add new module, do the following steps. - -Create a file with the _.sup_ extension that will contain module's -configuration for supervisor. The supervisor configuration file is -described in supervisor's README. However, the format is in XML-style, -users can inspire themselves by reading existing _.sup_ files. - -Note: the content of the _.sup_ file is placed into template of new -supervisor's configuration file. Therefore, it need not to have only one -root element as normal XML must have. - -If the added module need to use existing one or more directories e.g. for -logs or outputs, create a second file with _.mkdir_ extension instead of _.sup_ -for the module. The content of _.mkdir_ is simply read and passed to mkdir(1) -and chmod(1). This script does NOT support white spaces in the paths that -are listed in _.mkdir_ files. - -To generate new supervisor config files and to create needed directories, -run the *prepare_default_config.sh* script. The output configuration file -can be found as *supervisor_config.xml*. It must be moved into default path -e.g. /etc/nemea/. - -Note: if there is a special need of permissions and owners setting, -it should be tuned manually. Patches are welcome... - diff --git a/config-examples/amplification_detection.mkdir b/config-examples/amplification_detection.mkdir deleted file mode 100644 index f2b7ffad..00000000 --- a/config-examples/amplification_detection.mkdir +++ /dev/null @@ -1 +0,0 @@ -/data/dns_amplification_detection/ diff --git a/config-examples/amplification_detection.sup b/config-examples/amplification_detection.sup deleted file mode 100644 index 86ae6524..00000000 --- a/config-examples/amplification_detection.sup +++ /dev/null @@ -1,48 +0,0 @@ - -dns_amplification -true -/usr/bin/nemea/amplification_detection --d /data/dns_amplification_detection/ - - - - UNIXSOCKET - IN - flow_data_source - - - - UNIXSOCKET - OUT - dns_amp_reports - - - - SERVICE - SERVICE - service_dns_amp - - - - - -dns_amplification_logger -true -/usr/bin/nemea/logger --t -T -a /data/dns_amplification_detection/detected.log <AMPLIFICATION_ALERT> - - - - UNIXSOCKET - IN - localhost,dns_amp_reports - - - - SERVICE - SERVICE - service_dns_amp_logger - - - - diff --git a/config-examples/hoststats.mkdir b/config-examples/hoststats.mkdir deleted file mode 100644 index 7c144aed..00000000 --- a/config-examples/hoststats.mkdir +++ /dev/null @@ -1 +0,0 @@ -/data/hoststatsnemea/ diff --git a/config-examples/hoststats.sup b/config-examples/hoststats.sup deleted file mode 100644 index 9e717fba..00000000 --- a/config-examples/hoststats.sup +++ /dev/null @@ -1,47 +0,0 @@ - -hoststatsnemea -true -/usr/bin/nemea/hoststatsnemea --c /etc/nemea/hoststats.conf - - - - UNIXSOCKET - IN - flow_data_source - - - - UNIXSOCKET - OUT - hs_nemea_reports - - - - SERVICE - SERVICE - service_hs_nemea - - - - - -hs_nemea_logger -true -/usr/bin/nemea/logger --t -T -a /data/hoststatsnemea/detected.log EVENT_TYPE,TIME_FIRST,TIME_LAST,SRC_IP,DST_IP,SRC_PORT,DST_PORT,PROTOCOL,EVENT_SCALE,NOTE - - - - UNIXSOCKET - IN - localhost,hs_nemea_reports - - - - SERVICE - SERVICE - service_hs_nemea_logger - - - diff --git a/config-examples/ipblacklistfilter.mkdir b/config-examples/ipblacklistfilter.mkdir deleted file mode 100644 index fde12bb3..00000000 --- a/config-examples/ipblacklistfilter.mkdir +++ /dev/null @@ -1 +0,0 @@ -/data/ipblacklistfilter diff --git a/config-examples/ipblacklistfilter.sup b/config-examples/ipblacklistfilter.sup deleted file mode 100644 index a0d1b5d4..00000000 --- a/config-examples/ipblacklistfilter.sup +++ /dev/null @@ -1,48 +0,0 @@ - -ipblacklistfilter -true -/usr/bin/nemea/ipblacklistfilter --D - - - - UNIXSOCKET - IN - flow_data_source - - - - UNIXSOCKET - OUT - ipblacklistfilter_reports - - - - SERVICE - SERVICE - service_ipblacklistfilter_detector - - - - - -ipblf_logger -true -/usr/bin/nemea/logger --t -T -a /data/ipblacklistfilter/detected.log <COLLECTOR_FLOW>,SRC_BLACKLIST,DST_BLACKLIST,EVENT_SCALE - - - - UNIXSOCKET - IN - localhost,ipblacklistfilter_reports - - - - SERVICE - SERVICE - service_ipblacklistfilter_logger - - - - diff --git a/config-examples/prepare_default_config.sh b/config-examples/prepare_default_config.sh deleted file mode 100755 index 022fa215..00000000 --- a/config-examples/prepare_default_config.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh - -# name of the output file: -conffile=supervisor_config.xml - -cat > "$conffile" < - - - false - 4 - /var/log/nemea-supervisor/ - - - - detection modules - true - -END - -cat *.sup >> "$conffile" - -cat >> "$conffile" < - -END - -xmllint -format "$conffile" > "$conffile.tmp" && mv "$conffile.tmp" "$conffile" - -# creation of needed directories and permissions setup -for f in *.mkdir; do - for d in `cat "$f"`; do - mkdir -p "$d" - # This is not secure enough! It grants full access to the directory: - chmod 777 "$d" - done -done - diff --git a/config-examples/supervisor.mkdir b/config-examples/supervisor.mkdir deleted file mode 100644 index 557bee92..00000000 --- a/config-examples/supervisor.mkdir +++ /dev/null @@ -1 +0,0 @@ -/var/log/nemea-supervisor/ diff --git a/config-examples/traffic_repeater.sup b/config-examples/traffic_repeater.sup deleted file mode 100644 index a5c88bf6..00000000 --- a/config-examples/traffic_repeater.sup +++ /dev/null @@ -1,27 +0,0 @@ - -traffic_repeater -true -/usr/bin/nemea/traffic_repeater - - - - - TCP - IN - localhost,7600 - - - - UNIXSOCKET - OUT - flow_data_source - - - - SERVICE - SERVICE - service_traffic_repeater - - - - diff --git a/configure.ac b/configure.ac index 85114697..d55773a2 100644 --- a/configure.ac +++ b/configure.ac @@ -2,19 +2,52 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.63]) -AC_INIT([nemea], [2.1.0], [traffic-analysis@cesnet.cz]) +AC_INIT([nemea], [2.6.0], [nemea@cesnet.cz]) AM_INIT_AUTOMAKE([foreign subdir-objects]) +RELEASE=1 +AC_SUBST(RELEASE) +USERNAME=`git config --get user.name` +USERMAIL=`git config --get user.email` +AC_SUBST(USERNAME) +AC_SUBST(USERMAIL) +# Check for rpmbuild +AC_CHECK_PROG(RPMBUILD, rpmbuild, rpmbuild, [""]) -# Checks for programs. +AC_SUBST(RPMBUILD) +if test -z "$RPMBUILD"; then + AC_MSG_WARN([Due to missing rpmbuild you will not able to generate RPM package.]) +fi -# Checks for libraries. +RPM_RELEASE=1 +AC_SUBST(RPM_RELEASE) +AM_CONDITIONAL(MAKE_RPMS, test x$RPMBUILD != x) -# Checks for header files. +AC_ARG_ENABLE([supervisor], AC_HELP_STRING([--disable-supervisor], [Disable supervisor.]), + [AM_CONDITIONAL([ENABLE_SUPERVISOR], [test "x$enableval" = xyes])], + [AM_CONDITIONAL([ENABLE_SUPERVISOR], [true])]) -# Checks for typedefs, structures, and compiler characteristics. +AC_ARG_ENABLE([modules], AC_HELP_STRING([--disable-modules], [Disable NEMEA modules.]), + [AM_CONDITIONAL([ENABLE_MODULES], [test "x$enableval" = xyes])], + [AM_CONDITIONAL([ENABLE_MODULES], [true])]) -# Checks for library functions. +AC_ARG_ENABLE([detectors], AC_HELP_STRING([--disable-detectors], [Disable NEMEA detectors.]), + [AM_CONDITIONAL([ENABLE_DETECTORS], [test "x$enableval" = xyes])], + [AM_CONDITIONAL([ENABLE_DETECTORS], [true])]) -AC_CONFIG_FILES([Makefile]) +AC_CONFIG_FILES([Makefile + nemea.spec]) AC_CONFIG_SUBDIRS([nemea-framework nemea-supervisor modules detectors]) + +# By default, users expects "repobuild" behavior, i.e., use relative path to NEMEA framework. +# If there is some "repobuild" arg of configure, do not add "--enable-repobuild". +if test -z "$ac_configure_args" || echo "$ac_configure_args" | grep -q -v -e 'repobuild'; then +ac_configure_args="$ac_configure_args --enable-repobuild" +fi + AC_OUTPUT + +echo "Components:" +echo " NEMEA modules....................: `if [[ -z "$ENABLE_MODULES_TRUE" ]]; then echo enabled; else echo disabled; fi`" +echo " NEMEA detectors..................: `if [[ -z "$ENABLE_DETECTORS_TRUE" ]]; then echo enabled; else echo disabled; fi`" +echo " NEMEA supervisor.................: `if [[ -z "$ENABLE_SUPERVISOR_TRUE" ]]; then echo enabled; else echo disabled; fi`" + diff --git a/detectors b/detectors index fb487a74..84f6f1b1 160000 --- a/detectors +++ b/detectors @@ -1 +1 @@ -Subproject commit fb487a74e6d21168684d1987b20ad57ec4a1b8c9 +Subproject commit 84f6f1b15f43f0b81e34c4859ff1ada6fb6e8270 diff --git a/doc/NEMEA-parts.png b/doc/NEMEA-parts.png new file mode 100644 index 00000000..be70b2dd Binary files /dev/null and b/doc/NEMEA-parts.png differ diff --git a/generate-rpm.sh b/generate-rpm.sh new file mode 100755 index 00000000..7ea9b836 --- /dev/null +++ b/generate-rpm.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# +# Copyright (C) 2015 CESNET +# +# LICENSE TERMS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Company nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# ALTERNATIVELY, provided that this notice is retained in full, this +# product may be distributed under the terms of the GNU General Public +# License (GPL) version 2 or later, in which case the provisions +# of the GPL apply INSTEAD OF those given above. +# +# This software is provided ``as is'', and any express or implied +# warranties, including, but not limited to, the implied warranties of +# merchantability and fitness for a particular purpose are disclaimed. +# In no event shall the company or contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential +# damages (including, but not limited to, procurement of substitute +# goods or services; loss of use, data, or profits; or business +# interruption) however caused and on any theory of liability, whether +# in contract, strict liability, or tort (including negligence or +# otherwise) arising in any way out of the use of this software, even +# if advised of the possibility of such damage. + +pwd +./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q && make -j4 && make rpm + diff --git a/jenkins/Centos-Dockerfile b/jenkins/Centos-Dockerfile new file mode 100644 index 00000000..cb1127a8 --- /dev/null +++ b/jenkins/Centos-Dockerfile @@ -0,0 +1,16 @@ +FROM centos:7 + +RUN rpm -ih http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm; \ + yum install -y autoconf automake gcc gcc-c++ libtool libxml2-devel m4 make \ + openssl openssl-devel pkg-config libxslt-devel graphviz \ + xsltproc git bc libxml2-python libxslt-python doxygen \ + libpcap-devel bison flex python python-devel python-pip \ + python-setuptools python34 python34-devel python34-pip python34-setuptools python-yaml python34-ply python34-yaml \ + rpm-build epel-rpm-macros; \ + curl -s 'https://copr.fedorainfracloud.org/coprs/g/CESNET/NEMEA/repo/epel-7/group_CESNET-NEMEA-epel-7.repo' > /etc/yum.repos.d/group_CESNET-NEMEA-epel-7.repo; \ + yum install -y python-pynspect python34-pynspect + +RUN chmod u+s,o+rx /usr/sbin/useradd /usr/sbin/groupadd; yum install -y sudo; sed -i "\$aALL ALL=(ALL) NOPASSWD:ALL" /etc/sudoers + +CMD ["/usr/bin/cat"] + diff --git a/jenkins/Debian-Dockerfile b/jenkins/Debian-Dockerfile new file mode 100644 index 00000000..4123e9bc --- /dev/null +++ b/jenkins/Debian-Dockerfile @@ -0,0 +1,15 @@ +FROM debian:stable + +RUN apt-get update; apt-get install -y gawk autoconf automake gcc g++ libtool make pkg-config libpcap-dev \ + libxml2-dev libidn11-dev bison flex \ + libssl-dev pkg-config libxslt-dev xsltproc doxygen graphviz \ + bc git \ + python python-dev python-pip python-setuptools python3 python3-dev python3-pip python3-setuptools \ + debmake devscripts debhelper python3-all-dev + +RUN pip install stdeb; pip3 install stdeb + +RUN chmod u+s,o+rx /usr/sbin/useradd /usr/sbin/groupadd; apt-get install -y sudo; sed -i "\$aALL ALL=(ALL) NOPASSWD:ALL" /etc/sudoers + +CMD ["/bin/bash"] + diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile new file mode 100644 index 00000000..3c9b4ca0 --- /dev/null +++ b/jenkins/Jenkinsfile @@ -0,0 +1,53 @@ +pipeline { + agent none + stages { + stage('Build Centos') { + agent { + dockerfile { + filename 'Centos-Dockerfile' + dir 'jenkins' + } + } + steps { + //sh './bootstrap.sh && ./configure --prefix=$PWD/install --libdir=$PWD/install/lib -q --enable-debug CXXFLAGS=-coverage CFLAGS=-coverage LDFLAGS=-lgcov && make rpm make install && export LD_LIBRARY_PATH=$PWD/install/lib && export CPPFLAGS=-I$PWD/install/include && (cd pytrap && make coverage; python setup.py install;)' + + sh '/usr/sbin/groupadd -f -g `id -g` jenkins; /usr/sbin/useradd jenkins -u `id -u` -g `id -g`; true' + + sh '( cd nemea-framework/libtrap; ./bootstrap.sh && ./configure -q && make rpm && sudo yum install -y ./RPMBUILD/RPMS/*/libtrap*.rpm;)' + sh '( cd nemea-framework/unirec; ./bootstrap.sh && ./configure -q && make rpm && sudo yum install -y ./RPMBUILD/RPMS/*/unirec*.rpm;)' + sh '( cd nemea-framework/common; ./bootstrap.sh && ./configure -q && make rpm && sudo yum install -y ./RPMBUILD/RPMS/*/nemea-common*.rpm;)' + sh '( ./bootstrap.sh && ./configure -q;)' + sh '( cd nemea-framework/pytrap; make rpm; sudo yum install -y ./dist/*.rpm;)' + sh '( cd nemea-framework/pycommon; make rpm; sudo yum install -y ./dist/*.rpm;)' + sh '( cd modules; ./bootstrap.sh && ./configure -q && make rpm; sudo yum install -y ./RPMBUILD/RPMS/*/*.rpm;)' + sh '( cd detectors; ./bootstrap.sh && ./configure -q && make rpm; sudo yum install -y ./RPMBUILD/RPMS/*/*.rpm;)' + sh '( cd supervisor; ./bootstrap.sh && ./configure -q && make rpm; sudo yum install -y ./RPMBUILD/RPMS/*/*.rpm;)' + sh 'mkdir -p rpms; find libtrap unirec common pytrap pycommon -name "*.rpm" -exec mv {} rpms ";"' + archiveArtifacts artifacts: 'rpms/*', onlyIfSuccessful: true + } + } + stage('Build Debian packages') { + agent { + dockerfile { + filename 'Debian-Dockerfile' + dir 'jenkins' + } + } + steps { + sh '/usr/sbin/groupadd -f -g `id -g` jenkins; /usr/sbin/useradd jenkins -u `id -u` -g `id -g`; true' + + sh '( cd nemea-framework/libtrap; ./bootstrap.sh && ./configure -q && make deb && sudo dpkg -i ./libtrap*.deb;)' + sh '( cd nemea-framework/unirec; ./bootstrap.sh && ./configure -q && make deb && sudo dpkg -i ./unirec*.deb;)' + sh '( cd nemea-framework/common; ./bootstrap.sh && ./configure -q && make deb && sudo dpkg -i ./nemea-common*.deb;)' + sh '( cd nemea-framework/pytrap; python setup.py --command-packages=stdeb.command bdist_deb; python3 setup.py --command-packages=stdeb.command bdist_deb; sudo dpkg -i ./deb_dist/*.deb;)' + sh '( cd nemea-framework/pycommon; python setup.py --command-packages=stdeb.command bdist_deb; python3 setup.py --command-packages=stdeb.command bdist_deb; sudo dpkg -i ./deb_dist/*.deb;)' + sh '( cd modules; ./bootstrap.sh && ./configure -q && make rpm; sudo dpkg -i ./nemea-modules*.deb;)' + sh '( cd detectors; ./bootstrap.sh && ./configure -q && make rpm; sudo dpkg -i ./nemea-detectors*.deb;)' + sh '( cd supervisor; ./bootstrap.sh && ./configure -q && make rpm; sudo dpkg -i ./nemea-supervisor*.deb;)' + sh 'mkdir -p debs; find libtrap unirec common pytrap pycommon -name "*.deb" -exec mv {} debs ";"' + archiveArtifacts artifacts: 'debs/*', onlyIfSuccessful: true + } + } + } +} + diff --git a/modules b/modules index 99dc4c98..c1676670 160000 --- a/modules +++ b/modules @@ -1 +1 @@ -Subproject commit 99dc4c9890a69742bb742adc2265cd351bad2497 +Subproject commit c16766702209b4b899cf1ee101ae97c08e0e0b05 diff --git a/nemea-framework b/nemea-framework index a5f21343..a554edfb 160000 --- a/nemea-framework +++ b/nemea-framework @@ -1 +1 @@ -Subproject commit a5f21343d764a1aafd25cafd344f1bf0d9f6a643 +Subproject commit a554edfb3bb834218ba4a1268188a641cd07ae6f diff --git a/nemea-supervisor b/nemea-supervisor index 0fb16cbe..89afda17 160000 --- a/nemea-supervisor +++ b/nemea-supervisor @@ -1 +1 @@ -Subproject commit 0fb16cbe490c736e81b4380d7e1ccecc19548b26 +Subproject commit 89afda171fc119db253235f7a8e0c71614296dec diff --git a/nemea.spec.in b/nemea.spec.in new file mode 100644 index 00000000..8f5d10eb --- /dev/null +++ b/nemea.spec.in @@ -0,0 +1,19 @@ +Summary: NEMEA system +Name: nemea +Version: @PACKAGE_VERSION@ +Release: @RELEASE@ +URL: https://github.com/CESNET/nemea +#Source: https://www.liberouter.org/ +Group: Liberouter +License: BSD +Vendor: CESNET, z.s.p.o. +Packager: @USERNAME@ <@USERMAIL@> +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} +BuildArch: noarch +Requires: nemea-framework nemea-modules nemea-detectors nemea-supervisor + +%description +Meta-package of the NEMEA system, the distributed modular system for network traffic analysis and anomaly detection. + +%files + diff --git a/packer/README.md b/packer/README.md new file mode 100644 index 00000000..3b3b0788 --- /dev/null +++ b/packer/README.md @@ -0,0 +1,17 @@ +Creation of image using Packer +============================== + +Get Packer from https://www.packer.io + +Set PATH to its directory in order to use the binaries. + +``` +cd centos-7.1/ +packer build template.json +``` + +will create a directory with VirtualBox ovf and virtual disk. + +In case Vagrant box is needed, use `template.json-vagrant-box` +instead of `template.json`. + diff --git a/packer/centos-7.1/http/ks.cfg b/packer/centos-7.1/http/ks.cfg new file mode 100644 index 00000000..208b11b8 --- /dev/null +++ b/packer/centos-7.1/http/ks.cfg @@ -0,0 +1,42 @@ +install +cdrom +lang en_US.UTF-8 +keyboard us +network --onboot yes --device eth0 --bootproto dhcp --noipv6 +rootpw --plaintext vagrant +firewall --enabled --service=ssh +authconfig --enableshadow --passalgo=sha512 +selinux --disabled +timezone Europe/Prague +bootloader --location=mbr --driveorder=sda --append="crashkernel=auto rhgb quiet" + +text +skipx +zerombr + +clearpart --all --initlabel +autopart + +auth --useshadow --enablemd5 +firstboot --disabled +reboot + +%packages --ignoremissing +@core +bzip2 +kernel-devel +kernel-headers +-ipw2100-firmware +-ipw2200-firmware +-ivtv-firmware +%end + +%post +/usr/bin/yum -y install sudo +/usr/sbin/groupadd -g 501 vagrant +/usr/sbin/useradd vagrant -u 501 -g vagrant -G wheel +echo "vagrant"|passwd --stdin vagrant +echo "vagrant ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/vagrant +echo "Defaults:vagrant !requiretty" >> /etc/sudoers.d/vagrant +chmod 0440 /etc/sudoers.d/vagrant +%end diff --git a/packer/centos-7.1/scripts/base.sh b/packer/centos-7.1/scripts/base.sh new file mode 100644 index 00000000..f3e482c2 --- /dev/null +++ b/packer/centos-7.1/scripts/base.sh @@ -0,0 +1,3 @@ +sed -i "s/^.*requiretty/#Defaults requiretty/" /etc/sudoers +yum -y install gcc make gcc-c++ kernel-devel-`uname -r` perl + diff --git a/packer/centos-7.1/scripts/cleanup.sh b/packer/centos-7.1/scripts/cleanup.sh new file mode 100644 index 00000000..c7ee1a1b --- /dev/null +++ b/packer/centos-7.1/scripts/cleanup.sh @@ -0,0 +1,5 @@ +yum -y erase gtk2 libX11 hicolor-icon-theme avahi freetype bitstream-vera-fonts +yum -y clean all +rm -rf VBoxGuestAdditions_*.iso +rm -rf /tmp/rubygems-* + diff --git a/packer/centos-7.1/scripts/nemea-system.sh b/packer/centos-7.1/scripts/nemea-system.sh new file mode 100644 index 00000000..b8002856 --- /dev/null +++ b/packer/centos-7.1/scripts/nemea-system.sh @@ -0,0 +1,277 @@ +rpm -ivh https://homeproj.cesnet.cz/rpm/liberouter/devel/x86_64/liberouter-devel-1.0.0-1.noarch.rpm +yum install -y nemea nemea-cesnet-modules ipfixcol ipfixcol-unirec-output + +echo '[mongodb-org-3.2] +name=MongoDB Repository +baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.2/x86_64/ +gpgcheck=0 +enabled=1 +' > /etc/yum.repos.d/mongodb-org-3.2.repo + +# install dependencies +yum install -y https://centos7.iuscommunity.org/ius-release.rpm +yum -y install mongodb-org git wget python34 python34-devel httpd +yum groupinstall -y "Development Tools" +wget -q https://bootstrap.pypa.io/get-pip.py +python3.4 get-pip.py + +# start services +systemctl enable httpd +systemctl enable mongod +service httpd start +service mongod start + +# clone Nemea-Dashboard +mkdir -p /var/www/html +cd /var/www/html +git clone --depth 1 https://github.com/CESNET/Nemea-Dashboard +cd Nemea-Dashboard + +# install dashboard dependencies +pip install -r requirements.txt + +mongo 127.0.0.1/nemeadb --eval ' +db.users.drop(); +var d = { + "password" : "$2a$12$MuAuralQ8p01q8eXPtTm4eZsflipBKx6cK0HZMpQbCNyDgcX4YPse", + "username" : "nemea", + "name" : "Default", + "surname" : "User", + "settings" : [ { "settings" : { "title" : "Basic", "timeshift" : "0", "interval" : "60" }, "items" : [ { "content" : "Click the menu icon to select edit", "sizeY" : 3, "row" : 0, "type" : "piechart", "col" : 0, "loading" : false, "config" : { "type" : "piechart", "metric" : "Category", "period" : "24" }, "sizeX" : 4, "title" : "24h shares" } ] } ] +}; +db.users.insert(d); +' + +# start backend on background +cd /var/log; +nohup python3.4 /var/www/html/Nemea-Dashboard/apiv2.py > nemea-dashboard.out 2> nemea-dashboard.err < /dev/null & + + +# Prepare IPFIXcol +echo '--- startup.xml 2016-02-25 14:30:15.595229902 +0100 ++++ startup_new.xml 2016-02-25 14:26:54.543236208 +0100 +@@ -1,148 +1,79 @@ + + + +- +- +- +- UDP collector +- +- +- +- Listening port 4739 +- +- 4739 +- +- 1800 +- +- 1800 +- +- +- +- +- +- 127.0.0.1 +- +- +- File writer UDP +- +- /tmp/ipfixcol_stat.log +- ++ ++ UDP collector ++ ++ Listening port 4739 ++ 4739 ++ ++ ++ UniRec output ++ ++ ++ ++ UniRec output ++ ++ Make unirec from the flow data ++ ++ unirec ++ ++ ++ t ++ 7500 ++ 10000 ++ 1000000 ++ 1 ++ DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,PACKETS,?DST_PORT,?SRC_PORT,DIR_BIT_FIELD,PROTOCOL,?TCP_FLAGS,?TOS,?TTL ++ ++ ++ ++ t ++ 7501 ++ 1000 ++ 1000000 ++ 1 ++ DST_IP,?INVEA_SIP_RTP_IP4,?INVEA_SIP_RTP_IP6,SRC_IP,BYTES,?INVEA_RTCP_OCTETS,?INVEA_RTCP_PACKETS,?INVEA_SIP_BYE_TIME,?INVEA_SIP_INVITE_RINGING_TIME,?INVEA_SIP_OK_TIME,?INVEA_SIP_STATS,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,?INVEA_RTCP_LOST,?INVEA_RTP_JITTER,PACKETS,?DST_PORT,?INVEA_SIP_RTP_AUDIO,?INVEA_SIP_RTP_VIDEO,?SRC_PORT,DIR_BIT_FIELD,?INVEA_RTCP_SOURCE_COUNT,?INVEA_RTP_CODEC,?INVEA_VOIP_PACKET_TYPE,PROTOCOL,?TCP_FLAGS,?TOS,?TTL,?INVEA_SIP_CALLED_PARTY,?INVEA_SIP_CALLING_PARTY,?INVEA_SIP_CALL_ID,INVEA_SIP_REQUEST_URI,?INVEA_SIP_USER_AGENT,?INVEA_SIP_VIA ++ ++ ++ ++ t ++ 7502 ++ 1000 ++ 1000000 ++ 1 ++ DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,PACKETS,?SMTP_2XX_STAT_CODE_COUNT,?SMTP_3XX_STAT_CODE_COUNT,?SMTP_4XX_STAT_CODE_COUNT,?SMTP_5XX_STAT_CODE_COUNT,?SMTP_COMMAND_FLAGS,?SMTP_MAIL_CMD_COUNT,?SMTP_RCPT_CMD_COUNT,?SMTP_STAT_CODE_FLAGS,DST_PORT,SRC_PORT,DIR_BIT_FIELD,PROTOCOL,TCP_FLAGS,?TOS,?TTL,?SMTP_DOMAIN,?SMTP_FIRST_RECIPIENT,?SMTP_FIRST_SENDER ++ ++ ++ ++ t ++ 7503 ++ 1000 ++ 1000000 ++ 1 ++ DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,?HTTP_REQUEST_AGENT_ID,?HTTP_REQUEST_METHOD_ID,?HTTP_RESPONSE_STATUS_CODE,PACKETS,DST_PORT,SRC_PORT,DIR_BIT_FIELD,PROTOCOL,TCP_FLAGS,?TOS,?TTL,?HTTP_REQUEST_AGENT,?HTTP_REQUEST_HOST,?HTTP_REQUEST_REFERER,?HTTP_REQUEST_URL,?HTTP_RESPONSE_CONTENT_TYPE ++ ++ ++ ++ t ++ 7504 ++ 1000 ++ 1000000 ++ 1 ++ DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,PACKETS,?DST_PORT,?SRC_PORT,DIR_BIT_FIELD,?IPV6_TUN_TYPE,PROTOCOL,?TCP_FLAGS,?TOS,?TTL ++ ++ ++ ++ t ++ 7505 ++ 1000 ++ 1000000 ++ 1 ++ DST_IP,SRC_IP,BYTES,LINK_BIT_FIELD,TIME_FIRST,TIME_LAST,DNS_RR_TTL,PACKETS,DNS_ANSWERS,DNS_CLASS,DNS_ID,?DNS_PSIZE,DNS_QTYPE,DNS_RLENGTH,?DST_PORT,?SRC_PORT,DIR_BIT_FIELD,?DNS_DO,DNS_RCODE,PROTOCOL,?TCP_FLAGS,?TOS,?TTL,?DNS_NAME,?DNS_RDATA ++ ++ ++ ++ yes ++ + +- +- TCP collector +- +- Listening port 4739 +- 4739 +- 127.0.0.1 +- +- File writer TCP +- +- +- +- SCTP collector +- +- Listening port 4739 +- 4739 +- +- 127.0.0.1 +- ::1 +- +- File writer SCTP +- +- +- +- +- +- File writer UDP +- +- +- +- Write to /tmp folder +- +- 1 +- +- +- +- ipfix +- +- file://tmp/collected-records-udp_1.ipfix +- +- +- +- Write to /tmp folder +- +- ipfix +- file://tmp/collected-records-udp.ipfix +- +- +- +- +- +- File writer TCP +- +- Write to /tmp folder +- +- ipfix +- file://tmp/collected-records-tcp.ipfix +- +- +- +- +- +- +- +- +- File writer SCTP +- +- Store data using ipfix file format +- +- ipfix +- file://tmp/collected-records-sctp.ipfix +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- + ' > /etc/ipfixcol/startup.diff +patch /etc/ipfixcol/startup.xml /etc/ipfixcol/startup.diff +rm -f /etc/ipfixcol/startup.diff diff --git a/packer/centos-7.1/scripts/vagrant.sh b/packer/centos-7.1/scripts/vagrant.sh new file mode 100644 index 00000000..c3e2631a --- /dev/null +++ b/packer/centos-7.1/scripts/vagrant.sh @@ -0,0 +1,6 @@ +date > /etc/vagrant_box_build_time + +mkdir -pm 700 /home/vagrant/.ssh +curl -L https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub -o /home/vagrant/.ssh/authorized_keys +chmod 0600 /home/vagrant/.ssh/authorized_keys +chown -R vagrant:vagrant /home/vagrant/.ssh diff --git a/packer/centos-7.1/scripts/virtualbox.sh b/packer/centos-7.1/scripts/virtualbox.sh new file mode 100644 index 00000000..7c9dcd1f --- /dev/null +++ b/packer/centos-7.1/scripts/virtualbox.sh @@ -0,0 +1,11 @@ +VBOX_VERSION=$(cat /home/vagrant/.vbox_version) + +# required for VirtualBox 4.3.26 +yum install -y bzip2 + +cd /tmp +mount -o loop /home/vagrant/VBoxGuestAdditions_$VBOX_VERSION.iso /mnt +sh /mnt/VBoxLinuxAdditions.run +umount /mnt +rm -rf /home/vagrant/VBoxGuestAdditions_*.iso + diff --git a/packer/centos-7.1/scripts/vmware.sh b/packer/centos-7.1/scripts/vmware.sh new file mode 100644 index 00000000..1de6d1ea --- /dev/null +++ b/packer/centos-7.1/scripts/vmware.sh @@ -0,0 +1 @@ +yum install -y fuse-libs open-vm-tools diff --git a/packer/centos-7.1/scripts/zerodisk.sh b/packer/centos-7.1/scripts/zerodisk.sh new file mode 100644 index 00000000..45f45ed6 --- /dev/null +++ b/packer/centos-7.1/scripts/zerodisk.sh @@ -0,0 +1,3 @@ +dd if=/dev/zero of=/EMPTY bs=1M +rm -f /EMPTY + diff --git a/packer/centos-7.1/template.json b/packer/centos-7.1/template.json new file mode 100644 index 00000000..ffbb67ea --- /dev/null +++ b/packer/centos-7.1/template.json @@ -0,0 +1,45 @@ +{ + "provisioners": [ + { + "type": "shell", + "execute_command": "echo 'vagrant'|sudo -S sh '{{.Path}}'", + "override": { + "virtualbox-iso": { + "scripts": [ + "scripts/base.sh", + "scripts/vagrant.sh", + "scripts/virtualbox.sh", + "scripts/nemea-system.sh", + "scripts/cleanup.sh" + ] + } + } + } + ], + "builders": [ + { + "type": "virtualbox-iso", + "boot_command": [ + " text ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ks.cfg" + ], + "boot_wait": "10s", + "disk_size": 40520, + "guest_os_type": "RedHat_64", + "http_directory": "http", + "iso_checksum": "4c6c65b5a70a1142dadb3c65238e9e97253c0d3a", + "iso_checksum_type": "sha1", + "iso_url": "http://ftp.cvut.cz/centos/7/isos/x86_64/CentOS-7-x86_64-DVD-1511.iso", + "ssh_username": "vagrant", + "ssh_password": "vagrant", + "ssh_port": 22, + "ssh_wait_timeout": "10000s", + "shutdown_command": "echo '/sbin/halt -h -p' > /tmp/shutdown.sh; echo 'vagrant'|sudo -S sh '/tmp/shutdown.sh'", + "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso", + "virtualbox_version_file": ".vbox_version", + "vboxmanage": [ + [ "modifyvm", "{{.Name}}", "--memory", "512" ], + [ "modifyvm", "{{.Name}}", "--cpus", "1" ] + ] + } + ] +} diff --git a/packer/centos-7.1/template.json-vagrant-box b/packer/centos-7.1/template.json-vagrant-box new file mode 100644 index 00000000..e69de29b diff --git a/prepare-package.sh b/prepare-package.sh new file mode 100755 index 00000000..03df4e70 --- /dev/null +++ b/prepare-package.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# +# Copyright (C) 2018 CESNET +# +# LICENSE TERMS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Company nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# ALTERNATIVELY, provided that this notice is retained in full, this +# product may be distributed under the terms of the GNU General Public +# License (GPL) version 2 or later, in which case the provisions +# of the GPL apply INSTEAD OF those given above. +# +# This software is provided ``as is'', and any express or implied +# warranties, including, but not limited to, the implied warranties of +# merchantability and fitness for a particular purpose are disclaimed. +# In no event shall the company or contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential +# damages (including, but not limited to, procurement of substitute +# goods or services; loss of use, data, or profits; or business +# interruption) however caused and on any theory of liability, whether +# in contract, strict liability, or tort (including negligence or +# otherwise) arising in any way out of the use of this software, even +# if advised of the possibility of such damage. +# + +function prompt() +{ + read -p "$1" -n 1 p + echo "" + if [ "$p" != "y" -a "$p" != "Y" ]; then + return 1 + else + return 0 + fi +} + +function cont_prompt() +{ + if prompt "Continue? [y/n] "; then + return 0 + else + exit 1 + fi +} + + +# Look up commit of last releasing +lastreleased=$(git log -n 1 --grep="released\? \(RPM \)\?package" --format=%h -- ./configure.ac) + +if [ -z "$lastreleased" ]; then + echo "I didn't find the last commit that released new package." + exit 1 +else + echo "The package was lastly released by $lastreleased." + git show -q $lastreleased + echo "" + + echo "I found these changes since the last release:" + git shortlog $lastreleased..HEAD -- ./ + cont_prompt +fi + +# Look up version of the latest package and the current version +lastversion=$(git show $lastreleased:./configure.ac | sed -n 's/AC_INIT([^,]*, \?\([^,]*\),.*/\1/p;' |tr -d '[]') +currversion=$(sed -n 's/AC_INIT([^,]*, \?\([^,]*\),.*/\1/p;' configure.ac |tr -d '[]') +name=$(sed -n 's/AC_INIT(\([^,]*\),.*/\1/p;' configure.ac |tr -d '[]') + +echo "Version is changing from $lastversion to $currversion." + +if [ "$lastversion" = "$currversion" ]; then + if prompt "Should I increase version? [y/n] "; then + echo "" + read -p "Write a new version in x.y.z format, please: " currversion + sed -i '/AC_INIT/ s/'"$lastversion"'/'"$currversion"'/;' configure.ac + echo "Version was replaced." + fi +fi + +cont_prompt + +echo "Analysing Makefiles..." +for makefile in $(grep -Rl --include=*am -e '-version-info [0-9]\+:[0-9]\+:[0-9]\+' .); do + echo -e "\t$makefile" + lastlibvers="$(git show $lastreleased:$makefile | + sed -n 's/^\(.*\)_la_LDFLAGS\s*=.*-version-info \([0-9]\+\):\([0-9]\+\):\([0-9]\+\).*/\1:\2:\3:\4/pg')" + + currlibvers="$(sed -n 's/^\(.*\)_la_LDFLAGS\s*=.*-version-info \([0-9]\+\):\([0-9]\+\):\([0-9]\+\).*/\1:\2:\3:\4/pg' $makefile)" + + tmplist=$(mktemp) + paste -d: <(echo "$lastlibvers" | sort) <(echo "$currlibvers" | sort) > $tmplist + while read -u 3 line; do + IFS=":" read -r -a cols <<< "$line" + if [ "${cols[0]}" = "${cols[4]}" -a "${cols[1]}" = "${cols[5]}" -a \ + "${cols[2]}" = "${cols[6]}" -a "${cols[3]}" = "${cols[7]}" ]; then + + if prompt "Do You want to increase version of ${cols[0]}? [y/n]"; then + current="${cols[1]}" + release="${cols[2]}" + age="${cols[3]}" + echo "FYI: Old version was: $current:$release:$age" + echo "" + if prompt "2. Was the library source code changed at all since the last update? [y/n] "; then + ((release++)) + fi + if prompt "3. Was any interface added, removed, or changed since the last update? [y/n] "; then + ((current++)) + revision=0 + fi + if prompt "4. Was any interface added since the last public release? [y/n] "; then + ((age++)) + fi + if prompt "5. Was any interfaces removed or changed since the last public release (INcompatibility)? [y/n] "; then + age=0 + fi + echo "New version for ${cols[0]} is $current:$release:$age." + echo "" + if prompt "Should I update $makefile? [y/n] "; then + sed -i "s/\(${cols[0]}_la_LDFLAGS.*-version-info \)\([0-9]\+:[0-9]\+:[0-9]\+\)\(.*\))/\1$current:$release:$age\3/" $makefile + fi + fi + fi + done 3< $tmplist + rm $tmplist +done + +changelog="$(date "+%F $name-$currversion" +git log --pretty='subject:%s%nbody:%b' $lastreleased..master -- .| sed -n 's/subject:\([^:]*\):.*/\t* \1:/p; s/\(body:\)\?\s*changelog: \(.*\)/\t\t\2/p;' +echo "" +echo "" +)" + +echo "$changelog" > ChangeLog.tmp +$EDITOR ChangeLog.tmp + +cat ChangeLog.tmp + +echo "I will update ChangeLog with Your entry..." +cont_prompt + +cat ChangeLog >> ChangeLog.tmp +mv ChangeLog.tmp ChangeLog + +git log --date=short --no-merges --format="%cd (%an): %s" $lastreleased..HEAD -- ./>> NEWS.tmp +echo "" >> NEWS.tmp +$EDITOR NEWS.tmp + +echo "I will Insert this NEWS..." +cont_prompt + +cat NEWS >> NEWS.tmp +mv NEWS.tmp NEWS + +echo "I will prepare a commit about released package..." +cont_prompt + +git commit -a -m "$name: increased version, updated ChangeLog, released RPM package" -e + +echo "I will build the new RPM to get source package (SRPM)" +if [ "$name" = ipfixprobe ]; then +# ipfixprobe has no bootstrap.sh +autoreconf -i &&./configure -q --enable-coprrpm && make rpm +else +./bootstrap.sh &&./configure -q && make rpm +fi + +echo "I will upload the new RPM to copr" +cont_prompt + +copr build @CESNET/NEMEA RPMBUILD/SRPMS/$name-$currversion-1.src.rpm + +echo "Last commit:" +git log -1 + +if prompt "Push to git origin? [y/n] "; then + git push +fi + diff --git a/rpms.sh b/rpms.sh new file mode 100755 index 00000000..a06cbdf8 --- /dev/null +++ b/rpms.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# +# Copyright (C) 2015 CESNET +# +# LICENSE TERMS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Company nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# ALTERNATIVELY, provided that this notice is retained in full, this +# product may be distributed under the terms of the GNU General Public +# License (GPL) version 2 or later, in which case the provisions +# of the GPL apply INSTEAD OF those given above. +# +# This software is provided ``as is'', and any express or implied +# warranties, including, but not limited to, the implied warranties of +# merchantability and fitness for a particular purpose are disclaimed. +# In no event shall the company or contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential +# damages (including, but not limited to, procurement of substitute +# goods or services; loss of use, data, or profits; or business +# interruption) however caused and on any theory of liability, whether +# in contract, strict liability, or tort (including negligence or +# otherwise) arising in any way out of the use of this software, even +# if advised of the possibility of such damage. + +#set -x + +if [ -x "`which dnf`" ]; then + export pkginst=dnf +elif [ -x "`which yum`" ]; then + export pkginst=yum +else + echo "Unsupported package manager (dnf/yum)" >&2 + exit 1 +fi + +echo "Warning: You must have 'rpmbuild' in order to generate RPM package." +echo "If you want to abort this script, press CTRL+C (i.e. send SIGINT signal)" +sleep 5 + +if [ "x`whoami`" != xroot ]; then + echo "Run this script as root, since it must install RPM packages continuously" + exit 1 +fi + +read -p "Enter the name of user who will compile packages: " chuser + +read -p "Are You sure You want to continue? [yn]" -n1 ans + +if [ "x$ans" != xy ]; then + exit 0 +fi + +echo "Remove previously installed packages" +$pkginst remove -q -y libtrap\* unirec\* nemea\* +$pkginst install -q -y libnf-devel libpcap-devel libidn-devel bison flex + +export topdir=$PWD +export chuser + +( + cd nemea-framework + ( + cd libtrap + su $chuser -p -c "$topdir/generate-rpm.sh" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) + ) + ( + cd common + su $chuser -p -c "$topdir/generate-rpm.sh" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) + ) + ( + cd unirec + su $chuser -p -c "$topdir/generate-rpm.sh" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) + ) + su $chuser -p -c "./bootstrap.sh >/dev/null 2>/dev/null&& ./configure -q" + ( + cd python + su $chuser -p -c "make && make rpm" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) + ) + ( + cd pycommon + su $chuser -p -c "make && make rpm" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) + ) +) +( + cd modules + su $chuser -p -c "$topdir/generate-rpm.sh" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) +) +( + cd detectors + su $chuser -p -c "$topdir/generate-rpm.sh" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) +) +( + cd nemea-supervisor + su $chuser -p -c "$topdir/generate-rpm.sh" + $pkginst install -y -q $(find \( -name '*noarch.rpm' -o -name '*64.rpm' \)) +) + +su $chuser -p -c "$topdir/bootstrap.sh >/dev/null 2>/dev/null&& $topdir/configure -q" +mkdir -p "`pwd`/RPMBUILD" +rpmbuild -ba nemea.spec --define "_topdir `pwd`/RPMBUILD" +mkdir -p "`pwd`/rpms" +find -name *.rpm -not -path "./rpms/*" -exec mv {} rpms/ \; +chown -R $chuser rpms/ + diff --git a/unirec_fields.md b/unirec_fields.md new file mode 100644 index 00000000..a7405284 --- /dev/null +++ b/unirec_fields.md @@ -0,0 +1,140 @@ +# About this file +This file contains a list of UniRec fields collected from all parts of project (including git submodules). +The part of this file is generated automatically, so be careful during any editing. + + +# List of UniRec fields +| Field data type | Field name | Description | +| ----- | ----- | ----- | +| uint32 | ADDR_CNT | Number of probed destination addresses. | +| bytes | ARP_DST_HA | ARP destination hardware address. | +| bytes | ARP_DST_PA | ARP destination protocol address. | +| uint16 | ARP_HA_FORMAT | Type of ARP hardware address. | +| uint16 | ARP_OPCODE | Type of ARP message. | +| uint16 | ARP_PA_FORMAT | Type of ARP protocol address. | +| bytes | ARP_SRC_HA | ARP source hardware address. | +| bytes | ARP_SRC_PA | ARP source protocol address. | +| uint32 | BAR | Generic field containing 32bit unsigned integer (used for testing and example purposes). | +| uint32 | BAZ | Generic field containing 32bit unsigned integer (used for testing and example purposes). | +| uint8 | BLACKLIST_TYPE | Type of the used blacklist (spam, C&C, malware, etc.). | +| uint64 | BYTES | Total number of bytes transferred by the flow. | +| uint64 | CALLEE_CNT | VoIP Fraud Detection specific field for counting number of unique calling parties. | +| uint64 | CALLER_CNT | VoIP Fraud Detection specific field for counting number of unique called parties. | +| time | DETECTION_TIME | Timestamp of the detection of some event. | +| uint8 | DIR_BIT_FIELD | Bit field used for detemining incomming/outgoing flow. | +| uint8 | DIRECTION_FLAGS | Bit field for identification of flow direction. | +| uint16 | DNS_ANSWERS | Number of DNS answer records. | +| uint8 | DNS_BLACKLIST | ID of blacklist which contains suspicious domain name. | +| uint16 | DNS_CLASS | Class field from DNS question. | +| uint8 | DNS_DO | DNSSEC OK bit. | +| uint16 | DNS_ID | DNS transaction ID. | +| string | DNS_NAME | DNS question domain name. | +| uint16 | DNS_PSIZE | Requestor's payload size (RFC 6891). | +| uint16 | DNS_QTYPE | DNS question type field. | +| uint8 | DNS_RCODE | DNS response code field. | +| bytes | DNS_RDATA | Resource record specific data. | +| uint16 | DNS_RLENGTH | Length of DNS_RDATA. | +| uint32 | DNS_RR_TTL | DNS resource record TTL field. | +| uint64 | DST_BLACKLIST | Bit field of blacklists IDs which contains the destination address of the flow. | +| ipaddr | DST_IP | Destination IP address. | +| bytes | DST_MAC | Destination MAC address (L2). | +| uint16 | DST_PORT | Destination port of Transport layer (L4), e.g. TCP, UDP. | +| uint16 | ETHERTYPE | Protocol encapsulated in payload of L2 frame. | +| uint32 | EVENT_ID | Identification number of reported event. | +| uint32 | EVENT_SCALE | Attack intensity. | +| uint8 | EVENT_TYPE | Type of detected event. | +| uint64 | FLOWS | Number of flows, used after aggregation. | +| uint32 | FOO | Generic field containing 32bit unsigned integer (used for testing and example purposes). | +| string | HTTP_CONTENT_TYPE | Content type field from HTTP response message. | +| string | HTTP_HOST | Host field from HTTP request message. | +| string | HTTP_METHOD | Method field from HTTP request message. | +| string | HTTP_REFERER | Referer field from HTTP request message. | +| string | HTTP_REQUEST_HOST | Host field from HTTP request message. | +| string | HTTP_REQUEST_REFERER | Referer field from HTTP request message. | +| string | HTTP_REQUEST_URL | URL field from HTTP request message. | +| uint16 | HTTP_RESPONSE_CODE | Response code from HTTP response message. | +| string | HTTP_SDM_REQUEST_HOST | | +| string | HTTP_SDM_REQUEST_REFERER | | +| string | HTTP_SDM_REQUEST_URL | | +| string | HTTP_URL | URL field from HTTP request message. | +| string | HTTP_USER_AGENT | User agent field from HTTP request message. | +| uint64 | INVITE_CNT | VoIP Fraud Detection specific field for counting number INVITE requests observed. | +| ipaddr | IP | IP address. | +| uint64 | LINK_BIT_FIELD | Bit field where each bit marks whether a flow was captured on corresponding link. | +| string | NOTE | Generic string note. | +| uint32 | NTP_DELAY | NTP root delay. | +| uint32 | NTP_DISPERSION | NTP root dispersion. | +| uint8 | NTP_LEAP | NTP leap field. | +| uint8 | NTP_MODE | NTP mode field. | +| string | NTP_ORIG | NTP origin timestamp. | +| uint8 | NTP_POLL | NTP poll interval. | +| uint8 | NTP_PRECISION | NTP precision field. | +| string | NTP_RECV | NTP receive timestamp. | +| string | NTP_REF | NTP reference timestamp. | +| string | NTP_REF_ID | NTP reference ID. | +| string | NTP_SENT | NTP transmit timestamp. | +| uint8 | NTP_STRATUM | NTP stratum field. | +| uint8 | NTP_VERSION | NTP message version. | +| uint32 | PACKETS | Number of packets of the flow. | +| uint32 | PORT_CNT | Number of probed destination ports. | +| uint8 | PROTOCOL | Transport protocol identification (e.g. 6 for TCP, 17 for UDP, https://en.wikipedia.org/wiki/List_of_IP_protocol_numbers). | +| uint64 | REQ_BYTES | Number of bytes in a flow or in an interval (requests). | +| uint32 | REQ_FLOWS | Number of flows in an interval (requests). | +| uint32 | REQ_PACKETS | Number of packets in a flow or in an interval (requests). | +| uint64 | RSP_BYTES | Number of bytes in a flow or in an interval (responses). | +| uint32 | RSP_FLOWS | Number of flows in an interval (responses). | +| uint32 | RSP_PACKETS | Number of packets in a flow or in an interval (responses). | +| uint32 | SBFD_ATTEMPTS | Total number of attack messages received. | +| uint32 | SBFD_AVG_ATTEMPTS | Average count of attack messages received (relevant in case of user scan attack type). | +| time | SBFD_BREACH_TIME | Time of breach occurrence (can be 0 if the breach did not occur). | +| time | SBFD_CEASE_TIME | Time of the last attack message received. | +| uint64 | SBFD_EVENT_ID | A unique number of an alert. | +| time | SBFD_EVENT_TIME | Time of the first attack message received. | +| uint8 | SBFD_EVENT_TYPE | Type of an alert (0 - simple brute-force, 1 - distributed brute-force, 2 - user scan). | +| uint64 | SBFD_LINK_BIT_FIELD | Indicator of the particular monitoring probe. | +| uint8 | SBFD_PROTOCOL | Protocol used to perform the attack (TCP or UDP). | +| ipaddr | SBFD_SOURCE | IP address of the attacker. | +| ipaddr | SBFD_TARGET | IP address of the targeted server. | +| string | SBFD_USER | Name of the targeted user (can be empty in case of user scan alert). | +| string | SDM_CAPTURE_FILE_ID | ID of file for sdmcap. | +| string | SIP_CALLED_PARTY | SIP "To" header field. | +| string | SIP_CALL_ID | SIP "Call-ID" header field. | +| string | SIP_CALLING_PARTY | SIP "From" header field. | +| string | SIP_CSEQ | SIP "Cseq" header field. | +| uint16 | SIP_MSG_TYPE | SIP message type (see flow_meter documentation). | +| string | SIP_REQUEST_URI | SIP "Request-URI" header field. | +| uint16 | SIP_STATUS_CODE | SIP status code (see flow_meter documentation). | +| string | SIP_USER_AGENT | SIP "User-Agent" header field. | +| string | SIP_VIA | SIP "Via" header fied. | +| uint64 | SRC_BLACKLIST | Bit field of blacklists IDs which contains the source address of the flow. | +| ipaddr | SRC_IP | Source IP address. | +| bytes | SRC_MAC | Destination MAC address (L2). | +| uint16 | SRC_PORT | Source port of Transport layer (L4), e.g. TCP, UDP. | +| string | STR1 | | +| string | STR2 | | +| uint8 | TCP_FLAGS | TCP flags of all packets from the flow - flag bits are added bitwise. | +| time | TIME | Timestamp of packet capture. | +| time | TIME_FIRST | Timestamp of the first packet of the flow. | +| time | TIME_LAST | Timestamp of the last packet of the flow. | +| uint32 | TIMEOUT | Timeout for capture rule. (related to time machine). | +| uint8 | TOS | Type of service field from IP header. | +| uint8 | TTL | Time-To-Live value from IP header (https://en.wikipedia.org/wiki/Time_to_live). | +| uint32 | TUNNEL_CNT_PACKET | Number of packets which were recorded recognized like anomaly. | +| string | TUNNEL_DOMAIN | Anomaly domain name. | +| float | TUNNEL_PER_NEW_DOMAIN | Percent of new domains (searched just ones). | +| float | TUNNEL_PER_SUBDOMAIN | Percent of subdomains in the most used domain for tunnel type, for another anomaly it is percent of different domains. | +| uint8 | TUNNEL_TYPE | Type of detected event. | +| string | URL | | +| string | VOIP_FRAUD_COUNTRY_CODE | Country identification (ISO 3166, 2 char). | +| uint32 | VOIP_FRAUD_INVITE_COUNT | Total number of INVITE requests in the context of prefix examination. | +| uint32 | VOIP_FRAUD_PREFIX_EXAMINATION_COUNT | Number of unique SIP TO that was evaluated as prefix examination attack. | +| uint16 | VOIP_FRAUD_PREFIX_LENGTH | Prefix length of VOIP_FRAUD_SIP_TO (in prefix examination attack). | +| string | VOIP_FRAUD_SIP_FROM | SIP FROM header. | +| string | VOIP_FRAUD_SIP_TO | SIP TO header. | +| uint32 | VOIP_FRAUD_SUCCESSFUL_CALL_COUNT | Number of successful calls initiation to unique SIP TO in the context of prefix examination. | +| string | VOIP_FRAUD_USER_AGENT | SIP User-Agent header. | +| uint8 | WARDEN_TYPE | Type of event. | + +This table can be updated using `ur_dict_updater.sh` which completes missing fields. +Description of the fields must be filled manually. + diff --git a/ur_dict_updater.sh b/ur_dict_updater.sh new file mode 100755 index 00000000..2a2006c5 --- /dev/null +++ b/ur_dict_updater.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +DEBUG=1 +EXISTING_FILE="`mktemp`" +NEW_FILE="`mktemp`" +TARGET_FILE=unirec_fields.md +# Check for md file +if ! [ -s "$TARGET_FILE" ]; then + echo "Target MD file does not exist or its empty." + echo "Generating new one.." + echo "# About this file +This file contains a list of UniRec fields collected from all parts of project (including git submodules). +The part of this file is generated automatically, so be careful during any editing. +# List of UniRec fields + +" >"$TARGET_FILE" +fi + +if fgrep -q "# List of UniRec fields" "$TARGET_FILE"; then + existing=1 +else + existing=0 +fi + +if [ "$existing" -eq 1 ]; then + # we need to retrieve the list of fields before any updates + if [ "$DEBUG" -eq 1 ]; then + echo "Existing section, loading..." + fi + # Find the head of the list + sed -n "/^\# List of UniRec fields\s*$/,/^$/p" "$TARGET_FILE" | tail -n +3 | sed 's/| *\([^ ]*\) *| *\([^ ]*\) *| *\([^|]*\)|/\1 \2 \3/g' >"$EXISTING_FILE" + +fi + +if [ "$DEBUG" -eq 1 ]; then + echo "Searching for files with UR_FIELDS..." +fi + +find . \( -name '*.c' -o -name '*.h' -o -name '*.cpp' \) -exec grep -l "\s*UR_FIELDS\s*" {} \; | #tee /dev/stderr | +# remove line and block comments + xargs -I{} sed 's,\s*//.*$,,;:a; s%\(.*\)/\*.*\*/%\1%; ta; /\/\*/ !b; N; ba' {} | +# print contents of UR_FIELDS + sed -n '/^\s*UR_FIELDS\s*([^)]*$/,/)/p; /^\s*UR_FIELDS\s*([^)]*$/,/)/p' 2>/dev/null | +# clean output to get fields only + sed 's/^\s*UR_FIELDS\s*(\s*//g; s/)//g; s/,/\n/g; /^\s*$/d; s/^\s*//; s/\s\s*/ /g; s/\s\s*$//' | +# sort by name + sort -k2 -t' ' | uniq >> "$EXISTING_FILE" + +#merge temporary files together and sort them +cat "$EXISTING_FILE" |tail -n+2 |sort -bk2,2 -bk3,3r | +awk ' +BEGIN { + print "# List of UniRec fields" + print "| Field data type | Field name | Description |" + print "| ----- | ----- | ----- |" +} +/^..*$/ { + if (name != $2) { + type=$1 + name=$2 + desc=$3 + for (i=4; i<=NF; i++) { + desc=desc" "$i + } + print "| "type" | "name" | "desc" |" + } else if (type != $1) { + printf("Conflicting types (%s, %s) of UniRec field (%s)\n", type, $1, name) > "/dev/stderr"; + exit 1; + } +} +END { + print "" +}' > "$NEW_FILE" + +sed -i "/^\# List of UniRec fields\s*$/r $NEW_FILE +/^\# List of UniRec fields\s*$/,/^$/d;" unirec_fields.md + +rm "$EXISTING_FILE" "$NEW_FILE" 2> /dev/null + +exit 0; + + diff --git a/use-cases/Makefile.am b/use-cases/Makefile.am index 629ef8f6..2fcba959 100644 --- a/use-cases/Makefile.am +++ b/use-cases/Makefile.am @@ -1 +1 @@ -EXTRA_DIST=traffic-filtering-stats.sh logger-repeater.sh +EXTRA_DIST=traffic-filtering-stats.sh logger-repeater.sh flow_meter.sh basic_modules.sh diff --git a/use-cases/README.md b/use-cases/README.md index 70c54f24..64792702 100644 --- a/use-cases/README.md +++ b/use-cases/README.md @@ -4,7 +4,7 @@ Use-Cases - Example scripts This directory contains example scripts that show usage of Nemea modules. To get more information, run a script without parameters -to show description explaining functionality of a script. +to show description explaining functionality of the script. The source codes of scripts may be used for educational purposes to understand how to use the Nemea modules. diff --git a/use-cases/basic_modules.sh b/use-cases/basic_modules.sh new file mode 100755 index 00000000..ab763b93 --- /dev/null +++ b/use-cases/basic_modules.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Author: Marek Svepes +# Copyright (C) 2015 CESNET +# +# LICENSE TERMS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Company nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# ALTERNATIVELY, provided that this notice is retained in full, this +# product may be distributed under the terms of the GNU General Public +# License (GPL) version 2 or later, in which case the provisions +# of the GPL apply INSTEAD OF those given above. +# +# This software is provided ``as is'', and any express or implied +# warranties, including, but not limited to, the implied warranties of +# merchantability and fitness for a particular purpose are disclaimed. +# In no event shall the company or contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential +# damages (including, but not limited to, procurement of substitute +# goods or services; loss of use, data, or profits; or business +# interruption) however caused and on any theory of liability, whether +# in contract, strict liability, or tort (including negligence or +# otherwise) arising in any way out of the use of this software, even +# if advised of the possibility of such damage. +# + +if [ $# -eq 0 ] || [ $# -gt 1 ]; then + printf " Usage: + \t\"$0 nfdump_file\" + \tor + \t\"$0 default\" default input file with 6000 flows is used + + This script shows connection of the basic Nemea modules + (nfreader, merger, logger, logreplay and flowcounter). Three nfreader + modules read netflow data from nfdump file (specified by program argument) + and send them in unirec format to merger module which sends incoming + data from all input interfaces to one output interface. The data from merger + are stored into csv file by logger module. The csv file can be read by logreplay + module which sends them again in unirec format to flowcounter module. + Result of this module is number of received flows (it should be + number of flows from nfreader input file x 3). + + Step 1: + +----------+ + nfdump_file ---> | nfreader |----------- + +----------+ | + v + +----------+ +--------+ +--------+ + nfdump_file ---> | nfreader |----> | merger |----> | logger |---> file.csv + +----------+ +--------+ +--------+ + ^ + +----------+ | + nfdump_file ---> | nfreader |----------- + +----------+ + + Step 2: + +-----------+ +-------------+ + file.csv ---> | logreplay |---> | flowcounter |---> Result (number of flows) + +-----------+ +-------------+ + + Note: To run this script, ../modules/ repository has to be compiled. + \"libnf\" or \"libnfdump\" library is need for nfreader compilation.\n" + + exit 0 +fi + +# INPUT_FILE - Input file for nfreader modules (netflow data) +if [ "$1" = "default" ]; then + INPUT_FILE="./nfcap_6000_gener_flows.dat" +else + INPUT_FILE="$1" +fi + +if [ ! -r "$INPUT_FILE" ]; then + echo "Input file \"$INPUT_FILE\" cannot be opened. Exiting..." + exit 1 +fi + +# Logger output file (csv format) +LOGGER_OUTPUT_FILE="./logger_test_out" + +printf ">>> Starting 3 nfreaders, merger and logger...\n>Nfreaders sent:\n" +../modules/nfreader/nfdump_reader -i u:nfr1_test_out $INPUT_FILE & +../modules/nfreader/nfdump_reader -i u:nfr2_test_out $INPUT_FILE & +../modules/nfreader/nfdump_reader -i u:nfr3_test_out $INPUT_FILE & +../modules/merger/merger -n 3 -i u:nfr1_test_out,u:nfr2_test_out,u:nfr3_test_out,u:merger_test_out & +../modules/logger/logger -i u:merger_test_out -w $LOGGER_OUTPUT_FILE -t + +printf ">>> Starting logreplay and flowcounter...\n>Flowcounter received:\n" +../modules/logreplay/logreplay -i u:logrep_test_out -f $LOGGER_OUTPUT_FILE & +../modules/flowcounter/flowcounter -i u:logrep_test_out + +# Cleanup +rm -f $LOGGER_OUTPUT_FILE diff --git a/use-cases/flow_meter.sh b/use-cases/flow_meter.sh new file mode 100755 index 00000000..bc436191 --- /dev/null +++ b/use-cases/flow_meter.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Author: Jiri Havranek +# Copyright (C) 2015 CESNET +# +# LICENSE TERMS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Company nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# ALTERNATIVELY, provided that this notice is retained in full, this +# product may be distributed under the terms of the GNU General Public +# License (GPL) version 2 or later, in which case the provisions +# of the GPL apply INSTEAD OF those given above. +# +# This software is provided ``as is'', and any express or implied +# warranties, including, but not limited to, the implied warranties of +# merchantability and fitness for a particular purpose are disclaimed. +# In no event shall the company or contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential +# damages (including, but not limited to, procurement of substitute +# goods or services; loss of use, data, or profits; or business +# interruption) however caused and on any theory of liability, whether +# in contract, strict liability, or tort (including negligence or +# otherwise) arising in any way out of the use of this software, even +# if advised of the possibility of such damage. + + +# The purpose of this script is to show how to execute nemea module +# flow_meter and send it's output to another module (logger in this +# case). flow_meter is able to capture from interface (-I option) or +# file (-r option). In case of capture from interface it needs to be +# executed with root permission otherwise it will not be able to run. +# If you want to capture specific number of packets you can use -c +# option. + +# Check if interface is specified. +if [ "$1" = "" ]; then + echo "Specify the network interface." + echo "Usage: $0 interface-name" + exit 2 +fi + +if [ "$EUID" != 0 ]; then + echo "You should run this script as root." + exit 1 +fi + +flow_meter="../modules/flow_meter/flow_meter" +logger="../modules/logger/logger" + +# Check if modules are compiled. +if [ ! -e "$flow_meter" ]; then + echo "$flow_meter does not exist. Compile flow_meter first." + exit 3 +fi + +if [ ! -e "$logger" ]; then + echo "$logger does not exist. Compile logger first." + exit 3 +fi + +# Start capture from given interface. Quit after 10 packets are captured. +./"$flow_meter" -i u:my_socket -I "$1" -c 10 >/dev/null & + +# Start logger and quit when flow_meter exits. +./"$logger" -i u:my_socket -t + diff --git a/use-cases/logger-repeater.sh b/use-cases/logger-repeater.sh index 2f0cba27..d49e956f 100755 --- a/use-cases/logger-repeater.sh +++ b/use-cases/logger-repeater.sh @@ -50,7 +50,7 @@ fi if [ "$1" = "generate" ]; then file=`mktemp` cat > $file < +# Copyright (C) 2015 CESNET +# +# LICENSE TERMS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Company nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# ALTERNATIVELY, provided that this notice is retained in full, this +# product may be distributed under the terms of the GNU General Public +# License (GPL) version 2 or later, in which case the provisions +# of the GPL apply INSTEAD OF those given above. +# +# This software is provided ``as is'', and any express or implied +# warranties, including, but not limited to, the implied warranties of +# merchantability and fitness for a particular purpose are disclaimed. +# In no event shall the company or contributors be liable for any +# direct, indirect, incidental, special, exemplary, or consequential +# damages (including, but not limited to, procurement of substitute +# goods or services; loss of use, data, or profits; or business +# interruption) however caused and on any theory of liability, whether +# in contract, strict liability, or tort (including negligence or +# otherwise) arising in any way out of the use of this software, even +# if advised of the possibility of such damage. +# + +if [ $# -eq 0 ]; then + echo "$0 nfdump_file... + +This script expects one or more NFDUMP files. It replays the flows +and sends them into unirecfilter. Unirecfilter has 4 output IFCs +that are connected with 4 flowcounters. + +As a result, 4 statistics are computed: + # of flows with src or dst port 25 + # of flows with src or dst port 80 + # of flows with src or dst port 22 + # of flows that don'ŧ have ports 22, 25, or 80 +These numbers are summed. +" + exit 0 +fi + +# Prepare files and filenames +services="ssh web smtp other" +conffile=`mktemp` + +nfifc="u:cejkat2" +urifc=$nfifc +for i in $services; do + urifc="$urifc,u:$i" + flcnt="$flcnt $i" +done + +cat > $conffile < /dev/null& +../modules/unirecfilter/unirecfilter -i $urifc -f $conffile > /dev/null& +for i in $services; do + ( ../modules/flowcounter/flowcounter -i "u:$i" | sed -n 's/Flows:\s*\([0-1]*\)/\1/p' > ${i}; )& + pids="$pids $!" +done + +echo "Running, wait a moment please..." + +# Wait until children die +for i in $pids; do + wait $i +done + +# Statistics: +awk '{s+=$1; sub(".cnt", "", FILENAME); print FILENAME":", $1;} END{print "Total:", s;}' $flcnt + +# Cleanup +rm $flcnt $conffile + diff --git a/vagrant/CentOS7/README.md b/vagrant/CentOS7/README.md new file mode 100644 index 00000000..88f2534d --- /dev/null +++ b/vagrant/CentOS7/README.md @@ -0,0 +1,29 @@ +Installation of NEMEA using Vagrant on CentOS7 +============================================== + +If you do not have a vagrant box for CentOS7 box yet, use: + +``` +vagrant box add centos/7 --provider=virtualbox +``` + +Installation - Final Step +========================= + +Start the VM Installation using (this will take few minutes): +``` +vagrant up +``` + +Once the installation is complete, SSH into the VM: +``` +vagrant ssh +``` + +Content of VM +============= + +* all NEMEA packages installed from RPM from homeproj.cesnet.cz +* NEMEA-Dashboard +* ipfixcol with ipfixcol-unirec-output + diff --git a/vagrant/CentOS7/Vagrantfile b/vagrant/CentOS7/Vagrantfile new file mode 100644 index 00000000..a23f2076 --- /dev/null +++ b/vagrant/CentOS7/Vagrantfile @@ -0,0 +1,54 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +load '../common/nemea-dashboard.rb' + +$tools = <