diff --git a/CMakeLists.txt b/CMakeLists.txt index 114bee3c..12d29831 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.7.2) +cmake_minimum_required(VERSION 3.12) project(darwin) @@ -29,6 +29,8 @@ if (NOT DEFINED FILTER) CONTENT_INSPECTION BUFFER YARA + PYTHON + TEST ) else (NOT DEFINED FILTER) set( @@ -72,12 +74,15 @@ set( include_directories( toolkit + toolkit/thread-pool-cpp/include/ samples/base + samples/base/network samples/ ${HIREDIS_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ) +add_compile_options(-Wno-unknown-pragmas) #################### # CORE SOURCES # @@ -96,9 +101,22 @@ set( samples/base/ThreadGroup.cpp samples/base/ThreadGroup.hpp samples/base/AlertManager.cpp samples/base/AlertManager.hpp - samples/base/Server.cpp samples/base/Server.hpp + samples/base/DarwinPacket.cpp samples/base/DarwinPacket.hpp + samples/base/network/ANextFilterConnector.cpp samples/base/network/ANextFilterConnector.hpp + samples/base/network/UnixNextFilterConnector.cpp samples/base/network/UnixNextFilterConnector.hpp + samples/base/network/TcpNextFilterConnector.cpp samples/base/network/TcpNextFilterConnector.hpp + samples/base/network/UdpNextFilterConnector.cpp samples/base/network/UdpNextFilterConnector.hpp + + samples/base/network/AServer.cpp samples/base/network/AServer.hpp + samples/base/network/UnixServer.cpp samples/base/network/UnixServer.hpp + samples/base/network/TcpServer.cpp samples/base/network/TcpServer.hpp + samples/base/network/UdpServer.cpp samples/base/network/UdpServer.hpp samples/base/Manager.cpp samples/base/Manager.hpp - samples/base/Session.cpp samples/base/Session.hpp + samples/base/network/ASession.cpp samples/base/network/ASession.hpp + samples/base/network/UnixSession.cpp samples/base/network/UnixSession.hpp + samples/base/network/TcpSession.cpp samples/base/network/TcpSession.hpp + samples/base/network/UdpSession.cpp samples/base/network/UdpSession.hpp + samples/base/ATask.cpp samples/base/ATask.hpp toolkit/Network.cpp toolkit/Network.hpp toolkit/Validators.cpp toolkit/Validators.hpp @@ -168,3 +186,7 @@ endif() if("VAML" IN_LIST FILTERS) include(fvaml) endif() + +if("PYTHON" IN_LIST FILTERS) +include(fpython) +endif() diff --git a/cmake/fpython.cmake b/cmake/fpython.cmake new file mode 100644 index 00000000..773794ef --- /dev/null +++ b/cmake/fpython.cmake @@ -0,0 +1,32 @@ +set(PYTHON_NAME darwin_python) + +####################### +# FILTER DEPENDENCIES # +####################### + +find_package(Python3 REQUIRED COMPONENTS Development) + +include_directories(SYSTEM ${Python3_INCLUDE_DIRS}) +link_directories(${Python3_LIBRARY_DIRS}) + +################### +# EXECUTABLE # +################### + + +add_executable( + ${PYTHON_NAME} + ${DARWIN_SOURCES} + samples/fpython/PythonTask.cpp samples/fpython/PythonTask.hpp + samples/fpython/Generator.cpp samples/fpython/Generator.hpp +) + +target_link_libraries( + ${PYTHON_NAME} + ${DARWIN_LIBRARIES} + ${Python3_LIBRARIES} + # DL libs are used for loading shared objects + ${CMAKE_DL_LIBS} +) + +target_include_directories(${PYTHON_NAME} PUBLIC samples/fpython/) diff --git a/manager/HeartBeat.py b/manager/HeartBeat.py index f5e7c768..264ab13d 100644 --- a/manager/HeartBeat.py +++ b/manager/HeartBeat.py @@ -7,7 +7,7 @@ __doc__ = 'The heartbeat functions and main class' from os import kill, access, F_OK - +import socket class HeartBeat: """ @@ -15,7 +15,7 @@ class HeartBeat: """ @staticmethod - def check_socket(file): + def check_unix_socket(file): try: if access(file, F_OK): return True @@ -23,6 +23,30 @@ def check_socket(file): except Exception: return False + def check_network_socket(filter_network): + if filter_network['socket_type'] == 'UNIX': + return HeartBeat.check_unix_socket(filter_network['address_path']) + + elif filter_network['socket_type'] == 'TCP': + # parse ip address field + splitted_address = filter_network['address_path'].rsplit(':', 1) + if '[' in splitted_address[0]: # ipv6 address + host_address = splitted_address[0][1:-1] + else: + host_address = splitted_address[0] + host_port = int(splitted_address[1]) + #test connection to socket + if ':' in host_address: # ipv6 + s = socket.socket(socket.AF_INET6) + else: + s = socket.socket(socket.AF_INET) + s.settimeout(2) + res = s.connect_ex((host_address,host_port)) + s.close() + return res == 0 + else: #No check for UDP + return True + @staticmethod def check_pid_file(file): """ diff --git a/manager/Services.py b/manager/Services.py index 1fe01dcb..54f22e73 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -50,9 +50,10 @@ def start_all(self): self.stop_one(filter, no_lock=True) self.clean_one(filter, no_lock=True) else: - logger.debug("Linking UNIX sockets...") - filter['status'] = psutil.STATUS_RUNNING - call(['ln', '-s', filter['socket'], filter['socket_link']]) + if filter['network']['socket_type'] == 'UNIX': + logger.debug("Linking UNIX sockets...") + filter['status'] = psutil.STATUS_RUNNING + call(['ln', '-s', filter['socket'], filter['socket_link']]) def rotate_logs_all(self): """ @@ -89,6 +90,14 @@ def _build_cmd(filt): cmd = [filt['exec_path']] + # Flags MUST be before positional arguments as the parsing on HardenedBSD is not done on all the arguments + # On BSD getopt stops at the first argument which is not in the specified flags + if filt['network']['socket_type'] == 'UDP': + cmd.append('-u') + + if filt['next_filter_network']['socket_type'] == 'UDP': + cmd.append('-v') + try: if filt['log_level'] not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DEVELOPER"]: logger.warning( @@ -99,15 +108,15 @@ def _build_cmd(filt): cmd.append(filt['log_level']) except KeyError: pass - + cmd += [ filt['name'], - filt['socket'], + filt['network']['address_path'], filt['config_file'], filt['monitoring'], filt['pid_file'], filt['output'], - filt['next_filter_unix_socket'], + filt['next_filter_network']['address_path'], str(filt['nb_thread']), str(filt['cache_size']), str(filt['threshold']), @@ -369,6 +378,9 @@ def update(self, names, prefix, suffix): prefix=prefix, suffix=suffix, name=n, extension=new[n]['extension'] ) + # TODO Handle TCP case? + if new[n]['network']['socket_type'] == 'UNIX': + new[n]['network']['address_path'] = new[n]['socket'] new[n]['monitoring'] = '{prefix}/sockets{suffix}/{name}_mon{extension}.sock'.format( prefix=prefix, suffix=suffix, @@ -531,7 +543,7 @@ def _wait_process_ready(content): status = "Process not running" continue - if not HeartBeat.check_socket(content['monitoring']): + if not HeartBeat.check_unix_socket(content['monitoring']): status = "Monitoring socket not created" continue @@ -550,7 +562,7 @@ def _wait_process_ready(content): continue if "running" in resp: - if not HeartBeat.check_socket(content['socket']): + if not HeartBeat.check_network_socket(content['network']): status = "Main socket not created" continue else: @@ -574,10 +586,10 @@ def hb_one(self, filter): if not HeartBeat.check_process(pid): raise Exception('Process not running') - if not HeartBeat.check_socket(filter['socket']): + if not HeartBeat.check_network_socket(filter['network']): raise Exception('Socket not accessible') - if not HeartBeat.check_socket(filter['monitoring']): + if not HeartBeat.check_unix_socket(filter['monitoring']): raise Exception('Monitoring socket not accessible') except Exception as e: diff --git a/manager/config.py b/manager/config.py index 405f6cdf..158d7637 100644 --- a/manager/config.py +++ b/manager/config.py @@ -189,10 +189,35 @@ def set_defaults(validator, properties, instance, schema): "enum": ["NONE", "RAW", "LOG", "PARSED"], "default": "NONE" }, - "next_filter": {"type": "string"}, + "next_filter": { + "OneOf":[ + {"type": "string"}, + {"type": "object", + "properties": { + "socket_type":{ + "type":"string", + "enum": ["NONE", "UNIX", "TCP", "UDP"], + "default":"NONE" + }, + "address_path": {"type": "string"} + } + } + ] + }, "threshold": { "type": "integer", "default": 100 + }, + "network": { + "type": "object", + "properties": { + "socket_type":{ + "type":"string", + "enum": ["UNIX", "TCP", "UDP"], + "default":"UNIX" + }, + "address_path": {"type": "string"} + } } }, "required": ["name", "exec_path", "config_file"], @@ -268,19 +293,34 @@ def complete_filters_conf(prefix, suffix): filter['failures'] = 0 filter['extension'] = '.1' filter['pid_file'] = '{prefix}/run{suffix}/{filter}{extension}.pid'.format(prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension']) - - if not filter['next_filter']: - filter['next_filter_unix_socket'] = 'no' - else: - filter['next_filter_unix_socket'] = '{prefix}/sockets{suffix}/{next_filter}.sock'.format( - prefix=prefix, suffix=suffix, - next_filter=filter['next_filter'] - ) - + filter['socket'] = '{prefix}/sockets{suffix}/{filter}{extension}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension']) filter['socket_link'] = '{prefix}/sockets{suffix}/{filter}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name']) + if 'network' not in filter: + filter['network'] = { "socket_type":"UNIX", "address_path":filter['socket'] } + elif filter['network']['socket_type'] == "UNIX": + filter['network']['address_path'] = filter['socket'] + filter['monitoring'] = '{prefix}/sockets{suffix}/{filter}_mon{extension}.sock'.format( prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension'] ) + + # Next filter is setup with a second loop (we need network information already setup) + for _, filter in filters.items(): + if 'next_filter' not in filter or filter['next_filter'] == '': + filter['next_filter_network'] = { "socket_type":"NONE", "address_path":"no" } + else: + if isinstance(filter['next_filter'], str): + #check other filters + modified = False + for _, other_filter in filters.items(): + if filter['next_filter'] == other_filter['name']: + filter['next_filter_network'] = other_filter['network'] + modified = True + if not modified: + raise ConfParseError("Filter '{}' had next_filter configured to '{}' but it was not found in the configuration" + .format(filter['name'], filter['next_filter'])) + else: + filter['next_filter_network'] = filter['next_filter'] diff --git a/manager/manager.py b/manager/manager.py index 44270d9a..a7798081 100644 --- a/manager/manager.py +++ b/manager/manager.py @@ -33,7 +33,7 @@ def create_dirs(dirs, prefix, suffix): for d in dirs: path = '{}/{}{}'.format(prefix, d, suffix) if not os.path.exists(path): - os.mkdir(path) + os.makedirs(path, exist_ok=True) # Argparse parser = argparse.ArgumentParser() diff --git a/samples/base/AGenerator.cpp b/samples/base/AGenerator.cpp index 57c8a6fa..773ab3ed 100644 --- a/samples/base/AGenerator.cpp +++ b/samples/base/AGenerator.cpp @@ -13,6 +13,17 @@ #include "AGenerator.hpp" #include "AlertManager.hpp" +AGenerator::AGenerator(size_t nb_task_threads): _threadPool{GetThreadPoolOptions(nb_task_threads)} +{ + ; +} + +tp::ThreadPoolOptions AGenerator::GetThreadPoolOptions(size_t nb_task_threads){ + auto opts = tp::ThreadPoolOptions(); + opts.setThreadCount(nb_task_threads); + return opts; +} + bool AGenerator::Configure(std::string const& configFile, const std::size_t cache_size) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("AGenerator:: Configuring..."); @@ -116,3 +127,7 @@ bool AGenerator::ReadConfig(const std::string &configuration_file_path) { conf_file_stream.close(); return true; } + +tp::ThreadPool& AGenerator::GetTaskThreadPool() { + return this->_threadPool; +} diff --git a/samples/base/AGenerator.hpp b/samples/base/AGenerator.hpp index aa975c9e..657b61e7 100644 --- a/samples/base/AGenerator.hpp +++ b/samples/base/AGenerator.hpp @@ -11,25 +11,27 @@ #include #include #include +#include -#include "Session.hpp" +#include "ATask.hpp" #include "../toolkit/rapidjson/document.h" class AGenerator { public: - AGenerator() = default; + AGenerator(size_t nb_task_threads); virtual ~AGenerator() = default; + // Methods to be implemented by the child public: - /// Create a new task. /// - /// \param socket The Session's socket. - /// \param manager The Session manager. - /// \return A pointer to a new session. - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept = 0; + /// \brief Create a Task object + /// + /// \param s a shred pointer to the sessions creating the task + /// \return std::shared_ptr a shared pointer to the created task + /// + virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept = 0; + virtual bool ConfigureNetworkObject(boost::asio::io_context &context); protected: @@ -59,6 +61,8 @@ class AGenerator { Configure(std::string const& configFile, const std::size_t cache_size) final; + virtual tp::ThreadPool& GetTaskThreadPool() final; + private: /// Open and read the configuration file. /// Try to load the json format of the configuration. @@ -76,7 +80,18 @@ class AGenerator { virtual bool ExtractCustomAlertingTags(const rapidjson::Document &configuration, std::string& tags); + /// + /// \brief Get the configuration object of the task thread pool + /// + /// \param nb_task_threads number of workers (threads) to spawn for the tasks + /// \return tp::ThreadPoolOptions + /// + static tp::ThreadPoolOptions GetThreadPoolOptions(size_t nb_task_threads); + protected: std::shared_ptr> _cache; //!< The cache for already processed request std::mutex _cache_mutex; + + tp::ThreadPool _threadPool; + }; diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp new file mode 100755 index 00000000..7cf1c720 --- /dev/null +++ b/samples/base/ATask.cpp @@ -0,0 +1,107 @@ +#include +#include +#include + +#include "Logger.hpp" +#include "Manager.hpp" +#include "ASession.hpp" +#include "ATask.hpp" +#include "errors.hpp" +#include "DarwinPacket.hpp" + +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" + +namespace darwin { + + ATask::ATask(std::string name, + std::shared_ptr> cache, + std::mutex& cache_mutex, session_ptr_t s, + DarwinPacket& packet) + : _filter_name(name), _cache{cache}, _cache_mutex{cache_mutex}, _s{s}, + _packet{std::move(packet)}, _threshold{_s->GetThreshold()} + { + ; + } + + void ATask::SetStartingTime() { + _starting_time = std::chrono::high_resolution_clock::now(); + } + + double ATask::GetDurationMs() { + return std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - _starting_time + ).count() / ((double)1000); + } + + xxh::hash64_t ATask::GenerateHash() { + // could be easily overridden in the children + return xxh::xxhash<64>(_packet.GetBody()); + } + + void ATask::SaveToCache(const xxh::hash64_t &hash, const unsigned int certitude) const { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("SaveToCache:: Saving certitude " + std::to_string(certitude) + " to cache"); + std::unique_lock lck{_cache_mutex}; + _cache->insert(hash, certitude); + } + + bool ATask::GetCacheResult(const xxh::hash64_t &hash, unsigned int& certitude) { + DARWIN_LOGGER; + boost::optional cached_certitude; + + { + std::unique_lock lck{_cache_mutex}; + cached_certitude = _cache->get(hash); + } + + if (cached_certitude != boost::none) { + certitude = cached_certitude.get(); + DARWIN_LOG_DEBUG("GetCacheResult:: Already processed request. Cached certitude is " + + std::to_string(certitude)); + + return true; + } + + return false; + } + + std::string ATask::GetFilterName() { + return _filter_name; + } + + bool ATask::ParseBody() { + DARWIN_LOGGER; + try { + _body.Parse(_packet.GetBody().c_str()); + + if (!_body.IsArray()) { + DARWIN_LOG_ERROR("ATask:: ParseBody: You must provide a list"); + return false; + } + } catch (...) { + DARWIN_LOG_CRITICAL("ATask:: ParseBody: Could not parse raw body"); + return false; + } + // DARWIN_LOG_DEBUG("ATask:: ParseBody:: parsed body : " + _raw_body); + return true; + } + + void ATask::run() { + DARWIN_LOGGER; + if( ! this->ParseBody()) { + DARWIN_LOG_DEBUG("ASession::ReadBodyCallback Something went wrong while parsing the body"); + _s->SendErrorResponse("Error receiving body: Something went wrong while parsing the body", DARWIN_RESPONSE_CODE_REQUEST_ERROR); + return; + } + (*this)(); + + _packet.SetFilterCode(this->GetFilterCode()); + auto& body = _packet.GetMutableBody(); + body.clear(); + body.append(_response_body); + _s->SendNext(_packet); + } + +} \ No newline at end of file diff --git a/samples/base/ATask.hpp b/samples/base/ATask.hpp new file mode 100755 index 00000000..6e925246 --- /dev/null +++ b/samples/base/ATask.hpp @@ -0,0 +1,117 @@ +/// +/// \file ATask.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Task to be extended by filter's tasks +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" +#include "../../toolkit/rapidjson/document.h" +#include "../../toolkit/rapidjson/writer.h" +#include "../../toolkit/rapidjson/stringbuffer.h" +#include "ASession.fwd.hpp" +#include "DarwinPacket.hpp" +#include "config.hpp" +#include "Time.hpp" + +namespace darwin { + + class ATask : public std::enable_shared_from_this { + public: + /// + /// \brief Construct a new ATask object + /// + /// \param name name of the filter + /// \param cache shared pointer to the cache to use + /// \param cache_mutex mutex of the cache + /// \param session shared pointer to the session spawning the task + /// \param packet DarwinPacket parsed from the session, moved to the task + /// + ATask(std::string name, + std::shared_ptr> cache, + std::mutex& cache_mutex, session_ptr_t session, + DarwinPacket& packet); + + virtual ~ATask() = default; + + // Make the ATask non copyable & non movable + ATask(ATask const&) = delete; + + ATask(ATask const&&) = delete; + + ATask& operator=(ATask const&) = delete; + + ATask& operator=(ATask const&&) = delete; + + /// Entry point of the execution. + /// MUST be overloaded. + /// WARNING This method will be executed by a thread. + virtual void operator()() = 0; + + void run(); + + protected: + /// Return filter code. + virtual long GetFilterCode() noexcept = 0; + + /// Generate the hash + virtual xxh::hash64_t GenerateHash(); + + /// Set starting time + void SetStartingTime(); + + /// Get total time elapsed since the starting time + double GetDurationMs(); + + /// Save the result to cache + virtual void SaveToCache(const xxh::hash64_t &hash, unsigned int certitude) const; + + /// Get the result from the cache + virtual bool GetCacheResult(const xxh::hash64_t &hash, unsigned int &certitude); + + /// Get the name of the filter + std::string GetFilterName(); + + /// Parse a line in the body. + /// This function should be implemented in each child, + /// and should be called between every entry to check validity (no early parsing). + virtual bool ParseLine(rapidjson::Value &line) = 0; + + /// Parse the body received. + /// This is the default function, trying to get a JSON array from the _raw_body, + /// if you wan't to recover something else (full/complex JSON, custom data), + /// override the function in the child class. + virtual bool ParseBody() ; + + private: + std::string _filter_name; //!< name of the filter + + protected: + //!< Cache received from the Generator + std::shared_ptr> _cache; + std::mutex& _cache_mutex; + + std::chrono::time_point _starting_time; + + bool _is_cache = false; + + session_ptr_t _s; //!< associated session + + DarwinPacket _packet; //!< Ref to the Header received from the session. + rapidjson::Document _body; //!< Ref to the Body received from session (if any). + std::string _response_body; //!< Ref to the body to send back to the client + + std::size_t _threshold; //!< Threshold, overriden at creation by the session + + }; + +} \ No newline at end of file diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index baa23bc1..69ebc396 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -15,16 +15,23 @@ #include "Generator.hpp" #include "Logger.hpp" -#include "Server.hpp" +#include "UnixServer.hpp" +#include "TcpServer.hpp" +#include "UdpServer.hpp" #include "Core.hpp" #include "Stats.hpp" +#include "StringUtils.hpp" +#include "UnixNextFilterConnector.hpp" +#include "TcpNextFilterConnector.hpp" +#include "UdpNextFilterConnector.hpp" + +#include namespace darwin { Core::Core() - : _name{}, _socketPath{}, _modConfigPath{}, _monSocketPath{}, - _pidPath{}, _nbThread{0}, - _threadpool{}, daemon{true} {} + : _name{}, _modConfigPath{}, _monSocketPath{}, + _pidPath{}, _nbThread{0}, _socketPath{}, daemon{true} {} int Core::run() { DARWIN_LOGGER; @@ -32,47 +39,76 @@ namespace darwin { SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::starting); + std::unique_ptr server; + try { Monitor monitor{_monSocketPath}; std::thread t{std::bind(&Monitor::Run, std::ref(monitor))}; + std::thread t_nextfilter; + if(_next_filter_connector){ + t_nextfilter = std::thread{std::bind(&ANextFilterConnector::Run, std::ref(*_next_filter_connector))}; + } SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::configuring); - Generator gen{}; + Generator gen{_nbThread}; if (not gen.Configure(_modConfigPath, _cacheSize)) { DARWIN_LOG_CRITICAL("Core:: Run:: Unable to configure the filter"); raise(SIGTERM); - t.join(); + if(t_nextfilter.joinable()) + t_nextfilter.join(); + if(t.joinable()) + t.join(); return 1; } DARWIN_LOG_DEBUG("Core::run:: Configured generator"); + switch(_net_type) { + case network::NetworkSocketType::Unix: + DARWIN_LOG_DEBUG("Core::run:: Unix socket configured on path " + _socketPath); + server = std::make_unique(_socketPath, _output, _threshold, gen); + break; + case network::NetworkSocketType::Tcp: + DARWIN_LOG_DEBUG("Core::run:: TCP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); + server = std::make_unique(_net_address, _net_port, _output, _threshold, gen); + break; + case network::NetworkSocketType::Udp: + DARWIN_LOG_DEBUG("Core::run:: UDP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); + server = std::make_unique(_net_address, _net_port, _output, _threshold, gen); + break; + default: + DARWIN_LOG_CRITICAL("Core:: Run:: Network Configuration problem"); + raise(SIGTERM); + if(t_nextfilter.joinable()) + t_nextfilter.join(); + if (t.joinable()) + t.join(); + return 1; + } + try { - Server server{_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen}; - if (not gen.ConfigureNetworkObject(server.GetIOContext())) { + if (not gen.ConfigureNetworkObject(server->GetIOContext())) { raise(SIGTERM); - t.join(); + if(t_nextfilter.joinable()) + t_nextfilter.join(); + if (t.joinable()) + t.join(); return 1; } SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::running); - DARWIN_LOG_DEBUG("Core::run:: Creating threads..."); - // Starting from 1 because the current process will run - // the io_context too. - for (std::size_t i = 1; i < _nbThread; ++i) { - _threadpool.CreateThread( - std::bind(&Server::Run, std::ref(server)) - ); - } - server.Run(); + DARWIN_LOG_DEBUG("Core::run:: Launching server..."); + server->Run(); DARWIN_LOG_DEBUG("Core::run:: Joining threads..."); - _threadpool.JoinAll(); - server.Clean(); + + server->Clean(); } catch (const std::exception& e) { DARWIN_LOG_CRITICAL(std::string("Core::run:: Cannot open unix socket: ") + e.what()); ret = 1; raise(SIGTERM); } DARWIN_LOG_DEBUG("Core::run:: Joining monitoring thread..."); + if(t_nextfilter.joinable()) + t_nextfilter.join(); if (t.joinable()) t.join(); } catch (const std::exception& e) { @@ -101,12 +137,16 @@ namespace darwin { int opt; std::string log_level; + bool is_udp = false; + bool is_next_filter_udp = false; // OPTIONS log.setLevel(logger::Warning); // Log level by default opt = -1; - while((opt = getopt(ac, av, ":l:h")) != -1) + // Flags MUST be before positional arguments as the parsing on HardenedBSD is not done on all the arguments + // On BSD getopt stops at the first argument which is not in the specified flags + while((opt = getopt(ac, av, ":l:huv")) != -1) { - DARWIN_LOG_DEBUG("OPT : " + std::to_string(opt)); + DARWIN_LOG_DEBUG(std::string("OPT : ") + (char)opt); DARWIN_LOG_DEBUG("OPTIND : " + std::to_string(optind)); switch(opt) { @@ -130,6 +170,12 @@ namespace darwin { DARWIN_LOG_ERROR("Core:: Program Arguments:: Unknown option"); Core::Usage(); return false; + case 'u': + is_udp = true; + break; + case 'v': + is_next_filter_udp = true; + break; } } @@ -143,12 +189,15 @@ namespace darwin { // MANDATORY ARGUMENTS log.setName(av[optind]); _name = av[optind]; - _socketPath = av[optind + 1]; + if (! network::ParseSocketAddress(std::string(av[optind + 1]), is_udp, _net_type, _net_address, _net_port, _socketPath)) + return false; _modConfigPath = av[optind + 2]; _monSocketPath = av[optind + 3]; _pidPath = av[optind + 4]; _output = av[optind + 5]; - _nextFilterUnixSocketPath = av[optind + 6]; + + if (!SetNextFilterConnector(std::string(av[optind + 6]), is_next_filter_udp)) + return false; if (!GetULArg(_nbThread, av[optind + 7])) return false; if (!GetULArg(_cacheSize, av[optind + 8])) @@ -159,6 +208,39 @@ namespace darwin { return true; } + bool Core::SetNextFilterConnector(std::string const& path_address, bool is_udp) { + DARWIN_LOGGER; + if(path_address == "no"){ + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: No Next Filter set"); + return true; + } + network::NetworkSocketType net_type; + std::string path; + int port; + boost::asio::ip::address addr; + if (! network::ParseSocketAddress(path_address, is_udp, net_type, addr, port, path)) + return false; + + switch(net_type) { + case network::NetworkSocketType::Unix: + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: Next Filter UNIX set on " + path); + _next_filter_connector = std::make_unique(path); + return true; + case network::NetworkSocketType::Tcp: + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: Next Filter TCP set on " + addr.to_string() + ":" + std::to_string(port)); + _next_filter_connector = std::make_unique(addr, port); + return true; + case network::NetworkSocketType::Udp: + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: Next Filter UDP set on " + addr.to_string() + ":" + std::to_string(port)); + _next_filter_connector = std::make_unique(addr, port); + return true; + default: + DARWIN_LOG_CRITICAL("Core:: SetNextFilterConnector:: Next Filter Configuration error : unrecognized type"); + return false; + } + return false; + } + void Core::Usage() { std::cout << "Usage: ./darwin filter_name socket_path config_file" << " monitoring_socket_path pid_file output next_filter_socket_path nb_thread " @@ -247,4 +329,16 @@ namespace darwin { return this->_name; } + bool Core::IsDaemon() const { + return this->daemon; + } + + ANextFilterConnector* Core::GetNextFilterconnector() { + return _next_filter_connector.get(); + } + + const std::string& Core::GetName() const { + return this->_name; + } + } diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 57f6aa74..166536a9 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -11,6 +11,10 @@ #include #include "Monitor.hpp" #include "ThreadGroup.hpp" +#include "Network.hpp" +#include "ANextFilterConnector.hpp" + +#include #ifndef PID_PATH # define PID_PATH "/var/run/darwin/" @@ -78,22 +82,49 @@ namespace darwin { /// \return true on success, false otherwise. bool SetLogLevel(std::string level); + /// + /// \brief returns the daemon attribute, it is set ine the Configure method + /// + /// \return true If the filter is configured to be ran as deamon + bool IsDaemon() const; + + /// + /// \brief Set the Next Filter Connector object + /// + /// \param path_address string to be parsed, can be a socket path, + /// an ipv4 address with the port specified 'x.x.x.x:pppp' or + /// an ipv6 address with the port specified '[xx:xx:xx:xx:xx]:ppp' + /// \param is_udp true if we should use udp, the parameter is unused if path_address is a socket + /// \return true if there was no error setting up the Next FilterConnector + /// + bool SetNextFilterConnector(std::string const& path_address, bool is_udp); + + /// + /// \brief Get a pointer to the NextFilterConnector, it may be a nullptr if no NextFilter was set during the configuration + /// If it is not nullptr, it has the lifetime of the darwin::Core instance (most of the duration of the program) + /// + /// \return ANextFilterConnector* a pointer to the next filter connector or nullptr + ANextFilterConnector* GetNextFilterconnector(); + + const std::string& GetName() const; + private: std::string _name; - std::string _socketPath; std::string _modConfigPath; std::string _monSocketPath; std::string _pidPath; - std::string _nextFilterUnixSocketPath; std::string _output; std::size_t _nbThread; std::size_t _cacheSize; std::size_t _threshold; - ThreadGroup _threadpool; + std::unique_ptr _next_filter_connector; + + network::NetworkSocketType _net_type; + std::string _socketPath; + boost::asio::ip::address _net_address; + int _net_port; - public: - // TODO Maybe a getter is a better idea... bool daemon; }; } diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp new file mode 100644 index 00000000..ebc3846f --- /dev/null +++ b/samples/base/DarwinPacket.cpp @@ -0,0 +1,191 @@ +#include "DarwinPacket.hpp" +#include +#include +#include +#include +#include "../../toolkit/rapidjson/writer.h" +#include "../../toolkit/rapidjson/stringbuffer.h" +#include "Logger.hpp" + +namespace darwin { + + DarwinPacket::DarwinPacket( + darwin_packet_type type, + darwin_filter_response_type response, + long filter_code, + unsigned char event_id[16], + size_t certitude_size, + size_t body_size) + :_type{type}, _response{response}, _filter_code{filter_code}, + _parsed_certitude_size{certitude_size}, _parsed_body_size{body_size} + { + std::memcpy(this->_evt_id, event_id, 16); + + } + + DarwinPacket::DarwinPacket(DarwinPacket&& other) noexcept + : _type{std::move(other._type)}, _response{std::move(other._response)}, _filter_code{std::move(other._filter_code)}, + _parsed_certitude_size{std::move(other._parsed_certitude_size)}, _parsed_body_size{std::move(other._parsed_body_size)}, + _certitude_list{std::move(other._certitude_list)}, _body{std::move(other._body)}, + _parsed_body{other._parsed_body.release()}, _logs{std::move(other._logs)} + { + std::memcpy(this->_evt_id, other._evt_id, sizeof(this->_evt_id)); + std::memset(other._evt_id, 0, sizeof(other._evt_id)); + } + + DarwinPacket& DarwinPacket::operator=(DarwinPacket&& other) noexcept { + this->clear(); + _type = std::move(other._type); + _response = std::move(other._response); + _filter_code = std::move(other._filter_code); + std::memcpy(this->_evt_id, other._evt_id, sizeof(this->_evt_id)); + std::memset(other._evt_id, 0, sizeof(other._evt_id)); + _parsed_certitude_size = std::move(other._parsed_certitude_size); + _parsed_body_size = std::move(other._parsed_body_size); + _certitude_list = std::move(other._certitude_list); + _body = std::move(other._body); + _parsed_body.reset(other._parsed_body.release()); + _logs = std::move(other._logs); + return *this; + } + + DarwinPacket::DarwinPacket(darwin_filter_packet_t& input) + : _type{input.type}, _response{input.response}, _filter_code{input.filter_code}, + _parsed_certitude_size{input.certitude_size}, _parsed_body_size{input.body_size} + { + std::memcpy(this->_evt_id, input.evt_id, 16); + } + + std::vector DarwinPacket::Serialize() const { + darwin_filter_packet_t header { + this->_type, + this->_response, + this->_filter_code, + this->_body.size(), + {0}, + this->_certitude_list.size(), + {} + }; + + std::memcpy(header.evt_id, this->_evt_id, 16); + + size_t size = sizeof(header) + _body.size() + _certitude_list.size() * sizeof(unsigned int); + + std::vector ret(size, 0); + + unsigned char * pt = ret.data(); + + std::memcpy(pt, &header, sizeof(header)); + pt += sizeof(header); + //certitudes + for(size_t i=0; i < _certitude_list.size(); i++) { + unsigned int certitude = _certitude_list[i]; + std::memcpy(pt,(void*)(&certitude), sizeof(certitude)); + pt += sizeof(certitude); + } + + //body + std::memcpy(pt, _body.data(), _body.length()); + return ret; + } + + void DarwinPacket::clear() { + this->_type = DARWIN_PACKET_OTHER; + this->_response = DARWIN_RESPONSE_SEND_NO; + this->_filter_code = 0; + this->_parsed_body_size = 0; + std::memset(this->_evt_id, 0, sizeof(_evt_id)); + this->_parsed_certitude_size = 0; + _certitude_list.clear(); + _body.clear(); + this->_parsed_body.reset(nullptr); + } + + rapidjson::Document& DarwinPacket::JsonBody() { + if(! this->_parsed_body) { + this->_parsed_body = std::make_unique(); + } + this->_parsed_body->Parse(this->_body.c_str()); + return *(this->_parsed_body); + } + + std::string DarwinPacket::Evt_idToString() const { + DARWIN_LOGGER; + std::ostringstream oss; + auto default_flags = oss.flags(); + + for(size_t i=0; i(_evt_id[i]) & 0xFF); + } + DARWIN_LOG_DEBUG("DarwinPacket::Evt_idToString:: UUID - " + oss.str()); + return oss.str(); + } + + enum darwin_packet_type DarwinPacket::GetType() const { + return this->_type; + } + + enum darwin_filter_response_type DarwinPacket::GetResponseType() const { + return this->_response; + } + + void DarwinPacket::SetResponseType(enum darwin_filter_response_type response) { + this->_response = response; + } + + long DarwinPacket::GetFilterCode() const { + return this->_filter_code; + } + + void DarwinPacket::SetFilterCode(long filter_code) { + this->_filter_code = filter_code; + } + + const unsigned char * DarwinPacket::GetEventId() const { + return this->_evt_id; + } + + size_t DarwinPacket::GetEventIdSize() const { + return sizeof(this->_evt_id); + } + + size_t DarwinPacket::GetParsedCertitudeSize() const { + return this->_parsed_certitude_size; + } + + size_t DarwinPacket::GetParsedBodySize() const { + return this->_parsed_body_size; + } + + const std::string& DarwinPacket::GetBody() const { + return this->_body; + } + + std::string& DarwinPacket::GetMutableBody() { + return this->_body; + } + + const std::vector& DarwinPacket::GetCertitudeList() const { + return this->_certitude_list; + } + + std::vector& DarwinPacket::GetMutableCertitudeList() { + return this->_certitude_list; + } + + const std::string& DarwinPacket::GetLogs() const { + return this->_logs; + } + + std::string& DarwinPacket::GetMutableLogs() { + return this->_logs; + } + + void DarwinPacket::SetLogs(std::string& logs) { + this->_logs = logs; + } +} \ No newline at end of file diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp new file mode 100644 index 00000000..76609f7d --- /dev/null +++ b/samples/base/DarwinPacket.hpp @@ -0,0 +1,243 @@ +/// +/// \file DarwinPacket.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief A Description of a darwin packet with serializing and deserializing capacity +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include +#include +#include "protocol.h" +#include "../../toolkit/rapidjson/document.h" + + +namespace darwin { + + class DarwinPacket { + public: + /// + /// \brief Construct a new Darwin Packet object + /// + /// \param type type of the packet + /// \param response response type of the packet + /// \param filter_code filter code + /// \param event_id event id + /// \param certitude_size size of the certitude + /// \param body_size size of the body + /// + DarwinPacket( + enum darwin_packet_type type, + enum darwin_filter_response_type response, + long filter_code, + unsigned char event_id[16], + size_t certitude_size, + size_t body_size); + + /// + /// \brief Construct a new Darwin Packet object from a darwin_filter_packet_t + /// + /// \param input the native struct to be parsed + /// + DarwinPacket(darwin_filter_packet_t& input); + + DarwinPacket() = default; + + DarwinPacket(DarwinPacket&& other) noexcept; + + DarwinPacket& operator=(DarwinPacket&& other) noexcept; + + virtual ~DarwinPacket() = default; + + // No copy allowed of a DarwinPacket, only moves + // It may be implemented in the future as a deep copy if this is needed + DarwinPacket(const DarwinPacket&) = delete; + DarwinPacket& operator=(const DarwinPacket&) = delete; + + /// + /// \brief Serialize the packet + /// + /// \return std::vector a vector holding the serialized packet + /// + std::vector Serialize() const; + + /// + /// \brief Get the Minimal Size of a packet + /// + /// \return constexpr size_t size of the packet + /// + constexpr static size_t getMinimalSize() { + return sizeof(_type) + sizeof(_response) + sizeof(_filter_code) + + sizeof(_parsed_body_size) + sizeof(_evt_id) + sizeof(_parsed_certitude_size); + }; + + /// + /// \brief Clears the packet + /// + /// + void clear(); + + /// + /// \brief Parse the body as a json document and returns a reference to it + /// While the parsed Body is kept inside the DarwinPacket, subsequent calls to JsonBody() + /// will re-parse the body as the body may have been modified + /// + /// \return rapidjson::Document& the reference to the parsed body + /// + rapidjson::Document& JsonBody(); + + /// + /// \brief Get the Type + /// + /// \return enum darwin_packet_type + /// + enum darwin_packet_type GetType() const; + + /// + /// \brief Get the Response Type + /// + /// \return enum darwin_filter_response_type + /// + enum darwin_filter_response_type GetResponseType() const; + + /// + /// \brief Set the Response Type + /// + /// + void SetResponseType(enum darwin_filter_response_type); + + /// + /// \brief Get the Filter Code + /// + /// \return long filter code + /// + long GetFilterCode() const; + + /// + /// \brief Set the Filter Code + /// + /// \param filter_code + /// + void SetFilterCode(long filter_code); + + /// + /// \brief Get the Event Id pointer + /// + /// \return const unsigned char* pointer of a 16 bytes buffer + /// + const unsigned char * GetEventId() const; + /// + /// \brief Get the Event Id Size + /// + /// \return size_t 16 + /// + size_t GetEventIdSize() const; + + /// + /// \brief Serialize the event id as a UUID String + /// + /// \return std::string UUID encoded event id + /// + std::string Evt_idToString() const; + + /// + /// \brief Get the Parsed Certitude Size, it may differs from GetCertitudeList().size() + /// + /// \return size_t Parsed Certitude Size + /// + size_t GetParsedCertitudeSize() const; + + /// + /// \brief Get the Parsed Body Size, it may differ from GetBody().size() + /// + /// \return size_t Parsed Body size + /// + size_t GetParsedBodySize() const; + + /// + /// \brief Get a const ref to the Body + /// + /// \return const std::string& body + /// + const std::string& GetBody() const; + + /// + /// \brief Get a mutable ref to the Body + /// + /// \return std::string& body + /// + std::string& GetMutableBody(); + + /// + /// \brief Get a const ref to the Certitude List + /// + /// \return const std::vector& certitude list + /// + const std::vector& GetCertitudeList() const; + + /// + /// \brief Get a mutable ref to the Certitude List + /// + /// \return std::vector& certitude list + /// + std::vector& GetMutableCertitudeList(); + + /// + /// \brief Add a certitude to the list + /// + /// \param certitude to add + /// + inline void AddCertitude(unsigned int certitude) { + this->_certitude_list.push_back(certitude); + }; + + /// + /// \brief Get a const ref to the Logs + /// + /// \return const std::string& logs + /// + const std::string& GetLogs() const; + + /// + /// \brief Get a mutable ref to the Logs + /// + /// \return std::string& logs + /// + std::string& GetMutableLogs(); + + /// + /// \brief Set the Logs + /// + /// \param logs + /// + void SetLogs(std::string& logs); + + protected: + + private: + + darwin_packet_type _type; //!< The type of information sent. + darwin_filter_response_type _response; //!< Whom the response will be sent to. + long _filter_code; //!< The unique identifier code of a filter. + unsigned char _evt_id[16]; //!< An array containing the event ID + size_t _parsed_certitude_size; + size_t _parsed_body_size; + std::vector _certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. + std::string _body; + + // Parsed Body is stored as late-initiliaized pointer because is cannot be moved, nor assigned + // For move semantics, we want to be able to move this, hence the std::unique_ptr + std::unique_ptr _parsed_body; + + std::string _logs; + + }; + + +} \ No newline at end of file diff --git a/samples/base/Logger.cpp b/samples/base/Logger.cpp index 7a284976..ee66e2af 100644 --- a/samples/base/Logger.cpp +++ b/samples/base/Logger.cpp @@ -84,6 +84,10 @@ namespace darwin { return false; } + log_type Logger::getLevel() const { + return this->_logLevel; + } + void Logger::setName(std::string const& name) { _name = name; } diff --git a/samples/base/Logger.hpp b/samples/base/Logger.hpp index 352a741b..41c23b05 100644 --- a/samples/base/Logger.hpp +++ b/samples/base/Logger.hpp @@ -95,6 +95,13 @@ namespace darwin { /// \return true if success, false otherwise. bool setLevel(std::string level); + /// + /// \brief Get the Log Level + /// + /// \return log_type + /// + log_type getLevel() const; + /// Set the name of the module in the logger. /// /// \param name The name to set as the module name. diff --git a/samples/base/Manager.cpp b/samples/base/Manager.cpp index 56de739a..13b05377 100644 --- a/samples/base/Manager.cpp +++ b/samples/base/Manager.cpp @@ -8,6 +8,7 @@ #include "Logger.hpp" #include "Stats.hpp" #include "Manager.hpp" +#include "ASession.hpp" namespace darwin { diff --git a/samples/base/Manager.hpp b/samples/base/Manager.hpp index 93e0e092..8d17ae02 100644 --- a/samples/base/Manager.hpp +++ b/samples/base/Manager.hpp @@ -9,7 +9,7 @@ #include #include -#include "Session.hpp" +#include "ASession.fwd.hpp" namespace darwin { class Manager { diff --git a/samples/base/Server.cpp b/samples/base/Server.cpp deleted file mode 100644 index 4039debf..00000000 --- a/samples/base/Server.cpp +++ /dev/null @@ -1,119 +0,0 @@ -/// \file Server.cpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 02/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - -#include -#include -#include -#include -#include "Server.hpp" -#include "Logger.hpp" -#include "Stats.hpp" - -namespace darwin { - - Server::Server(std::string const& socket_path, - std::string const& output, - std::string const& next_filter_socket, - std::size_t threshold, - Generator& generator) - : _socket_path{socket_path}, _socket_next{next_filter_socket}, _output{output}, - _io_context{}, _threshold{threshold}, _signals{_io_context}, - _acceptor{_io_context, - boost::asio::local::stream_protocol::endpoint( - socket_path)}, - _new_connection{_io_context}, _generator{generator} { - - // Setting the stopping signals for the service - _signals.add(SIGINT); - _signals.add(SIGTERM); -#ifdef SIGQUIT - _signals.add(SIGQUIT); -#endif // !SIGQUIT - - // Start the waiting for a stopping signal - AwaitStop(); - - // Start accepting connections - Accept(); - } - - boost::asio::io_context &Server::GetIOContext() { - return this->_io_context; - } - - void Server::Run() { - // The io_context::run() call will block until all asynchronous operations - // have finished. While the server is running, there is always at least one - // asynchronous operation outstanding: the asynchronous accept call waiting - // for new incoming connections. - - // By using multiple threads, a program can call run() multiple times. - // Once an asynchronous operation is complete, - // the I/O service object will execute the handler in one of these threads. - // If a second operation is completed shortly after the first one, - // the I/O service object can execute the handler in a different thread. - // Now, not only can operations outside of a process be executed concurrently, - // but handlers within the process can be executed concurrently, too. - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Server::Run:: Running..."); - _io_context.run(); - } - - void Server::Clean() { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); - _manager.StopAll(); - unlink(_socket_path.c_str()); - } - - void Server::AwaitStop() { - _signals.async_wait( - boost::bind(&Server::HandleStop, this, - boost::asio::placeholders::error, - boost::asio::placeholders::signal_number)); - } - - void Server::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { - // The server is stopped by cancelling all outstanding asynchronous - // operations. Once all operations have finished the io_context::run() - // call will exit. - DARWIN_LOGGER; - - SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); - DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); - _acceptor.close(); - _io_context.stop(); - } - - void Server::Accept() { - _acceptor.async_accept(_new_connection, - boost::bind(&Server::HandleAccept, this, - boost::asio::placeholders::error)); - } - - void Server::HandleAccept(boost::system::error_code const& e) { - DARWIN_LOGGER; - - if (!_acceptor.is_open()) { - DARWIN_LOG_INFO("Server::HandleAccept:: Acceptor closed, closing server..."); - return; - } - - if (!e) { - DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); - auto task = _generator.CreateTask(_new_connection, _manager); - task->SetNextFilterSocketPath(_socket_next); - task->SetOutputType(_output); - task->SetThreshold(_threshold); - _manager.Start(task); - Accept(); - } else { - DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); - } - } - -} \ No newline at end of file diff --git a/samples/base/Server.hpp b/samples/base/Server.hpp deleted file mode 100644 index 595c5a67..00000000 --- a/samples/base/Server.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/// \file Server.hpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 02/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - -#pragma once - -#include -#include -#include -#include "Session.hpp" -#include "Manager.hpp" -#include "Generator.hpp" - -namespace darwin { - - class Server { - public: - /// Create an async UNIX stream socket server. - /// The server runs on nb_threads thread. - /// - /// \param socket_path Path of the UNIX socket to listen on. - /// \param output Filters' output type - /// \param next_filter_socket Path of the UNIX socket of the filter to send data to. - /// \param threshold Threshold at which the filter will raise a log. - Server(std::string const& socket_path, - std::string const& output, - std::string const& next_filter_socket, - std::size_t threshold, - Generator& generator); - - ~Server() = default; - - // Make the server non copyable & non movable - Server(Server const&) = delete; - - Server(Server const&&) = delete; - - Server& operator=(Server const&) = delete; - - Server& operator=(Server const&&) = delete; - - public: - /// Start the server and the threads. - void Run(); - - /// Clean the server's ressources (sessions, socket) - void Clean(); - - boost::asio::io_context &GetIOContext(); - - private: - /// Start async waiting for the stopping signals. - void AwaitStop(); - - /// Handler called when signal is received by the async wait. - /// - /// \param sig The received signal. - void HandleStop(boost::system::error_code const& error, int sig); - - /// Start an async connection acceptation on _new_session's socket. - void Accept(); - - /// Handler called on async accept trigger. - void HandleAccept(boost::system::error_code const& e); - - private: - std::string _socket_path; //!< Path to the UNIX socket to listen on. - std::string _socket_next; //!< Path to the next filter's UNIX socket. - std::string _output; //!< Filter's output type - boost::asio::io_context _io_context; //!< The async io context. - std::size_t _threshold; //!< Filter's threshold - boost::asio::signal_set _signals; //!< Set of the stopping signals. - boost::asio::local::stream_protocol::acceptor _acceptor; //!< Acceptor for the incoming connections. - boost::asio::local::stream_protocol::socket _new_connection; //!< Socket used to accept a new connection. - Generator& _generator; //!< Generator used to create new Sessions. - Manager _manager; //!< Server's session manager. - }; -} \ No newline at end of file diff --git a/samples/base/Session.cpp b/samples/base/Session.cpp deleted file mode 100644 index 034f6b6f..00000000 --- a/samples/base/Session.cpp +++ /dev/null @@ -1,481 +0,0 @@ -/// \file Session.cpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 05/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - -#include -#include -#include -#include -#include -#include -#include -#include "Logger.hpp" -#include "Manager.hpp" -#include "Session.hpp" -#include "errors.hpp" - -#include "../../toolkit/lru_cache.hpp" -#include "../../toolkit/xxhash.h" -#include "../../toolkit/xxhash.hpp" - -namespace darwin { - Session::Session(std::string name, boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex) - : _filter_name(name), _connected{false}, _socket{std::move(socket)}, - _filter_socket{socket.get_executor()}, - _manager{manager}, _cache{cache}, _cache_mutex{cache_mutex} {} - - void Session::SetStartingTime() { - _starting_time = std::chrono::high_resolution_clock::now(); - } - - double Session::GetDurationMs() { - return std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - _starting_time - ).count() / ((double)1000); - } - - void Session::Start() { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::Start::"); - ReadHeader(); - } - - void Session::Stop() { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::Stop::"); - _socket.close(); - if(_connected) _filter_socket.close(); - } - - void Session::SetThreshold(std::size_t const& threshold) { - DARWIN_LOGGER; - //If the threshold is negative, we keep the actual default threshold - if(threshold>100){ - DARWIN_LOG_DEBUG("Session::SetThreshold:: Default threshold " + std::to_string(_threshold) + "applied"); - return; - } - _threshold = threshold; - } - - void Session::SetNextFilterSocketPath(std::string const& path){ - _next_filter_path = path; - } - - void Session::SetOutputType(std::string const& output) { - _output = config::convert_output_string(output); - } - - config::output_type Session::GetOutputType() { - return _output; - } - - std::string Session::GetLogs(){ - return _logs; - } - - std::string Session::GetDataToSendToFilter(){ - switch (GetOutputType()){ - case config::output_type::RAW: - return _raw_body; - case config::output_type::PARSED: - return JsonStringify(_body); - case config::output_type::NONE: - return ""; - case config::output_type::LOG: - return GetLogs(); - } - return ""; - } - - void Session::ReadHeader() { - DARWIN_LOGGER; - - DARWIN_LOG_DEBUG("Session::ReadHeader:: Starting to read incoming header..."); - boost::asio::async_read(_socket, - boost::asio::buffer(&_header, sizeof(_header)), - boost::bind(&Session::ReadHeaderCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - } - - void Session::ReadHeaderCallback(const boost::system::error_code& e, - std::size_t size) { - DARWIN_LOGGER; - - _raw_body.clear(); - _certitudes.clear(); - _logs.clear(); - - DARWIN_LOG_DEBUG("Session::ReadHeaderCallback:: Reading header"); - if (!e) { - if (size != sizeof(_header)) { - DARWIN_LOG_ERROR("Session::ReadHeaderCallback:: Mismatching header size"); - goto header_callback_stop_session; - } - if (_header.body_size == 0) { - ExecuteFilter(); - return; - } // Else the ReadBodyCallback will call ExecuteFilter - ReadBody(_header.body_size); - return; - } - - if (boost::asio::error::eof == e) { - DARWIN_LOG_DEBUG("Session::ReadHeaderCallback:: " + e.message()); - } else { - DARWIN_LOG_WARNING("Session::ReadHeaderCallback:: " + e.message()); - } - - header_callback_stop_session: - _manager.Stop(shared_from_this()); - } - - void Session::ReadBody(std::size_t size) { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::ReadBody:: Starting to read incoming body..."); - - _socket.async_read_some(boost::asio::buffer(_buffer, - size), - boost::bind(&Session::ReadBodyCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - } - - void Session::ReadBodyCallback(const boost::system::error_code& e, - std::size_t size) { - DARWIN_LOGGER; - - if (!e) { - _raw_body.append(_buffer.data(), size); - DARWIN_LOG_DEBUG("Session::ReadBodyCallback:: Body len (" + - std::to_string(_raw_body.length()) + - ") - Header body size (" + - std::to_string(_header.body_size) + - ")"); - unsigned int bodyLength = _raw_body.length(); - unsigned int totalBodyLength = _header.body_size; - if (bodyLength < totalBodyLength) { - ReadBody(totalBodyLength - bodyLength); - } else { - if (_raw_body.empty()) { - DARWIN_LOG_WARNING("Session::ReadBodyCallback Empty body retrieved"); - this->SendErrorResponse("Error receiving body: Empty body retrieved", DARWIN_RESPONSE_CODE_REQUEST_ERROR); - return; - } - if (_header.body_size <= 0) { - DARWIN_LOG_ERROR( - "Session::ReadBodyCallback Body is not empty, but the header appears to be invalid" - ); - this->SendErrorResponse("Error receiving body: Body is not empty, but the header appears to be invalid", DARWIN_RESPONSE_CODE_REQUEST_ERROR); - return; - } - - if (!ParseBody()) { - DARWIN_LOG_DEBUG("Session::ReadBodyCallback Something went wrong while parsing the body"); - this->SendErrorResponse("Error receiving body: Something went wrong while parsing the body", DARWIN_RESPONSE_CODE_REQUEST_ERROR); - return; - } - - ExecuteFilter(); - } - return; - } - DARWIN_LOG_ERROR("Session::ReadBodyCallback:: " + e.message()); - _manager.Stop(shared_from_this()); - } - - void Session::ExecuteFilter() { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::ExecuteFilter::"); - - (*this)(); - this->SendNext(); - } - - void Session::SendNext() { - switch(_header.response) { - case DARWIN_RESPONSE_SEND_BOTH: - if(this->SendToFilter()) break; - case DARWIN_RESPONSE_SEND_BACK: - if(not this->SendToClient()) Start(); - break; - case DARWIN_RESPONSE_SEND_DARWIN: - if(not this->SendToFilter()) Start(); - break; - default: - Start(); - } - } - - bool Session::ParseBody() { - DARWIN_LOGGER; - try { - _body.Parse(_raw_body.c_str()); - - if (!_body.IsArray()) { - DARWIN_LOG_ERROR("Session:: ParseBody: You must provide a list"); - return false; - } - } catch (...) { - DARWIN_LOG_CRITICAL("Session:: ParseBody: Could not parse raw body"); - return false; - } - // DARWIN_LOG_DEBUG("Session:: ParseBody:: parsed body : " + _raw_body); - return true; - } - - std::string Session::JsonStringify(rapidjson::Document &json) { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - - json.Accept(writer); - - return std::string(buffer.GetString()); - } - - bool Session::SendToClient() noexcept { - DARWIN_LOGGER; - const std::size_t certitude_size = _certitudes.size(); - - /* - * Allocate the header + - * the size of the certitude - - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size - */ - std::size_t packet_size = 0; - if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { - packet_size = sizeof(darwin_filter_packet_t) + - (certitude_size - DEFAULT_CERTITUDE_LIST_SIZE) * sizeof(unsigned int); - } else { - packet_size = sizeof(darwin_filter_packet_t); - } - - if (not this->_response_body.empty()) { - packet_size += this->_response_body.size(); - } - - DARWIN_LOG_DEBUG("Session::SendToClient: Computed packet size: " + std::to_string(packet_size)); - - darwin_filter_packet_t* packet = reinterpret_cast(malloc(packet_size)); - - if (packet == nullptr) { - DARWIN_LOG_CRITICAL("Session::SendToClient: Could not create a Darwin packet"); - return false; - } - - /* - * Initialisation of the structure for the padding bytes because of - * missing __attribute__((packed)) in the protocol structure. - */ - memset(packet, 0, packet_size); - - for (std::size_t index = 0; index < certitude_size; ++index) { - packet->certitude_list[index] = _certitudes[index]; - } - - packet->type = DARWIN_PACKET_FILTER; - packet->response = _header.response; - packet->certitude_size = certitude_size; - packet->filter_code = GetFilterCode(); - packet->body_size = this->_response_body.size(); - memcpy(packet->evt_id, _header.evt_id, 16); - memcpy((char*)(packet) + sizeof(darwin_filter_packet_t), _response_body.c_str(), _response_body.length()); - - boost::asio::async_write(_socket, - boost::asio::buffer(packet, packet_size), - boost::bind(&Session::SendToClientCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - - free(packet); - this->_response_body.clear(); - return true; - } - - bool Session::SendToFilter() noexcept { - DARWIN_LOGGER; - - if (!_next_filter_path.compare("no")) { - DARWIN_LOG_NOTICE("Session::SendToFilter:: No next filter provided. Ignoring..."); - return false; - } - - if (!_connected) { - DARWIN_LOG_DEBUG("Session::SendToFilter:: Trying to connect to: " + - _next_filter_path); - try { - _filter_socket.connect( - boost::asio::local::stream_protocol::endpoint( - _next_filter_path.c_str())); - _connected = true; - } catch (std::exception const& e) { - DARWIN_LOG_ERROR(std::string("Session::SendToFilter:: " - "Unable to connect to next filter: ") + - e.what()); - return false; - } - } - - std::string data = GetDataToSendToFilter(); - DARWIN_LOG_DEBUG("Session::SendToFilter:: data to send: " + data); - DARWIN_LOG_DEBUG("Session::SendToFilter:: data size: " + std::to_string(data.size())); - - const std::size_t certitude_size = _certitudes.size(); - - /* - * Allocate the header + - * the size of the certitude - - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size - */ - std::size_t packet_size = 0; - if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { - packet_size = sizeof(darwin_filter_packet_t) + - (certitude_size - DEFAULT_CERTITUDE_LIST_SIZE) * sizeof(unsigned int); - } else { - packet_size = sizeof(darwin_filter_packet_t); - } - - packet_size += data.size(); - - DARWIN_LOG_DEBUG("Session::SendToFilter:: Computed packet size: " + std::to_string(packet_size)); - - darwin_filter_packet_t* packet; - packet = (darwin_filter_packet_t *) malloc(packet_size); - - if (!packet) { - DARWIN_LOG_CRITICAL("Session:: SendToFilter:: Could not create a Darwin packet"); - return false; - } - - /* - * Initialisation of the structure for the padding bytes because of - * missing __attribute__((packed)) in the protocol structure. - */ - memset(packet, 0, packet_size); - - for (std::size_t index = 0; index < certitude_size; ++index) { - packet->certitude_list[index] = _certitudes[index]; - } - - if(data.size() != 0) { - // TODO: set a proper pointer in protocol.h for the body - // Yes We Hack... - memcpy(&packet->certitude_list[certitude_size+1], data.c_str(), data.size()); - } - - packet->type = DARWIN_PACKET_FILTER; - packet->response = _header.response == DARWIN_RESPONSE_SEND_BOTH ? DARWIN_RESPONSE_SEND_DARWIN : _header.response; - packet->certitude_size = certitude_size; - packet->filter_code = GetFilterCode(); - packet->body_size = data.size(); - memcpy(packet->evt_id, _header.evt_id, 16); - - DARWIN_LOG_DEBUG("Session:: SendToFilter:: Sending header + data"); - boost::asio::async_write(_filter_socket, - boost::asio::buffer(packet, packet_size), - boost::bind(&Session::SendToFilterCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - - free(packet); - return true; - } - - void Session::SendToClientCallback(const boost::system::error_code& e, - std::size_t size __attribute__((unused))) { - DARWIN_LOGGER; - - if (e) { - DARWIN_LOG_ERROR("Session::SendToClientCallback:: " + e.message()); - _manager.Stop(shared_from_this()); - DARWIN_LOG_DEBUG("Session::SendToClientCallback:: Stopped session in manager"); - return; - } - - Start(); - } - - void Session::SendToFilterCallback(const boost::system::error_code& e, - std::size_t size __attribute__((unused))) { - DARWIN_LOGGER; - - if (e) { - DARWIN_LOG_ERROR("Session::SendToFilterCallback:: " + e.message()); - _filter_socket.close(); - _connected = false; - } - - if(_header.response == DARWIN_RESPONSE_SEND_BOTH) { - this->SendToClient(); - } - else { - Start(); - } - } - - xxh::hash64_t Session::GenerateHash() { - // could be easily overridden in the children - return xxh::xxhash<64>(_raw_body); - } - - void Session::SaveToCache(const xxh::hash64_t &hash, const unsigned int certitude) const { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("SaveToCache:: Saving certitude " + std::to_string(certitude) + " to cache"); - std::unique_lock lck{_cache_mutex}; - _cache->insert(hash, certitude); - } - - bool Session::GetCacheResult(const xxh::hash64_t &hash, unsigned int& certitude) { - DARWIN_LOGGER; - boost::optional cached_certitude; - - { - std::unique_lock lck{_cache_mutex}; - cached_certitude = _cache->get(hash); - } - - if (cached_certitude != boost::none) { - certitude = cached_certitude.get(); - DARWIN_LOG_DEBUG("GetCacheResult:: Already processed request. Cached certitude is " + - std::to_string(certitude)); - - return true; - } - - return false; - } - - std::string Session::Evt_idToString() { - DARWIN_LOGGER; - char str[37] = {}; - snprintf(str, - 37, - "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", - _header.evt_id[0], _header.evt_id[1], _header.evt_id[2], _header.evt_id[3], - _header.evt_id[4], _header.evt_id[5], _header.evt_id[6], _header.evt_id[7], - _header.evt_id[8], _header.evt_id[9], _header.evt_id[10], _header.evt_id[11], - _header.evt_id[12], _header.evt_id[13], _header.evt_id[14], _header.evt_id[15] - ); - std::string res(str); - DARWIN_LOG_DEBUG(std::string("Session::Evt_idToString:: UUID - ") + res); - return res; - } - - std::string Session::GetFilterName() { - return _filter_name; - } - - void Session::SendErrorResponse(const std::string& message, const unsigned int code) { - if (this->_header.response != DARWIN_RESPONSE_SEND_BACK && this->_header.response != DARWIN_RESPONSE_SEND_BOTH) - return; - this->_response_body.clear(); - this->_response_body += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; - this->SendToClient(); - } -} diff --git a/samples/base/Session.hpp b/samples/base/Session.hpp deleted file mode 100644 index daec6bcb..00000000 --- a/samples/base/Session.hpp +++ /dev/null @@ -1,230 +0,0 @@ -/// \file Session.hpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 05/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - -#pragma once - -#include -#include - -#include "config.hpp" -#include "protocol.h" -#include "../../toolkit/lru_cache.hpp" -#include "../../toolkit/xxhash.h" -#include "../../toolkit/xxhash.hpp" -#include "../../toolkit/rapidjson/document.h" -#include "../../toolkit/rapidjson/writer.h" -#include "../../toolkit/rapidjson/stringbuffer.h" -#include "Time.hpp" - -#define DARWIN_SESSION_BUFFER_SIZE 2048 -#define DARWIN_DEFAULT_THRESHOLD 80 -#define DARWIN_ERROR_RETURN 101 - -namespace darwin { - - class Manager; - - class Session : public std::enable_shared_from_this { - public: - Session(std::string name, - boost::asio::local::stream_protocol::socket& socket, - Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex); - - virtual ~Session() = default; - - // Make the manager non copyable & non movable - Session(Session const&) = delete; - - Session(Session const&&) = delete; - - Session& operator=(Session const&) = delete; - - Session& operator=(Session const&&) = delete; - - public: - /// Entry point of the execution. - /// MUST be overloaded. - /// WARNING This method will be executed by a thread. - virtual void operator()() = 0; - - public: - /// Start the session and the async read of the incoming packet. - virtual void Start() final; - - /// Stop the session and close the socket. - virtual void Stop() final; - - /// Set the filter's threshold - /// - /// \param threshold The threshold wanted. - virtual void SetThreshold(std::size_t const& threshold) final; - - /// Set the path to the associated decision module UNIX socket - /// - /// \param path Path to the UNIX socket. - virtual void SetNextFilterSocketPath(std::string const& path) final; - - /// Set the output's type of the filter - /// - /// \param name the string that represent the output type - virtual void SetOutputType(std::string const& output) final; - - protected: - /// Return filter code. - virtual long GetFilterCode() noexcept = 0; - - /// Save the result to cache - virtual void SaveToCache(const xxh::hash64_t &hash, unsigned int certitude) const; - - /// Get the result from the cache - virtual bool GetCacheResult(const xxh::hash64_t &hash, unsigned int &certitude); - - /// Generate the hash - virtual xxh::hash64_t GenerateHash(); - - /// Set starting time - void SetStartingTime(); - - /// Get total time elapsed since the starting time - double GetDurationMs(); - - /// Get the filter's output type - /// - /// \return The filter's output type - config::output_type GetOutputType(); - - /// Get the data to send to the next filter - /// according to the filter's output type - /// - /// \param size data's size - /// \param data data to send - std::string GetDataToSendToFilter(); - - /// Get the filter's result in a log form - /// - /// \return The filter's log - virtual std::string GetLogs(); - - - /// Transform the evt id in the header into a string - /// - /// \return evt_di as string - std::string Evt_idToString(); - - /// Get the name of the filter - std::string GetFilterName(); - - /// Get the string representation of a rapidjson document - std::string JsonStringify(rapidjson::Document &json); - - /// Parse the body received. - /// This is the default function, trying to get a JSON array from the _raw_body, - /// if you wan't to recover something else (full/complex JSON, custom data), - /// override the function in the child class. - virtual bool ParseBody(); - - /// Parse a line in the body. - /// This function should be implemented in each child, - /// and should be called between every entry to check validity (no early parsing). - virtual bool ParseLine(rapidjson::Value &line) = 0; - - /// Send - virtual void SendNext() final; - - /// Send result to the client. - /// - /// \return false if the function could not send the data, true otherwise. - virtual bool SendToClient() noexcept; - - /// Send result to next filter. - /// - /// \return false if the function could not send the data, true otherwise. - virtual bool SendToFilter() noexcept; - - /// Called when data is sent using Send() method. - /// Terminate the session on failure. - /// - /// \param size The number of byte sent. - virtual void - SendToClientCallback(const boost::system::error_code& e, - std::size_t size); - - /// Called when data is sent using SendToFilter() method. - /// Terminate the filter session on failure. - /// - /// \param size The number of byte sent. - virtual void - SendToFilterCallback(const boost::system::error_code& e, - std::size_t size); - -private: - /// Set the async read for the header. - /// - /// \return -1 on error, 0 on socket closed & sizeof(header) on success. - virtual void ReadHeader() final; - - /// Callback of async read for the header. - /// Terminate the session on failure. - virtual void ReadHeaderCallback(const boost::system::error_code& e, std::size_t size) final; - - /// Set the async read for the body. - /// - /// \return -1 on error, 0 on socket closed & sizeof(header) on success. - virtual void ReadBody(std::size_t size) final; - - /// Callback of async read for the body. - /// Terminate the session on failure. - virtual void - ReadBodyCallback(const boost::system::error_code& e, - std::size_t size) final; - - /// Execute the filter and - virtual void ExecuteFilter() final; - - /// Sends a response with a body containing an error message - /// - /// \param message The error message to send - /// \param code The error code to send - virtual void SendErrorResponse(const std::string& message, const unsigned int code) final; - - // Not accessible by children - private: - std::string _filter_name; //!< name of the filter - bool _connected; //!< True if the socket to the next filter is connected. - std::string _next_filter_path; //!< The socket path to the next filter. - config::output_type _output; //!< The filter's output. - std::array _buffer; //!< Reading buffer for the body. - - - // Accessible by children - protected: - boost::asio::local::stream_protocol::socket _socket; //!< Session's socket. - boost::asio::local::stream_protocol::socket _filter_socket; //!< Filter's socket. - Manager& _manager; //!< The associated connection manager. - darwin_filter_packet_t _header; //!< Header received from the session. - rapidjson::Document _body; //!< Body received from session (if any). - std::string _raw_body; //!< Body received from session (if any), that will not be parsed. - std::string _logs; //!< Represents data given in the logs by the Session - std::chrono::time_point _starting_time; - std::vector _certitudes; //!< The Darwin results obtained. - //!< Cache received from the Generator - std::shared_ptr> _cache; - std::mutex& _cache_mutex; - bool _is_cache = false; - std::size_t _threshold = DARWIN_DEFAULT_THRESHOLD; //! session_ptr_t; - -} - diff --git a/samples/base/main.cpp b/samples/base/main.cpp index 85cf59da..32e548f7 100644 --- a/samples/base/main.cpp +++ b/samples/base/main.cpp @@ -31,7 +31,7 @@ int main(int ac, char**av) { return 1; DARWIN_LOG_INFO("Configured"); - if (core.daemon) { + if (core.IsDaemon()) { daemon(1, 0); } diff --git a/samples/base/network/ANextFilterConnector.cpp b/samples/base/network/ANextFilterConnector.cpp new file mode 100644 index 00000000..58919865 --- /dev/null +++ b/samples/base/network/ANextFilterConnector.cpp @@ -0,0 +1,42 @@ +#include "ANextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + ANextFilterConnector::ANextFilterConnector() + : _io_context{}, _nb_attempts{0}, _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)}, _is_connected{false} + { + } + + void ANextFilterConnector::Run() { + _io_context.run(); + } + + void ANextFilterConnector::Send(DarwinPacket& packet) { + std::vector v = packet.Serialize(); + std::shared_ptr buf_ptr = std::make_shared(v.data(), v.size()); + _buffer_set.insert(buf_ptr); + this->Send(buf_ptr); + } + + void ANextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, std::shared_ptr buffer) { + DARWIN_LOGGER; + if(ec || bytes_transferred != buffer->size()) { + if(ec){ + _is_connected = false; + DARWIN_LOG_NOTICE("ANextFilterConnector::SendCallback Error when sending to Next Filter, retrying, error:" + ec.message()); + } else { + DARWIN_LOG_NOTICE("ANextFilterConnector::SendCallback Error when sending to Next Filter, retrying, error: mismatched size, was :" + + std::to_string(bytes_transferred) + ", expected:" + std::to_string(buffer->size())); + } + Send(buffer); + return; + } + + _buffer_set.erase(buffer); + _nb_attempts = 0; + } +} \ No newline at end of file diff --git a/samples/base/network/ANextFilterConnector.hpp b/samples/base/network/ANextFilterConnector.hpp new file mode 100644 index 00000000..e03cac16 --- /dev/null +++ b/samples/base/network/ANextFilterConnector.hpp @@ -0,0 +1,80 @@ +/// +/// \file ANextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Connector to the Next Filter to be extended for different protocols +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include +#include + +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class ANextFilterConnector { + public: + ANextFilterConnector(); + virtual ~ANextFilterConnector() = default; + + /// + /// \brief Runs the io_context from the class + /// + /// + virtual void Run() final; + + /// + /// \brief Tries to connect to the next filter + /// + /// \return true if the connection is established + /// \return false else + /// + virtual bool Connect() = 0; + + /// + /// \brief Attempt to send a DarwinPacket to the next filter, it will atttempt to connect by calling this->Connect() + /// The Packet will be copied to a boost::asio::const_buffer and kept in a set + /// There will be _max_attempts tries before aborting the packet + /// + /// \param packet packet to be sent + /// + virtual void Send(DarwinPacket& packet); + + protected: + + /// + /// \brief Attempt to send a packet to the next filter, it will atttempt to connect by calling this->Connect() + /// The Packet will be copied to a boost::asio::const_buffer and kept in a set + /// There will be _max_attempts tries before aborting the packet + /// + /// \param packet a shared pointer to a boost asio buffer to be sent + /// + virtual void Send(std::shared_ptr packet) = 0; + + /// + /// \brief Callback called by the io_context when the async 'Send' operation has ended + /// This Callback will call 'Send(buffer)' if there is an error + /// + /// \param e error code returned + /// \param size number of bytes sent + /// \param buffer shared pointer to buffer sent + /// + virtual void SendCallback(const boost::system::error_code& e, size_t size, std::shared_ptr buffer); + + boost::asio::io_context _io_context; + std::set> _buffer_set; + + size_t _nb_attempts; + size_t _max_attempts; + std::chrono::milliseconds _attempts_delay_ms; + bool _is_connected; + }; +} \ No newline at end of file diff --git a/samples/base/network/AServer.cpp b/samples/base/network/AServer.cpp new file mode 100644 index 00000000..9d0c2a83 --- /dev/null +++ b/samples/base/network/AServer.cpp @@ -0,0 +1,70 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include +#include "AServer.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + AServer::AServer(std::string const& output, + std::size_t threshold, + Generator& generator) + : _output{output}, _io_context{}, _threshold{threshold}, + _signals{_io_context}, _generator{generator} { + } + + void AServer::InitSignalsAndStart() { + // Setting the stopping signals for the service + _signals.add(SIGINT); + _signals.add(SIGTERM); +#ifdef SIGQUIT + _signals.add(SIGQUIT); +#endif // !SIGQUIT + + // Start the waiting for a stopping signal + AwaitStop(); + + // Start accepting connections + Accept(); + } + + boost::asio::io_context &AServer::GetIOContext() { + return this->_io_context; + } + + void AServer::Run() { + // The io_context::run() call will block until all asynchronous operations + // have finished. While the server is running, there is always at least one + // asynchronous operation outstanding: the asynchronous accept call waiting + // for new incoming connections. + + // By using multiple threads, a program can call run() multiple times. + // Once an asynchronous operation is complete, + // the I/O service object will execute the handler in one of these threads. + // If a second operation is completed shortly after the first one, + // the I/O service object can execute the handler in a different thread. + // Now, not only can operations outside of a process be executed concurrently, + // but handlers within the process can be executed concurrently, too. + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Server::Run:: Running..."); + _io_context.run(); + } + + void AServer::AwaitStop() { + _signals.async_wait( + boost::bind(&AServer::HandleStop, this, + boost::asio::placeholders::error, + boost::asio::placeholders::signal_number)); + } + +} \ No newline at end of file diff --git a/samples/base/network/AServer.hpp b/samples/base/network/AServer.hpp new file mode 100644 index 00000000..38deb70d --- /dev/null +++ b/samples/base/network/AServer.hpp @@ -0,0 +1,82 @@ +/// +/// \file AServer.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Server to be extended or different protocols +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include +#include "ASession.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +namespace darwin { + + class AServer { + public: + virtual ~AServer() = default; + + // Make the server non copyable & non movable + AServer(AServer const&) = delete; + + AServer(AServer const&&) = delete; + + AServer& operator=(AServer const&) = delete; + + AServer& operator=(AServer const&&) = delete; + + public: + /// Start the server and the threads. + void Run(); + + /// Clean the server's ressources (sessions, socket) + virtual void Clean() = 0; + + boost::asio::io_context &GetIOContext(); + + protected: + + /// + /// \brief Construct a new AServer object + /// + /// \param output Filter's output type + /// \param threshold Threshold at which the filter will raise a log + /// \param generator Reference to the Task Generator + /// + AServer(std::string const& output, + std::size_t threshold, + Generator& generator); + + /// Start async waiting for the stopping signals. + void AwaitStop(); + + /// Handler called when signal is received by the async wait. + /// + /// \param sig The received signal. + virtual void HandleStop(boost::system::error_code const& error, int sig) = 0; + + /// Start an async connection acceptation on _new_session's socket. + virtual void Accept() = 0; + + /// Handler called on async accept trigger. + virtual void HandleAccept(boost::system::error_code const& e) = 0; + + void InitSignalsAndStart(); + + protected: + std::string _output; //!< Filter's output type + boost::asio::io_context _io_context; //!< The async io context. + std::size_t _threshold; //!< Filter's threshold + boost::asio::signal_set _signals; //!< Set of the stopping signals. + Generator& _generator; //!< Generator used to create new Sessions. + Manager _manager; //!< Server's session manager. + + + }; +} \ No newline at end of file diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp new file mode 100755 index 00000000..08d0184b --- /dev/null +++ b/samples/base/network/ASession.cpp @@ -0,0 +1,242 @@ +/// \file ASession.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 05/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Logger.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +#include "ASession.hpp" +#include "errors.hpp" +#include "Core.hpp" + +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" + +namespace darwin { + ASession::ASession(darwin::Manager& manager, + Generator& generator) + : _manager{manager}, _generator{generator} {} + + + + void ASession::Start() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("ASession::Start::"); + ReadHeader(); + } + + void ASession::SetOutputType(std::string const& output) { + _output = config::convert_output_string(output); + } + + config::output_type ASession::GetOutputType() const { + return _output; + } + + std::string ASession::GetLogs(){ + return _logs; + } + + void ASession::ModifyDataToSendToFilter(DarwinPacket &packet) const { + std::string& body = packet.GetMutableBody(); + switch (this->_output){ + case config::output_type::RAW: + return; + case config::output_type::PARSED: + body = JsonStringify(packet.JsonBody()); + return; + case config::output_type::NONE: + body.clear(); + return; + case config::output_type::LOG: + body = packet.GetLogs(); + return; + } + } + + void ASession::ReadHeaderCallback(const boost::system::error_code& e, + std::size_t size) { + DARWIN_LOGGER; + + _packet.clear(); + _logs.clear(); + + DARWIN_LOG_DEBUG("ASession::ReadHeaderCallback:: Reading header"); + if (!e) { + if (size != sizeof(darwin_filter_packet_t)) { + DARWIN_LOG_ERROR("ASession::ReadHeaderCallback:: Mismatching header size"); + goto header_callback_stop_session; + } + _packet = DarwinPacket(_header); + if (_packet.GetParsedBodySize() == 0 && _packet.GetParsedCertitudeSize() == 0) { + ExecuteFilter(); + return; + } // Else the ReadBodyCallback willcall ExecuteFilter + + ReadBody(_packet.GetParsedBodySize() + _packet.GetParsedCertitudeSize()); + return; + } + + if (boost::asio::error::eof == e) { + DARWIN_LOG_DEBUG("ASession::ReadHeaderCallback:: " + e.message()); + } else { + DARWIN_LOG_WARNING("ASession::ReadHeaderCallback:: " + e.message()); + } + + header_callback_stop_session: + _manager.Stop(shared_from_this()); + } + + void ASession::ReadBodyCallback(const boost::system::error_code& e, + std::size_t size) { + DARWIN_LOGGER; + + if (!e) { + size_t already_parsed_size = 0; + if(_packet.GetParsedCertitudeSize() > 0){ + while(_packet.GetParsedCertitudeSize() != _packet.GetCertitudeList().size() + && already_parsed_size + sizeof(unsigned int) <= size) { + unsigned int cert = 0; + std::memcpy(&cert, _body_buffer.data() + already_parsed_size, sizeof(cert)); + already_parsed_size += sizeof(cert); + _packet.AddCertitude(cert); + } + } + + _packet.GetMutableBody().append(_body_buffer.data() + already_parsed_size, size - already_parsed_size); + DARWIN_LOG_DEBUG("ASession::ReadBodyCallback:: Body len (" + + std::to_string(_packet.GetBody().length()) + + ") - Header body size (" + + std::to_string(_packet.GetParsedBodySize()) + + ")"); + size_t bodyLength = _packet.GetBody().length(); + size_t totalBodyLength = _packet.GetParsedBodySize(); + if (bodyLength < totalBodyLength) { + ReadBody(totalBodyLength - bodyLength); + } else { + if (_packet.GetBody().empty()) { + DARWIN_LOG_WARNING("ASession::ReadBodyCallback Empty body retrieved"); + this->SendErrorResponse("Error receiving body: Empty body retrieved", DARWIN_RESPONSE_CODE_REQUEST_ERROR); + return; + } + if (_packet.GetParsedBodySize() == 0) { + DARWIN_LOG_ERROR( + "ASession::ReadBodyCallback Body is not empty, but the header appears to be invalid" + ); + this->SendErrorResponse("Error receiving body: Body is not empty, but the header appears to be invalid", DARWIN_RESPONSE_CODE_REQUEST_ERROR); + return; + } + + ExecuteFilter(); + } + return; + } + DARWIN_LOG_ERROR("ASession::ReadBodyCallback:: " + e.message()); + _manager.Stop(shared_from_this()); + } + + void ASession::ExecuteFilter() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("ASession::ExecuteFilter::"); + auto t = _generator.CreateTask(shared_from_this()); + + _generator.GetTaskThreadPool().post(std::bind(&ATask::run, t)); + } + + void ASession::SendNext(DarwinPacket& packet) { + switch(packet.GetResponseType()) { + case DARWIN_RESPONSE_SEND_BOTH: + // We try to send to filter, then send back to client anyway + this->SendToFilter(packet); + __attribute((fallthrough)); + case DARWIN_RESPONSE_SEND_BACK: + this->SendToClient(packet); + // We don't resume the session by calling Start() because the SendToClientCallback already does + break; + case DARWIN_RESPONSE_SEND_DARWIN: + this->SendToFilter(packet); + Start(); + break; + default: + Start(); + } + } + + + std::string ASession::JsonStringify(rapidjson::Document &json) const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + + json.Accept(writer); + + return std::string(buffer.GetString()); + } + + void ASession::SendToClient(DarwinPacket& packet) noexcept { + DARWIN_LOGGER; + std::vector serialized_packet = packet.Serialize(); + DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(serialized_packet.size())); + this->WriteToClient(serialized_packet); + } + + void ASession::SendToFilter(DarwinPacket& packet) noexcept { + DARWIN_LOGGER; + + ANextFilterConnector* connector_ptr = Core::instance().GetNextFilterconnector(); + + if (!connector_ptr) { + DARWIN_LOG_NOTICE("ASession::SendToFilter:: No next filter provided. Ignoring..."); + return; + } + this->ModifyDataToSendToFilter(packet); + connector_ptr->Send(packet); + } + + void ASession::SendToClientCallback(const boost::system::error_code& e, + std::size_t size __attribute__((unused))) { + DARWIN_LOGGER; + + if (e) { + DARWIN_LOG_ERROR("ASession::SendToClientCallback:: " + e.message()); + _manager.Stop(shared_from_this()); + DARWIN_LOG_DEBUG("ASession::SendToClientCallback:: Stopped session in manager"); + return; + } + + Start(); + } + + void ASession::SendErrorResponse(const std::string& message, const unsigned int code) { + if (this->_packet.GetResponseType() != DARWIN_RESPONSE_SEND_BACK && this->_packet.GetResponseType() != DARWIN_RESPONSE_SEND_BOTH) + return; + _packet.GetMutableBody().clear(); + _packet.GetMutableBody() += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; + this->SendToClient(_packet); + } + + void ASession::SetThreshold(std::size_t const& threshold) { + DARWIN_LOGGER; + //If the threshold is negative, we keep the actual default threshold + if(threshold>100){ + DARWIN_LOG_DEBUG("Session::SetThreshold:: Default threshold " + std::to_string(_threshold) + "applied"); + return; + } + _threshold = threshold; + } + + size_t ASession::GetThreshold() { + return _threshold; + } +} diff --git a/samples/base/network/ASession.fwd.hpp b/samples/base/network/ASession.fwd.hpp new file mode 100644 index 00000000..b9a44173 --- /dev/null +++ b/samples/base/network/ASession.fwd.hpp @@ -0,0 +1,18 @@ +#pragma once +#include +#include + +// Forward Declaration mandatory because of the circular dependency between Tasks, Sessions and Generators +namespace darwin { + class ASession; + + /// Definition of a session's self-managing pointer. + /// + /// \typedef session_ptr_t + typedef std::shared_ptr session_ptr_t; + + /// Definition of a UDP buffer + /// + /// \typedef udp_buffer_t + typedef std::array udp_buffer_t; +} \ No newline at end of file diff --git a/samples/base/network/ASession.hpp b/samples/base/network/ASession.hpp new file mode 100755 index 00000000..f4f5c1a9 --- /dev/null +++ b/samples/base/network/ASession.hpp @@ -0,0 +1,173 @@ +/// +/// \file ASession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Session to be extended for different protocols +/// It is created by Servers and managed by a Manager +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include "config.hpp" +#include "Generator.hpp" +#include "Manager.hpp" +#include "DarwinPacket.hpp" +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" +#include "../../toolkit/rapidjson/document.h" +#include "../../toolkit/rapidjson/writer.h" +#include "../../toolkit/rapidjson/stringbuffer.h" +#include "Time.hpp" + +#include "ASession.fwd.hpp" + +#define DARWIN_SESSION_BUFFER_SIZE 2048 +#define DARWIN_DEFAULT_THRESHOLD 80 +#define DARWIN_ERROR_RETURN 101 + +namespace darwin { + + class ASession : public std::enable_shared_from_this { + public: + /// + /// \brief Construct a new ASession object + /// + /// \param manager reference to the manager that will manage this session + /// \param generator referece to the task generator + /// + ASession(Manager& manager, + Generator& generator); + + virtual ~ASession() = default; + + // Make the Session non copyable & non movable + ASession(ASession const&) = delete; + + ASession(ASession const&&) = delete; + + ASession& operator=(ASession const&) = delete; + + ASession& operator=(ASession const&&) = delete; + + public: + /// Start the session and the async read of the incoming packet. + virtual void Start(); + + /// Stop the session and close the socket. + virtual void Stop() = 0; + + /// Set the filter's threshold + /// + /// \param threshold The threshold wanted. + virtual void SetThreshold(std::size_t const& threshold) final; + + /// Set the output's type of the filter + /// + /// \param name the string that represent the output type + virtual void SetOutputType(std::string const& output) final; + + /// Get the filter's output type + /// + /// \return The filter's output type + config::output_type GetOutputType() const; + + /// Get the filter's result in a log form + /// + /// \return The filter's log + virtual std::string GetLogs(); + + /// Get the filter's threshold + /// + /// \return The filter's threshold + size_t GetThreshold(); + + /// Send + virtual void SendNext(DarwinPacket& packet) final; + + protected: + + + /// Get the data to send to the next filter + /// according to the filter's output type + /// + /// \param packet packet to modify (the field _body may be modified) + void ModifyDataToSendToFilter(DarwinPacket& packet) const; + + virtual void WriteToClient(std::vector& packet) = 0; + + /// Send result to the client. + /// Possible failures must be handled in SendToClientCallback + /// + virtual void SendToClient(DarwinPacket& packet) noexcept; + + /// Send result to next filter. + /// Possible failures must be handled in SendToFilterCallback + /// + virtual void SendToFilter(DarwinPacket& packet) noexcept; + + /// Called when data is sent using Send() method. + /// Terminate the session on failure. + /// + /// \param size The number of byte sent. + virtual void + SendToClientCallback(const boost::system::error_code& e, + std::size_t size); + + std::string JsonStringify(rapidjson::Document &json) const; + + /// Set the async read for the header. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadHeader() = 0; + + /// Callback of async read for the header. + /// Terminate the session on failure. + virtual void ReadHeaderCallback(const boost::system::error_code& e, std::size_t size) final; + + /// Set the async read for the body. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadBody(std::size_t size) = 0; + + /// Callback of async read for the body. + /// Terminate the session on failure. + virtual void + ReadBodyCallback(const boost::system::error_code& e, + std::size_t size) final; + + /// Execute the filter and + virtual void ExecuteFilter(); + + /// Sends a response with a body containing an error message + /// + /// \param message The error message to send + /// \param code The error code to send + virtual void SendErrorResponse(const std::string& message, const unsigned int code) final; + + friend std::shared_ptr Generator::CreateTask(std::shared_ptr s) noexcept; + + friend void ATask::run(); + + // Not accessible by children + private: + std::string _filter_name; //!< name of the filter + config::output_type _output; //!< The filter's output. + + std::size_t _threshold = DARWIN_DEFAULT_THRESHOLD; + + // Accessible by children + protected: + darwin_filter_packet_t _header; //!< Reading Header + std::array _body_buffer; //!< Reading buffer for the body. + Manager& _manager; //!< The associated connection manager. + Generator& _generator; //!< The Task Generator. + DarwinPacket _packet; + std::string _logs; //!< Represents data given in the logs by the Session + }; +} + diff --git a/samples/base/network/TcpNextFilterConnector.cpp b/samples/base/network/TcpNextFilterConnector.cpp new file mode 100644 index 00000000..5916a348 --- /dev/null +++ b/samples/base/network/TcpNextFilterConnector.cpp @@ -0,0 +1,51 @@ +#include "TcpNextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + TcpNextFilterConnector::TcpNextFilterConnector(boost::asio::ip::address const& net_address, int port) + : ANextFilterConnector{}, _net_address{net_address}, _net_port{port}, _socket{_io_context} + { + + } + + + bool TcpNextFilterConnector::Connect() { + DARWIN_LOGGER; + boost::system::error_code ec; + + _socket.connect(boost::asio::ip::tcp::endpoint(_net_address, _net_port), ec); + + if(ec) { + _nb_attempts++; + DARWIN_LOG_ERROR("TcpNextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + return false; + } + // Reset the counter on success + _nb_attempts = 0; + _is_connected = true; + return true; + } + + void TcpNextFilterConnector::Send(std::shared_ptr packet) { + DARWIN_LOGGER; + if(! _is_connected){ + while( ! this->Connect()) { + if(_nb_attempts >= _max_attempts){ + DARWIN_LOG_ERROR("TcpNextFilterConnector::Send:: Maximal number of attempts reached, aborting packet"); + return; + } + std::this_thread::sleep_for(_attempts_delay_ms); + } + } + + boost::asio::async_write(_socket, *packet, boost::bind(&TcpNextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + + } +} \ No newline at end of file diff --git a/samples/base/network/TcpNextFilterConnector.hpp b/samples/base/network/TcpNextFilterConnector.hpp new file mode 100644 index 00000000..5c87de92 --- /dev/null +++ b/samples/base/network/TcpNextFilterConnector.hpp @@ -0,0 +1,51 @@ +/// +/// \file TcpNextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Connection to NExt Filter (TCP Protocol) +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include +#include + +#include "ANextFilterConnector.hpp" +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class TcpNextFilterConnector: public ANextFilterConnector { + public: + /// + /// \brief Construct a new Tcp Next Filter Connector object + /// + /// \param net_address parsed ip address of the next filter + /// \param port port of the next filter + /// + TcpNextFilterConnector(boost::asio::ip::address const& net_address, int port); + virtual ~TcpNextFilterConnector() = default; + + /// + /// \brief Attempts to connect to the next Filter + /// + /// \return true if the attempt is successful + /// \return false else + /// + virtual bool Connect(); + + private: + + virtual void Send(std::shared_ptr packet); + + boost::asio::ip::address _net_address; + int _net_port; + boost::asio::ip::tcp::socket _socket; + }; +} \ No newline at end of file diff --git a/samples/base/network/TcpServer.cpp b/samples/base/network/TcpServer.cpp new file mode 100755 index 00000000..c330ba7e --- /dev/null +++ b/samples/base/network/TcpServer.cpp @@ -0,0 +1,78 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include +#include "TcpServer.hpp" +#include "TcpSession.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + TcpServer::TcpServer(boost::asio::ip::address const& address, + int port_nb, + std::string const& output, + std::size_t threshold, + Generator& generator) + : AServer(output, threshold, generator), + _address{address}, _port_nb{port_nb}, + _acceptor{_io_context, boost::asio::ip::tcp::endpoint(_address, _port_nb)}, + _new_connection{_io_context} { + + this->InitSignalsAndStart(); + } + + + void TcpServer::Clean() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("TcpServer::Clean:: Cleaning server..."); + _manager.StopAll(); + } + + void TcpServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { + // The server is stopped by cancelling all outstanding asynchronous + // operations. Once all operations have finished the io_context::run() + // call will exit. + DARWIN_LOGGER; + + SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); + DARWIN_LOG_DEBUG("TcpServer::Handle:: Closing acceptor"); + _acceptor.close(); + _io_context.stop(); + } + + void TcpServer::Accept() { + _acceptor.async_accept(_new_connection, + boost::bind(&TcpServer::HandleAccept, this, + boost::asio::placeholders::error)); + } + + void TcpServer::HandleAccept(boost::system::error_code const& e) { + DARWIN_LOGGER; + + if (!_acceptor.is_open()) { + DARWIN_LOG_INFO("TcpServer::HandleAccept:: Acceptor closed, closing server..."); + return; + } + + if (!e) { + DARWIN_LOG_DEBUG("TcpServer::HandleAccept:: New connection accepted"); + auto sess = std::make_shared(_new_connection, _manager, _generator); + sess->SetOutputType(_output); + sess->SetThreshold(_threshold); + _manager.Start(sess); + Accept(); + } else { + DARWIN_LOG_ERROR("TcpServer::HandleAccept:: Error accepting connection, no longer accepting"); + } + } + +} \ No newline at end of file diff --git a/samples/base/network/TcpServer.hpp b/samples/base/network/TcpServer.hpp new file mode 100755 index 00000000..1583a001 --- /dev/null +++ b/samples/base/network/TcpServer.hpp @@ -0,0 +1,66 @@ +/// +/// \file TcpServer.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Server for TCP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include +#include +#include "ASession.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +#include "AServer.hpp" + +namespace darwin { + + class TcpServer : public AServer { + public: + /// + /// \brief Construct a new Tcp Server object + /// + /// \param address parsed ip address of this filter + /// \param port port of this filter + /// \param output filter's output type + /// \param threshold filter's threshold + /// \param generator ref to the task generator + /// + TcpServer(boost::asio::ip::address const& address, + int port, + std::string const& output, + std::size_t threshold, + Generator& generator); + + virtual ~TcpServer() = default; + + // Make the TcpServer non copyable & non movable + TcpServer(TcpServer const&) = delete; + + TcpServer(TcpServer const&&) = delete; + + TcpServer& operator=(TcpServer const&) = delete; + + TcpServer& operator=(TcpServer const&&) = delete; + + void Clean(); + + void HandleStop(boost::system::error_code const& error, int sig); + + void Accept(); + + void HandleAccept(boost::system::error_code const& e); + + private: + boost::asio::ip::address _address; + int _port_nb; //!< Path to the Tcp socket to listen on. + boost::asio::ip::tcp::acceptor _acceptor; //!< Acceptor for the incoming connections. + boost::asio::ip::tcp::socket _new_connection; //!< Socket used to accept a new connection. + }; +} \ No newline at end of file diff --git a/samples/base/network/TcpSession.cpp b/samples/base/network/TcpSession.cpp new file mode 100755 index 00000000..eb2bb329 --- /dev/null +++ b/samples/base/network/TcpSession.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +#include "Logger.hpp" +#include "TcpSession.hpp" + +namespace darwin { + TcpSession::TcpSession( + boost::asio::ip::tcp::socket& socket, + Manager& manager, Generator& generator) + : ASession(manager, generator), _socket{std::move(socket)} + { + ; + } + + void TcpSession::Stop() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("TcpSession::Stop::"); + _socket.close(); + } + + void TcpSession::ReadHeader() { + DARWIN_LOGGER; + + DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); + boost::asio::async_read(_socket, + boost::asio::buffer(&_header, sizeof(_header)), + boost::bind(&TcpSession::ReadHeaderCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void TcpSession::ReadBody(std::size_t size) { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); + + _socket.async_read_some(boost::asio::buffer(_body_buffer, + size), + boost::bind(&TcpSession::ReadBodyCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void TcpSession::WriteToClient(std::vector& packet) { + boost::asio::async_write(_socket, + boost::asio::buffer(packet), + boost::bind(&TcpSession::SendToClientCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } +} + diff --git a/samples/base/network/TcpSession.hpp b/samples/base/network/TcpSession.hpp new file mode 100755 index 00000000..67cd1c44 --- /dev/null +++ b/samples/base/network/TcpSession.hpp @@ -0,0 +1,52 @@ +/// +/// \file TcpSession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Session for TCP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include "ASession.hpp" +#include "Generator.hpp" + +namespace darwin { + + class TcpSession: public ASession { + public: + /// + /// \brief Construct a new Tcp Session object + /// + /// \param socket ref to the socket of a connection, the socket will be moved + /// \param manager ref to the sessions' manager + /// \param generator ref to the task generator + /// + TcpSession(boost::asio::ip::tcp::socket& socket, + Manager& manager, + Generator& generator); + + ~TcpSession() = default; + + void Stop() override final; + + protected: + + /// Set the async read for the header. + /// + virtual void ReadHeader() override final; + + /// Set the async read for the body. + /// + virtual void ReadBody(std::size_t size) override final; + + /// + virtual void WriteToClient(std::vector& packet) override final; + + private: + boost::asio::ip::tcp::socket _socket; //!< Session's socket. + }; + +} \ No newline at end of file diff --git a/samples/base/network/UdpNextFilterConnector.cpp b/samples/base/network/UdpNextFilterConnector.cpp new file mode 100644 index 00000000..d3c937f1 --- /dev/null +++ b/samples/base/network/UdpNextFilterConnector.cpp @@ -0,0 +1,44 @@ +#include "UdpNextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + UdpNextFilterConnector::UdpNextFilterConnector(boost::asio::ip::address const& net_address, int port) + : ANextFilterConnector{}, _net_address{net_address}, _net_port{port}, _endpoint{_net_address, static_cast(_net_port)}, + _socket{_io_context} + { + + } + + + bool UdpNextFilterConnector::Connect() { + DARWIN_LOGGER; + boost::system::error_code ec; + + _socket.connect(_endpoint, ec); + + if(ec) { + // We handle the case but, since UDP is stateless, this _socket.connect call does nothing and never fails + DARWIN_LOG_ERROR("UdpNextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + return false; + } + return true; + } + + void UdpNextFilterConnector::Send(std::shared_ptr packet) { + DARWIN_LOGGER; + // The Connect call cannot fail, we don't have to check and we don't have to retry it + this->Connect(); + + DARWIN_LOG_DEBUG("UdpNextFilterConnector::Send : Sending packet to UDP endpoint, no retries"); + + _socket.async_send(*packet, + boost::bind(&UdpNextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + } +} \ No newline at end of file diff --git a/samples/base/network/UdpNextFilterConnector.hpp b/samples/base/network/UdpNextFilterConnector.hpp new file mode 100644 index 00000000..e9815444 --- /dev/null +++ b/samples/base/network/UdpNextFilterConnector.hpp @@ -0,0 +1,46 @@ +/// +/// \file UdpNextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Connector to Next Filter for UDP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include +#include + +#include "ANextFilterConnector.hpp" +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class UdpNextFilterConnector: public ANextFilterConnector { + public: + /// + /// \brief Construct a new Udp Next Filter Connector object + /// + /// \param net_address parsed IP address of the next filter + /// \param port port of the next filter + /// + UdpNextFilterConnector(boost::asio::ip::address const& net_address, int port); + virtual ~UdpNextFilterConnector() = default; + + virtual bool Connect(); + + private: + + virtual void Send(std::shared_ptr packet); + + boost::asio::ip::address _net_address; + int _net_port; + boost::asio::ip::udp::endpoint _endpoint; + boost::asio::ip::udp::socket _socket; + }; +} \ No newline at end of file diff --git a/samples/base/network/UdpServer.cpp b/samples/base/network/UdpServer.cpp new file mode 100755 index 00000000..1ea96181 --- /dev/null +++ b/samples/base/network/UdpServer.cpp @@ -0,0 +1,84 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include +#include "UdpServer.hpp" +#include "UdpSession.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + UdpServer::UdpServer(boost::asio::ip::address const& address, + int port_nb, + std::string const& output, + std::size_t threshold, + Generator& generator) + : AServer(output, threshold, generator), + _address{address}, _port_nb{port_nb}, + _endpoint{_address, static_cast(_port_nb)}, + _new_connection{_io_context,_endpoint} + { + this->InitSignalsAndStart(); + } + + + + + void UdpServer::Clean() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); + _manager.StopAll(); + } + + void UdpServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { + // The server is stopped by cancelling all outstanding asynchronous + // operations. Once all operations have finished the io_context::run() + // call will exit. + DARWIN_LOGGER; + + SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); + DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); + _io_context.stop(); + } + + void UdpServer::Accept() { + _new_connection.async_receive(boost::asio::buffer(_buffer), + boost::bind(&UdpServer::HandleReceive, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void UdpServer::HandleReceive(boost::system::error_code const& e, size_t bytes_received) { + DARWIN_LOGGER; + + if (!e) { + if(bytes_received >= sizeof(darwin_filter_packet_t)){ + DARWIN_LOG_DEBUG("UdpServer::HandleReceive:: New connection accepted"); + auto sess = std::make_shared(_buffer, _manager, _generator); + sess->SetOutputType(_output); + sess->SetThreshold(_threshold); + sess->ReadBody(bytes_received); + } else { + DARWIN_LOG_ERROR("UdpServer::HandleReceive:: packet too small (in bytes) : " + std::to_string(bytes_received)); + } + } else { + DARWIN_LOG_ERROR("UdpServer::HandleReceive:: Error accepting connection, no longer accepting"); + } + Accept(); + } + + void UdpServer::HandleAccept(boost::system::error_code const& e __attribute__((unused))) { + return; + } + + +} \ No newline at end of file diff --git a/samples/base/network/UdpServer.hpp b/samples/base/network/UdpServer.hpp new file mode 100755 index 00000000..acce378f --- /dev/null +++ b/samples/base/network/UdpServer.hpp @@ -0,0 +1,68 @@ +/// +/// \file UdpServer.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Server for TCP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include +#include +#include +#include +#include "ASession.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +#include "AServer.hpp" + +namespace darwin { + + class UdpServer : public AServer { + public: + /// + /// \brief Construct a new Udp Server object + /// + /// \param address parsed ip address of this filter + /// \param port port of this filter + /// \param output filter's output type + /// \param threshold filter's threshold + /// \param generator ref to the task generator + /// + UdpServer(boost::asio::ip::address const& address, + int port, + std::string const& output, + std::size_t threshold, + Generator& generator); + + virtual ~UdpServer() = default; + + // Make the UdpServer non copyable & non movable + UdpServer(UdpServer const&) = delete; + + UdpServer(UdpServer const&&) = delete; + + UdpServer& operator=(UdpServer const&) = delete; + + UdpServer& operator=(UdpServer const&&) = delete; + + void Clean(); + + void HandleStop(boost::system::error_code const& error, int sig); + + void Accept(); + + void HandleAccept(boost::system::error_code const& e); + + void HandleReceive(boost::system::error_code const& e, size_t bytes_received); + private: + boost::asio::ip::address _address; + int _port_nb; //!< Path to the Udp socket to listen on. + boost::asio::ip::udp::endpoint _endpoint; + boost::asio::ip::udp::socket _new_connection; //!< Socket used to accept a new connection. + udp_buffer_t _buffer; + }; +} \ No newline at end of file diff --git a/samples/base/network/UdpSession.cpp b/samples/base/network/UdpSession.cpp new file mode 100755 index 00000000..4d02fa78 --- /dev/null +++ b/samples/base/network/UdpSession.cpp @@ -0,0 +1,61 @@ +#include +#include +#include + +#include "Logger.hpp" +#include "UdpSession.hpp" + +namespace darwin { + UdpSession::UdpSession(const udp_buffer_t& buffer, + Manager& manager, Generator& generator) + : ASession(manager, generator), _buffer{buffer} + { + ; + } + + void UdpSession::Stop() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("UdpSession::Stop::"); + } + + void UdpSession::ReadHeader() { + //NOP + return; + } + + void UdpSession::ReadBody(std::size_t size) { + DARWIN_LOGGER; + + if(size < sizeof(_header)){ + DARWIN_LOG_ERROR("Error parsing header, buffer is less than expected : " + std::to_string(size)); + return; + } + + std::memcpy(&_header, _buffer.data(), sizeof(_header)); + + //The header can hold one certitude, if there are more, we have to parsed them accordingly + size_t cert_size = _header.certitude_size * sizeof(unsigned int); + if(size != sizeof(_header) + _header.body_size + cert_size){ + DARWIN_LOG_ERROR("Error parsing header sizes, expected " + std::to_string(sizeof(_header) + _header.body_size + cert_size) + " but buffer is " + std::to_string(size)); + return; + } + + _packet = DarwinPacket(_header); + + for(size_t i=0; i < _header.certitude_size; i++){ + unsigned int cert = 0; + std::memcpy(&cert, _buffer.data() + sizeof(_header) + i*sizeof(unsigned int), sizeof(unsigned int)); + _packet.AddCertitude(cert); + } + + std::string& body = _packet.GetMutableBody(); + body.append(_buffer.begin() + sizeof(_header) + cert_size, _header.body_size); + + this->ExecuteFilter(); + } + + void UdpSession::WriteToClient(std::vector& packet __attribute((unused))) { + return; + } +} + diff --git a/samples/base/network/UdpSession.hpp b/samples/base/network/UdpSession.hpp new file mode 100755 index 00000000..da469771 --- /dev/null +++ b/samples/base/network/UdpSession.hpp @@ -0,0 +1,54 @@ +/// +/// \file UdpSession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Session for UDP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include "ASession.hpp" +#include "Generator.hpp" + +namespace darwin { + + class UdpSession: public ASession { + public: + /// + /// \brief Construct a new Udp Session object + /// + /// \param buffer ref to a buffer containing a udp datagram + /// \param manager ref to the sessions' manager + /// \param generator ref to the task generator + /// + UdpSession(const udp_buffer_t& buffer, + Manager& manager, + Generator& generator); + + ~UdpSession() = default; + + void Stop() override final; + + void SetNextFilterPort(int port); + + /// Set the async read for the body. + /// + virtual void ReadBody(std::size_t size) override; + + protected: + + /// Set the async read for the header. + /// + virtual void ReadHeader() override; + + virtual void WriteToClient(std::vector& packet) override; + + private: + + const udp_buffer_t& _buffer; + }; + +} \ No newline at end of file diff --git a/samples/base/network/UnixNextFilterConnector.cpp b/samples/base/network/UnixNextFilterConnector.cpp new file mode 100644 index 00000000..371e7c0d --- /dev/null +++ b/samples/base/network/UnixNextFilterConnector.cpp @@ -0,0 +1,49 @@ +#include "UnixNextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + UnixNextFilterConnector::UnixNextFilterConnector(std::string const& path) + : ANextFilterConnector{}, _socket_path{path}, _socket{_io_context} + { + } + + + bool UnixNextFilterConnector::Connect() { + DARWIN_LOGGER; + boost::system::error_code ec; + + _socket.connect(boost::asio::local::stream_protocol::endpoint(_socket_path), ec); + + if(ec) { + _nb_attempts++; + DARWIN_LOG_ERROR("UnixNextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + return false; + } + // Reset the counter on success + _nb_attempts = 0; + _is_connected = true; + return true; + } + + void UnixNextFilterConnector::Send(std::shared_ptr packet) { + DARWIN_LOGGER; + if(! _is_connected){ + while( ! this->Connect()) { + if(_nb_attempts >= _max_attempts){ + DARWIN_LOG_ERROR("UnixNextFilterConnector::Send:: Maximal number of attempts reached, aborting packet"); + return; + } + std::this_thread::sleep_for(_attempts_delay_ms); + } + } + boost::asio::async_write(_socket, *packet, boost::bind(&UnixNextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + + } +} \ No newline at end of file diff --git a/samples/base/network/UnixNextFilterConnector.hpp b/samples/base/network/UnixNextFilterConnector.hpp new file mode 100644 index 00000000..af10504e --- /dev/null +++ b/samples/base/network/UnixNextFilterConnector.hpp @@ -0,0 +1,42 @@ +/// +/// \file UnixNextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Connector to the NExt filter for Unix socket Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// + +#include +#include +#include +#include + +#include "ANextFilterConnector.hpp" +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class UnixNextFilterConnector: public ANextFilterConnector { + public: + /// + /// \brief Construct a new Unix Next Filter Connector object + /// + /// \param path path of the unix socket of the next filter + /// + UnixNextFilterConnector(std::string const& path); + virtual ~UnixNextFilterConnector() = default; + + virtual bool Connect(); + + private: + + virtual void Send(std::shared_ptr packet); + + std::string _socket_path; + boost::asio::local::stream_protocol::socket _socket; + }; +} \ No newline at end of file diff --git a/samples/base/network/UnixServer.cpp b/samples/base/network/UnixServer.cpp new file mode 100644 index 00000000..a88a428a --- /dev/null +++ b/samples/base/network/UnixServer.cpp @@ -0,0 +1,77 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include "UnixServer.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + UnixServer::UnixServer(std::string const& socket_path, + std::string const& output, + std::size_t threshold, + Generator& generator) + : AServer(output, threshold, generator), + _socket_path{socket_path}, + _acceptor{_io_context, boost::asio::local::stream_protocol::endpoint( + socket_path)}, + _new_connection{_io_context} { + + this->InitSignalsAndStart(); + } + + + void UnixServer::Clean() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); + _manager.StopAll(); + unlink(_socket_path.c_str()); + } + + void UnixServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { + // The server is stopped by cancelling all outstanding asynchronous + // operations. Once all operations have finished the io_context::run() + // call will exit. + DARWIN_LOGGER; + + SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); + DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); + _acceptor.close(); + _io_context.stop(); + } + + void UnixServer::Accept() { + _acceptor.async_accept(_new_connection, + boost::bind(&UnixServer::HandleAccept, this, + boost::asio::placeholders::error)); + } + + void UnixServer::HandleAccept(boost::system::error_code const& e) { + DARWIN_LOGGER; + + if (!_acceptor.is_open()) { + DARWIN_LOG_INFO("Server::HandleAccept:: Acceptor closed, closing server..."); + return; + } + + if (!e) { + DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); + auto sess = std::make_shared(_new_connection, _manager, _generator); + sess->SetOutputType(_output); + sess->SetThreshold(_threshold); + _manager.Start(sess); + Accept(); + } else { + DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); + } + } + +} \ No newline at end of file diff --git a/samples/base/network/UnixServer.hpp b/samples/base/network/UnixServer.hpp new file mode 100644 index 00000000..995ae205 --- /dev/null +++ b/samples/base/network/UnixServer.hpp @@ -0,0 +1,59 @@ +/// \file UnixServer.hpp +/// \authors hsoszynski +/// \version 2.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018-2021 Advens. All rights reserved. + +#pragma once + +#include +#include +#include +#include "UnixSession.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +#include "AServer.hpp" + +namespace darwin { + + class UnixServer : public AServer { + public: + /// + /// \brief Construct a new Unix Server object + /// + /// \param socket_path Path of the UNIX socket to listen on + /// \param output Filters' output type + /// \param threshold Threshold at which the filter will raise a log. + /// \param generator ref to the task generator + /// + UnixServer(std::string const& socket_path, + std::string const& output, + std::size_t threshold, + Generator& generator); + + virtual ~UnixServer() = default; + + // Make the UnixServer non copyable & non movable + UnixServer(UnixServer const&) = delete; + + UnixServer(UnixServer const&&) = delete; + + UnixServer& operator=(UnixServer const&) = delete; + + UnixServer& operator=(UnixServer const&&) = delete; + + void Clean() override; + + void HandleStop(boost::system::error_code const& error, int sig) override; + + void Accept() override; + + void HandleAccept(boost::system::error_code const& e) override; + + private: + std::string _socket_path; //!< Path to the UNIX socket to listen on. + boost::asio::local::stream_protocol::acceptor _acceptor; //!< Acceptor for the incoming connections. + boost::asio::local::stream_protocol::socket _new_connection; //!< Socket used to accept a new connection. + }; +} \ No newline at end of file diff --git a/samples/base/network/UnixSession.cpp b/samples/base/network/UnixSession.cpp new file mode 100755 index 00000000..8a47138c --- /dev/null +++ b/samples/base/network/UnixSession.cpp @@ -0,0 +1,53 @@ +#include +#include + +#include "Logger.hpp" +#include "UnixSession.hpp" + +namespace darwin { + + UnixSession::UnixSession( + boost::asio::local::stream_protocol::socket& socket, + Manager& manager, Generator& generator) + : ASession(manager, generator), _socket{std::move(socket)} + { + ; + } + + void UnixSession::Stop() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("UnixSession::Stop::"); + _socket.close(); + } + + void UnixSession::ReadHeader() { + DARWIN_LOGGER; + + DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); + boost::asio::async_read(_socket, + boost::asio::buffer(&_header, sizeof(_header)), + boost::bind(&UnixSession::ReadHeaderCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void UnixSession::ReadBody(std::size_t size) { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); + + _socket.async_read_some(boost::asio::buffer(_body_buffer, + size), + boost::bind(&UnixSession::ReadBodyCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void UnixSession::WriteToClient(std::vector& packet) { + boost::asio::async_write(_socket, + boost::asio::buffer(packet), + boost::bind(&UnixSession::SendToClientCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } +} + diff --git a/samples/base/network/UnixSession.hpp b/samples/base/network/UnixSession.hpp new file mode 100755 index 00000000..7c6a3e27 --- /dev/null +++ b/samples/base/network/UnixSession.hpp @@ -0,0 +1,53 @@ +/// +/// \file UnixSession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Session for Unix Socket Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// +#pragma once + +#include "ASession.hpp" + +namespace darwin { + + class UnixSession: public ASession { + public: + /// + /// \brief Construct a new Unix Session object + /// + /// \param socket ref to the unix socket, it will be moved to the object + /// \param manager ref to the sessions' manager + /// \param generator ref to the task generator + /// + UnixSession(boost::asio::local::stream_protocol::socket& socket, + Manager& manager, + Generator& generator); + + ~UnixSession() = default; + + void Stop() override final; + + protected: + + /// Set the async read for the header. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadHeader() override final; + + /// Set the async read for the body. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadBody(std::size_t size) override final; + + /// + virtual void WriteToClient(std::vector& packet) override final; + + private: + boost::asio::local::stream_protocol::socket _socket; //!< Session's socket. + }; + +} \ No newline at end of file diff --git a/samples/fanomaly/AnomalyTask.cpp b/samples/fanomaly/AnomalyTask.cpp index c2213a21..c3c05f03 100644 --- a/samples/fanomaly/AnomalyTask.cpp +++ b/samples/fanomaly/AnomalyTask.cpp @@ -14,14 +14,14 @@ #include "AnomalyTask.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" +#include "ASession.hpp" #include "AlertManager.hpp" -AnomalyTask::AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex) - : Session{"anomaly", socket, manager, cache, cache_mutex}{} +AnomalyTask::AnomalyTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet) + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet){} void AnomalyTask::operator()() { DARWIN_LOGGER; @@ -40,7 +40,7 @@ void AnomalyTask::operator()() { } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } @@ -77,7 +77,7 @@ bool AnomalyTask::Detection(){ index_anomalies = arma::find(assignments == SIZE_MAX); // index of noises find by the clustering in the _matrix if (index_anomalies.is_empty()){ DARWIN_LOG_DEBUG("AnomalyTask::Detection:: No anomalies found"); - _certitudes.push_back(0); + _packet.AddCertitude(0); return false; } no_anomalies.shed_cols(index_anomalies); // suppress noisy columns, keep only normals data @@ -94,10 +94,10 @@ bool AnomalyTask::Detection(){ alerts.insert_rows(DISTANCE, distances); STAT_MATCH_INC; - _certitudes.push_back(100); + _packet.AddCertitude(100); GenerateAlerts(_ips, index_anomalies, alerts); - if(GetOutputType() == darwin::config::output_type::LOG){ + if(_s->GetOutputType() == darwin::config::output_type::LOG){ GenerateLogs(_ips, index_anomalies, alerts); } return true; @@ -114,15 +114,15 @@ void AnomalyTask::GenerateAlerts(std::vector ips, arma::uvec index_ details += R"("icmp_nb_host": )" + std::to_string(alerts(ICMP_NB_HOST, i)) + ","; details += R"("distance": )" + std::to_string(alerts(DISTANCE, i)); details += "}"; - DARWIN_ALERT_MANAGER.Alert(ips[index_anomalies(i)], 100, Evt_idToString(), details); + DARWIN_ALERT_MANAGER.Alert(ips[index_anomalies(i)], 100, _packet.Evt_idToString(), details); } } void AnomalyTask::GenerateLogs(std::vector ips, arma::uvec index_anomalies, arma::mat alerts){ - + auto logs = _packet.GetMutableLogs(); for(unsigned int i=0; i ips, arma::uvec index_an alert_log += R"("icmp_nb_host": )" + std::to_string(alerts(ICMP_NB_HOST, i)) + ","; alert_log += R"("distance": )" + std::to_string(alerts(DISTANCE, i)); alert_log += "}}"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } }; diff --git a/samples/fanomaly/AnomalyTask.hpp b/samples/fanomaly/AnomalyTask.hpp index 177c91cc..de64e9d5 100644 --- a/samples/fanomaly/AnomalyTask.hpp +++ b/samples/fanomaly/AnomalyTask.hpp @@ -13,8 +13,9 @@ #include #include -#include "protocol.h" -#include "Session.hpp" +#include "DarwinPacket.hpp" +#include "ATask.hpp" +#include "ASession.fwd.hpp" #include "../../toolkit/lru_cache.hpp" @@ -27,12 +28,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class AnomalyTask: public darwin::Session { +class AnomalyTask: public darwin::ATask { public: - explicit AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex); + explicit AnomalyTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet); ~AnomalyTask() override = default; public: diff --git a/samples/fanomaly/Generator.cpp b/samples/fanomaly/Generator.cpp index 4c2dcf1a..ece2ca51 100644 --- a/samples/fanomaly/Generator.cpp +++ b/samples/fanomaly/Generator.cpp @@ -12,8 +12,15 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "AnomalyTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -35,9 +42,13 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration __attribute_ return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_ANOMALY; } \ No newline at end of file diff --git a/samples/fanomaly/Generator.hpp b/samples/fanomaly/Generator.hpp index c165c478..2b0e184c 100644 --- a/samples/fanomaly/Generator.hpp +++ b/samples/fanomaly/Generator.hpp @@ -12,19 +12,20 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; + virtual long GetFilterCode() const; + public: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; virtual bool ConfigureAlerting(const std::string& tags) override final; - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; }; \ No newline at end of file diff --git a/samples/fbuffer/BufferTask.cpp b/samples/fbuffer/BufferTask.cpp index f3d237e9..013c5420 100644 --- a/samples/fbuffer/BufferTask.cpp +++ b/samples/fbuffer/BufferTask.cpp @@ -12,18 +12,18 @@ #include "../../toolkit/Validators.hpp" #include "../toolkit/rapidjson/document.h" #include "BufferTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "AlertManager.hpp" -BufferTask::BufferTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, +BufferTask::BufferTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::vector> &inputs, std::vector> &connectors) - : Session{"buffer", socket, manager, cache, cache_mutex}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _inputs_format(inputs), _connectors(connectors) { } @@ -41,11 +41,11 @@ void BufferTask::operator()() { for (rapidjson::Value &line : array) { STAT_INPUT_INC; if (ParseLine(line)) { - this->_certitudes.push_back(0); + this->_packet.AddCertitude(0); this->AddEntries(); } else { STAT_PARSE_ERROR_INC; - this->_certitudes.push_back(DARWIN_ERROR_RETURN); + this->_packet.AddCertitude(DARWIN_ERROR_RETURN); } DARWIN_LOG_DEBUG("BufferTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms"); } diff --git a/samples/fbuffer/BufferTask.hpp b/samples/fbuffer/BufferTask.hpp index 7f63b0e4..3fd54000 100644 --- a/samples/fbuffer/BufferTask.hpp +++ b/samples/fbuffer/BufferTask.hpp @@ -12,12 +12,14 @@ #include "../../toolkit/RedisManager.hpp" #include "BufferThreadManager.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #define DARWIN_FILTER_BUFFER 0x62756672 // Made from: bufr +#define DARWIN_FILTER_NAME "buffer" -class BufferTask final : public darwin::Session { +class BufferTask final : public darwin::ATask { /// This class inherits from Session (see Session.hpp) /// This class handles Tasks for Buffer Filter. /// It parses the body of incoming messages and splits data into several REDIS lists depending on @@ -34,10 +36,10 @@ class BufferTask final : public darwin::Session { ///\param cache_mutex Transfered to Session constructor ///\param inputs This vector holds the name and types of data in input (in the correct order) ///\param connectors This vector holds all the Connectors needed depending on the output Filters in config file. - BufferTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, + BufferTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::vector> &inputs, std::vector> &connectors); diff --git a/samples/fbuffer/Connectors/AConnector.cpp b/samples/fbuffer/Connectors/AConnector.cpp index a4c76145..4e8d3e20 100644 --- a/samples/fbuffer/Connectors/AConnector.cpp +++ b/samples/fbuffer/Connectors/AConnector.cpp @@ -215,14 +215,14 @@ bool AConnector::SendToFilter(std::vector &logs) { } // Prepare packet - DARWIN_LOG_DEBUG("AConnector::SendToFilter:: data to send: " + data + ", data size: " + std::to_string(data.size())); + DARWIN_LOG_DEBUG("AConnector::SendToFilter:: data to send: size:" + std::to_string(data.size()) + ", data : " + data); /* * Allocate the header + * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size */ std::size_t certitude_size = 1; - std::size_t packet_size = sizeof(darwin_filter_packet_t) + data.size(); + std::size_t packet_size = sizeof(darwin_filter_packet_t) + data.size() + certitude_size*sizeof(unsigned int); DARWIN_LOG_DEBUG("AConnector::SendToFilter:: Computed packet size: " + std::to_string(packet_size)); darwin_filter_packet_t* packet; @@ -240,7 +240,7 @@ bool AConnector::SendToFilter(std::vector &logs) { if(data.size() != 0) { // TODO: set a proper pointer in protocol.h for the body // Yes We Hack... - memcpy(&packet->certitude_list[certitude_size + 1], data.c_str(), data.size()); + memcpy(&packet->certitude_list[certitude_size], data.c_str(), data.size()); } packet->type = DARWIN_PACKET_FILTER; packet->response = DARWIN_RESPONSE_SEND_DARWIN; diff --git a/samples/fbuffer/Connectors/SumConnector.hpp b/samples/fbuffer/Connectors/SumConnector.hpp index 622ae245..11924882 100644 --- a/samples/fbuffer/Connectors/SumConnector.hpp +++ b/samples/fbuffer/Connectors/SumConnector.hpp @@ -35,7 +35,7 @@ class SumConnector final : public AConnector { ///\brief test every Redis Key in _redis_lists to ensure keys are of the correct type if they exist in Redis /// ///\return true if the keys are not in redis, or are of the good type, false otherwise. - virtual bool PrepareKeysInRedis(); + virtual bool PrepareKeysInRedis() override; ///\brief This function sends data to the REDIS storage. It overrides default pure virtual one as each filter doesn't need the same data. /// @@ -60,7 +60,7 @@ class SumConnector final : public AConnector { ///\param sum_name unused parameter /// ///\return always true - virtual bool REDISReinsertLogs(std::vector &logs __attribute__((unused)), const std::string &sum_name __attribute__((unused))); + virtual bool REDISReinsertLogs(std::vector &logs __attribute__((unused)), const std::string &sum_name __attribute__((unused))) override; ///\brief Get the sum from Redis /// @@ -85,5 +85,5 @@ class SumConnector final : public AConnector { ///\param format The string to fill with the result /// ///\return True on success (formatting successful), False otherwise. - virtual bool FormatDataToSendToFilter(std::vector &logs, std::string &formatted); + virtual bool FormatDataToSendToFilter(std::vector &logs, std::string &formatted) override; }; \ No newline at end of file diff --git a/samples/fbuffer/Generator.cpp b/samples/fbuffer/Generator.cpp index e86692d5..2e0f584a 100644 --- a/samples/fbuffer/Generator.cpp +++ b/samples/fbuffer/Generator.cpp @@ -9,9 +9,16 @@ #include "../../toolkit/lru_cache.hpp" #include "base/Logger.hpp" #include "BufferTask.hpp" +#include "ASession.hpp" #include "Generator.hpp" #include "Connectors.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags __attribute__((unused))) { return true; } @@ -246,11 +253,15 @@ std::vector> Generator::FormatRedisListVecto return vector; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Generator::CreateTask:: Creating a new task"); - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, this->_inputs, this->_outputs)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + this->_inputs, this->_outputs + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_BUFFER; } \ No newline at end of file diff --git a/samples/fbuffer/Generator.hpp b/samples/fbuffer/Generator.hpp index 2287ef47..192b2e5c 100644 --- a/samples/fbuffer/Generator.hpp +++ b/samples/fbuffer/Generator.hpp @@ -10,7 +10,7 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "BufferThreadManager.hpp" #include "AConnector.hpp" @@ -28,7 +28,7 @@ class Generator: public AGenerator { ///\brief Unique default constructor /// DOES NOT fill the attributes members. /// They are filled by ConfigureNetworkObject LoadConfig, LoadInputs and LoadOutputs - Generator() = default; + Generator(size_t nb_task_threads); ///\brief default destructor virtual ~Generator() = default; @@ -40,9 +40,10 @@ class Generator: public AGenerator { ///\param manager Transfered to BufferTask constructor /// ///\return A shared ptr on the newly created BufferTask (in the form of a Session) - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; ///\brief Creates the Connectors depending on the _output_configs vector. /// diff --git a/samples/fconnection/ConnectionSupervisionTask.cpp b/samples/fconnection/ConnectionSupervisionTask.cpp index 78765fb2..e9a24a79 100644 --- a/samples/fconnection/ConnectionSupervisionTask.cpp +++ b/samples/fconnection/ConnectionSupervisionTask.cpp @@ -13,22 +13,22 @@ #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "AlertManager.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" #include "../../toolkit/lru_cache.hpp" #include "ConnectionSupervisionTask.hpp" +#include "ASession.hpp" #include "../../toolkit/RedisManager.hpp" #include "../toolkit/rapidjson/document.h" -ConnectionSupervisionTask::ConnectionSupervisionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, - unsigned int expire) - : Session{"connection", socket, manager, cache, cache_mutex}, +ConnectionSupervisionTask::ConnectionSupervisionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + unsigned int expire) + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _redis_expire{expire}{} long ConnectionSupervisionTask::GetFilterCode() noexcept { @@ -37,7 +37,8 @@ long ConnectionSupervisionTask::GetFilterCode() noexcept { void ConnectionSupervisionTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); unsigned int certitude; // Should not fail, as the Session body parser MUST check for validity ! @@ -52,21 +53,21 @@ void ConnectionSupervisionTask::operator()() { if(certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_connection, certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_connection, certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "connection": ")" + _connection + R"(", "certitude": )" + std::to_string(certitude) + "}"; - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("ConnectionSupervisionTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } @@ -113,7 +114,6 @@ bool ConnectionSupervisionTask::ParseLine(rapidjson::Value& line){ unsigned int ConnectionSupervisionTask::REDISLookup(const std::string& connection) noexcept { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ConnectionSupervisionTask:: Looking up '" + connection + "' in the Redis"); - int redisReplyCode; darwin::toolkit::RedisManager& redis = darwin::toolkit::RedisManager::GetInstance(); long long int result; diff --git a/samples/fconnection/ConnectionSupervisionTask.hpp b/samples/fconnection/ConnectionSupervisionTask.hpp index dbd545ce..03378501 100644 --- a/samples/fconnection/ConnectionSupervisionTask.hpp +++ b/samples/fconnection/ConnectionSupervisionTask.hpp @@ -8,8 +8,9 @@ #pragma once #include -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/RedisManager.hpp" @@ -24,13 +25,13 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class ConnectionSupervisionTask : public darwin::Session { +class ConnectionSupervisionTask : public darwin::ATask { public: - explicit ConnectionSupervisionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, - unsigned int expire); + explicit ConnectionSupervisionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + unsigned int expire); ~ConnectionSupervisionTask() override = default; public: diff --git a/samples/fconnection/Generator.cpp b/samples/fconnection/Generator.cpp index a3e7bd67..9c6e9315 100644 --- a/samples/fconnection/Generator.cpp +++ b/samples/fconnection/Generator.cpp @@ -14,8 +14,15 @@ #include "Generator.hpp" #include "base/Logger.hpp" #include "ConnectionSupervisionTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -139,11 +146,18 @@ bool Generator::ConfigRedis(const std::string &redis_socket_path, const std::str return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _redis_expire)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _redis_expire + ) + ); } +long Generator::GetFilterCode() const { + return DARWIN_FILTER_CONNECTION; +} + + Generator::~Generator() = default; diff --git a/samples/fconnection/Generator.hpp b/samples/fconnection/Generator.hpp index 7a5762a5..cf30a215 100644 --- a/samples/fconnection/Generator.hpp +++ b/samples/fconnection/Generator.hpp @@ -13,18 +13,19 @@ #include "../toolkit/rapidjson/document.h" #include "../../toolkit/RedisManager.hpp" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator(); public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fdga/DGATask.cpp b/samples/fdga/DGATask.cpp index 6cf8af3c..ea730f69 100644 --- a/samples/fdga/DGATask.cpp +++ b/samples/fdga/DGATask.cpp @@ -19,21 +19,21 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "DGATask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "tensorflow/core/framework/tensor.h" #include "AlertManager.hpp" -DGATask::DGATask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +DGATask::DGATask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens) - : Session{"dga", socket, manager, cache, cache_mutex}, _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, _faup_options(faup_options) { _is_cache = _cache != nullptr; } @@ -48,11 +48,10 @@ long DGATask::GetFilterCode() noexcept { void DGATask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; - + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); // Should not fail, as the Session body parser MUST check for validity ! rapidjson::GenericArray array = _body.GetArray(); - for (rapidjson::Value &value : array) { STAT_INPUT_INC; SetStartingTime(); @@ -71,14 +70,14 @@ void DGATask::operator()() { if (GetCacheResult(hash, certitude)) { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_domain, certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("DGATask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -88,14 +87,14 @@ void DGATask::operator()() { certitude = Predict(); if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_domain, certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) { SaveToCache(hash, certitude); } @@ -104,7 +103,7 @@ void DGATask::operator()() { } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } diff --git a/samples/fdga/DGATask.hpp b/samples/fdga/DGATask.hpp index dba6df2d..e66ca5d1 100644 --- a/samples/fdga/DGATask.hpp +++ b/samples/fdga/DGATask.hpp @@ -14,8 +14,9 @@ #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "tensorflow/core/public/session.h" #define DARWIN_FILTER_DGA 0x64676164 @@ -27,12 +28,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class DGATask : public darwin::Session { +class DGATask : public darwin::ATask { public: - explicit DGATask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit DGATask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens = 50); diff --git a/samples/fdga/Generator.cpp b/samples/fdga/Generator.cpp index ed0e4925..1c683803 100644 --- a/samples/fdga/Generator.cpp +++ b/samples/fdga/Generator.cpp @@ -14,9 +14,16 @@ #include "base/Logger.hpp" #include "DGATask.hpp" #include "Generator.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" #include "tensorflow/core/framework/graph.pb.h" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -168,12 +175,18 @@ bool Generator::LoadFaupOptions() { } return true; } +// Method is friend with ASession +std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, + s->_packet, + _session, _faup_options, _token_map, _max_tokens + ) + ); +} -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _session, _faup_options, _token_map, _max_tokens)); +long Generator::GetFilterCode() const { + return DARWIN_FILTER_DGA; } Generator::~Generator() { diff --git a/samples/fdga/Generator.hpp b/samples/fdga/Generator.hpp index 08a45305..947e11e8 100644 --- a/samples/fdga/Generator.hpp +++ b/samples/fdga/Generator.hpp @@ -12,21 +12,21 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "tensorflow/core/public/session.h" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator(); public: static constexpr int DEFAULT_MAX_TOKENS = 75; - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fhostlookup/Generator.cpp b/samples/fhostlookup/Generator.cpp index 16a3d0ab..e9e3892e 100644 --- a/samples/fhostlookup/Generator.cpp +++ b/samples/fhostlookup/Generator.cpp @@ -12,12 +12,18 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "HostLookupTask.hpp" +#include "ASession.hpp" #include "tsl/hopscotch_map.h" #include "tsl/hopscotch_set.h" #include "../toolkit/rapidjson/document.h" #include "../toolkit/rapidjson/istreamwrapper.h" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -224,9 +230,15 @@ bool Generator::LoadRsyslogEntry(const rapidjson::Value& entry) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _database, _feed_name)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _database, _feed_name + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_HOSTLOOKUP; } diff --git a/samples/fhostlookup/Generator.hpp b/samples/fhostlookup/Generator.hpp index a4bc1ade..04973e7e 100644 --- a/samples/fhostlookup/Generator.hpp +++ b/samples/fhostlookup/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "HostLookupTask.hpp" #include "tsl/hopscotch_map.h" @@ -21,13 +21,13 @@ class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; -public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; protected: enum class db_type { diff --git a/samples/fhostlookup/HostLookupTask.cpp b/samples/fhostlookup/HostLookupTask.cpp index a7b0b0da..6604aa46 100644 --- a/samples/fhostlookup/HostLookupTask.cpp +++ b/samples/fhostlookup/HostLookupTask.cpp @@ -14,18 +14,18 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "HostLookupTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "AlertManager.hpp" -HostLookupTask::HostLookupTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +HostLookupTask::HostLookupTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, tsl::hopscotch_map>& db, const std::string& feed_name) - : Session{"hostlookup", socket, manager, cache, cache_mutex}, _database{db}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _database{db}, _feed_name{feed_name} { _is_cache = _cache != nullptr; } @@ -40,8 +40,8 @@ long HostLookupTask::GetFilterCode() noexcept { void HostLookupTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; - + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); // Should not fail, as the Session body parser MUST check for validity ! auto array = _body.GetArray(); @@ -58,13 +58,13 @@ void HostLookupTask::operator()() { if (GetCacheResult(hash, certitude)) { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN) { STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_host, certitude, Evt_idToString(), this->AlertDetails()); + DARWIN_ALERT_MANAGER.Alert(_host, certitude, _packet.Evt_idToString(), this->AlertDetails()); if (is_log) { std::string alert_log = this->BuildAlert(_host, certitude); - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("HostLookupTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -75,24 +75,24 @@ void HostLookupTask::operator()() { certitude = DBLookup(description); if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN) { STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_host, certitude, Evt_idToString(), this->AlertDetails(description)); + DARWIN_ALERT_MANAGER.Alert(_host, certitude, _packet.Evt_idToString(), this->AlertDetails(description)); if (is_log){ std::string alert_log = this->BuildAlert(_host, certitude); - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) SaveToCache(hash, certitude); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } DARWIN_LOG_DEBUG("HostLookupTask:: processed entry in " - + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_certitudes.back())); + + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_packet.GetCertitudeList().back())); } } @@ -110,7 +110,7 @@ const std::string HostLookupTask::AlertDetails(std::string const& description) { const std::string HostLookupTask::BuildAlert(const std::string& host, unsigned int certitude) { std::string alert_log = - R"({"evt_id": ")" + Evt_idToString() + + R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "entry": ")" + host + diff --git a/samples/fhostlookup/HostLookupTask.hpp b/samples/fhostlookup/HostLookupTask.hpp index 9b4a84a5..d0ad572b 100644 --- a/samples/fhostlookup/HostLookupTask.hpp +++ b/samples/fhostlookup/HostLookupTask.hpp @@ -10,8 +10,9 @@ #include #include "../../toolkit/lru_cache.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "tsl/hopscotch_map.h" #include "tsl/hopscotch_set.h" @@ -24,12 +25,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class HostLookupTask : public darwin::Session { +class HostLookupTask : public darwin::ATask { public: - explicit HostLookupTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit HostLookupTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, tsl::hopscotch_map>& db, const std::string& feed_name); diff --git a/samples/finspection/ContentInspectionTask.cpp b/samples/finspection/ContentInspectionTask.cpp index bfd5efae..f8061703 100644 --- a/samples/finspection/ContentInspectionTask.cpp +++ b/samples/finspection/ContentInspectionTask.cpp @@ -13,15 +13,15 @@ #include "ContentInspectionTask.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" +#include "ASession.hpp" #include "AlertManager.hpp" -ContentInspectionTask::ContentInspectionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +ContentInspectionTask::ContentInspectionTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, Configurations& configurations) - : Session{"content_inspection", socket, manager, cache, cache_mutex} { + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet) { _is_cache = _cache != nullptr; _configurations = configurations; } @@ -33,7 +33,8 @@ long ContentInspectionTask::GetFilterCode() noexcept { void ContentInspectionTask::operator()() { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ContentInspectionTask:: started task"); - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); for(Packet *pkt : _packetList) { unsigned int certitude = 0; @@ -89,19 +90,19 @@ void ContentInspectionTask::operator()() { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; DARWIN_ALERT_MANAGER.SetTags(tagListJson); - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, Evt_idToString(), details); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _packet.Evt_idToString(), details); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + R"(, "rules": )" + ruleListJson + R"(, "tags": )" + tagListJson + "}"; - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_INFO("ContentInspectionTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); freePacket(pkt); @@ -113,36 +114,37 @@ void ContentInspectionTask::operator()() { bool ContentInspectionTask::ParseBody() { DARWIN_LOGGER; _packetList.clear(); - DARWIN_LOG_DEBUG("ContentInspectionTask:: ParseBody: _raw_body: " + _raw_body); + auto raw_body = _packet.GetBody(); + DARWIN_LOG_DEBUG("ContentInspectionTask:: ParseBody: _raw_body: " + raw_body); try { - _logs.clear(); + auto logs = _packet.GetMutableLogs(); + logs.clear(); std::size_t packetMeta = 0, packetMetaEnd; std::size_t packetData, packetDataEnd = 0; - std::size_t openingBracket; do { - packetMeta = _raw_body.find("\"{", packetDataEnd + 1); + packetMeta = raw_body.find("\"{", packetDataEnd + 1); if(packetMeta == std::string::npos) { break; } STAT_INPUT_INC; - packetMetaEnd = _raw_body.find("}\",", packetMeta); + packetMetaEnd = raw_body.find("}\",", packetMeta); if(packetMetaEnd == std::string::npos) { DARWIN_LOG_WARNING("ContentInspectionTask:: parse fail 1"); STAT_PARSE_ERROR_INC; break; } - packetData = _raw_body.find("\"{", packetMetaEnd); + packetData = raw_body.find("\"{", packetMetaEnd); if(packetData == std::string::npos) { DARWIN_LOG_WARNING("ContentInspectionTask:: parse fail 2"); STAT_PARSE_ERROR_INC; break; } - packetDataEnd = _raw_body.find("}\"", packetData); + packetDataEnd = raw_body.find("}\"", packetData); if(packetDataEnd == std::string::npos) { DARWIN_LOG_WARNING("ContentInspectionTask:: parse fail 3"); STAT_PARSE_ERROR_INC; @@ -150,8 +152,8 @@ bool ContentInspectionTask::ParseBody() { } _packetList.push_back(getImpcapData( - _raw_body.substr(packetMeta + 1, packetMetaEnd - packetMeta), - _raw_body.substr(packetData + 1, packetDataEnd - packetData) + raw_body.substr(packetMeta + 1, packetMetaEnd - packetMeta), + raw_body.substr(packetData + 1, packetDataEnd - packetData) )); } while(true); diff --git a/samples/finspection/ContentInspectionTask.hpp b/samples/finspection/ContentInspectionTask.hpp index e2f636cc..a95f6e88 100644 --- a/samples/finspection/ContentInspectionTask.hpp +++ b/samples/finspection/ContentInspectionTask.hpp @@ -13,8 +13,9 @@ #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/rapidjson/stringbuffer.h" #include "../../toolkit/rapidjson/writer.h" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "tcp_sessions.hpp" #include "stream_buffer.hpp" @@ -37,12 +38,12 @@ typedef struct Configurations_t { // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class ContentInspectionTask : public darwin::Session { +class ContentInspectionTask : public darwin::ATask { public: - explicit ContentInspectionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& _cache_mutex, + explicit ContentInspectionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, Configurations& configurations); ~ContentInspectionTask() override = default; @@ -67,7 +68,7 @@ class ContentInspectionTask : public darwin::Session { std::string GetJsonListFromSet(std::set &input); // Implemented but not used. - bool ParseLine(rapidjson::Value& line __attribute__((unsused))) final {return true;} + bool ParseLine(rapidjson::Value& line __attribute__((unused))) final {return true;} private: Configurations _configurations; diff --git a/samples/finspection/Generator.cpp b/samples/finspection/Generator.cpp index 55db860f..bfaf57c6 100644 --- a/samples/finspection/Generator.cpp +++ b/samples/finspection/Generator.cpp @@ -12,9 +12,13 @@ #include "../../toolkit/rapidjson/document.h" #include "base/Logger.hpp" #include "Generator.hpp" +#include "ContentInspectionTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" -Generator::Generator() { +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ poolStorage = initPoolStorage(); _configurations.flowCnf = (FlowCnf *)calloc(1, sizeof(FlowCnf)); _configurations.streamsCnf = (StreamsCnf *)calloc(1, sizeof(StreamsCnf)); @@ -168,13 +172,20 @@ bool Generator::LoadConfig(const rapidjson::Document &config) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _configurations)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _configurations + ) + ); } +long Generator::GetFilterCode() const { + return DARWIN_FILTER_CONTENT_INSPECTION; +} + + Generator::~Generator() { destroyTCPPools(); yaraDeleteConfig(_configurations.yaraCnf); diff --git a/samples/finspection/Generator.hpp b/samples/finspection/Generator.hpp index 1eed042e..80693043 100644 --- a/samples/finspection/Generator.hpp +++ b/samples/finspection/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "data_pool.hpp" #include "AGenerator.hpp" #include "ContentInspectionTask.hpp" @@ -19,13 +19,14 @@ class Generator: public AGenerator { public: - Generator(); + Generator(size_t nb_task_threads); ~Generator(); public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/finspection/data_pool.cpp b/samples/finspection/data_pool.cpp index 68a54137..872f3084 100644 --- a/samples/finspection/data_pool.cpp +++ b/samples/finspection/data_pool.cpp @@ -202,7 +202,7 @@ DataObject *getOrCreateAvailableObject(DataPool *pool) { return object; } -DataPool *createPool(char *poolName, constructor_t objectConstructor, +DataPool *createPool(const char *poolName, constructor_t objectConstructor, destructor_t objectDestructor, resetor_t objectResetor, uint32_t minAvailableElems) { DARWIN_LOGGER; @@ -346,8 +346,6 @@ void *memoryManagerDoWork(void *pData) { DARWIN_LOG_DEBUG("memory manager: cleanup finished, memory freed: " + std::to_string(totalMemFreed) + "," " total memory used: " + std::to_string(poolStorage->totalDataSize)); - uint32_t ratio = (unsigned long int)((float)poolStorage->totalDataSize/(float)poolStorage->maxDataSize*100); - DARWIN_LOG_DEBUG("memory manager: starting TCP session cleanup"); DataObject *sessObject; TcpSession *session; @@ -403,7 +401,7 @@ void startMemoryManager(MemManagerParams *memManagerParams) { pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); if(pthread_create(&(memManagerParams->thread), &attr, memoryManagerDoWork, (void *)memManagerParams) != 0) { - memManagerParams->thread = NULL; + memManagerParams->thread = 0; pthread_attr_destroy(&attr); return; } diff --git a/samples/finspection/data_pool.hpp b/samples/finspection/data_pool.hpp index bfdc4043..804d340e 100644 --- a/samples/finspection/data_pool.hpp +++ b/samples/finspection/data_pool.hpp @@ -101,7 +101,7 @@ uint32_t deleteDataObjectFromPool(DataObject *, DataPool *); void setObjectAvailable(DataObject *); void updateDataObjectSize(DataObject *, int); DataObject *getOrCreateAvailableObject(DataPool *); -DataPool *createPool(char *, constructor_t, destructor_t, resetor_t, uint32_t); +DataPool *createPool(const char *, constructor_t, destructor_t, resetor_t, uint32_t); void destroyPool(DataPool *); PoolStorage *initPoolStorage(); void deletePoolStorage(PoolStorage *); diff --git a/samples/finspection/extract_impcap.cpp b/samples/finspection/extract_impcap.cpp index 94ab9ea5..345edaf1 100644 --- a/samples/finspection/extract_impcap.cpp +++ b/samples/finspection/extract_impcap.cpp @@ -31,8 +31,6 @@ #include "../toolkit/rapidjson/document.h" Packet *getImpcapData(std::string impcapMeta, std::string impcapData) { - DARWIN_LOGGER; - int localret; uint32_t contentLength; const char *content; uint16_t ethType; @@ -85,9 +83,8 @@ Packet *getImpcapData(std::string impcapMeta, std::string impcapData) { uint8_t *ImpcapDataDecode(const char *hex, uint32_t length) { uint8_t *retBuf = (uint8_t *)malloc(length/2*sizeof(uint8_t)); - int i; - for(i = 0; i < length; ++i) { + for(uint32_t i = 0; i < length; ++i) { if(i%2) { retBuf[i/2] <<= 4; if(hex[i] >= '0' && hex[i] <= '9') { diff --git a/samples/finspection/flow.cpp b/samples/finspection/flow.cpp index d303dea4..9514e963 100644 --- a/samples/finspection/flow.cpp +++ b/samples/finspection/flow.cpp @@ -135,6 +135,7 @@ static inline int addFlowToList(Flow *flow, FlowList *flowList) { } } +__attribute((unused)) static inline int removeFlowFromList(Flow *flow, FlowList *flowList) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("removeFlowFromList"); diff --git a/samples/finspection/stream_buffer.cpp b/samples/finspection/stream_buffer.cpp index afb84cd7..1a180d91 100644 --- a/samples/finspection/stream_buffer.cpp +++ b/samples/finspection/stream_buffer.cpp @@ -197,7 +197,7 @@ int streamBufferExtend(StreamBuffer *sb, uint32_t extLength) { return -1; } -static inline void streamBufferShift(StreamBuffer *sb, int amount) { +static inline void streamBufferShift(StreamBuffer *sb, uint32_t amount) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("streamBufferShift, amount=" + std::to_string(amount)); diff --git a/samples/finspection/yara_utils.cpp b/samples/finspection/yara_utils.cpp index 7298118d..6f1ede66 100644 --- a/samples/finspection/yara_utils.cpp +++ b/samples/finspection/yara_utils.cpp @@ -287,7 +287,7 @@ YaraResults yaraScan(uint8_t *buffer, uint32_t buffLen, StreamBuffer *sb) { } #if YR_MAJOR_VERSION == 3 -void yaraErrorCallback(int errorLevel, const char *fileName, int lineNumber, const char *message, void *userData) { +void yaraErrorCallback(int errorLevel, const char *fileName, int lineNumber, const char *message, void *userData __attribute((unused))) { DARWIN_LOGGER; char errStr[2048]; if(fileName) { diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp new file mode 100644 index 00000000..ddd654bd --- /dev/null +++ b/samples/fpython/Generator.cpp @@ -0,0 +1,493 @@ +/// \file Generator.cpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#include +#include +#include + +#include "../../toolkit/lru_cache.hpp" +#include "base/Logger.hpp" +#include "Generator.hpp" +#include "PythonTask.hpp" +#include "ASession.hpp" +#include "AlertManager.hpp" +#include "fpython.hpp" +#include "PythonObject.hpp" + +/// +/// \brief This expects the thread to hold the GIL +/// +/// \param prefix_log +/// \param log_type +/// \return true if an exception was found and handled +/// \return false if no exception were risen +/// +bool Generator::PyExceptionCheckAndLog(std::string const& prefix_log, darwin::logger::log_type log_type){ + if(PyErr_Occurred() == nullptr){ + return false; + } + DARWIN_LOGGER; + PyObject *errType, *err, *tb; + PyObjectOwner pErrStr, pTypeStr; + const char * pErrChars, *pTypeChars; + ssize_t errCharsLen = 0, typeCharsLen = 0; + std::string errString, typeString; + PyErr_Fetch(&errType, &err, &tb); + + // Early return if we don't have to log + if (log_type < log.getLevel()) { + return true; + } + + if((pErrStr = PyObject_Str(err)) != nullptr){ + if((pErrChars = PyUnicode_AsUTF8AndSize(*pErrStr, &errCharsLen)) != nullptr) { + errString = std::string(pErrChars, errCharsLen); + } + } + + if((pTypeStr = PyObject_Str(errType)) != nullptr){ + if((pTypeChars = PyUnicode_AsUTF8AndSize(*pTypeStr, &typeCharsLen)) != nullptr) { + typeString = std::string(pTypeChars, typeCharsLen); + } + } + std::ostringstream oss; + oss << prefix_log; + oss << "Python error '" << typeString << "' : " << errString; + log.log(log_type, oss.str()); + return true; +} + +static PyObject* darwin_log(PyObject *self __attribute__((__unused__)), PyObject* args) { + DARWIN_LOGGER; + int tag = 0; + const char* msg = nullptr; + ssize_t msgLen = 0; + if(PyArg_ParseTuple(args, "is#", &tag, &msg, &msgLen) != 1){ + Generator::PyExceptionCheckAndLog("darwin_log: Error parsing arguments :"); + Py_RETURN_NONE; + } + + log.log(static_cast(tag), std::string(msg, msgLen)); + + Py_RETURN_NONE; +} + +static PyMethodDef logMethod[] = { + {"log", darwin_log, METH_VARARGS, "sends log to darwin"}, + {nullptr, nullptr, 0, nullptr} /* Sentinel */ +}; + +static PyModuleDef darwin_log_mod = { + PyModuleDef_HEAD_INIT, + "darwin_logger", + "Python interface for the fdarwin C++ logger", + -1, + logMethod, + nullptr, + nullptr, + nullptr, + nullptr +}; + +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) + { } + +bool Generator::ConfigureAlerting(const std::string& tags) { + DARWIN_LOGGER; + + DARWIN_LOG_DEBUG("Python:: ConfigureAlerting:: Configuring Alerting"); + DARWIN_ALERT_MANAGER_SET_FILTER_NAME(DARWIN_FILTER_NAME); + DARWIN_ALERT_MANAGER_SET_RULE_NAME(DARWIN_ALERT_RULE_NAME); + if (tags.empty()) { + DARWIN_LOG_DEBUG("Python:: ConfigureAlerting:: No alert tags provided in the configuration. Using default."); + DARWIN_ALERT_MANAGER_SET_TAGS(DARWIN_ALERT_TAGS); + } else { + DARWIN_ALERT_MANAGER_SET_TAGS(tags); + } + return true; +} + +bool Generator::LoadConfig(const rapidjson::Document &configuration) { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Python:: Generator:: Loading configuration..."); + + std::string python_script_path, shared_library_path, python_venv_folder; + + if (!configuration.HasMember("python_script_path")) { + DARWIN_LOG_CRITICAL("Python:: Generator:: Missing parameter: 'python_script_path'"); + return false; + } + + if (!configuration["python_script_path"].IsString()) { + DARWIN_LOG_CRITICAL("Python:: Generator:: 'python_script_path' needs to be a string"); + return false; + } + + python_script_path = configuration["python_script_path"].GetString(); + + if (!configuration.HasMember("shared_library_path")) { + DARWIN_LOG_CRITICAL("Python:: Generator:: Missing parameter: 'shared_library_path'"); + return false; + } + + if (!configuration["shared_library_path"].IsString()) { + DARWIN_LOG_CRITICAL("Python:: Generator:: 'shared_library_path' needs to be a string"); + return false; + } + + shared_library_path = configuration["shared_library_path"].GetString(); + + if(configuration.HasMember("python_venv_folder")){ + if (!configuration["python_venv_folder"].IsString()) { + DARWIN_LOG_CRITICAL("Python:: Generator:: 'python_venv_folder' needs to be a string"); + return false; + } else { + python_venv_folder = configuration["python_venv_folder"].GetString(); + } + } + + DARWIN_LOG_DEBUG("Python:: Generator:: Loading configuration..."); + + bool pythonLoad = LoadPythonScript(python_script_path, python_venv_folder); + + if(Py_IsInitialized() != 0) { + // Release the GIL, This is safe to call because we just initialized the python interpreter + PyEval_SaveThread(); + } + + return pythonLoad && LoadSharedLibrary(shared_library_path) && CheckConfig() && SendConfig(configuration); +} + +bool Generator::SendConfig(rapidjson::Document const& config) { + DARWIN_LOGGER; + bool pyConfigRes = false, soConfigRes = false; + switch(_functions.configPyFunc.origin) { + case FunctionOrigin::none: + DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in python script"); + pyConfigRes = true; + break; + case FunctionOrigin::python_module: + pyConfigRes = this->SendPythonConfig(config); + break; + case FunctionOrigin::shared_library: + DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: Incoherent configuration"); + return false; + } + + switch(_functions.configSoFunc.origin) { + case FunctionOrigin::none: + DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in shared object"); + soConfigRes = true; + break; + case FunctionOrigin::python_module: + DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: Incoherent configuration"); + return false; + case FunctionOrigin::shared_library: + try{ + soConfigRes = _functions.configSoFunc.so(config); + } catch(std::exception const& e){ + DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: error while sending config :" + std::string(e.what())); + soConfigRes = false; + } + break; + } + + return pyConfigRes && soConfigRes; +} + +bool Generator::SendPythonConfig(rapidjson::Document const& config) { + DARWIN_LOGGER; + rapidjson::StringBuffer buffer; + buffer.Clear(); + + rapidjson::Writer writer(buffer); + config.Accept(writer); + + PythonLock lock; + if(PyRun_SimpleString("import json") != 0) { + DARWIN_LOG_CRITICAL("Python:: Generator:: SendPythonConfig: import json failed"); + return false; + } + + std::string cmd = "json.loads(\"\"\""; + cmd += buffer.GetString(); + cmd += "\"\"\")"; + PyObject* globalDictionary = PyModule_GetDict(*_pModule); + + PyObjectOwner pConfig = PyRun_StringFlags(cmd.c_str(), Py_eval_input, globalDictionary, globalDictionary, nullptr); + if(Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: Converting config to Py Object :")){ + return false; + } + PyObjectOwner res; + if ((res = PyObject_CallFunctionObjArgs(_functions.configPyFunc.py, *pConfig, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: "); + return false; + } + int resTruthiness = -1; + if((resTruthiness = PyObject_IsTrue(*res)) == 1) { + // 1 : true + return true; + } else if (resTruthiness == 0){ + // 0 : false + DARWIN_LOG_CRITICAL("Python:: Generator:: SendPythonConfig: filter_config returned false"); + } else { + // -1 : error while evaluating truthiness + DARWIN_LOG_CRITICAL("Python:: Generator:: SendPythonConfig: truthiness evaluation of the returned value of 'filter_config' failed"); + Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: "); + } + + return false; +} + +bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { + DARWIN_LOGGER; + + if(shared_library_path.empty()){ + DARWIN_LOG_INFO("Generator::LoadSharedLibrary : No shared library to load"); + return true; + } + + void* handle = dlopen(shared_library_path.c_str(), RTLD_NOW | RTLD_GLOBAL); + if(handle == nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadSharedLibrary : Error loading the shared library : failed to open " + shared_library_path + " : " + std::string(dlerror())); + return false; + } + + LoadFunctionFromSO(handle, _functions.configSoFunc, "filter_config"); + LoadFunctionFromSO(handle, _functions.parseBodyFunc, "parse_body"); + LoadFunctionFromSO(handle, _functions.preProcessingFunc, "filter_pre_process"); + LoadFunctionFromSO(handle, _functions.processingFunc, "filter_process"); + LoadFunctionFromSO(handle, _functions.contextualizeFunc, "filter_contextualize"); + LoadFunctionFromSO(handle, _functions.alertContextualizeFunc, "alert_contextualize"); + LoadFunctionFromSO(handle, _functions.alertFormatingFunc, "alert_formating"); + LoadFunctionFromSO(handle, _functions.outputFormatingFunc, "output_formating"); + LoadFunctionFromSO(handle, _functions.responseFormatingFunc, "response_formating"); + + return true; +} +template +inline void Generator::LoadFunctionFromSO(void * lib_handle, FunctionPySo &function_holder, const std::string& function_name) { + DARWIN_LOGGER; + F func = (F)dlsym(lib_handle, function_name.c_str()); + if(func == nullptr){ + DARWIN_LOG_INFO("Generator::LoadPythonScript : No '" + function_name + "' function in the shared library"); + } else { + function_holder.origin = FunctionOrigin::shared_library; + function_holder.so = func; + } +} + +bool Generator::CheckConfig() const { + if(_functions.preProcessingFunc.origin == FunctionOrigin::none + || _functions.processingFunc.origin == FunctionOrigin::none + || _functions.contextualizeFunc.origin == FunctionOrigin::none + || _functions.alertContextualizeFunc.origin == FunctionOrigin::none + || _functions.alertFormatingFunc.origin == FunctionOrigin::none + || _functions.outputFormatingFunc.origin == FunctionOrigin::none + || _functions.responseFormatingFunc.origin == FunctionOrigin::none) + { + DARWIN_LOGGER; + DARWIN_LOG_CRITICAL("Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library"); + return false; + } + return true; +} + +PythonThread& Generator::GetPythonThread(){ + thread_local PythonThread thread; + if(! thread.IsInit()){ + PyGILState_STATE lock = PyGILState_Ensure(); + thread.Init(PyInterpreterState_Main()); + PyGILState_Release(lock); + } + return thread; +} + +bool Generator::LoadPythonScript(const std::string& python_script_path, const std::string& python_venv_folder) { + DARWIN_LOGGER; + + if(python_script_path.empty()){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : No python script to load"); + return false; + } + + std::ifstream f(python_script_path.c_str()); + if(! f.is_open()) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : failed to open " + python_script_path); + return false; + } + f.close(); + + std::size_t pos = python_script_path.rfind("/"); + auto folders = python_script_path.substr(0, pos); + auto filename = python_script_path.substr(pos+1, std::string::npos); + if((pos = filename.rfind(".py")) != std::string::npos) { + filename.erase(pos, std::string::npos); + } + + Py_Initialize(); + PyEval_InitThreads(); + + if(PyExceptionCheckAndLog("Generator::LoadPythonScript : Error during python init :")) { + return false; + } + PyObjectOwner pName; + if((pName = PyUnicode_DecodeFSDefault(filename.c_str())) == nullptr){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : error importing string " + filename + " : "); + return false; + } + + if(PyImport_AddModule("darlog") == nullptr){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to import the logger module : "); + return false; + } + + PyObjectOwner logmod; + if((logmod = PyModule_Create(&darwin_log_mod))== nullptr){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the logger module : "); + return false; + } + + PyObject* sys_modules = PyImport_GetModuleDict(); + if (PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while calling PyImport_GetModuleDict(): ")){ + return false; + } + PyDict_SetItemString(sys_modules, "darwin_logger", *logmod); + if (PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while calling PyDict_SetItemString(): ")){ + return false; + } + + if (PyRun_SimpleString("import sys") != 0) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : An error occurred while loading the 'sys' module"); + return false; + } + + std::string command = "sys.path.append(\"" + folders + "\")"; + if (PyRun_SimpleString(command.c_str()) != 0) { + DARWIN_LOG_CRITICAL( + "Generator::LoadPythonScript : An error occurred while appending the custom path '" + + folders + + "' to the Python path" + ); + return false; + } + + if(!python_venv_folder.empty()){ + std::filesystem::path venv_path(python_venv_folder); + venv_path /= std::filesystem::path("lib") / (std::string("python") + PYTHON_VERSION) / "site-packages"; + std::error_code _ec; + if(!std::filesystem::exists(venv_path, _ec)) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Virtual Env does not exist : " + venv_path.string()); + return false; + } + command = "sys.path.insert(0, \"" + venv_path.string() + "\")"; + // PyRun_SimpleString("sys.path.insert(0, \"/home/myadvens.lan/tcartegnie/workspace/darwin/.venv/lib/python3.8/site-packages\")"); + if (PyRun_SimpleString(command.c_str()) != 0) { + DARWIN_LOG_CRITICAL( + "Generator::LoadPythonScript : An error occurred while inserting the custom venv '" + + venv_path.string() + + "' to the Python path" + ); + return false; + } + } + + if((_pModule = PyImport_Import(*pName)) == nullptr) { + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python script module '" + filename + "' : "); + return false; + } + pName.Decref(); + + _pClass = PyObject_GetAttrString(*_pModule, "Execution"); + if (PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python class 'Execution' : ")){ + return false; + } + if(*_pClass == nullptr || ! PyType_Check(*_pClass)) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error while attempting to load the python class 'Execution' : Not a Class"); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.configPyFunc, "filter_config")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_config' : "); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.parseBodyFunc, "parse_body")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'parse_body' : "); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.preProcessingFunc, "filter_pre_process")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_pre_process' : "); + return false; + } + if( ! LoadFunctionFromPython(*_pClass, _functions.processingFunc, "filter_process")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_process' : "); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.contextualizeFunc, "filter_contextualize")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_contextualize' : "); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.alertContextualizeFunc, "alert_contextualize")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_contextualize' : "); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.alertFormatingFunc, "alert_formating")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_formating' : "); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.outputFormatingFunc, "output_formating")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'output_formating' : "); + return false; + } + + if( ! LoadFunctionFromPython(*_pClass, _functions.responseFormatingFunc, "response_formating")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'response_formating' : "); + return false; + } + + return true; +} + +template +inline bool Generator::LoadFunctionFromPython(PyObject* _pClass, FunctionPySo& function_holder, const std::string& function_name){ + DARWIN_LOGGER; + function_holder.py = PyObject_GetAttrString(_pClass, function_name.c_str()); + if(function_holder.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No '" + function_name + "' method in the python script"); + } else if (! PyCallable_Check(function_holder.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : '" + function_name + "' symbol exists but is not callable"); + return false; + } else { + function_holder.origin = FunctionOrigin::python_module; + } + return true; +} + +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + *_pClass, _functions) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_PYTHON; +} + +Generator::~Generator() { + PyGILState_Ensure(); + Py_Finalize(); + // Python environment is destroyed, we can't release the GIL +} diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp new file mode 100644 index 00000000..89c73c63 --- /dev/null +++ b/samples/fpython/Generator.hpp @@ -0,0 +1,171 @@ +/// \file Generator.hpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#pragma once + +#include + +#include "Logger.hpp" +#include "../../toolkit/PythonUtils.hpp" +#include "ATask.hpp" +#include "../../toolkit/RedisManager.hpp" +#include "../toolkit/rapidjson/document.h" +#include "AGenerator.hpp" +#include "fpython.hpp" +#include "PythonThread.hpp" +#include "PythonObject.hpp" + +// Preprocessor hack to extract the major and minor version of python +#define PY_VER_STR_(maj, min) #maj "." #min +#define PY_VER_STR(maj, min) PY_VER_STR_(maj, min) +#define PYTHON_VERSION PY_VER_STR(PY_MAJOR_VERSION,PY_MINOR_VERSION) + +/// +/// \brief Tag for the FunctionPySo struct +/// It is declared outside the struct to stay unaffected by the template grammar +/// +enum class FunctionOrigin { + none, + python_module, + shared_library, +}; + +/// +/// \brief tagged union containing either a function pointer from a shared object or a python function reference +/// along with a tag FunctionOrigin +/// +/// \tparam F prototype of the function if loaded from a shared object +/// +template +struct FunctionPySo { +public: + FunctionOrigin origin; + union { + PyObject* py; + F so; + }; + FunctionPySo(): origin{FunctionOrigin::none}, py {nullptr} { } + ~FunctionPySo() { + if(origin == FunctionOrigin::python_module){ + //We use PyObjectOwner to decrement the reference counter of the function + PyObjectOwner _{py}; + } + }; +}; + +/// +/// \brief Struct holding the function pointers for the python filter +/// It should be unique and passed by reference +/// +/// +struct FunctionHolder{ + FunctionHolder() = default; + ~FunctionHolder() = default; + + FunctionHolder(FunctionHolder const &) = delete; + FunctionHolder(FunctionHolder &&) = delete; + FunctionHolder& operator=(FunctionHolder const &) = delete; + FunctionHolder& operator=(FunctionHolder &&) = delete; + + typedef bool(*config_t)(rapidjson::Document const &); + typedef PyObject*(*parse_body_t)(PyObject*, PyObject*, const std::string&); + typedef PyObject*(*process_t)(PyObject*, PyObject*, PyObject*); + typedef std::vector(*alert_format_t)(PyObject*, PyObject*, PyObject*); + typedef DarwinResponse(*resp_format_t)(PyObject*, PyObject*, PyObject*); + + // There are 2 config functions as we accept to pass informations to both the python module and the shared library + // Note that the shared library has a direct access to the python module + FunctionPySo configPyFunc; + FunctionPySo configSoFunc; + + FunctionPySo parseBodyFunc; + + FunctionPySo processingFunc; + FunctionPySo preProcessingFunc; + + FunctionPySo contextualizeFunc; + FunctionPySo alertContextualizeFunc; + + FunctionPySo alertFormatingFunc; + FunctionPySo outputFormatingFunc; + FunctionPySo responseFormatingFunc; +}; + +class Generator: public AGenerator { +public: + Generator(size_t nb_task_threads); + ~Generator(); + + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; + static PythonThread& GetPythonThread(); + static bool PyExceptionCheckAndLog(std::string const& prefix_log, darwin::logger::log_type log_type=darwin::logger::Critical); + +private: + virtual bool LoadConfig(const rapidjson::Document &configuration) override final; + virtual bool ConfigureAlerting(const std::string& tags) override final; + + /// + /// \brief loads the python script which path is given and attempts to load all defined functions + /// If the path is not empty, a python interpreter will be initialized (by a call to Py_Initialize()) + /// + /// \param python_script_path + /// \param python_venv_folder + /// \return true if the script was correctly loaded or if the path is empty + /// \return false if the path can't be read, if loading the module triggered an error + /// or if a symbol with the name of a defined function is found but not callable + /// + bool LoadPythonScript(const std::string& python_script_path, const std::string& python_venv_folder); + + /// + /// \brief loads the shared library which path is given and attempts to load all defined functions + /// + /// \param shared_library_path + /// \return true if the SO was correctly loaded or if the path is empty + /// \return false if the shared library can't be read or if loading the SO triggered an error + /// + bool LoadSharedLibrary(const std::string& shared_library_path); + + /// + /// \brief Verifies that the configuration if valid + /// filter_preprocess, filter_process, alert_formating, + /// response_formating and output_formating must be set + /// parse_body and filter_config are optional + /// + /// \return true the configuration is valid + /// + bool CheckConfig() const; + + /// + /// \brief Dispatches the configuration to the Shared object (if any) and the python module (if any) + /// + /// \param config parsed configuration + /// \return true the module and/or the SO sent back true + /// + bool SendConfig(rapidjson::Document const& config); + + /// + /// \brief Sends the configuration to the Python module + /// + /// \param config + /// \return false an error occurred while calling the python function or it sent back false + /// + bool SendPythonConfig(rapidjson::Document const& config); + + template + inline void LoadFunctionFromSO(void* lib_handle, FunctionPySo& function_holder, const std::string& function_name); + + template + inline bool LoadFunctionFromPython(PyObject* pClass, FunctionPySo& function_holder, const std::string& function_name); + + PyObjectOwner _pModule; + PyObjectOwner _pClass; + FunctionHolder _functions; + +}; \ No newline at end of file diff --git a/samples/fpython/PythonObject.hpp b/samples/fpython/PythonObject.hpp new file mode 100644 index 00000000..283df2c4 --- /dev/null +++ b/samples/fpython/PythonObject.hpp @@ -0,0 +1,70 @@ +#pragma once +#include + +/// +/// \brief Helper class that takes the ownership of a PyObject pointer +/// when detroyed, if it still holds a reference, it will attempt to DECREF it to the interpreter +/// Calls to destructor or Move operator or Decref will query the GIL if it does not have it. +/// These methods are always safe to call +/// +class PyObjectOwner { + public: + // no copy, move ok + PyObjectOwner(): handle{nullptr} {} + PyObjectOwner(PyObject* o): handle{o} {} + ~PyObjectOwner(){ + this->Decref(); + } + + PyObjectOwner(PyObjectOwner const&) = delete; + PyObjectOwner(PyObjectOwner&& other): handle{other.handle} { + other.handle = nullptr; + } + + PyObjectOwner& operator=(PyObjectOwner const&) = delete; + PyObjectOwner& operator=(PyObjectOwner&& other){ + this->Decref(); + this->handle = other.handle; + other.handle = nullptr; + return *this; + } + + bool operator==(std::nullptr_t){ + return handle == nullptr; + } + + bool operator!=(std::nullptr_t){ + return handle != nullptr; + } + + PyObjectOwner& operator=(PyObject* o){ + this->Decref(); + this->handle = o; + return *this; + } + + PyObject* operator* () + { + return handle; + } + + PyObject& operator-> () + { + return *handle; + } + + void Decref(){ + if(handle != nullptr && Py_IsInitialized() != 0){ + if(PyGILState_Check() == 1){ + Py_DECREF(handle); + } else { + PythonLock lock; + Py_DECREF(handle); + } + handle = nullptr; + } + } + +private: + PyObject* handle; +}; \ No newline at end of file diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp new file mode 100644 index 00000000..b655c098 --- /dev/null +++ b/samples/fpython/PythonTask.cpp @@ -0,0 +1,384 @@ +/// \file PythonExample.cpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#include + +#include "../toolkit/rapidjson/document.h" +#include "PythonTask.hpp" +#include "Logger.hpp" +#include "AlertManager.hpp" +#include "PythonObject.hpp" +#include "Stats.hpp" + +PythonTask::PythonTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, PyObject* pClass, FunctionHolder& functions) + : ATask{DARWIN_FILTER_NAME, cache, cache_mutex, s, packet}, _pClass{pClass}, _functions{functions} +{ + _is_cache = _cache != nullptr; + PythonLock lock; + _pSelf = PyObject_CallObject(_pClass, nullptr); + Generator::PyExceptionCheckAndLog("PythonTask:: PythonTask:: Error creating Execution object: ", darwin::logger::Error); +} + +long PythonTask::GetFilterCode() noexcept{ + return DARWIN_FILTER_PYTHON; +} + +bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { + return true; +} + +void PythonTask::operator()() { + DARWIN_LOGGER; + PyObjectOwner preProc_res, proc_res, context_res, alert_context_res; + if (*_pSelf == nullptr) { + DARWIN_LOG_ERROR("No Execution object was created, Lines skipped."); + return; + } + switch(_functions.preProcessingFunc.origin) { + case FunctionOrigin::python_module:{ + PythonLock pylock; + if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, *_pSelf, *_parsed_body, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Preprocess: ", darwin::logger::Error); + return; + } + break; + } + case FunctionOrigin::shared_library:{ + try { + if ((preProc_res = _functions.preProcessingFunc.so(_pClass, *_pSelf, *_parsed_body)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_pre_process function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO filter_pre_process function : " + std::string(e.what())); + return; + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); + } + + switch(_functions.processingFunc.origin) { + case FunctionOrigin::python_module:{ + PythonLock pylock; + if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, *_pSelf, *preProc_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Process: ", darwin::logger::Error); + return; + } + break; + } + case FunctionOrigin::shared_library:{ + try{ + if ((proc_res = _functions.processingFunc.so(_pClass, *_pSelf, *preProc_res)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_process function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO filter_process function : " + std::string(e.what())); + return; + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); + } + + switch(_functions.contextualizeFunc.origin) { + case FunctionOrigin::python_module:{ + PythonLock pylock; + if ((context_res = PyObject_CallFunctionObjArgs(_functions.contextualizeFunc.py, *_pSelf, *proc_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Contextualize: ", darwin::logger::Error); + return; + } + break; + } + case FunctionOrigin::shared_library:{ + try{ + if ((context_res = _functions.contextualizeFunc.so(_pClass, *_pSelf, *proc_res)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_contextualize function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO filter_contextualize function : " + std::string(e.what())); + return; + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); + } + + switch(_functions.alertContextualizeFunc.origin) { + case FunctionOrigin::python_module:{ + PythonLock pylock; + if ((alert_context_res = PyObject_CallFunctionObjArgs(_functions.alertContextualizeFunc.py, *_pSelf, *context_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Alert Contextualize: ", darwin::logger::Error); + return; + } + break; + } + case FunctionOrigin::shared_library:{ + try{ + if ((alert_context_res = _functions.alertContextualizeFunc.so(_pClass, *_pSelf, *context_res)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO alert_contextualize function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO alert_contextualize function : " + std::string(e.what())); + return; + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); + } + + + std::vector alerts = GetFormatedAlerts(_functions.alertFormatingFunc, *alert_context_res); + // DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, *context_res); + DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, *context_res); + + for(auto& alert: alerts) { + if( ! alert.empty()) { + DARWIN_ALERT_MANAGER.Alert(alert); + } + } + for(auto cert: resp.certitudes) { + if(cert > _threshold) { + STAT_MATCH_INC; + } + _packet.AddCertitude(cert); + } + + if( ! resp.body.empty()){ + _response_body.clear(); + _response_body.append(resp.body); + } +} + +bool PythonTask::ParseBody() { + DARWIN_LOGGER; + PyObjectOwner raw_body; + + if (*_pSelf == nullptr) { + DARWIN_LOG_ERROR("No Execution object was created, Lines skipped."); + return false; + } + + bool ret = false; + switch(_functions.parseBodyFunc.origin){ + case FunctionOrigin::none:{ + ret = ATask::ParseBody(); + if(!ret){ + return false; + } + PythonLock pylock; + if((_parsed_body = PyList_New(_body.Size())) == nullptr){ + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); + return false; + } + + for(ssize_t i = 0; i < _body.Size(); i++){ + PyObject *str; // This reference is "stolen" by PyList_SetItem, we don't have to DECREF it + if(! _body.GetArray()[i].IsString()){ + DARWIN_LOG_ERROR("PythonTask::ParseBody: Item on the list is not a string : " + std::to_string(i)); + return false; + } + if((str = PyUnicode_DecodeFSDefault(_body.GetArray()[i].GetString())) == nullptr) { + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); + return false; + } + if(PyList_SetItem(*_parsed_body, i, str) != 0){ + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); + return false; + } + } + return ret; + } + case FunctionOrigin::python_module:{ + // This lock is valid because we use only ONE interpreter + // If at some point, we decide to use multiple interpreters + // (one by thread for example) + // We must switch to PythonThread utility class + PythonLock pylock; + if((raw_body = PyUnicode_DecodeFSDefault(_packet.GetBody().c_str())) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); + return false; + } + + if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.py, *_pSelf, *raw_body, nullptr)) == nullptr) { + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: An error occurred while calling the Python function"); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); + return false; + } + + return true; + } + case FunctionOrigin::shared_library:{ + try{ + if((_parsed_body = _functions.parseBodyFunc.so(_pClass, *_pSelf, _packet.GetBody()))== nullptr){ + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: An error occurred while calling the SO function"); + return false; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: Error while calling the SO parse_body function : " + std::string(e.what())); + return false; + } + + return true; + } + + } + return false; +} + + +std::vector PythonTask::GetFormatedAlerts(FunctionPySo& func, PyObject* processedData){ + DARWIN_LOGGER; + ssize_t pySize = 0, strSize=0; + const char * out = nullptr; + std::vector ret; + switch(func.origin) { + case FunctionOrigin::python_module:{ + PythonLock pylock; + PyObjectOwner pyRes; + if ((pyRes = PyObject_CallFunctionObjArgs(func.py, *_pSelf, processedData, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: ", darwin::logger::Error); + return ret; + } + if(PyList_Check(*pyRes) == 0){ + DARWIN_LOG_ERROR("PythonTask::GetFormatedAlerts : 'format_alert' did not return a list"); + return ret; + } + pySize = PyList_Size(*pyRes); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: ", darwin::logger::Error)) { + return ret; + } + + for(ssize_t i = 0; i < pySize; i++) { + // str is a borrowed reference, we don't have to DECREF it + PyObject* str = PyList_GetItem(*pyRes, i); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: ", darwin::logger::Error)){ + continue; + } + out = PyUnicode_AsUTF8AndSize(str, &strSize); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: ", darwin::logger::Error)){ + continue; + } + ret.push_back(std::string(out, strSize)); + } + return ret; + } + case FunctionOrigin::shared_library:{ + try{ + return func.so(_pClass, *_pSelf, processedData); + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: GetFormatedAlerts:: Error while calling the SO alert_format function : " + std::string(e.what())); + return ret; + } + } + + default: + return ret; + } + + +} + +DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo& func, PyObject* processedData) { + DARWIN_LOGGER; + DarwinResponse ret; + switch(func.origin) { + case FunctionOrigin::python_module:{ + PythonLock pylock; + + PyObjectOwner pBody, pCertitudes, pyRes; + ssize_t certSize = 0; + if ((pyRes = PyObject_CallFunctionObjArgs(func.py, *_pSelf, processedData, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedResponse:: ", darwin::logger::Error); + return ret; + } + if((pBody = PyObject_GetAttrString(*pyRes, "body")) == nullptr){ + Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedResponse:: data.body :", darwin::logger::Error); + return ret; + } + + const char* cBody = nullptr; + ssize_t bodySize = 0; + + if((cBody = PyUnicode_AsUTF8AndSize(*pBody, &bodySize)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedResponse:: str(data.body) :", darwin::logger::Error); + return ret; + } + + ret.body = std::string(cBody, bodySize); + if((pCertitudes = PyObject_GetAttrString(*pyRes, "certitudes")) == nullptr){ + Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedResponse:: data.certitudes :", darwin::logger::Error); + return ret; + } + + if(PyList_Check(*pCertitudes) == 0){ + DARWIN_LOG_ERROR("PythonTask::GetFormatedResponse : 'data.certitudes' is not a list"); + return ret; + } + + certSize = PyList_Size(*pCertitudes); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedResponse:: ", darwin::logger::Error)) { + return ret; + } + + for(ssize_t i = 0; i < certSize; i++) { + PyObject* cert = nullptr; // cert is a borrowed reference, no need to DECREF it + if((cert = PyList_GetItem(*pCertitudes, i)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedResponse:: ", darwin::logger::Error); + continue; + } + + if(PyLong_Check(cert) == 0) { + DARWIN_LOG_ERROR("PythonTask:: GetFormatedResponse:: an object from the 'certitudes' list is not a long"); + continue; + } + long lCert = PyLong_AS_LONG(cert); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedResponse:: ", darwin::logger::Error)) { + continue; + } + if (lCert < 0 || lCert > UINT_MAX) { + DARWIN_LOG_DEBUG("PythonTask:: GetFormatedResponse:: out of range certitude"); + continue; + } + ret.certitudes.push_back(static_cast(lCert)); + } + return ret; + } + case FunctionOrigin::shared_library:{ + try { + return func.so(_pClass, *_pSelf, processedData); + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: GetFormatedResponse:: Error while calling the SO response_format function : " + std::string(e.what())); + return ret; + } + + } + default: + return ret; + } +} + +PythonTask::~PythonTask() { + /** + * The next lines are not necessaray as exception in destructors are normally ignored by the python interpreter + * Just in case they are not we may avoid crashes + */ + PythonLock lock; + _pSelf.Decref(); + Generator::PyExceptionCheckAndLog("PythonTask::~PythonTask : Erreur while calling 'Execution' destructor", darwin::logger::Error); +} \ No newline at end of file diff --git a/samples/fpython/PythonTask.hpp b/samples/fpython/PythonTask.hpp new file mode 100644 index 00000000..bfe2f576 --- /dev/null +++ b/samples/fpython/PythonTask.hpp @@ -0,0 +1,63 @@ +/// \file PythonExample.hpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#pragma once + +#include + +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/PythonUtils.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" +#include "Generator.hpp" +#include "fpython.hpp" + +#define DARWIN_FILTER_PYTHON 0x70797468 +#define DARWIN_FILTER_NAME "python" +#define DARWIN_ALERT_RULE_NAME "Python" +#define DARWIN_ALERT_TAGS "[]" + +class PythonTask : public darwin::ATask { +public: + explicit PythonTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + PyObject * pClass, + FunctionHolder& functions); + + ~PythonTask() override; + +public: + // You need to override the functor to compile and be executed by the thread + void operator()() override; + +protected: + /// Get the result from the cache + // xxh::hash64_t GenerateHash() override; + /// Return filter code + long GetFilterCode() noexcept override; + +private: + + bool ParseLine(rapidjson::Value& line) override; + + /// Parse the body received. + bool ParseBody() override; + std::vector GetFormatedAlerts(FunctionPySo& func, PyObject* processedData); + DarwinResponse GetFormatedResponse(FunctionPySo& func, PyObject* processedData); + +private: + PyObject* _pClass; + FunctionHolder& _functions; + + PyObjectOwner _pSelf; + PyObjectOwner _parsed_body; +}; diff --git a/samples/fpython/PythonThread.hpp b/samples/fpython/PythonThread.hpp new file mode 100644 index 00000000..e8d3caa9 --- /dev/null +++ b/samples/fpython/PythonThread.hpp @@ -0,0 +1,63 @@ +#include + +class PythonThread { +public: + PythonThread(): _init{false}, _pThreadState{nullptr} {} + void Init(PyInterpreterState * interp) { + if (! _init){ + _pThreadState = PyThreadState_New(interp); + _init = true; + } + } + bool IsInit(){ + return _init; + } + + PyThreadState* GetThreadState(){ + return _pThreadState; + } + + void SetThreadState(PyThreadState* state){ + _pThreadState = state; + } + + ~PythonThread() { + if(_pThreadState != nullptr){ + PyThreadState_Clear(_pThreadState); + PyThreadState_Delete(_pThreadState); + } + } + + class Use { + public: + Use(PythonThread& thread): _thread{thread} { + PyEval_RestoreThread(_thread.GetThreadState()); + } + + ~Use(){ + _thread.SetThreadState(PyEval_SaveThread()); + } + + private: + PythonThread& _thread; + }; + +private: + + bool _init; + PyThreadState * _pThreadState; + +}; + +class PythonLock { +public: + PythonLock(){ + _lock = PyGILState_Ensure(); + } + ~PythonLock(){ + PyGILState_Release(_lock); + } +private: + PyGILState_STATE _lock; + +}; \ No newline at end of file diff --git a/samples/fpython/algorithm_module.py b/samples/fpython/algorithm_module.py new file mode 100644 index 00000000..5bf7d433 --- /dev/null +++ b/samples/fpython/algorithm_module.py @@ -0,0 +1,143 @@ +from abc import abstractmethod +from typing import List, Union +from dataclasses import dataclass +from enum import IntEnum + +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level: int, msg: str): + import sys + outpipe = sys.stderr if level >= 4 else sys.stdout + print(DarwinLogLevel(level).name + ' : ' + msg,file=outpipe) + + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + """ + Function to call to push logs to Darwin + the module 'darwin_logger' is automatically loaded by Darwin + """ + darwin_logger.log(int(level), msg) + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +##################### +## ## +## API ## +## ## +##################### +class Execution: + + @staticmethod + @abstractmethod + def filter_config(config: dict) -> bool: + """ + Function called with the filter configuration + This function is called only once when starting the filter + """ + pass + + @abstractmethod + def parse_body(self, body: str) -> Union[list, CustomData]: + """ + Function called when receiving a new packet + This implementation is optional, the default implementation parses a list of objects + """ + pass + + @abstractmethod + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + """ + Function called after parse_body with the output of parse_body + """ + pass + + @abstractmethod + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + """ + Function called after filter_pre_process with the output of filter_pre_process + This output will be forwarded to alert_formating, output_formating and response_formating + """ + pass + + @abstractmethod + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + """ + Function called after filter_process with the output of filter_process + It should usually call query_context to gather information about the processed_date in order to contextualize it + The output of this function can extend PythonFilterResponse or may be any structure that has the + attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' + The output of the functions is forwarded to alert_contextualize, output_formating and response_formating + """ + pass + + @abstractmethod + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + """ + Function called between filter_contextualize and alert_formating + It should be used to contextualize the alerts that should be risen by darwin + The output of this function can extend PythonFilterResponse or may be any structure that has the + attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' + """ + pass + + + @abstractmethod + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + """ + Function called after filter_process, each 'str' in the output will raise an alert + """ + pass + + @abstractmethod + # WIP This function is not wired, use only response_formating for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + """ + Function called after filter_process, the output will be sent to the next filter as the body + with the certitudes returned by filter_process + """ + pass + + @abstractmethod + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + """ + Function called after filter_process, the output will be sent to the sender as the body with + the certitudes returned by filter_process + """ + pass + + +# Context API + +def write_context(key, value): + """Writes an information about the current task performed""" + pass + +def query_context(key): + """Query information about the current task performed""" + pass \ No newline at end of file diff --git a/samples/fpython/fpython.hpp b/samples/fpython/fpython.hpp new file mode 100644 index 00000000..28f3f00e --- /dev/null +++ b/samples/fpython/fpython.hpp @@ -0,0 +1,42 @@ +#pragma once +#include +#include "../../toolkit/rapidjson/document.h" + +#ifdef __cplusplus +#include +#include +struct DarwinResponse{ + std::vector certitudes; + std::string body; +}; +#endif +struct DarwinResponseC{ + unsigned int *certitudes; + size_t certitudes_size; + char* body; + size_t body_size; +}; + +extern "C" { + // C++ linkage +#ifdef __cplusplus + bool filter_config(rapidjson::Document const&); + PyObject* parse_body(PyObject* pClass, PyObject* pSelf, const std::string&); + + std::vector alert_formating(PyObject* pClass, PyObject* pSelf, PyObject*); + DarwinResponse output_formating(PyObject* pClass, PyObject* pSelf, PyObject*); + DarwinResponse response_formating(PyObject* pClass, PyObject* pSelf, PyObject*); +#endif + // C compatible linkage + PyObject* filter_pre_process(PyObject* pClass, PyObject* pSelf, PyObject*); + PyObject* filter_process(PyObject* pClass, PyObject* pSelf, PyObject*); + PyObject* filter_contextualize(PyObject* pClass, PyObject* pSelf, PyObject*); + PyObject* alert_contextualize(PyObject* pClass, PyObject* pSelf, PyObject*); + + + bool filter_config_c(const char*, const size_t); + PyObject* parse_body_c(PyObject* pClass, PyObject* pSelf, const char*, const size_t); + void alert_formating_c(PyObject* pClass, PyObject* pSelf, PyObject* input, char ** alerts, size_t nb_alerts, size_t* alerts_sizes); + DarwinResponseC* output_formating_c(PyObject* pClass, PyObject* pSelf, PyObject*); + DarwinResponseC* response_formating_c(PyObject* pClass, PyObject* pSelf, PyObject*); +} \ No newline at end of file diff --git a/samples/fsession/Generator.cpp b/samples/fsession/Generator.cpp index 61d23e46..e9159e81 100644 --- a/samples/fsession/Generator.cpp +++ b/samples/fsession/Generator.cpp @@ -13,8 +13,15 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "SessionTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -53,9 +60,14 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { return redis.FindAndConnect(); } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet + ) + ); } + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_SESSION; +} \ No newline at end of file diff --git a/samples/fsession/Generator.hpp b/samples/fsession/Generator.hpp index d1c16970..78e868fd 100644 --- a/samples/fsession/Generator.hpp +++ b/samples/fsession/Generator.hpp @@ -15,20 +15,20 @@ extern "C" { #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "../../toolkit/RedisManager.hpp" #include "../toolkit/rapidjson/document.h" #include "AGenerator.hpp" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; -public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fsession/SessionTask.cpp b/samples/fsession/SessionTask.cpp index cc5eb858..6a47ef85 100644 --- a/samples/fsession/SessionTask.cpp +++ b/samples/fsession/SessionTask.cpp @@ -23,14 +23,14 @@ extern "C" { #include "Logger.hpp" #include "Stats.hpp" #include "SessionTask.hpp" -#include "protocol.h" +#include "ASession.hpp" -SessionTask::SessionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex) - : Session{"session", socket, manager, cache, cache_mutex}{ +SessionTask::SessionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet) + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet){ } long SessionTask::GetFilterCode() noexcept { @@ -52,14 +52,14 @@ void SessionTask::operator()() { certitude = ReadFromSession(_token, _repo_ids); if(certitude) STAT_MATCH_INC; - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("SessionTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } @@ -91,7 +91,7 @@ std::string SessionTask::JoinRepoIDs(const std::vector &repo_ids) { } -bool SessionTask::REDISResetExpire(const std::string &token, const std::string &repo_id) { +bool SessionTask::REDISResetExpire(const std::string &token, const std::string &repo_id __attribute((unused))) { DARWIN_LOGGER; long long int ttl; darwin::toolkit::RedisManager& redis = darwin::toolkit::RedisManager::GetInstance(); diff --git a/samples/fsession/SessionTask.hpp b/samples/fsession/SessionTask.hpp index 98154b2e..8af419f6 100644 --- a/samples/fsession/SessionTask.hpp +++ b/samples/fsession/SessionTask.hpp @@ -19,8 +19,9 @@ extern "C" { #include #include -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/RedisManager.hpp" @@ -34,12 +35,12 @@ extern "C" { // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class SessionTask : public darwin::Session { +class SessionTask : public darwin::ATask { public: - explicit SessionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex); + explicit SessionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet); ~SessionTask() override = default; diff --git a/samples/ftanomaly/Generator.cpp b/samples/ftanomaly/Generator.cpp index 1f1da826..441be2f9 100644 --- a/samples/ftanomaly/Generator.cpp +++ b/samples/ftanomaly/Generator.cpp @@ -13,9 +13,16 @@ #include "base/Core.hpp" #include "Generator.hpp" #include "TAnomalyTask.hpp" +#include "ASession.hpp" #include "TAnomalyThreadManager.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -92,10 +99,15 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, - _anomaly_thread_manager, _redis_internal)); -} \ No newline at end of file +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _anomaly_thread_manager, _redis_internal + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_TANOMALY; +} diff --git a/samples/ftanomaly/Generator.hpp b/samples/ftanomaly/Generator.hpp index 088c891c..99903530 100644 --- a/samples/ftanomaly/Generator.hpp +++ b/samples/ftanomaly/Generator.hpp @@ -13,7 +13,7 @@ #include "../toolkit/rapidjson/document.h" #include "../toolkit/FileManager.hpp" -#include "Session.hpp" +#include "ATask.hpp" #include "TAnomalyThreadManager.hpp" #include "AGenerator.hpp" @@ -21,13 +21,13 @@ class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; -public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/ftanomaly/TAnomalyTask.cpp b/samples/ftanomaly/TAnomalyTask.cpp index 42bd2d2b..d8f90876 100644 --- a/samples/ftanomaly/TAnomalyTask.cpp +++ b/samples/ftanomaly/TAnomalyTask.cpp @@ -13,17 +13,17 @@ #include "../../toolkit/RedisManager.hpp" #include "../../toolkit/lru_cache.hpp" #include "TAnomalyTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" -AnomalyTask::AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +AnomalyTask::AnomalyTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr vat, std::string redis_list_name) - : Session{"tanomaly", socket, manager, cache, cache_mutex}, _redis_list_name{std::move(redis_list_name)}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _redis_list_name{std::move(redis_list_name)}, _anomaly_thread_manager{std::move(vat)} { } @@ -43,11 +43,11 @@ void AnomalyTask::operator()() { STAT_INPUT_INC; if(ParseLine(line)) { REDISAddEntry(); - _certitudes.push_back(0); + _packet.AddCertitude(0); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } diff --git a/samples/ftanomaly/TAnomalyTask.hpp b/samples/ftanomaly/TAnomalyTask.hpp index e257a70f..a5b16f43 100644 --- a/samples/ftanomaly/TAnomalyTask.hpp +++ b/samples/ftanomaly/TAnomalyTask.hpp @@ -8,8 +8,9 @@ #pragma once #include -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "TAnomalyThreadManager.hpp" #include "../../toolkit/RedisManager.hpp" @@ -25,14 +26,14 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class AnomalyTask: public darwin::Session { +class AnomalyTask: public darwin::ATask { public: - explicit AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, - std::shared_ptr vat, - std::string redis_list_name); + explicit AnomalyTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + std::shared_ptr vat, + std::string redis_list_name); ~AnomalyTask() override = default; public: diff --git a/samples/ftanomaly/TAnomalyThreadManager.cpp b/samples/ftanomaly/TAnomalyThreadManager.cpp index f7e5918c..e4ea21ee 100644 --- a/samples/ftanomaly/TAnomalyThreadManager.cpp +++ b/samples/ftanomaly/TAnomalyThreadManager.cpp @@ -110,7 +110,7 @@ void AnomalyThreadManager::PreProcess(std::vector logs) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("AnomalyThread::PreProcess:: Starting the pre-process..."); - size_t size, pos, i; + size_t size, i; char delimiter = ';'; std::array values{}; std::string ip, ip_dst, port, protocol; diff --git a/samples/ftest/Generator.cpp b/samples/ftest/Generator.cpp index 191b9492..171e360b 100644 --- a/samples/ftest/Generator.cpp +++ b/samples/ftest/Generator.cpp @@ -12,10 +12,17 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "TestTask.hpp" +#include "ASession.hpp" #include "tsl/hopscotch_map.h" #include "tsl/hopscotch_set.h" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -79,9 +86,15 @@ bool Generator::ConfigRedis(std::string redis_socket_path) { return redis.FindAndConnect(); } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _redis_list_name, _redis_channel_name)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _redis_list_name, _redis_channel_name + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_TEST; } diff --git a/samples/ftest/Generator.hpp b/samples/ftest/Generator.hpp index 46dd6f6f..364d42c0 100644 --- a/samples/ftest/Generator.hpp +++ b/samples/ftest/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "../toolkit/Files.hpp" #include "../toolkit/rapidjson/document.h" @@ -19,13 +19,14 @@ class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/ftest/TestTask.cpp b/samples/ftest/TestTask.cpp index ee128d8b..2baf8abe 100644 --- a/samples/ftest/TestTask.cpp +++ b/samples/ftest/TestTask.cpp @@ -13,17 +13,18 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "TestTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" -#include "protocol.h" #include "AlertManager.hpp" +#include "Core.hpp" -TestTask::TestTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +TestTask::TestTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::string& redis_list, std::string& redis_channel) - : Session{"test", socket, manager, cache, cache_mutex}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _redis_list{redis_list}, _redis_channel{redis_channel} { _is_cache = _cache != nullptr; @@ -51,29 +52,37 @@ void TestTask::operator()() { if(_line == "trigger_redis_list") { DARWIN_LOG_DEBUG("TestTask:: triggered redis list action"); if(REDISAddList(_redis_list, _line)) { - _certitudes.push_back(0); + _packet.AddCertitude(0); } else { - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } else if(_line == "trigger_redis_channel") { DARWIN_LOG_DEBUG("TestTask:: triggered redis channel action"); if(REDISPublishChannel(_redis_channel, _line)) { - _certitudes.push_back(0); + _packet.AddCertitude(0); } else { - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } - else { + else if (_line == "load_test") { + DARWIN_LOG_DEBUG("TestTask:: load_test triggered, doing work for some times (2 000 000 mults)"); + int n = _line.size(); + for(int i=0; i< 2000000;i++){ + n *=i; + } + _packet.AddCertitude(n & 1); + } else { DARWIN_LOG_DEBUG("TestTask:: not triggered specific action, generating alert by default"); - DARWIN_ALERT_MANAGER.Alert(_line, 100, Evt_idToString()); - _certitudes.push_back(0); + DARWIN_ALERT_MANAGER.Alert(_line, 100, _packet.Evt_idToString(), "{\"filter_name\":\"" + darwin::Core::instance().GetName() + "\"}"); + _packet.AddCertitude(0); + _response_body.append(_packet.GetBody()); } } else { - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } diff --git a/samples/ftest/TestTask.hpp b/samples/ftest/TestTask.hpp index aa8a1400..62c80a9a 100644 --- a/samples/ftest/TestTask.hpp +++ b/samples/ftest/TestTask.hpp @@ -10,8 +10,9 @@ #include #include "../../toolkit/lru_cache.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #define DARWIN_FILTER_TEST 0x74657374 #define DARWIN_FILTER_NAME "test" @@ -22,12 +23,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class TestTask : public darwin::Session { +class TestTask : public darwin::ATask { public: - explicit TestTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit TestTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::string& list, std::string& channel); diff --git a/samples/fuseragent/Generator.cpp b/samples/fuseragent/Generator.cpp index 266b83c7..35806138 100644 --- a/samples/fuseragent/Generator.cpp +++ b/samples/fuseragent/Generator.cpp @@ -13,6 +13,13 @@ #include "Generator.hpp" #include "tensorflow/core/framework/graph.pb.h" #include "UserAgentTask.hpp" +#include "ASession.hpp" + +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} bool Generator::LoadConfig(const rapidjson::Document &configuration) { DARWIN_LOGGER; @@ -112,11 +119,17 @@ bool Generator::LoadModel(const std::string &model_path) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _session, _token_map, _max_tokens)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _session, _token_map, _max_tokens + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_USER_AGENT; } Generator::~Generator() { diff --git a/samples/fuseragent/Generator.hpp b/samples/fuseragent/Generator.hpp index 70a87320..33776092 100644 --- a/samples/fuseragent/Generator.hpp +++ b/samples/fuseragent/Generator.hpp @@ -11,21 +11,22 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "tensorflow/core/public/session.h" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator(); public: static constexpr int DEFAULT_MAX_TOKENS = 50; - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fuseragent/UserAgentTask.cpp b/samples/fuseragent/UserAgentTask.cpp index 4595cc9e..cccab942 100644 --- a/samples/fuseragent/UserAgentTask.cpp +++ b/samples/fuseragent/UserAgentTask.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -16,20 +15,20 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "Logger.hpp" -#include "protocol.h" -#include "tensorflow/core/framework/tensor.h" #include "UserAgentTask.hpp" +#include "ASession.hpp" +#include "tensorflow/core/framework/tensor.h" const std::vector UserAgentTask::USER_AGENT_CLASSES({"Desktop", "Tool", "Libraries", "Good bot", "Bad bot", "Mail", "IOT", "Mobile"}); -UserAgentTask::UserAgentTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +UserAgentTask::UserAgentTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr &session, std::map &token_map, const unsigned int max_tokens) - : Session{"user_agent", socket, manager, cache, cache_mutex}, _session{session}, _max_tokens{max_tokens}, _token_map{token_map} { + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _session{session}, _max_tokens{max_tokens}, _token_map{token_map} { _is_cache = _cache != nullptr; } @@ -44,7 +43,7 @@ long UserAgentTask::GetFilterCode() noexcept { void UserAgentTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; for (const std::string &user_agent : _user_agents) { SetStartingTime(); @@ -61,10 +60,10 @@ void UserAgentTask::operator()() { if (GetCacheResult(hash, certitude)) { if (is_log && (certitude>=_threshold)){ - _logs += R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + _logs += R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"user_agent\": \"" + user_agent + "\", \"ua_classification\": " + std::to_string(certitude) + "}\n"; } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("UserAgentTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -73,10 +72,10 @@ void UserAgentTask::operator()() { certitude = Predict(user_agent); if (is_log && (certitude>=_threshold)){ - _logs += R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + _logs += R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"user_agent\": \"" + user_agent + "\", \"ua_classification\": " + std::to_string(certitude) + "}\n"; } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) { SaveToCache(hash, certitude); @@ -183,7 +182,7 @@ bool UserAgentTask::ParseBody() { try { rapidjson::Document document; - document.Parse(body.c_str()); + document.Parse(_packet.GetBody().c_str()); if (!document.IsArray()) { DARWIN_LOG_ERROR("UserAgentTask:: ParseBody: You must provide a list"); diff --git a/samples/fuseragent/UserAgentTask.hpp b/samples/fuseragent/UserAgentTask.hpp index 69434efb..a0ba424d 100644 --- a/samples/fuseragent/UserAgentTask.hpp +++ b/samples/fuseragent/UserAgentTask.hpp @@ -13,8 +13,9 @@ #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "tensorflow/core/public/session.h" #define DARWIN_FILTER_USER_AGENT 0x75736572 @@ -23,12 +24,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class UserAgentTask : public darwin::Session { +class UserAgentTask : public darwin::ATask { public: - explicit UserAgentTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit UserAgentTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr &session, std::map &token_map, const unsigned int max_tokens = 50); ~UserAgentTask() override; diff --git a/samples/fyara/Generator.cpp b/samples/fyara/Generator.cpp index a80f047f..9c1f5b84 100644 --- a/samples/fyara/Generator.cpp +++ b/samples/fyara/Generator.cpp @@ -11,8 +11,15 @@ #include "../toolkit/lru_cache.hpp" #include "base/Logger.hpp" #include "YaraTask.hpp" +#include "ASession.hpp" #include "Generator.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::LoadConfig(const rapidjson::Document &configuration) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("Yara:: Generator:: Loading configuration..."); @@ -114,9 +121,15 @@ bool Generator::ConfigureAlerting(const std::string& tags) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _yaraCompiler->GetEngine(_fastmode, _timeout))); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _yaraCompiler->GetEngine(_fastmode, _timeout) + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_YARA_SCAN; } diff --git a/samples/fyara/Generator.hpp b/samples/fyara/Generator.hpp index 8ede14b1..251b7e00 100644 --- a/samples/fyara/Generator.hpp +++ b/samples/fyara/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "AlertManager.hpp" #include "../../toolkit/rapidjson/document.h" @@ -19,13 +19,14 @@ class Generator : public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; public: - darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fyara/YaraTask.cpp b/samples/fyara/YaraTask.cpp index c3b22c2e..95b91665 100644 --- a/samples/fyara/YaraTask.cpp +++ b/samples/fyara/YaraTask.cpp @@ -11,17 +11,18 @@ #include #include "YaraTask.hpp" +#include "ASession.hpp" #include "Stats.hpp" #include "Logger.hpp" #include "AlertManager.hpp" -YaraTask::YaraTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +YaraTask::YaraTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr yaraEngine) - : Session{"yara", socket, manager, cache, cache_mutex}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _yaraEngine{yaraEngine} { _is_cache = _cache != nullptr; } @@ -36,7 +37,8 @@ long YaraTask::GetFilterCode() noexcept { void YaraTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); // Should not fail, as the Session body parser MUST check for validity ! auto array = _body.GetArray(); @@ -57,16 +59,16 @@ void YaraTask::operator()() { if (certitude>=_threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + "}\n"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("YaraTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -78,7 +80,7 @@ void YaraTask::operator()() { if(_yaraEngine->ScanData(data, certitude) == -1) { DARWIN_LOG_WARNING("YaraTask:: error while scanning, ignoring chunk"); - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); continue; } @@ -90,16 +92,16 @@ void YaraTask::operator()() { std::string tagListJson = YaraTask::GetJsonListFromSet(results.tags); std::string details = "{\"rules\": " + ruleListJson + "}"; - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, Evt_idToString(), details, tagListJson); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _packet.Evt_idToString(), details, tagListJson); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + R"(, "rules": )" + ruleListJson + R"(, "tags": )" + tagListJson + "}\n"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) { SaveToCache(hash, certitude); @@ -107,11 +109,11 @@ void YaraTask::operator()() { } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } DARWIN_LOG_DEBUG("YaraTask:: processed entry in " - + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_certitudes.back())); + + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_packet.GetCertitudeList().back())); } } @@ -137,6 +139,7 @@ bool YaraTask::ParseLine(rapidjson::Value& line) { else { encoding = fields[1].GetString(); } + __attribute((fallthrough)); // No break here! case 1: if(not fields[0].IsString()){ diff --git a/samples/fyara/YaraTask.hpp b/samples/fyara/YaraTask.hpp index 9ccdae7f..08983126 100644 --- a/samples/fyara/YaraTask.hpp +++ b/samples/fyara/YaraTask.hpp @@ -18,7 +18,9 @@ #include "Encoders.h" #include "Yara.hpp" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #define DARWIN_FILTER_YARA_SCAN 0x79617261 #define DARWIN_FILTER_NAME "yara" @@ -29,12 +31,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class YaraTask : public darwin::Session { +class YaraTask : public darwin::ATask { public: - explicit YaraTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit YaraTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr yaraEngine); ~YaraTask() override = default; diff --git a/samples/protocol.h b/samples/protocol.h index c0ce485f..c02b8d21 100644 --- a/samples/protocol.h +++ b/samples/protocol.h @@ -24,7 +24,6 @@ extern "C" { #define DARWIN_FILTER_CODE_NO 0x00000000 // the default certitude list size, which is 1, to allow FMAs (see flexible array members on C99) for both C and C++ code -#define DEFAULT_CERTITUDE_LIST_SIZE 1 /// Represent the receiver of the results. /// @@ -54,8 +53,9 @@ typedef struct { size_t body_size; //!< The complete size of the the parameters to be sent (if needed). unsigned char evt_id[16]; //!< An array containing the event ID size_t certitude_size; //!< The size of the list containing the certitudes. - unsigned int certitude_list[DEFAULT_CERTITUDE_LIST_SIZE]; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. -} darwin_filter_packet_t; + unsigned int certitude_list[0]; //!< DEPRECATED, certitudes are no longer passed through this array, + //!< they must be appened after the header, before the body +} __attribute__((packed)) darwin_filter_packet_t; #ifdef __cplusplus }; diff --git a/tests/conf.py b/tests/conf.py index 18f1956a..73e8a278 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -8,6 +8,9 @@ DEFAULT_MANAGER_PATH = '/home/darwin/manager/manager.py' DEFAULT_FILTER_PATH = '/home/darwin/filters/' +DEFAULT_PROTOCOL = 'tcp' +DEFAULT_ADDRESS = '127.0.0.1:12121' + # TEST CONFIG VALGRIND_MEMCHECK = False \ No newline at end of file diff --git a/tests/core/alert.py b/tests/core/alert.py index d5bb7e46..b6b114a9 100644 --- a/tests/core/alert.py +++ b/tests/core/alert.py @@ -43,7 +43,7 @@ def clean_files(self): pass def send(self, data): - api = DarwinApi(socket_type="unix", socket_path=self.socket) + api = self.get_darwin_api() api.call([data], response_type='no') def get_redis_alerts(self): diff --git a/tests/core/base.py b/tests/core/base.py index 61cc0b35..29432f18 100644 --- a/tests/core/base.py +++ b/tests/core/base.py @@ -5,7 +5,6 @@ from tools.filter import Filter from tools.output import print_result from core.utils import DEFAULT_PATH, FTEST_CONFIG, RESP_MON_STATUS_RUNNING -from darwin import DarwinApi @@ -14,8 +13,14 @@ def run(): tests = [ check_start_stop, check_pid_file, - check_socket_create_delete, - check_socket_connection, + check_unix_socket_create_delete, + check_unix_socket_connection, + check_tcp_socket_create_delete, + check_tcp_socket_connection, + check_tcp6_socket_create_delete, + check_tcp6_socket_connection, + check_udp_socket_connection, + check_udp6_socket_connection, check_socket_monitor_create_delete, check_socket_monitor_connection, check_start_wrong_conf, @@ -78,39 +83,182 @@ def check_pid_file(): return True -def check_socket_create_delete(): - filter = Filter(filter_name="test") - pid = -1 +def check_unix_socket_create_delete(): + filter = Filter(filter_name="test", socket_type='unix') filter.configure(FTEST_CONFIG) filter.valgrind_start() if not access(filter.socket, F_OK): - logging.error("check_socket_create_delete: Socket file not accesible") + logging.error("check_unix_socket_create_delete: Socket file not accesible") return False filter.stop() if access(filter.socket, F_OK): - logging.error("check_socket_create_delete: Socket file not deleted") + logging.error("check_unix_socket_create_delete: Socket file not deleted") return False return True -def check_socket_connection(): - filter = Filter(filter_name="test") - pid = -1 +def check_unix_socket_connection(): + filter = Filter(filter_name="test", socket_type='unix') filter.configure(FTEST_CONFIG) filter.valgrind_start() try: - api = DarwinApi(socket_path=filter.socket, socket_type="unix") + api = filter.get_darwin_api() api.call("test\n", filter_code=0x74657374, response_type="back") api.close() except Exception as e: - logging.error("check_socket_connection_back: Error connecting to socket: {}".format(e)) + logging.error("check_unix_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + +def check_tcp_socket_create_delete(): + filter = Filter(filter_name="test", socket_type='tcp', socket_path='127.0.0.1:12323') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + with socket.socket(socket.AF_INET) as s: + s.settimeout(2) + res = s.connect_ex(('127.0.0.1',12323)) + s.close() + + if res != 0: + logging.error("check_tcp_socket_create_delete: Socket file not accesible") + return False + filter.stop() + + with socket.socket(socket.AF_INET) as s: + s.settimeout(2) + res = s.connect_ex(('127.0.0.1',12323)) + s.close() + + if res == 0: + logging.error("check_tcp_socket_create_delete: Socket file not deleted") + return False + + return True + + +def check_tcp_socket_connection(): + filter = Filter(filter_name="test", socket_type='tcp', socket_path='127.0.0.1:12123') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("test\n", filter_code=0x74657374, response_type="back") + api.close() + except Exception as e: + logging.error("check_tcp_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + +def check_tcp6_socket_create_delete(): + filter = Filter(filter_name="test", socket_type='tcp', socket_path='[::1]:12123') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + with socket.socket(socket.AF_INET6) as s: + s.settimeout(2) + res = s.connect_ex(('::1',12123)) + s.close() + + if res != 0: + logging.error("check_tcp6_socket_create_delete: Socket file not accesible") + return False + + filter.stop() + + with socket.socket(socket.AF_INET6) as s: + s.settimeout(2) + res = s.connect_ex(('::1',12123)) + s.close() + + if res == 0: + logging.error("check_tcp6_socket_create_delete: Socket file not deleted") + return False + + return True + + +def check_tcp6_socket_connection(): + filter = Filter(filter_name="test", socket_type='tcp', socket_path='[::1]:1111') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("test\n", filter_code=0x74657374, response_type="back") + api.close() + except Exception as e: + logging.error("check_tcp6_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + +# These tests are not done as there is no reliable way to check if a udp socket is open and listening +# unreliable ways include listening for a icmp packet back but it depends on the +# system and if the packet is blocked by firewalls + +# def check_udp_socket_create_delete() +# def check_udp6_socket_create_delete() + +def check_udp_socket_connection(): + filter = Filter(filter_name="test", socket_type='udp', socket_path='127.0.0.1:12123') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("udp test", filter_code=0x74657374, response_type="no") + # sleep to let the filter process the call + #TODO check alert + api.call("udp test2", filter_code=0x74657374, response_type="no") + api.call("udp test3", filter_code=0x74657374, response_type="no") + api.call("udp test4", filter_code=0x74657374, response_type="no") + api.call("udp test5", filter_code=0x74657374, response_type="no") + sleep(2) + + api.close() + except Exception as e: + logging.error("check_udp_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + +def check_udp6_socket_connection(): + filter = Filter(filter_name="test", socket_type='udp', socket_path='[::1]:1111') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("test\n", filter_code=0x74657374, response_type="no") + sleep(1) #let filter process + # todo check alert + api.close() + except Exception as e: + logging.error("check_udp6_socket_connection: Error connecting to socket: {}".format(e)) return False filter.stop() @@ -119,7 +267,6 @@ def check_socket_connection(): def check_socket_monitor_create_delete(): filter = Filter(filter_name="test") - pid = -1 filter.configure(FTEST_CONFIG) filter.valgrind_start() @@ -139,7 +286,6 @@ def check_socket_monitor_create_delete(): def check_socket_monitor_connection(): filter = Filter(filter_name="test") - pid = -1 filter.configure(FTEST_CONFIG) filter.valgrind_start() diff --git a/tests/core/redis.py b/tests/core/redis.py index d6243483..df71d62d 100644 --- a/tests/core/redis.py +++ b/tests/core/redis.py @@ -82,13 +82,13 @@ def master_replica(): message = master.channel_get_message() if message is '': - logging.error("master_replica: expected to get a message in channel {} " + - "but got nothing".format(REDIS_CHANNEL_NAME)) + logging.error(("master_replica: expected to get a message in channel {} " + + "but got nothing").format(REDIS_CHANNEL_NAME)) return False if message != REDIS_CHANNEL_TRIGGER: - logging.error("master_replica: expected to get a message in channel {} saying '{}' " + - "but got '{}' instead".format(REDIS_CHANNEL_NAME, REDIS_CHANNEL_TRIGGER, message)) + logging.error(("master_replica: expected to get a message in channel {} saying '{}' " + + "but got '{}' instead").format(REDIS_CHANNEL_NAME, REDIS_CHANNEL_TRIGGER, message)) return False return True @@ -143,8 +143,8 @@ def master_replica_master_temp_fail(): num_list_entries = master_connection.llen(REDIS_LIST_NAME) if num_list_entries != 1: - logging.error("master_replica_master_temp_fail: wrong number of entries in the redis list {}: " + - "expected 1 but got {}".format(REDIS_LIST_NAME, num_list_entries)) + logging.error(("master_replica_master_temp_fail: wrong number of entries in the redis list {}: " + + "expected 1 but got {}").format(REDIS_LIST_NAME, num_list_entries)) return False return True @@ -185,16 +185,14 @@ def master_replica_transfer(function_name, healthcheck): return False if return_code != 0: - logging.error("{}: Filter didn't return correct code, " + - "waited for 0 but got {}".format(function_name, return_code)) + logging.error("{}: Filter didn't return correct code, waited for 0 but got {}".format(function_name, return_code)) return False with replica.connect() as new_master_connection: num_entries = new_master_connection.llen(REDIS_LIST_NAME) if num_entries != 2: - logging.error("{}: Wrong number of entries in {}, " + - "expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) + logging.error("{}: Wrong number of entries in {}, expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) return False return True @@ -242,16 +240,14 @@ def master_replica_failover(function_name, healthcheck): return False if return_code != 0: - logging.error("{}: Filter didn't return correct code, " + - "waited for 0 but got {}".format(function_name, return_code)) + logging.error("{}: Filter didn't return correct code, waited for 0 but got {}".format(function_name, return_code)) return False with replica.connect() as new_master_connection: num_entries = new_master_connection.llen(REDIS_LIST_NAME) if num_entries != 2: - logging.error("{}: Wrong number of entries in {}, " + - "expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) + logging.error("{}: Wrong number of entries in {}, expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) return False return True @@ -275,7 +271,7 @@ def multi_thread_master(): thread_list = [] def thread_brute(filter, count_log): - for count in range(0, count_log): + for _ in range(0, count_log): try: filter.send_single(REDIS_LIST_TRIGGER) except: @@ -283,7 +279,7 @@ def thread_brute(filter, count_log): return True - for num in range(0, 5): + for _ in range(0, 5): thread_list.append(threading.Thread(target=thread_brute, args=(filter, 500))) for thread in thread_list: @@ -296,9 +292,9 @@ def thread_brute(filter, count_log): number = master.get_number_of_connections() - # 5 threads - if number != 5: - logging.error("multi_thread_master: wrong number of active connections: expected 5 but got " + str(number)) + # 6 threads : 5 task threads + the main thread for configuring the redis socket + if number != 6: + logging.error("multi_thread_master: wrong number of active connections: expected 6 but got {}".format(number)) return False return True @@ -317,7 +313,7 @@ def master_replica_discovery_rate_limiting(): # success filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: - logging.error("master_replica_discovery_rate_limiting: Could not connect to test filter: {}".format(function_name, e)) + logging.error("master_replica_discovery_rate_limiting: Could not connect to test filter: {}".format(e)) return False # master shuts down @@ -337,7 +333,7 @@ def thread_brute_time(filter, time_end): thread_list = [] - for num in range(0, 5): + for _ in range(0, 5): thread_list.append(threading.Thread(target=thread_brute_time, args=(filter, 9))) # ought to crash if command fails @@ -352,9 +348,14 @@ def thread_brute_time(filter, time_end): # new generated connections generated, minus the one generated by the call to get it new_connections = replica.connect().info()['total_connections_received'] - initial_connections_num - 1 - if new_connections > 10: + # In darwin::RedisManager, there is at most 2 connections attempts per 8 seconds per thread + # We try de post something in a stopped redis and expect a controlled number of connection attempts + # The limit is calculated as : + # 20 connections : 5 threads * 2 attempts * 2 (the 9 seconds span let the discover happen twice) + # We measure generally only 19 because there was usually already one attempt done in the first part of the test + if new_connections > 20: logging.error("master_replica_discovery_rate_limiting: Wrong number of new connection attempts, " + - "was supposed to have 10 new at most, but got ".format(new_connections)) + "was supposed to have 10 new at most, but got {}".format(new_connections)) return False return True \ No newline at end of file diff --git a/tests/filters/data/bad_example.py b/tests/filters/data/bad_example.py new file mode 100644 index 00000000..8cc73695 --- /dev/null +++ b/tests/filters/data/bad_example.py @@ -0,0 +1,122 @@ +import json +import os +import datetime +from enum import IntEnum +from typing import List, Union +from dataclasses import dataclass + +# DOES NOT WORK , Used in tests to check if missing methods are detected by the filter + +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level, msg): + print(level, msg) + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + +class PythonFilterError(Exception): + pass + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +threshold = 3 +class Execution: + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + # MISSING METHOD filter_process + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/data/bad_example_bad_import.py b/tests/filters/data/bad_example_bad_import.py new file mode 100644 index 00000000..05f969ac --- /dev/null +++ b/tests/filters/data/bad_example_bad_import.py @@ -0,0 +1,140 @@ +import json +import os +import datetime +from enum import IntEnum +from typing import List, Union +from dataclasses import dataclass + +# DOES NOT WORK , Used in tests to check if bad imports are detected by the filter +import trucmuche + + +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level, msg): + print(level, msg) + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + +class PythonFilterError(Exception): + pass + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +threshold = 3 +class Execution: + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: + if isinstance(line, str): + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) + + return resp + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/data/bad_example_fail_init.py b/tests/filters/data/bad_example_fail_init.py new file mode 100644 index 00000000..42b3a303 --- /dev/null +++ b/tests/filters/data/bad_example_fail_init.py @@ -0,0 +1,157 @@ +import json +import os +import datetime +from enum import IntEnum +from random import randint +from typing import List, Union +from dataclasses import dataclass + +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level: int, msg: str): + import sys + outpipe = sys.stderr if level >= 4 else sys.stdout + print(DarwinLogLevel(level).name, msg,file=outpipe) + + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + +class PythonFilterError(Exception): + pass + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +threshold = 3 +nb = 0 # Fail at 3 +class Execution: + + def __init__(self) -> None: + global nb + if nb == 3: + nb += 1 + raise Exception("Fail at init") + nb += 1 + + + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + if config.get('fail_conf', '') == 'yes': + raise Exception('FAILED PYTHON SCRIPT CONFIGURATION') + venv = config.get('python_venv_folder', '') + if len(venv) != 0: + import setuptools + if venv in str(setuptools): + darwin_log(DarwinLogLevel.Debug, 'WE ARE IN VIRTUAL ENV') + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: + if isinstance(line, str): + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) + + return resp + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/data/example.py b/tests/filters/data/example.py new file mode 100644 index 00000000..ec299375 --- /dev/null +++ b/tests/filters/data/example.py @@ -0,0 +1,149 @@ +import json +import os +import datetime +from enum import IntEnum +from random import randint +from typing import List, Union +from dataclasses import dataclass + +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level: int, msg: str): + import sys + outpipe = sys.stderr if level >= 4 else sys.stdout + print(DarwinLogLevel(level).name, msg,file=outpipe) + + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + +class PythonFilterError(Exception): + pass + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +threshold = 3 + +class Execution: + + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + if config.get('fail_conf', '') == 'yes': + raise Exception('FAILED PYTHON SCRIPT CONFIGURATION') + venv = config.get('python_venv_folder', '') + if len(venv) != 0: + import setuptools + if venv in str(setuptools): + darwin_log(DarwinLogLevel.Debug, 'WE ARE IN VIRTUAL ENV') + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: + if isinstance(line, str): + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) + + return resp + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/fanomaly.py b/tests/filters/fanomaly.py index 58206f35..27288370 100644 --- a/tests/filters/fanomaly.py +++ b/tests/filters/fanomaly.py @@ -45,8 +45,7 @@ def test(test_name, data, expected_certitudes, expected_certitudes_size): return False # SEND TEST - darwin_api = DarwinApi(socket_path=anomaly_filter.socket, - socket_type="unix", ) + darwin_api = anomaly_filter.get_darwin_api() results = darwin_api.bulk_call( data, diff --git a/tests/filters/fbuffer.py b/tests/filters/fbuffer.py index 43771bb3..664902c1 100644 --- a/tests/filters/fbuffer.py +++ b/tests/filters/fbuffer.py @@ -132,8 +132,7 @@ def data_to_bytes(data): return False # SEND TEST - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix", ) + darwin_api = buffer_filter.get_darwin_api() if bulk: darwin_api.bulk_call( @@ -413,7 +412,7 @@ def thread_working_test(): buffer_filter = Buffer() buffer_filter.configure(config_buffer) - test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock") + test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock", socket_type='unix') test_filter.configure(config_test) @@ -426,8 +425,7 @@ def thread_working_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix", ) + darwin_api = buffer_filter.get_darwin_api() data = buffer_filter.get_test_data() @@ -539,7 +537,7 @@ def fanomaly_connector_and_send_test(): buffer_filter = Buffer() buffer_filter.configure(config_buffer) - test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock") + test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock", socket_type='unix') test_filter.configure(config_test) # START FILTER @@ -554,8 +552,7 @@ def fanomaly_connector_and_send_test(): # SEND TEST data = buffer_filter.get_test_data() - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix") + darwin_api = buffer_filter.get_darwin_api() darwin_api.bulk_call( data, @@ -654,7 +651,7 @@ def sum_tests(test_name, values=[], required_log_lines=0, expected_alert=1, init buffer_filter = Buffer() buffer_filter.configure(config_buffer) - test_filter = Filter(filter_name="test", socket_path="/tmp/test.sock") + test_filter = Filter(filter_name="test", socket_path="/tmp/test.sock", socket_type='unix') test_filter.configure(config_test) # Potentially add init data to redis @@ -677,8 +674,7 @@ def sum_tests(test_name, values=[], required_log_lines=0, expected_alert=1, init # SEND values - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix") + darwin_api = buffer_filter.get_darwin_api() for test_value in values: darwin_api.call( ["", test_value], diff --git a/tests/filters/fconnection.py b/tests/filters/fconnection.py index 47a1e1a2..d527b01a 100644 --- a/tests/filters/fconnection.py +++ b/tests/filters/fconnection.py @@ -65,8 +65,7 @@ def new_connection_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=connection_filter.socket, - socket_type="unix", ) + darwin_api = connection_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -125,8 +124,7 @@ def known_connection_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=connection_filter.socket, - socket_type="unix", ) + darwin_api = connection_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -184,8 +182,7 @@ def new_connection_to_known_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=connection_filter.socket, - socket_type="unix", ) + darwin_api = connection_filter.get_darwin_api() darwin_api.bulk_call( [ diff --git a/tests/filters/fhostlookup.py b/tests/filters/fhostlookup.py index 8c23456f..2f484056 100644 --- a/tests/filters/fhostlookup.py +++ b/tests/filters/fhostlookup.py @@ -92,8 +92,7 @@ def test(test_name, init_data, data, expected_certitudes, db_type="json"): return False # SEND TEST - darwin_api = DarwinApi(socket_path=hostlookup_filter.socket, - socket_type="unix", ) + darwin_api = hostlookup_filter.get_darwin_api() results = darwin_api.bulk_call( data, diff --git a/tests/filters/fpython.py b/tests/filters/fpython.py new file mode 100644 index 00000000..52d65ec8 --- /dev/null +++ b/tests/filters/fpython.py @@ -0,0 +1,290 @@ +import logging +import json +import tempfile +import venv +from tools.filter import Filter +from tools.output import print_result + +class PythonFilter(Filter): + alert_file='/tmp/python_alerts.log' + threshold=3 + def __init__(self): + super().__init__(filter_name="python") + + def configure(self, content=None, venv='', other_conf=''): + if not content: + content = '{{\n' \ + '"python_script_path": "{python_path}",\n' \ + '"shared_library_path": "{so_path}",\n' \ + '"log_file_path": "{alert_file}",\n' \ + '"python_venv_folder": "{venv}",\n' \ + '"dummy":"",\n' \ + '{other_conf}' \ + '"threshold":{threshold}\n' \ + '}}'.format(python_path='tests/filters/data/example.py', + alert_file=self.alert_file, + threshold=self.threshold, + venv=venv, + other_conf=other_conf, + so_path='') + super(PythonFilter, self).configure(content) + + + + +def run(): + tests = [ + test_bad_config_no_script, + test_bad_config_empty_python_script_path, + test_bad_config_nothing, + test_bad_config_no_venv, + test_wrong_config_missing_method, + test_wrong_python_missing_requirement, + test_wrong_python_exception_during_conf, + test_wrong_python_exception_during_init, + test_wrong_python_exception_during_steps, + test_wrong_python_exception_during_delete, + test_venv, + full_python_functional_test, + ] + + for i in tests: + print_result("Python: " + i.__name__, i) + +def test_bad_config_no_script(): + f = PythonFilter() + + f.configure('{"python_script_path":"/tmp/no_script_there.bad_ext", "shared_library_path":""}') + if f.valgrind_start(): + logging.error('test_bad_config_no_script: Filter should not start') + return False + if not f.check_line_in_filter_log('Generator::LoadPythonScript : Error loading the python script : failed to open'): + logging.error('test_bad_config_no_script: Filter should have failed with a log') + return False + return True + +def test_bad_config_empty_python_script_path(): + f = PythonFilter() + + f.configure('{"shared_library_path":"/tmp/no_script_there.bad_ext", "python_script_path":""}') + if f.valgrind_start(): + logging.error('test_bad_config_no_shared_obj: Filter should not start') + return False + if not f.check_line_in_filter_log('Generator::LoadPythonScript : No python script to load'): + logging.error('test_bad_config_no_shared_obj: Filter should have failed with a log') + return False + return True + +def test_bad_config_nothing(): + f = PythonFilter() + + f.configure('{"shared_library_path":"", "python_script_path":""}') + if f.valgrind_start(): + logging.error('test_bad_config_nothing: Filter should not start') + return False + if not f.check_line_in_filter_log('Generator::LoadPythonScript : No python script to load'): + logging.error('test_bad_config_nothing: Filter should have failed with a log') + return False + return True + +def test_bad_config_no_venv(): + f = PythonFilter() + + f.configure(venv='/tmp/VenvPythonFilterDoesNotExist') + if f.valgrind_start(): + logging.error('test_bad_config_no_venv: Filter should not start') + return False + + if not f.check_line_in_filter_log('Generator::LoadPythonScript : Virtual Env does not exist : '): + logging.error('test_bad_config_no_venv: Filter should have failed with a log') + return False + return True + +def test_wrong_config_missing_method(): + f = PythonFilter() + + f.configure('{"python_script_path":"tests/filters/data/bad_example.py", "shared_library_path":""}') + if f.valgrind_start(): + logging.error('test_wrong_config_missing_method: Filter should not start') + return False + + if not f.check_line_in_filter_log('Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library'): + logging.error('test_wrong_config_missing_method: Filter should have failed with a log') + return False + return True + +def test_wrong_python_missing_requirement(): + f = PythonFilter() + + f.configure('{"python_script_path":"tests/filters/data/bad_example_bad_import.py", "shared_library_path":""}') + if f.valgrind_start(): + logging.error('test_wrong_python_missing_requirement: Filter should not start') + return False + + if not f.check_line_in_filter_log('Generator::'): + logging.error('test_wrong_python_missing_requirement: Filter should have failed with a log') + return False + return True + +def test_wrong_python_exception_during_conf(): + f = PythonFilter() + + f.configure(other_conf='"fail_conf":"yes",\n') + if f.valgrind_start(): + logging.error('test_wrong_python_exception_during_conf: Filter should not start') + return False + + if not f.check_line_in_filter_log('FAILED PYTHON SCRIPT CONFIGURATION'): + logging.error('test_wrong_python_exception_during_conf: Filter should have failed with a log') + return False + return True + +def test_wrong_python_exception_during_steps(): + f = PythonFilter() + + f.configure() + if not f.valgrind_start(): + logging.error('test_wrong_python_exception_during_steps: Filter should start') + return False + # Error: line too long + res = f.send_single("fail"*25) + if res != 101: + logging.error('test_wrong_python_exception_during_steps: Filter should fail with an error cerittude (101), but we received: ' + str(res)) + return False + + if not f.check_run(): + logging.error('test_wrong_python_exception_during_steps: Filter should still be running') + return False + + return True + +def test_wrong_python_exception_during_init(): + f = PythonFilter() + + f.configure('{"python_script_path":"tests/filters/data/bad_example_fail_init.py", "shared_library_path":"", "dummy":""}') + if not f.valgrind_start(): + logging.error('test_wrong_python_exception_during_init: Filter should start') + return False + # Error: exception raised on third init + is_test_ok = True + for i in range(5): + line = 'hello' + res = f.send_single(line) + if i == 3 and res != None: + is_test_ok = False + logging.error('test_wrong_python_exception_during_init: Filter should fail with an error cerittude (101), but we received: ' + str(res)) + if i == 3 and not f.check_line_in_filter_log("PythonTask:: PythonTask:: Error creating Execution object: Python error '' : Fail at init"): + is_test_ok = False + logging.error('test_wrong_python_exception_during_init: Filter should fail with a specific log, but was not found') + if i != 3 and res != len(line): + is_test_ok = False + logging.error(f'test_wrong_python_exception_during_init: Filter should succced with a certitude of ({str(len(line))}), but we received: {str(res)}') + + if not f.check_run(): + logging.error('test_wrong_python_exception_during_init: Filter should still be running') + return False + + return is_test_ok + +def test_wrong_python_exception_during_delete(): + """ + Cannot be tested as exceptions during __del__ are ignored by the python interpreter + """ + return True + + +def test_venv(): + with tempfile.TemporaryDirectory() as tmpdir: + e = venv.EnvBuilder(with_pip=True) + e.create(tmpdir) + + f = PythonFilter() + f.configure(venv=tmpdir) + if not f.valgrind_start(): + logging.error('test_venv: Filter should start') + return False + + if not f.check_line_in_filter_log('WE ARE IN VIRTUAL ENV'): + logging.error('test_venv: Filter should have logged') + return False + + return True + +# Functional test with exemple.py, it should go through all python steps +def full_python_functional_test(): + f = PythonFilter() + f.threshold=5 + f.configure() + f.valgrind_start() + api = f.get_darwin_api(verbose=False) + is_test_ok =True + for i in range(100): + line = 'hello ! ' + str(i) + ret = api.call(line , response_type="back") + if ret != len(line): + logging.error('line {} returned a bad certitude : {} instead of {}'.format(str(i), ret, len(line))) + is_test_ok = False + + line = 'alert:hello wow ! ' + str(i%10) + ret = api.call(line , response_type="back") + if ret != len(line): + logging.error('line {} returned a bad certitude : {} instead of {}'.format(str(i), ret, len(line))) + is_test_ok = False + # This next line should be too long, hence it should return a 101 certitude + ret = api.call('hello ! '*20 , response_type="back") + if ret != 101: + logging.error('line {} returned a bad certitude : {} instead of {}'.format(str(i), ret, 101)) + is_test_ok = False + if not is_test_ok: + return False + + lines = [] + for i in range(100): + lines.append('bulk hello ! ' + str(i)) + lines.append('alert:bulk hello wow ! ' + str(i%10)) + lines.append('hello ! '*20) + + # empty the log file + with open(f.alert_file, 'w') as _: + pass + + ret=api.bulk_call(lines,response_type='back') + predicted_alerts={} + for i, l in enumerate(lines): + if l.startswith('alert:'): + tmp = l.replace('alert:', '') + if tmp in predicted_alerts: + predicted_alerts[tmp] += 1 + else: + predicted_alerts[tmp] = 1 + + with open(f.alert_file, 'r') as file: + alerts_risen = file.readlines() + + is_test_ok = True + for alert in alerts_risen: + try: + json_alert = json.loads(alert) + except json.JSONDecodeError as e: + is_test_ok=False + logging.error('Malformed alert : should be a valid json', exc_info=e) + continue + if "date" not in json_alert or "alert" not in json_alert: + is_test_ok=False + logging.error('Malformed alert : "date" or "alert" not present ' + alert) + continue + if json_alert['alert'] not in predicted_alerts: + is_test_ok=False + logging.error('Malformed alert : alert should not have been triggered ' + alert) + continue + if predicted_alerts[json_alert['alert']] < f.threshold: + is_test_ok=False + logging.error('Malformed alert : alert should not have been triggered (below threshold) ' + alert) + continue + + api.close() + f.valgrind_stop() + + if not is_test_ok: + return False + return True diff --git a/tests/filters/fsession.py b/tests/filters/fsession.py index a8c9a0d4..cd822aa3 100644 --- a/tests/filters/fsession.py +++ b/tests/filters/fsession.py @@ -7,7 +7,6 @@ from tools.filter import Filter from tools.output import print_result from conf import TEST_FILES_DIR -from darwin import DarwinApi REDIS_SOCKET = f"{TEST_FILES_DIR}/redis_session.sock" @@ -143,8 +142,7 @@ def _input_tests(test_name, data, expected_certitudes, populate_redis=None, expe return False # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( data, @@ -346,8 +344,7 @@ def no_timeout_no_refresh(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -411,8 +408,7 @@ def timeout_but_no_change_to_ttl(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -477,8 +473,7 @@ def set_new_ttl(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -543,8 +538,7 @@ def timeout_with_change_to_ttl(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ diff --git a/tests/filters/ftanomaly.py b/tests/filters/ftanomaly.py index 435d83dc..7256a9d0 100644 --- a/tests/filters/ftanomaly.py +++ b/tests/filters/ftanomaly.py @@ -209,8 +209,7 @@ def data_to_bytes(data): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( data, @@ -353,8 +352,7 @@ def thread_working_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( [ @@ -413,8 +411,7 @@ def alert_in_redis_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() data = tanomaly_filter.get_test_data() darwin_api.bulk_call( @@ -466,9 +463,7 @@ def alert_published_test(): if not tanomaly_filter.valgrind_start(): return False # SEND TEST - - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( tanomaly_filter.get_test_data(), @@ -536,8 +531,7 @@ def alert_in_file_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( tanomaly_filter.get_test_data(), diff --git a/tests/filters/fyara.py b/tests/filters/fyara.py index 552d73a7..fbc72130 100644 --- a/tests/filters/fyara.py +++ b/tests/filters/fyara.py @@ -204,8 +204,7 @@ def input_with_several_rules(data, expected_certitudes=[0]): return False # SEND TEST - darwin_api = DarwinApi(socket_path=yara_filter.socket, - socket_type="unix") + darwin_api = yara_filter.get_darwin_api() diff --git a/tests/filters/test.py b/tests/filters/test.py index b63432a8..1485d180 100644 --- a/tests/filters/test.py +++ b/tests/filters/test.py @@ -9,6 +9,7 @@ import filters.fvast as fvast import filters.fvaml as fvaml import filters.fsession as fsession +import filters.fpython as fpython from tools.output import print_results @@ -27,6 +28,7 @@ def run(): fvast.run() fvaml.run() fsession.run() + fpython.run() print() print() diff --git a/tests/manager_socket/multi_filters_test.py b/tests/manager_socket/multi_filters_test.py new file mode 100644 index 00000000..e15ac40b --- /dev/null +++ b/tests/manager_socket/multi_filters_test.py @@ -0,0 +1,142 @@ +import functools +from manager_socket.utils import CONF_FTEST, PATH_CONF_FTEST +import time +from tools.output import print_result +from tools.darwin_utils import darwin_configure, darwin_stop, darwin_start, darwin_remove_configuration +from tools.filter import Filter +import conf +import psutil + +def run(): + scenarios = [ + ['unix', 'unix', 'unix'], + ['tcp', 'tcp', 'tcp'], + ['udp', 'udp', 'udp'], + ['udp', 'unix', 'unix'], + ['tcp', 'unix', 'unix'], + ['udp', 'tcp', 'tcp'], + ['tcp', 'udp', 'unix'], + ['unix', 'tcp', 'udp'], + ] + + for i in scenarios: + print_result("Multi Filters: Alerting tests : " + ' -> '.join(i), functools.partial(alerting_tests, i)) + +ONE_FILTER = """ +{{ + "name": "{name}", + "exec_path": "{filter_path}darwin_test", + "config_file": "/tmp/test.conf", + "output": "RAW", + {next_filter} + "nb_thread": 1, + "log_level": "DEBUG", + "cache_size": 0, + "network":{network} + }} +""" +UNIX_NETWORK = """{{ + "socket_type":"UNIX" +}}""" + +TCP_NETWORK = """{{ + "socket_type":"TCP", + "address_path":"[::1]:{port}" + }}""" +UDP_NETWORK = """{{ + "socket_type":"UDP", + "address_path":"[::1]:{port}" + }}""" +network_map = { + 'unix': UNIX_NETWORK, + 'tcp': TCP_NETWORK, + 'udp': UDP_NETWORK +} +CONFIG = """ +{{ + "version": 2, + "filters": [ + {filter_1}, + {filter_2}, + {filter_3} + ], + "report_stats": {{ + "file": {{ + "filepath": "/tmp/darwin-stats", + "permissions": 640 + }}, + "interval": 5 + }} +}} +""" + +# Test different combinations of tcp/udp/unix socket and verifies that everytime, 3 alerts are spawned +def alerting_tests(filters_list): + assert(len(filters_list) == 3) + filter_1 = ONE_FILTER.format(name='test_1', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='"next_filter": "test_2",', + network=network_map[filters_list[0]].format(port=8181)) + filter_2 = ONE_FILTER.format(name='test_2', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='"next_filter": "test_3",', + network=network_map[filters_list[1]].format(port=8282)) + filter_3 = ONE_FILTER.format(name='test_3', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='', + network=network_map[filters_list[2]].format(port=8383)) + config = CONFIG.format(filter_1=filter_1, filter_2=filter_2, filter_3=filter_3) + + darwin_configure(config) + darwin_configure(CONF_FTEST, path=PATH_CONF_FTEST) + + # Erasing last alerts + file = open("/tmp/test_test.log","w") + file.truncate(0) + file.close() + + p = darwin_start() + + # socket path is given according to how it is currently generated in the manager, + # this may have to change in the future + path_addr = '{}/sockets/test_1.1.sock'.format(conf.TEST_FILES_DIR) if filters_list[0] == 'unix' else '[::1]:8181' + f = Filter(socket_type=filters_list[0], socket_path=path_addr) + api = f.get_darwin_api() + + start = time.time() + nb_test_filters=0 + # We check that all 3 filters are running, on VM HardenedBSD, it may take some time + while nb_test_filters < 3 and time.time() - start < 6: + time.sleep(1) + nb_test_filters=0 + for proc in psutil.process_iter(): + try: + # Check if process name contains the given name string. + if 'darwin_test' in proc.name().lower(): + nb_test_filters += 1 + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass + + api.call("Hello", response_type="darwin") + time.sleep(0.2) + api.call("There", response_type="darwin") + + # arbitrary sleep length, should be enough on most systems + # We verify that One alert per filter were written + time.sleep(2) + line1, line2, line3 = 0, 0, 0 + file = open("/tmp/test_test.log","r") + for l in file.readlines(): + if 'test_1' in l: + line1 += 1 + elif 'test_2' in l: + line2 += 1 + elif 'test_3' in l: + line3 += 1 + file.close() + darwin_stop(p) + darwin_remove_configuration() + darwin_remove_configuration(path=PATH_CONF_FTEST) + + if line1 != 2 or line2 != 2 or line3 != 2: + print('test failed: ', filters_list) + print("test1 : {}, test2 : {}, test3 : {}".format(line1, line2, line3)) + return False + return True diff --git a/tests/manager_socket/test.py b/tests/manager_socket/test.py index d0815573..7a250e78 100644 --- a/tests/manager_socket/test.py +++ b/tests/manager_socket/test.py @@ -2,6 +2,7 @@ import manager_socket.monitor_test as monitor_test import manager_socket.update_test as update_test import manager_socket.reporting_test as reporting_test +import manager_socket.multi_filters_test as multi_filters_test from tools.output import print_results def run(): @@ -10,6 +11,7 @@ def run(): monitor_test.run() update_test.run() reporting_test.run() + multi_filters_test.run() print() print() \ No newline at end of file diff --git a/tests/test.py b/tests/test.py index db8214c3..6f7ecd06 100644 --- a/tests/test.py +++ b/tests/test.py @@ -1,9 +1,36 @@ import logging +import argparse from sys import stderr import manager_socket.test as manager_socket import core.test as core import filters.test as filters +# `conf` is imported whole because variables inside +# may be modified by the command line +import conf + +parser = argparse.ArgumentParser() +socket_arg_group = parser.add_mutually_exclusive_group(required=False) +socket_arg_group.add_argument("--tcp", action="store_true") +socket_arg_group.add_argument("--udp", action="store_true") +socket_arg_group.add_argument("--unix", action="store_true") +parser.add_argument("ip_address", nargs='?', + help="(For tcp and udp only) The IP address formatted like this: ipv4: 'x.x.x.x:port', ipv6: '[x:x:x:x:x:x:x]:port'") +parser.add_argument("--valgrind", action='store_true', help="Run valgrind on the filters") +args = parser.parse_args() + +if args.tcp: + conf.DEFAULT_PROTOCOL = 'tcp' +elif args.udp: + conf.DEFAULT_PROTOCOL = 'udp' +elif args.unix: + conf.DEFAULT_PROTOCOL = 'unix' + +if args.ip_address != None: + conf.DEFAULT_ADDRESS = args.ip_address + +if args.valgrind: + conf.VALGRIND_MEMCHECK = True if __name__ == "__main__": logging.basicConfig(filename="test_error.log", filemode='w', level=logging.ERROR) diff --git a/tests/tools/filter.py b/tests/tools/filter.py index 90e2ae4d..d680da5f 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -2,10 +2,11 @@ import logging import os import os.path -import uuid import subprocess -from conf import DEFAULT_FILTER_PATH, VALGRIND_MEMCHECK, TEST_FILES_DIR +# `conf` is imported whole because variables inside +# may be modified by the command line +import conf from darwin import DarwinApi from time import sleep from tools.redis_utils import RedisServer @@ -17,19 +18,44 @@ class Filter(): - def __init__(self, path=None, config_file=None, filter_name="filter", socket_path=None, monitoring_socket_path=None, pid_file=None, output="NONE", next_filter_socket_path="no", nb_threads=1, cache_size=0, threshold=101, log_level="DEVELOPER"): + def __init__(self, path=None, config_file=None, filter_name="filter", socket_path=None, monitoring_socket_path=None, pid_file=None, output="NONE", next_filter_socket_path="no", nb_threads=1, cache_size=0, threshold=101, log_level="DEVELOPER", socket_type=None): self.filter_name = filter_name - self.socket = socket_path if socket_path else "{}/{}.sock".format(TEST_FILES_DIR, filter_name) - self.config = config_file if config_file else "{}/{}.conf".format(TEST_FILES_DIR, filter_name) - self.path = path if path else "{}darwin_{}".format(DEFAULT_FILTER_PATH, filter_name) - self.monitor = monitoring_socket_path if monitoring_socket_path else "{}/{}_mon.sock".format(TEST_FILES_DIR, filter_name) - self.pid = pid_file if pid_file else "{}/{}.pid".format(TEST_FILES_DIR, filter_name) - self.cmd = [self.path, "-l", log_level, self.filter_name, self.socket, self.config, self.monitor, self.pid, output, next_filter_socket_path, str(nb_threads), str(cache_size), str(threshold)] self.process = None + self.set_socket_info(socket_type, socket_path) + self.config = config_file if config_file else "{}/{}.conf".format(conf.TEST_FILES_DIR, filter_name) + self.path = path if path else "{}darwin_{}".format(conf.DEFAULT_FILTER_PATH, filter_name) + self.monitor = monitoring_socket_path if monitoring_socket_path else "{}/{}_mon.sock".format(conf.TEST_FILES_DIR, filter_name) + self.pid = pid_file if pid_file else "{}/{}.pid".format(conf.TEST_FILES_DIR, filter_name) + self.cmd = [self.path, "-l", log_level, self.filter_name, self.socket, self.config, self.monitor, self.pid, output, next_filter_socket_path, str(nb_threads), str(cache_size), str(threshold)] + if self.socket_type.startswith('udp'): + # Flags MUST be before positional arguments as the parsing on HardenedBSD is not done on all the arguments + # On BSD getopt stops at the first argument which is not in the specified flags + self.cmd.insert(1, '-u') self.error_code = 99 # For valgrind testing self.pubsub = None self.prepare_log_file() + def set_socket_info(self, socket_type, socket_path_address): + self.socket_type = socket_type if socket_type else conf.DEFAULT_PROTOCOL + if self.socket_type == 'unix': + self.socket = socket_path_address if socket_path_address else "{}/{}.sock".format(conf.TEST_FILES_DIR, self.filter_name) + self.host = None + self.port = -1 + elif self.socket_type == 'tcp' or self.socket_type == 'udp': + self.socket = socket_path_address if socket_path_address else conf.DEFAULT_ADDRESS + splitted_address = self.socket.rsplit(':', 1) + if '[' in splitted_address[0]: # ipv6 address + self.socket_type += '6' + self.host = splitted_address[0][1:-1] + else: + self.host = splitted_address[0] + self.port = int(splitted_address[1]) + else: + raise NotImplementedError("Unrecognized protocol : '{}'".format(self.socket_type)) + + def get_darwin_api(self, verbose=False): + return DarwinApi(socket_type=self.socket_type, socket_path=self.socket, socket_host=self.host, socket_port=self.port, verbose=verbose) + def prepare_log_file(self): # TODO variabilize once path can be changed self.log_file = open("/var/log/darwin/darwin.log", 'a+') @@ -39,7 +65,7 @@ def prepare_log_file(self): def __del__(self): if self.process and self.process.poll() is None: - if VALGRIND_MEMCHECK is False: + if conf.VALGRIND_MEMCHECK is False: self.stop() else: self.valgrind_stop() @@ -89,7 +115,7 @@ def stop(self): return self.check_stop() def valgrind_start(self): - if VALGRIND_MEMCHECK is False: + if conf.VALGRIND_MEMCHECK is False: return self.start() command = ['valgrind', '--tool=memcheck', @@ -98,13 +124,12 @@ def valgrind_start(self): '-q', # Quiet mode so it doesn't pollute the output '--error-exitcode={}'.format(self.error_code), #If valgrind reports error(s) during the run, return this exit code. ] + self.cmd - self.process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sleep(3) return self.check_start() def valgrind_stop(self): - if VALGRIND_MEMCHECK is False: + if conf.VALGRIND_MEMCHECK is False: return self.stop() self.process.terminate() try: @@ -160,7 +185,7 @@ def send_single(self, line): """ Send a single line. """ - api = DarwinApi(socket_type="unix", socket_path=self.socket) + api = self.get_darwin_api() ret = api.call(line, response_type="back") api.close() diff --git a/toolkit/Network.cpp b/toolkit/Network.cpp index c6131c66..1067b500 100644 --- a/toolkit/Network.cpp +++ b/toolkit/Network.cpp @@ -20,7 +20,9 @@ extern "C" { #include #include #include +#include #include "Logger.hpp" +#include "StringUtils.hpp" #include "Network.hpp" @@ -91,5 +93,53 @@ namespace darwin { return str; } + + bool ParsePort(const std::string& str_port, int& out_port) { + DARWIN_LOGGER; + + long port = 0; + if(!darwin::strings::StrToLong(str_port.c_str(), port)){ + DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number, unrecognized number: '" + str_port+ "'"); + return false; + } + + if (port < 0 || port > 65353) { + DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number : out of bounds [0; 65353]: '" + str_port + "'"); + return false; + } + + out_port = static_cast(port); + + return true; + } + + bool ParseSocketAddress(const std::string& path_address, bool is_udp, + NetworkSocketType& out_net_type, boost::asio::ip::address& out_net_address, int& out_port, std::string& out_unix_path) + { + DARWIN_LOGGER; + size_t colon = path_address.rfind(':'); + if (colon != std::string::npos) { + boost::system::error_code e; + std::string addr = path_address.substr(0, colon); + if(addr.find('[') != std::string::npos && addr.rfind(']') != std::string::npos) { + addr.pop_back(); + addr.erase(0, 1); + } + std::string port = path_address.substr(colon+1, path_address.length()-1); + bool portRes = ParsePort(port.c_str(), out_port); + + if( ! portRes || e.failed()) { + DARWIN_LOG_CRITICAL("Network::ParseSocketAddress::Error while parsing the ip address: " + path_address); + return false; + } + out_net_address = boost::asio::ip::make_address(addr, e); + out_net_type= is_udp ? NetworkSocketType::Udp : NetworkSocketType::Tcp; + } else { + out_net_type = NetworkSocketType::Unix; + out_unix_path = path_address; + } + return true; + } + } } diff --git a/toolkit/Network.hpp b/toolkit/Network.hpp index eb3ea015..51afd9f0 100644 --- a/toolkit/Network.hpp +++ b/toolkit/Network.hpp @@ -18,6 +18,13 @@ extern "C" { namespace darwin { /// \namespace network namespace network { + + enum NetworkSocketType { + Unix, + Tcp, + Udp + }; + /// Get the IP address type. /// /// \param ip_address_string The IP address to get the type from. @@ -46,5 +53,28 @@ namespace darwin { /// /// \param sa The in6_addr structure to be stringified. std::string GetStringAddrFromSockAddrIn6(const in6_addr &addr); + + /// + /// \brief Parse a string to get the socket path if any or the ipv4/v6 address + /// + /// \param path_address the string to parse + /// \param is_udp true if we should use udp instead of tcp (it is not used) + /// \param out_net_type type of address/path parsed + /// \param out_net_address ip address to fill when parsed + /// \param out_port ip port to fill when parsed + /// \param out_unix_path unix sock path to fill when parsed + /// \return true if the parsing was successful + /// + bool ParseSocketAddress(const std::string& path_address, bool is_udp, + NetworkSocketType& out_net_type, boost::asio::ip::address& out_net_address, int& out_port, std::string& out_unix_path); + + /// + /// \brief parse an ip port, returns early if there is an error while parsing or if the port is not a valid port number + /// + /// \param str_port port to parse + /// \param out_port port to fill with the port + /// \return true if the parsing was successful + /// + bool ParsePort(const std::string& str_port, int& out_port); } } diff --git a/toolkit/RedisManager.cpp b/toolkit/RedisManager.cpp index 0b283927..87d55f3f 100644 --- a/toolkit/RedisManager.cpp +++ b/toolkit/RedisManager.cpp @@ -171,18 +171,30 @@ namespace darwin { // rate limit if(std::time(nullptr) > threadData->_lastDiscovery + this->_healthCheckInterval) { std::time(&(threadData->_lastDiscovery)); + threadData->_retryAttempts = 1; // this is the first attempt /* try first to connect to current active connection (potentially updated), then search master in case of failure */ - if(this->Connect() and this->IsMaster()) + if ((this->Connect() && this->IsMaster()) || this->FindAndConnect() || this->FindAndConnectWithRateLimiting()) { + threadData->_retryAttempts = 0; return true; - else - return this->FindAndConnect(); + } else { + return false; + } + } + else if(threadData->_retryAttempts < this ->_maxRetryAttempts) { + // less than 8 seconds since last discovery, retrying + threadData->_retryAttempts++; + if ((this->Connect() && this->IsMaster()) || this->FindAndConnect() || this->FindAndConnectWithRateLimiting()) { + threadData->_retryAttempts = 0; + return true; + } else { + return false; + } } else { DARWIN_LOG_DEBUG("RedisManager::FindAndConnectWithRateLimiting:: Rate limiting applied"); + return false; } - - return false; } diff --git a/toolkit/RedisManager.hpp b/toolkit/RedisManager.hpp index dded0657..19dd0392 100644 --- a/toolkit/RedisManager.hpp +++ b/toolkit/RedisManager.hpp @@ -40,6 +40,7 @@ namespace darwin { redisContext* _redisContext = nullptr; time_t _redisLastUse = 0; time_t _lastDiscovery = 0; + size_t _retryAttempts = 0; }; struct RedisConnectionInfo { @@ -112,6 +113,7 @@ namespace darwin { // default values static constexpr int HEALTH_CHECK_INTERVAL = 8; // more than 8 seconds without calling redis will trigger a health check static constexpr int CONNECTION_TIMEOUT = 5; // 5 seconds for connections' timeout + static constexpr size_t MAX_RETRY_ATTEMPTS = 2; public: RedisManager(const RedisManager&) = delete; @@ -331,6 +333,7 @@ namespace darwin { private: time_t _healthCheckInterval = HEALTH_CHECK_INTERVAL; // will execute a HealthCheck if the connection wasn't used for x seconds time_t _connectTimeout = 0; // timeout when trying to connect + size_t _maxRetryAttempts = MAX_RETRY_ATTEMPTS; std::set> _threadSet; // set containing each thread's own context std::mutex _threadSetMut; // set mutex RedisConnectionInfo _activeConnection; // current active connection diff --git a/toolkit/StringUtils.cpp b/toolkit/StringUtils.cpp index 6f8b4e2d..4bf947d9 100644 --- a/toolkit/StringUtils.cpp +++ b/toolkit/StringUtils.cpp @@ -6,6 +6,8 @@ /// \brief Copyright (c) 2020 Advens. All rights reserved. #include +#include +#include #include "StringUtils.hpp" @@ -19,3 +21,23 @@ std::vector darwin::strings::SplitString(const std::string& source, res.emplace_back(str); return res; } + +bool darwin::strings::StrToLong(const char * str, long& out) { + char *end; + long l; + errno = 0; + l = strtol(str, &end, 10); + + if ( + ((errno == ERANGE && l == LONG_MAX) || l > INT_MAX) + || ((errno == ERANGE && l == LONG_MIN) || l < INT_MIN) + || (*str == '\0' || *end != '\0') + ) + { + // Overflow, underflow or Error while parsing a number + return false; + } + + out = l; + return true; +} diff --git a/toolkit/StringUtils.hpp b/toolkit/StringUtils.hpp index 93b2fa3c..8e615a77 100644 --- a/toolkit/StringUtils.hpp +++ b/toolkit/StringUtils.hpp @@ -19,5 +19,16 @@ namespace darwin { /// ///\return The newly created vector of strings std::vector SplitString(const std::string& source, char delim); + + ///\brief Util to parse a 'long' from a char array, it returns true if it works, + /// false if it couldn't parse the number, 'str' must be null terminated just after the number : + /// -> "70\0" -> true + /// -> "70Hi\0" -> false, 'out' is not modified + /// + ///\param str source to parse (must be null terminated, it will not be checked) + ///\param out a reference to a 'long' which will be set by the function + /// + ///\return true if the parsing works, false otherwise + bool StrToLong(const char * str, long& out); } // namespace strings } // namespace darwin diff --git a/toolkit/thread-pool-cpp/LICENSE b/toolkit/thread-pool-cpp/LICENSE new file mode 100644 index 00000000..e102761c --- /dev/null +++ b/toolkit/thread-pool-cpp/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Andrey Kubarkov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/toolkit/thread-pool-cpp/README.md b/toolkit/thread-pool-cpp/README.md new file mode 100644 index 00000000..afce4503 --- /dev/null +++ b/toolkit/thread-pool-cpp/README.md @@ -0,0 +1,32 @@ +thread-pool-cpp +================= +[![Build Status](https://travis-ci.org/inkooboo/thread-pool-cpp.svg?branch=master)](https://travis-ci.org/inkooboo/thread-pool-cpp) +[![Codecov branch](https://img.shields.io/codecov/c/github/inkooboo/thread-pool-cpp/master.svg)](https://codecov.io/gh/inkooboo/thread-pool-cpp) +[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) + + * It is highly scalable and fast. + * It is header only. + * No external dependencies, only standard library needed. + * It implements both work-stealing and work-distribution balancing startegies. + * It implements cooperative scheduling strategy. + +Example run: +Post job to thread pool is much faster than for boost::asio based thread pool. + + Benchmark job reposting + ***thread pool cpp*** + reposted 1000001 in 61.6754 ms + reposted 1000001 in 62.0187 ms + reposted 1000001 in 62.8785 ms + reposted 1000001 in 70.2714 ms + ***asio thread pool*** + reposted 1000001 in 1381.58 ms + reposted 1000001 in 1390.35 ms + reposted 1000001 in 1391.84 ms + reposted 1000001 in 1393.19 ms + +See benchmark/benchmark.cpp for benchmark code. + +All code except [MPMCBoundedQueue](https://github.com/inkooboo/thread-pool-cpp/blob/master/include/thread_pool/mpmc_bounded_queue.hpp) +is under MIT license. + diff --git a/toolkit/thread-pool-cpp/include/thread_pool.hpp b/toolkit/thread-pool-cpp/include/thread_pool.hpp new file mode 100644 index 00000000..d8b0bc4b --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include diff --git a/toolkit/thread-pool-cpp/include/thread_pool/fixed_function.hpp b/toolkit/thread-pool-cpp/include/thread_pool/fixed_function.hpp new file mode 100644 index 00000000..5237052a --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/fixed_function.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include + +namespace tp +{ + +/** + * @brief The FixedFunction class implements + * functional object. + * This function is analog of 'std::function' with limited capabilities: + * - It supports only move semantics. + * - The size of functional objects is limited to storage size. + * Due to limitations above it is much faster on creation and copying than + * std::function. + */ +template +class FixedFunction; + +template +class FixedFunction +{ + + typedef R (*func_ptr_type)(ARGS...); + +public: + FixedFunction() + : m_function_ptr(nullptr), m_method_ptr(nullptr), + m_alloc_ptr(nullptr) + { + } + + /** + * @brief FixedFunction Constructor from functional object. + * @param object Functor object will be stored in the internal storage + * using move constructor. Unmovable objects are prohibited explicitly. + */ + template + FixedFunction(FUNC&& object) + : FixedFunction() + { + typedef typename std::remove_reference::type unref_type; + + static_assert(sizeof(unref_type) < STORAGE_SIZE, + "functional object doesn't fit into internal storage"); + static_assert(std::is_move_constructible::value, + "Should be of movable type"); + + m_method_ptr = []( + void* object_ptr, func_ptr_type, ARGS... args) -> R + { + return static_cast(object_ptr) + -> + operator()(args...); + }; + + m_alloc_ptr = [](void* storage_ptr, void* object_ptr) + { + if(object_ptr) + { + unref_type* x_object = static_cast(object_ptr); + new(storage_ptr) unref_type(std::move(*x_object)); + } + else + { + static_cast(storage_ptr)->~unref_type(); + } + }; + + m_alloc_ptr(&m_storage, &object); + } + + /** + * @brief FixedFunction Constructor from free function or static member. + */ + template + FixedFunction(RET (*func_ptr)(PARAMS...)) + : FixedFunction() + { + m_function_ptr = func_ptr; + m_method_ptr = [](void*, func_ptr_type f_ptr, ARGS... args) -> R + { + return static_cast(f_ptr)(args...); + }; + } + + FixedFunction(FixedFunction&& o) : FixedFunction() + { + moveFromOther(o); + } + + FixedFunction& operator=(FixedFunction&& o) + { + moveFromOther(o); + return *this; + } + + ~FixedFunction() + { + if(m_alloc_ptr) m_alloc_ptr(&m_storage, nullptr); + } + + /** + * @brief operator () Execute stored functional object. + * @throws std::runtime_error if no functional object is stored. + */ + R operator()(ARGS... args) + { + if(!m_method_ptr) throw std::runtime_error("call of empty functor"); + return m_method_ptr(&m_storage, m_function_ptr, args...); + } + +private: + FixedFunction& operator=(const FixedFunction&) = delete; + FixedFunction(const FixedFunction&) = delete; + + union + { + typename std::aligned_storage::type + m_storage; + func_ptr_type m_function_ptr; + }; + + typedef R (*method_type)( + void* object_ptr, func_ptr_type free_func_ptr, ARGS... args); + method_type m_method_ptr; + + typedef void (*alloc_type)(void* storage_ptr, void* object_ptr); + alloc_type m_alloc_ptr; + + void moveFromOther(FixedFunction& o) + { + if(this == &o) return; + + if(m_alloc_ptr) + { + m_alloc_ptr(&m_storage, nullptr); + m_alloc_ptr = nullptr; + } + else + { + m_function_ptr = nullptr; + } + + m_method_ptr = o.m_method_ptr; + o.m_method_ptr = nullptr; + + if(o.m_alloc_ptr) + { + m_alloc_ptr = o.m_alloc_ptr; + m_alloc_ptr(&m_storage, &o.m_storage); + } + else + { + m_function_ptr = o.m_function_ptr; + } + } +}; + +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/mpmc_bounded_queue.hpp b/toolkit/thread-pool-cpp/include/thread_pool/mpmc_bounded_queue.hpp new file mode 100644 index 00000000..b06ee083 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/mpmc_bounded_queue.hpp @@ -0,0 +1,242 @@ +// Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided +// that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list +// of conditions and the following disclaimer in the documentation and/or +// other materials +// provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT +// SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +// OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// +// The views and conclusions contained in the software and documentation are +// those of the authors and +// should not be interpreted as representing official policies, either expressed +// or implied, of Dmitry Vyukov. + +#pragma once + +#include +#include +#include +#include + +namespace tp +{ + +/** + * @brief The MPMCBoundedQueue class implements bounded + * multi-producers/multi-consumers lock-free queue. + * Doesn't accept non-movable types as T. + * Inspired by Dmitry Vyukov's mpmc queue. + * http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue + */ +template +class MPMCBoundedQueue +{ + static_assert( + std::is_move_constructible::value, "Should be of movable type"); + +public: + /** + * @brief MPMCBoundedQueue Constructor. + * @param size Power of 2 number - queue length. + * @throws std::invalid_argument if size is bad. + */ + explicit MPMCBoundedQueue(size_t size); + + /** + * @brief Move ctor implementation. + */ + MPMCBoundedQueue(MPMCBoundedQueue&& rhs) noexcept; + + /** + * @brief Move assignment implementaion. + */ + MPMCBoundedQueue& operator=(MPMCBoundedQueue&& rhs) noexcept; + + /** + * @brief push Push data to queue. + * @param data Data to be pushed. + * @return true on success. + */ + template + bool push(U&& data); + + /** + * @brief pop Pop data from queue. + * @param data Place to store popped data. + * @return true on sucess. + */ + bool pop(T& data); + +private: + struct Cell + { + std::atomic sequence; + T data; + + Cell() = default; + + Cell(const Cell&) = delete; + Cell& operator=(const Cell&) = delete; + + Cell(Cell&& rhs) + : sequence(rhs.sequence.load()), data(std::move(rhs.data)) + { + } + + Cell& operator=(Cell&& rhs) + { + sequence = rhs.sequence.load(); + data = std::move(rhs.data); + + return *this; + } + }; + +private: + typedef char Cacheline[64]; + + Cacheline pad0; + std::vector m_buffer; + /* const */ size_t m_buffer_mask; + Cacheline pad1; + std::atomic m_enqueue_pos; + Cacheline pad2; + std::atomic m_dequeue_pos; + Cacheline pad3; +}; + + +/// Implementation + +template +inline MPMCBoundedQueue::MPMCBoundedQueue(size_t size) + : m_buffer(size), m_buffer_mask(size - 1), m_enqueue_pos(0), + m_dequeue_pos(0) +{ + bool size_is_power_of_2 = (size >= 2) && ((size & (size - 1)) == 0); + if(!size_is_power_of_2) + { + throw std::invalid_argument("buffer size should be a power of 2"); + } + + for(size_t i = 0; i < size; ++i) + { + m_buffer[i].sequence = i; + } +} + +template +inline MPMCBoundedQueue::MPMCBoundedQueue(MPMCBoundedQueue&& rhs) noexcept +{ + *this = rhs; +} + +template +inline MPMCBoundedQueue& MPMCBoundedQueue::operator=(MPMCBoundedQueue&& rhs) noexcept +{ + if (this != &rhs) + { + m_buffer = std::move(rhs.m_buffer); + m_buffer_mask = std::move(rhs.m_buffer_mask); + m_enqueue_pos = rhs.m_enqueue_pos.load(); + m_dequeue_pos = rhs.m_dequeue_pos.load(); + } + return *this; +} + +template +template +inline bool MPMCBoundedQueue::push(U&& data) +{ + Cell* cell; + size_t pos = m_enqueue_pos.load(std::memory_order_relaxed); + for(;;) + { + cell = &m_buffer[pos & m_buffer_mask]; + size_t seq = cell->sequence.load(std::memory_order_acquire); + intptr_t dif = (intptr_t)seq - (intptr_t)pos; + if(dif == 0) + { + if(m_enqueue_pos.compare_exchange_weak( + pos, pos + 1, std::memory_order_relaxed)) + { + break; + } + } + else if(dif < 0) + { + return false; + } + else + { + pos = m_enqueue_pos.load(std::memory_order_relaxed); + } + } + + cell->data = std::forward(data); + + cell->sequence.store(pos + 1, std::memory_order_release); + + return true; +} + +template +inline bool MPMCBoundedQueue::pop(T& data) +{ + Cell* cell; + size_t pos = m_dequeue_pos.load(std::memory_order_relaxed); + for(;;) + { + cell = &m_buffer[pos & m_buffer_mask]; + size_t seq = cell->sequence.load(std::memory_order_acquire); + intptr_t dif = (intptr_t)seq - (intptr_t)(pos + 1); + if(dif == 0) + { + if(m_dequeue_pos.compare_exchange_weak( + pos, pos + 1, std::memory_order_relaxed)) + { + break; + } + } + else if(dif < 0) + { + return false; + } + else + { + pos = m_dequeue_pos.load(std::memory_order_relaxed); + } + } + + data = std::move(cell->data); + + cell->sequence.store( + pos + m_buffer_mask + 1, std::memory_order_release); + + return true; +} + +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/thread_pool.hpp b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool.hpp new file mode 100644 index 00000000..4c79fdbf --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool.hpp @@ -0,0 +1,161 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace tp +{ + +template class Queue> +class ThreadPoolImpl; +using ThreadPool = ThreadPoolImpl, + MPMCBoundedQueue>; + +/** + * @brief The ThreadPool class implements thread pool pattern. + * It is highly scalable and fast. + * It is header only. + * It implements both work-stealing and work-distribution balancing + * startegies. + * It implements cooperative scheduling strategy for tasks. + */ +template class Queue> +class ThreadPoolImpl { +public: + /** + * @brief ThreadPool Construct and start new thread pool. + * @param options Creation options. + */ + explicit ThreadPoolImpl( + const ThreadPoolOptions& options = ThreadPoolOptions()); + + /** + * @brief Move ctor implementation. + */ + ThreadPoolImpl(ThreadPoolImpl&& rhs) noexcept; + + /** + * @brief ~ThreadPool Stop all workers and destroy thread pool. + */ + ~ThreadPoolImpl(); + + /** + * @brief Move assignment implementaion. + */ + ThreadPoolImpl& operator=(ThreadPoolImpl&& rhs) noexcept; + + /** + * @brief post Try post job to thread pool. + * @param handler Handler to be called from thread pool worker. It has + * to be callable as 'handler()'. + * @return 'true' on success, false otherwise. + * @note All exceptions thrown by handler will be suppressed. + */ + template + bool tryPost(Handler&& handler); + + /** + * @brief post Post job to thread pool. + * @param handler Handler to be called from thread pool worker. It has + * to be callable as 'handler()'. + * @throw std::overflow_error if worker's queue is full. + * @note All exceptions thrown by handler will be suppressed. + */ + template + void post(Handler&& handler); + +private: + Worker& getWorker(); + + std::vector>> m_workers; + std::atomic m_next_worker; +}; + + +/// Implementation + +template class Queue> +inline ThreadPoolImpl::ThreadPoolImpl( + const ThreadPoolOptions& options) + : m_workers(options.threadCount()) + , m_next_worker(0) +{ + for(auto& worker_ptr : m_workers) + { + worker_ptr.reset(new Worker(options.queueSize())); + } + + for(size_t i = 0; i < m_workers.size(); ++i) + { + Worker* steal_donor = + m_workers[(i + 1) % m_workers.size()].get(); + m_workers[i]->start(i, steal_donor); + } +} + +template class Queue> +inline ThreadPoolImpl::ThreadPoolImpl(ThreadPoolImpl&& rhs) noexcept +{ + *this = rhs; +} + +template class Queue> +inline ThreadPoolImpl::~ThreadPoolImpl() +{ + for (auto& worker_ptr : m_workers) + { + worker_ptr->stop(); + } +} + +template class Queue> +inline ThreadPoolImpl& +ThreadPoolImpl::operator=(ThreadPoolImpl&& rhs) noexcept +{ + if (this != &rhs) + { + m_workers = std::move(rhs.m_workers); + m_next_worker = rhs.m_next_worker.load(); + } + return *this; +} + +template class Queue> +template +inline bool ThreadPoolImpl::tryPost(Handler&& handler) +{ + return getWorker().post(std::forward(handler)); +} + +template class Queue> +template +inline void ThreadPoolImpl::post(Handler&& handler) +{ + const auto ok = tryPost(std::forward(handler)); + if (!ok) + { + throw std::runtime_error("thread pool queue is full"); + } +} + +template class Queue> +inline Worker& ThreadPoolImpl::getWorker() +{ + auto id = Worker::getWorkerIdForCurrentThread(); + + if (id > m_workers.size()) + { + id = m_next_worker.fetch_add(1, std::memory_order_relaxed) % + m_workers.size(); + } + + return *m_workers[id]; +} +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/thread_pool_options.hpp b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool_options.hpp new file mode 100644 index 00000000..c1cde526 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool_options.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include +#include + +namespace tp +{ + +/** + * @brief The ThreadPoolOptions class provides creation options for + * ThreadPool. + */ +class ThreadPoolOptions +{ +public: + /** + * @brief ThreadPoolOptions Construct default options for thread pool. + */ + ThreadPoolOptions(); + + /** + * @brief setThreadCount Set thread count. + * @param count Number of threads to be created. + */ + void setThreadCount(size_t count); + + /** + * @brief setQueueSize Set single worker queue size. + * @param count Maximum length of queue of single worker. + */ + void setQueueSize(size_t size); + + /** + * @brief threadCount Return thread count. + */ + size_t threadCount() const; + + /** + * @brief queueSize Return single worker queue size. + */ + size_t queueSize() const; + +private: + size_t m_thread_count; + size_t m_queue_size; +}; + +/// Implementation + +inline ThreadPoolOptions::ThreadPoolOptions() + : m_thread_count(std::max(1u, std::thread::hardware_concurrency())) + , m_queue_size(1024u) +{ +} + +inline void ThreadPoolOptions::setThreadCount(size_t count) +{ + m_thread_count = std::max(1u, count); +} + +inline void ThreadPoolOptions::setQueueSize(size_t size) +{ + m_queue_size = std::max(1u, size); +} + +inline size_t ThreadPoolOptions::threadCount() const +{ + return m_thread_count; +} + +inline size_t ThreadPoolOptions::queueSize() const +{ + return m_queue_size; +} + +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/worker.hpp b/toolkit/thread-pool-cpp/include/thread_pool/worker.hpp new file mode 100644 index 00000000..91e67a37 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/worker.hpp @@ -0,0 +1,179 @@ +#pragma once + +#include +#include + +namespace tp +{ + +/** + * @brief The Worker class owns task queue and executing thread. + * In thread it tries to pop task from queue. If queue is empty then it tries + * to steal task from the sibling worker. If steal was unsuccessful then spins + * with one millisecond delay. + */ +template class Queue> +class Worker +{ +public: + /** + * @brief Worker Constructor. + * @param queue_size Length of undelaying task queue. + */ + explicit Worker(size_t queue_size); + + /** + * @brief Move ctor implementation. + */ + Worker(Worker&& rhs) noexcept; + + /** + * @brief Move assignment implementaion. + */ + Worker& operator=(Worker&& rhs) noexcept; + + /** + * @brief start Create the executing thread and start tasks execution. + * @param id Worker ID. + * @param steal_donor Sibling worker to steal task from it. + */ + void start(size_t id, Worker* steal_donor); + + /** + * @brief stop Stop all worker's thread and stealing activity. + * Waits until the executing thread became finished. + */ + void stop(); + + /** + * @brief post Post task to queue. + * @param handler Handler to be executed in executing thread. + * @return true on success. + */ + template + bool post(Handler&& handler); + + /** + * @brief steal Steal one task from this worker queue. + * @param task Place for stealed task to be stored. + * @return true on success. + */ + bool steal(Task& task); + + /** + * @brief getWorkerIdForCurrentThread Return worker ID associated with + * current thread if exists. + * @return Worker ID. + */ + static size_t getWorkerIdForCurrentThread(); + +private: + /** + * @brief threadFunc Executing thread function. + * @param id Worker ID to be associated with this thread. + * @param steal_donor Sibling worker to steal task from it. + */ + void threadFunc(size_t id, Worker* steal_donor); + + Queue m_queue; + std::atomic m_running_flag; + std::thread m_thread; +}; + + +/// Implementation + +namespace detail +{ + inline size_t* thread_id() + { + static thread_local size_t tss_id = -1u; + return &tss_id; + } +} + +template class Queue> +inline Worker::Worker(size_t queue_size) + : m_queue(queue_size) + , m_running_flag(true) +{ +} + +template class Queue> +inline Worker::Worker(Worker&& rhs) noexcept +{ + *this = rhs; +} + +template class Queue> +inline Worker& Worker::operator=(Worker&& rhs) noexcept +{ + if (this != &rhs) + { + m_queue = std::move(rhs.m_queue); + m_running_flag = rhs.m_running_flag.load(); + m_thread = std::move(rhs.m_thread); + } + return *this; +} + +template class Queue> +inline void Worker::stop() +{ + m_running_flag.store(false, std::memory_order_relaxed); + m_thread.join(); +} + +template class Queue> +inline void Worker::start(size_t id, Worker* steal_donor) +{ + m_thread = std::thread(&Worker::threadFunc, this, id, steal_donor); +} + +template class Queue> +inline size_t Worker::getWorkerIdForCurrentThread() +{ + return *detail::thread_id(); +} + +template class Queue> +template +inline bool Worker::post(Handler&& handler) +{ + return m_queue.push(std::forward(handler)); +} + +template class Queue> +inline bool Worker::steal(Task& task) +{ + return m_queue.pop(task); +} + +template class Queue> +inline void Worker::threadFunc(size_t id, Worker* steal_donor) +{ + *detail::thread_id() = id; + + Task handler; + + while (m_running_flag.load(std::memory_order_relaxed)) + { + if (m_queue.pop(handler) || steal_donor->steal(handler)) + { + try + { + handler(); + } + catch(...) + { + // suppress all exceptions + } + } + else + { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } +} + +}