From a4ea276361b8c38f29446d92cc415138a5815f6a Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 9 Jun 2021 17:51:08 +0200 Subject: [PATCH 01/54] separated unix socket logic from generic logic in the server class into 2 classes --- CMakeLists.txt | 3 +- samples/base/AServer.cpp | 70 +++++++++++++++++++++++++++++++ samples/base/AServer.hpp | 74 +++++++++++++++++++++++++++++++++ samples/base/Core.cpp | 6 +-- samples/base/UnixServer.cpp | 82 +++++++++++++++++++++++++++++++++++++ samples/base/UnixServer.hpp | 60 +++++++++++++++++++++++++++ 6 files changed, 291 insertions(+), 4 deletions(-) create mode 100644 samples/base/AServer.cpp create mode 100644 samples/base/AServer.hpp create mode 100644 samples/base/UnixServer.cpp create mode 100644 samples/base/UnixServer.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 114bee3..b87458f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,7 +96,8 @@ set( samples/base/ThreadGroup.cpp samples/base/ThreadGroup.hpp samples/base/AlertManager.cpp samples/base/AlertManager.hpp - samples/base/Server.cpp samples/base/Server.hpp + samples/base/AServer.cpp samples/base/AServer.hpp + samples/base/UnixServer.cpp samples/base/UnixServer.hpp samples/base/Manager.cpp samples/base/Manager.hpp samples/base/Session.cpp samples/base/Session.hpp diff --git a/samples/base/AServer.cpp b/samples/base/AServer.cpp new file mode 100644 index 0000000..9d0c2a8 --- /dev/null +++ b/samples/base/AServer.cpp @@ -0,0 +1,70 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include +#include "AServer.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + AServer::AServer(std::string const& output, + std::size_t threshold, + Generator& generator) + : _output{output}, _io_context{}, _threshold{threshold}, + _signals{_io_context}, _generator{generator} { + } + + void AServer::InitSignalsAndStart() { + // Setting the stopping signals for the service + _signals.add(SIGINT); + _signals.add(SIGTERM); +#ifdef SIGQUIT + _signals.add(SIGQUIT); +#endif // !SIGQUIT + + // Start the waiting for a stopping signal + AwaitStop(); + + // Start accepting connections + Accept(); + } + + boost::asio::io_context &AServer::GetIOContext() { + return this->_io_context; + } + + void AServer::Run() { + // The io_context::run() call will block until all asynchronous operations + // have finished. While the server is running, there is always at least one + // asynchronous operation outstanding: the asynchronous accept call waiting + // for new incoming connections. + + // By using multiple threads, a program can call run() multiple times. + // Once an asynchronous operation is complete, + // the I/O service object will execute the handler in one of these threads. + // If a second operation is completed shortly after the first one, + // the I/O service object can execute the handler in a different thread. + // Now, not only can operations outside of a process be executed concurrently, + // but handlers within the process can be executed concurrently, too. + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Server::Run:: Running..."); + _io_context.run(); + } + + void AServer::AwaitStop() { + _signals.async_wait( + boost::bind(&AServer::HandleStop, this, + boost::asio::placeholders::error, + boost::asio::placeholders::signal_number)); + } + +} \ No newline at end of file diff --git a/samples/base/AServer.hpp b/samples/base/AServer.hpp new file mode 100644 index 0000000..0a5a9b9 --- /dev/null +++ b/samples/base/AServer.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include +#include "Session.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +namespace darwin { + + class AServer { + public: + /// Create an async UNIX stream socket server. + /// The server runs on nb_threads thread. + /// + /// \param socket_path Path of the UNIX socket to listen on. + /// \param output Filters' output type + /// \param next_filter_socket Path of the UNIX socket of the filter to send data to. + /// \param threshold Threshold at which the filter will raise a log. + // AServer() = default; + + ~AServer() = default; + + // Make the server non copyable & non movable + AServer(AServer const&) = delete; + + AServer(AServer const&&) = delete; + + AServer& operator=(AServer const&) = delete; + + AServer& operator=(AServer const&&) = delete; + + public: + /// Start the server and the threads. + void Run(); + + /// Clean the server's ressources (sessions, socket) + virtual void Clean() = 0; + + boost::asio::io_context &GetIOContext(); + + protected: + + AServer(std::string const& output, + std::size_t threshold, + Generator& generator); + + /// Start async waiting for the stopping signals. + void AwaitStop(); + + /// Handler called when signal is received by the async wait. + /// + /// \param sig The received signal. + virtual void HandleStop(boost::system::error_code const& error, int sig) = 0; + + /// Start an async connection acceptation on _new_session's socket. + virtual void Accept() = 0; + + /// Handler called on async accept trigger. + virtual void HandleAccept(boost::system::error_code const& e) = 0; + + void InitSignalsAndStart(); + + protected: + std::string _output; //!< Filter's output type + boost::asio::io_context _io_context; //!< The async io context. + std::size_t _threshold; //!< Filter's threshold + boost::asio::signal_set _signals; //!< Set of the stopping signals. + Generator& _generator; //!< Generator used to create new Sessions. + Manager _manager; //!< Server's session manager. + + + }; +} \ No newline at end of file diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index baa23bc..1a6a431 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -15,7 +15,7 @@ #include "Generator.hpp" #include "Logger.hpp" -#include "Server.hpp" +#include "UnixServer.hpp" #include "Core.hpp" #include "Stats.hpp" @@ -47,7 +47,7 @@ namespace darwin { DARWIN_LOG_DEBUG("Core::run:: Configured generator"); try { - Server server{_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen}; + UnixServer server{_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen}; if (not gen.ConfigureNetworkObject(server.GetIOContext())) { raise(SIGTERM); t.join(); @@ -60,7 +60,7 @@ namespace darwin { // the io_context too. for (std::size_t i = 1; i < _nbThread; ++i) { _threadpool.CreateThread( - std::bind(&Server::Run, std::ref(server)) + std::bind(&AServer::Run, std::ref(server)) ); } server.Run(); diff --git a/samples/base/UnixServer.cpp b/samples/base/UnixServer.cpp new file mode 100644 index 0000000..d9bc5d7 --- /dev/null +++ b/samples/base/UnixServer.cpp @@ -0,0 +1,82 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include "UnixServer.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + +// tcp: boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 8000) + + UnixServer::UnixServer(std::string const& socket_path, + std::string const& output, + std::string const& next_filter_socket, + std::size_t threshold, + Generator& generator) + : AServer(output, threshold, generator), + _socket_path{socket_path}, _socket_next{next_filter_socket}, + _acceptor{_io_context, boost::asio::local::stream_protocol::endpoint( + socket_path)}, + _new_connection{_io_context} { + + this->InitSignalsAndStart(); + } + + + void UnixServer::Clean() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); + _manager.StopAll(); + unlink(_socket_path.c_str()); + } + + void UnixServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { + // The server is stopped by cancelling all outstanding asynchronous + // operations. Once all operations have finished the io_context::run() + // call will exit. + DARWIN_LOGGER; + + SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); + DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); + _acceptor.close(); + _io_context.stop(); + } + + void UnixServer::Accept() { + _acceptor.async_accept(_new_connection, + boost::bind(&UnixServer::HandleAccept, this, + boost::asio::placeholders::error)); + } + + void UnixServer::HandleAccept(boost::system::error_code const& e) { + DARWIN_LOGGER; + + if (!_acceptor.is_open()) { + DARWIN_LOG_INFO("Server::HandleAccept:: Acceptor closed, closing server..."); + return; + } + + if (!e) { + DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); + auto task = _generator.CreateTask(_new_connection, _manager); + task->SetNextFilterSocketPath(_socket_next); + task->SetOutputType(_output); + task->SetThreshold(_threshold); + _manager.Start(task); + Accept(); + } else { + DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); + } + } + +} \ No newline at end of file diff --git a/samples/base/UnixServer.hpp b/samples/base/UnixServer.hpp new file mode 100644 index 0000000..f8849c5 --- /dev/null +++ b/samples/base/UnixServer.hpp @@ -0,0 +1,60 @@ +/// \file Server.hpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#pragma once + +#include +#include +#include +#include "Session.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +#include "AServer.hpp" + +namespace darwin { + + class UnixServer : public AServer { + public: + /// Create an async UNIX stream socket server. + /// The server runs on nb_threads thread. + /// + /// \param socket_path Path of the UNIX socket to listen on. + /// \param output Filters' output type + /// \param next_filter_socket Path of the UNIX socket of the filter to send data to. + /// \param threshold Threshold at which the filter will raise a log. + UnixServer(std::string const& socket_path, + std::string const& output, + std::string const& next_filter_socket, + std::size_t threshold, + Generator& generator); + + ~UnixServer() = default; + + // Make the UnixServer non copyable & non movable + UnixServer(UnixServer const&) = delete; + + UnixServer(UnixServer const&&) = delete; + + UnixServer& operator=(UnixServer const&) = delete; + + UnixServer& operator=(UnixServer const&&) = delete; + + void Clean(); + + void HandleStop(boost::system::error_code const& error, int sig); + + void Accept(); + + void HandleAccept(boost::system::error_code const& e); + + private: + std::string _socket_path; //!< Path to the UNIX socket to listen on. + std::string _socket_next; //!< Path to the next filter's UNIX socket. + boost::asio::local::stream_protocol::acceptor _acceptor; //!< Acceptor for the incoming connections. + boost::asio::local::stream_protocol::socket _new_connection; //!< Socket used to accept a new connection. + }; +} \ No newline at end of file From 534ba48cb9939a844b6c040ff31b1f44df4a3266 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 14 Jun 2021 11:35:23 +0200 Subject: [PATCH 02/54] Separated logic between tasks and Sessions, the separation is only applied on the dga filter for now --- CMakeLists.txt | 4 +- samples/base/AGenerator.hpp | 9 +- samples/base/AServer.hpp | 2 +- samples/base/{Session.cpp => ASession.cpp} | 210 +++++++++------------ samples/base/ASession.fwd.hpp | 13 ++ samples/base/{Session.hpp => ASession.hpp} | 102 ++++------ samples/base/ATask.cpp | 83 ++++++++ samples/base/ATask.hpp | 104 ++++++++++ samples/base/Manager.hpp | 2 +- samples/base/Server.cpp | 119 ------------ samples/base/Server.hpp | 81 -------- samples/base/TcpServer.cpp | 82 ++++++++ samples/base/TcpServer.hpp | 61 ++++++ samples/base/UnixServer.cpp | 10 +- samples/base/UnixServer.hpp | 10 +- samples/fdga/DGATask.cpp | 25 ++- samples/fdga/DGATask.hpp | 16 +- samples/fdga/Generator.cpp | 18 +- samples/fdga/Generator.hpp | 8 +- 19 files changed, 528 insertions(+), 431 deletions(-) rename samples/base/{Session.cpp => ASession.cpp} (65%) mode change 100644 => 100755 create mode 100644 samples/base/ASession.fwd.hpp rename samples/base/{Session.hpp => ASession.hpp} (68%) mode change 100644 => 100755 create mode 100755 samples/base/ATask.cpp create mode 100755 samples/base/ATask.hpp delete mode 100644 samples/base/Server.cpp delete mode 100644 samples/base/Server.hpp create mode 100755 samples/base/TcpServer.cpp create mode 100755 samples/base/TcpServer.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b87458f..cfc470d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,8 +98,10 @@ set( samples/base/AServer.cpp samples/base/AServer.hpp samples/base/UnixServer.cpp samples/base/UnixServer.hpp + samples/base/TcpServer.cpp samples/base/TcpServer.hpp samples/base/Manager.cpp samples/base/Manager.hpp - samples/base/Session.cpp samples/base/Session.hpp + samples/base/ASession.cpp samples/base/ASession.hpp + samples/base/ATask.cpp samples/base/ATask.hpp toolkit/Network.cpp toolkit/Network.hpp toolkit/Validators.cpp toolkit/Validators.hpp diff --git a/samples/base/AGenerator.hpp b/samples/base/AGenerator.hpp index aa975c9..43199a3 100644 --- a/samples/base/AGenerator.hpp +++ b/samples/base/AGenerator.hpp @@ -12,7 +12,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "../toolkit/rapidjson/document.h" class AGenerator { @@ -27,9 +27,8 @@ class AGenerator { /// \param socket The Session's socket. /// \param manager The Session manager. /// \return A pointer to a new session. - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept = 0; + virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept = 0; + virtual bool ConfigureNetworkObject(boost::asio::io_context &context); protected: @@ -59,6 +58,8 @@ class AGenerator { Configure(std::string const& configFile, const std::size_t cache_size) final; + virtual long GetFilterCode() const = 0; + private: /// Open and read the configuration file. /// Try to load the json format of the configuration. diff --git a/samples/base/AServer.hpp b/samples/base/AServer.hpp index 0a5a9b9..b2bb05c 100644 --- a/samples/base/AServer.hpp +++ b/samples/base/AServer.hpp @@ -3,7 +3,7 @@ #include #include #include -#include "Session.hpp" +#include "ASession.hpp" #include "Manager.hpp" #include "Generator.hpp" namespace darwin { diff --git a/samples/base/Session.cpp b/samples/base/ASession.cpp old mode 100644 new mode 100755 similarity index 65% rename from samples/base/Session.cpp rename to samples/base/ASession.cpp index 034f6b6..324f5d5 --- a/samples/base/Session.cpp +++ b/samples/base/ASession.cpp @@ -1,4 +1,4 @@ -/// \file Session.cpp +/// \file ASession.cpp /// \authors hsoszynski /// \version 1.0 /// \date 05/07/18 @@ -14,7 +14,8 @@ #include #include "Logger.hpp" #include "Manager.hpp" -#include "Session.hpp" +#include "Generator.hpp" +#include "ASession.hpp" #include "errors.hpp" #include "../../toolkit/lru_cache.hpp" @@ -22,64 +23,45 @@ #include "../../toolkit/xxhash.hpp" namespace darwin { - Session::Session(std::string name, boost::asio::local::stream_protocol::socket& socket, + ASession::ASession(boost::asio::local::stream_protocol::socket& socket, darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex) - : _filter_name(name), _connected{false}, _socket{std::move(socket)}, + Generator& generator) + : _connected{false}, _socket{std::move(socket)}, _filter_socket{socket.get_executor()}, - _manager{manager}, _cache{cache}, _cache_mutex{cache_mutex} {} + _manager{manager}, _generator{generator} {} - void Session::SetStartingTime() { - _starting_time = std::chrono::high_resolution_clock::now(); - } - - double Session::GetDurationMs() { - return std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - _starting_time - ).count() / ((double)1000); - } + - void Session::Start() { + void ASession::Start() { DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::Start::"); + DARWIN_LOG_DEBUG("ASession::Start::"); ReadHeader(); } - void Session::Stop() { + void ASession::Stop() { DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::Stop::"); + DARWIN_LOG_DEBUG("ASession::Stop::"); _socket.close(); if(_connected) _filter_socket.close(); } - void Session::SetThreshold(std::size_t const& threshold) { - DARWIN_LOGGER; - //If the threshold is negative, we keep the actual default threshold - if(threshold>100){ - DARWIN_LOG_DEBUG("Session::SetThreshold:: Default threshold " + std::to_string(_threshold) + "applied"); - return; - } - _threshold = threshold; - } - - void Session::SetNextFilterSocketPath(std::string const& path){ + void ASession::SetNextFilterSocketPath(std::string const& path){ _next_filter_path = path; } - void Session::SetOutputType(std::string const& output) { + void ASession::SetOutputType(std::string const& output) { _output = config::convert_output_string(output); } - config::output_type Session::GetOutputType() { + config::output_type ASession::GetOutputType() { return _output; } - std::string Session::GetLogs(){ + std::string ASession::GetLogs(){ return _logs; } - std::string Session::GetDataToSendToFilter(){ + std::string ASession::GetDataToSendToFilter(){ switch (GetOutputType()){ case config::output_type::RAW: return _raw_body; @@ -93,18 +75,18 @@ namespace darwin { return ""; } - void Session::ReadHeader() { + void ASession::ReadHeader() { DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::ReadHeader:: Starting to read incoming header..."); + DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); boost::asio::async_read(_socket, boost::asio::buffer(&_header, sizeof(_header)), - boost::bind(&Session::ReadHeaderCallback, this, + boost::bind(&ASession::ReadHeaderCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); } - void Session::ReadHeaderCallback(const boost::system::error_code& e, + void ASession::ReadHeaderCallback(const boost::system::error_code& e, std::size_t size) { DARWIN_LOGGER; @@ -112,10 +94,10 @@ namespace darwin { _certitudes.clear(); _logs.clear(); - DARWIN_LOG_DEBUG("Session::ReadHeaderCallback:: Reading header"); + DARWIN_LOG_DEBUG("ASession::ReadHeaderCallback:: Reading header"); if (!e) { if (size != sizeof(_header)) { - DARWIN_LOG_ERROR("Session::ReadHeaderCallback:: Mismatching header size"); + DARWIN_LOG_ERROR("ASession::ReadHeaderCallback:: Mismatching header size"); goto header_callback_stop_session; } if (_header.body_size == 0) { @@ -127,33 +109,33 @@ namespace darwin { } if (boost::asio::error::eof == e) { - DARWIN_LOG_DEBUG("Session::ReadHeaderCallback:: " + e.message()); + DARWIN_LOG_DEBUG("ASession::ReadHeaderCallback:: " + e.message()); } else { - DARWIN_LOG_WARNING("Session::ReadHeaderCallback:: " + e.message()); + DARWIN_LOG_WARNING("ASession::ReadHeaderCallback:: " + e.message()); } header_callback_stop_session: _manager.Stop(shared_from_this()); } - void Session::ReadBody(std::size_t size) { + void ASession::ReadBody(std::size_t size) { DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::ReadBody:: Starting to read incoming body..."); + DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); _socket.async_read_some(boost::asio::buffer(_buffer, size), - boost::bind(&Session::ReadBodyCallback, this, + boost::bind(&ASession::ReadBodyCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); } - void Session::ReadBodyCallback(const boost::system::error_code& e, + void ASession::ReadBodyCallback(const boost::system::error_code& e, std::size_t size) { DARWIN_LOGGER; if (!e) { _raw_body.append(_buffer.data(), size); - DARWIN_LOG_DEBUG("Session::ReadBodyCallback:: Body len (" + + DARWIN_LOG_DEBUG("ASession::ReadBodyCallback:: Body len (" + std::to_string(_raw_body.length()) + ") - Header body size (" + std::to_string(_header.body_size) + @@ -164,20 +146,20 @@ namespace darwin { ReadBody(totalBodyLength - bodyLength); } else { if (_raw_body.empty()) { - DARWIN_LOG_WARNING("Session::ReadBodyCallback Empty body retrieved"); + DARWIN_LOG_WARNING("ASession::ReadBodyCallback Empty body retrieved"); this->SendErrorResponse("Error receiving body: Empty body retrieved", DARWIN_RESPONSE_CODE_REQUEST_ERROR); return; } if (_header.body_size <= 0) { DARWIN_LOG_ERROR( - "Session::ReadBodyCallback Body is not empty, but the header appears to be invalid" + "ASession::ReadBodyCallback Body is not empty, but the header appears to be invalid" ); this->SendErrorResponse("Error receiving body: Body is not empty, but the header appears to be invalid", DARWIN_RESPONSE_CODE_REQUEST_ERROR); return; } - if (!ParseBody()) { - DARWIN_LOG_DEBUG("Session::ReadBodyCallback Something went wrong while parsing the body"); + if (!PreParseBody()) { + DARWIN_LOG_DEBUG("ASession::ReadBodyCallback Something went wrong while parsing the body"); this->SendErrorResponse("Error receiving body: Something went wrong while parsing the body", DARWIN_RESPONSE_CODE_REQUEST_ERROR); return; } @@ -186,19 +168,20 @@ namespace darwin { } return; } - DARWIN_LOG_ERROR("Session::ReadBodyCallback:: " + e.message()); + DARWIN_LOG_ERROR("ASession::ReadBodyCallback:: " + e.message()); _manager.Stop(shared_from_this()); } - void Session::ExecuteFilter() { + void ASession::ExecuteFilter() { DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Session::ExecuteFilter::"); - - (*this)(); + DARWIN_LOG_DEBUG("ASession::ExecuteFilter::"); + auto t = _generator.CreateTask(shared_from_this()); //shared pt Task + // TODO ThreadPool.Start(t); + (*t)(); this->SendNext(); } - void Session::SendNext() { + void ASession::SendNext() { switch(_header.response) { case DARWIN_RESPONSE_SEND_BOTH: if(this->SendToFilter()) break; @@ -213,24 +196,24 @@ namespace darwin { } } - bool Session::ParseBody() { + bool ASession::PreParseBody() { DARWIN_LOGGER; try { _body.Parse(_raw_body.c_str()); if (!_body.IsArray()) { - DARWIN_LOG_ERROR("Session:: ParseBody: You must provide a list"); + DARWIN_LOG_ERROR("ASession:: ParseBody: You must provide a list"); return false; } } catch (...) { - DARWIN_LOG_CRITICAL("Session:: ParseBody: Could not parse raw body"); + DARWIN_LOG_CRITICAL("ASession:: ParseBody: Could not parse raw body"); return false; } - // DARWIN_LOG_DEBUG("Session:: ParseBody:: parsed body : " + _raw_body); + // DARWIN_LOG_DEBUG("ASession:: ParseBody:: parsed body : " + _raw_body); return true; } - std::string Session::JsonStringify(rapidjson::Document &json) { + std::string ASession::JsonStringify(rapidjson::Document &json) { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -239,7 +222,7 @@ namespace darwin { return std::string(buffer.GetString()); } - bool Session::SendToClient() noexcept { + bool ASession::SendToClient() noexcept { DARWIN_LOGGER; const std::size_t certitude_size = _certitudes.size(); @@ -260,12 +243,14 @@ namespace darwin { packet_size += this->_response_body.size(); } - DARWIN_LOG_DEBUG("Session::SendToClient: Computed packet size: " + std::to_string(packet_size)); + DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(packet_size)); + // TODO: to be corrected + // si certitudes.size() > default ET body not empty : on écrit sur les dernieres certitudes darwin_filter_packet_t* packet = reinterpret_cast(malloc(packet_size)); if (packet == nullptr) { - DARWIN_LOG_CRITICAL("Session::SendToClient: Could not create a Darwin packet"); + DARWIN_LOG_CRITICAL("ASession::SendToClient: Could not create a Darwin packet"); return false; } @@ -282,14 +267,15 @@ namespace darwin { packet->type = DARWIN_PACKET_FILTER; packet->response = _header.response; packet->certitude_size = certitude_size; - packet->filter_code = GetFilterCode(); + packet->filter_code = _generator.GetFilterCode(); packet->body_size = this->_response_body.size(); memcpy(packet->evt_id, _header.evt_id, 16); + // TODO: potentiellement corruption (effacement de certitudes) memcpy((char*)(packet) + sizeof(darwin_filter_packet_t), _response_body.c_str(), _response_body.length()); boost::asio::async_write(_socket, boost::asio::buffer(packet, packet_size), - boost::bind(&Session::SendToClientCallback, this, + boost::bind(&ASession::SendToClientCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); @@ -298,16 +284,16 @@ namespace darwin { return true; } - bool Session::SendToFilter() noexcept { + bool ASession::SendToFilter() noexcept { DARWIN_LOGGER; if (!_next_filter_path.compare("no")) { - DARWIN_LOG_NOTICE("Session::SendToFilter:: No next filter provided. Ignoring..."); + DARWIN_LOG_NOTICE("ASession::SendToFilter:: No next filter provided. Ignoring..."); return false; } if (!_connected) { - DARWIN_LOG_DEBUG("Session::SendToFilter:: Trying to connect to: " + + DARWIN_LOG_DEBUG("ASession::SendToFilter:: Trying to connect to: " + _next_filter_path); try { _filter_socket.connect( @@ -315,7 +301,7 @@ namespace darwin { _next_filter_path.c_str())); _connected = true; } catch (std::exception const& e) { - DARWIN_LOG_ERROR(std::string("Session::SendToFilter:: " + DARWIN_LOG_ERROR(std::string("ASession::SendToFilter:: " "Unable to connect to next filter: ") + e.what()); return false; @@ -323,8 +309,8 @@ namespace darwin { } std::string data = GetDataToSendToFilter(); - DARWIN_LOG_DEBUG("Session::SendToFilter:: data to send: " + data); - DARWIN_LOG_DEBUG("Session::SendToFilter:: data size: " + std::to_string(data.size())); + DARWIN_LOG_DEBUG("ASession::SendToFilter:: data to send: " + data); + DARWIN_LOG_DEBUG("ASession::SendToFilter:: data size: " + std::to_string(data.size())); const std::size_t certitude_size = _certitudes.size(); @@ -343,13 +329,13 @@ namespace darwin { packet_size += data.size(); - DARWIN_LOG_DEBUG("Session::SendToFilter:: Computed packet size: " + std::to_string(packet_size)); + DARWIN_LOG_DEBUG("ASession::SendToFilter:: Computed packet size: " + std::to_string(packet_size)); darwin_filter_packet_t* packet; packet = (darwin_filter_packet_t *) malloc(packet_size); if (!packet) { - DARWIN_LOG_CRITICAL("Session:: SendToFilter:: Could not create a Darwin packet"); + DARWIN_LOG_CRITICAL("ASession:: SendToFilter:: Could not create a Darwin packet"); return false; } @@ -372,14 +358,14 @@ namespace darwin { packet->type = DARWIN_PACKET_FILTER; packet->response = _header.response == DARWIN_RESPONSE_SEND_BOTH ? DARWIN_RESPONSE_SEND_DARWIN : _header.response; packet->certitude_size = certitude_size; - packet->filter_code = GetFilterCode(); + packet->filter_code = _generator.GetFilterCode(); packet->body_size = data.size(); memcpy(packet->evt_id, _header.evt_id, 16); - DARWIN_LOG_DEBUG("Session:: SendToFilter:: Sending header + data"); + DARWIN_LOG_DEBUG("ASession:: SendToFilter:: Sending header + data"); boost::asio::async_write(_filter_socket, boost::asio::buffer(packet, packet_size), - boost::bind(&Session::SendToFilterCallback, this, + boost::bind(&ASession::SendToFilterCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); @@ -387,26 +373,26 @@ namespace darwin { return true; } - void Session::SendToClientCallback(const boost::system::error_code& e, + void ASession::SendToClientCallback(const boost::system::error_code& e, std::size_t size __attribute__((unused))) { DARWIN_LOGGER; if (e) { - DARWIN_LOG_ERROR("Session::SendToClientCallback:: " + e.message()); + DARWIN_LOG_ERROR("ASession::SendToClientCallback:: " + e.message()); _manager.Stop(shared_from_this()); - DARWIN_LOG_DEBUG("Session::SendToClientCallback:: Stopped session in manager"); + DARWIN_LOG_DEBUG("ASession::SendToClientCallback:: Stopped session in manager"); return; } Start(); } - void Session::SendToFilterCallback(const boost::system::error_code& e, + void ASession::SendToFilterCallback(const boost::system::error_code& e, std::size_t size __attribute__((unused))) { DARWIN_LOGGER; if (e) { - DARWIN_LOG_ERROR("Session::SendToFilterCallback:: " + e.message()); + DARWIN_LOG_ERROR("ASession::SendToFilterCallback:: " + e.message()); _filter_socket.close(); _connected = false; } @@ -419,39 +405,9 @@ namespace darwin { } } - xxh::hash64_t Session::GenerateHash() { - // could be easily overridden in the children - return xxh::xxhash<64>(_raw_body); - } - - void Session::SaveToCache(const xxh::hash64_t &hash, const unsigned int certitude) const { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("SaveToCache:: Saving certitude " + std::to_string(certitude) + " to cache"); - std::unique_lock lck{_cache_mutex}; - _cache->insert(hash, certitude); - } - - bool Session::GetCacheResult(const xxh::hash64_t &hash, unsigned int& certitude) { - DARWIN_LOGGER; - boost::optional cached_certitude; - - { - std::unique_lock lck{_cache_mutex}; - cached_certitude = _cache->get(hash); - } - - if (cached_certitude != boost::none) { - certitude = cached_certitude.get(); - DARWIN_LOG_DEBUG("GetCacheResult:: Already processed request. Cached certitude is " + - std::to_string(certitude)); - - return true; - } - - return false; - } + - std::string Session::Evt_idToString() { + std::string ASession::Evt_idToString() { DARWIN_LOGGER; char str[37] = {}; snprintf(str, @@ -463,19 +419,31 @@ namespace darwin { _header.evt_id[12], _header.evt_id[13], _header.evt_id[14], _header.evt_id[15] ); std::string res(str); - DARWIN_LOG_DEBUG(std::string("Session::Evt_idToString:: UUID - ") + res); + DARWIN_LOG_DEBUG(std::string("ASession::Evt_idToString:: UUID - ") + res); return res; } - std::string Session::GetFilterName() { - return _filter_name; - } + - void Session::SendErrorResponse(const std::string& message, const unsigned int code) { + void ASession::SendErrorResponse(const std::string& message, const unsigned int code) { if (this->_header.response != DARWIN_RESPONSE_SEND_BACK && this->_header.response != DARWIN_RESPONSE_SEND_BOTH) return; this->_response_body.clear(); this->_response_body += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; this->SendToClient(); } + + void ASession::SetThreshold(std::size_t const& threshold) { + DARWIN_LOGGER; + //If the threshold is negative, we keep the actual default threshold + if(threshold>100){ + DARWIN_LOG_DEBUG("Session::SetThreshold:: Default threshold " + std::to_string(_threshold) + "applied"); + return; + } + _threshold = threshold; + } + + size_t ASession::GetThreshold() { + return _threshold; + } } diff --git a/samples/base/ASession.fwd.hpp b/samples/base/ASession.fwd.hpp new file mode 100644 index 0000000..f86d63f --- /dev/null +++ b/samples/base/ASession.fwd.hpp @@ -0,0 +1,13 @@ +#pragma once +#include + +// Forward Declaration mandatory because of the circular dependency between Tasks, Sessions and Generators +namespace darwin { + class ASession; + + /// Definition of a session's self-managing pointer. + /// + /// \typedef session_ptr_t + // typedef std::shared_ptr session_ptr_t; + typedef std::shared_ptr session_ptr_t; +} \ No newline at end of file diff --git a/samples/base/Session.hpp b/samples/base/ASession.hpp old mode 100644 new mode 100755 similarity index 68% rename from samples/base/Session.hpp rename to samples/base/ASession.hpp index daec6bc..fbf13e9 --- a/samples/base/Session.hpp +++ b/samples/base/ASession.hpp @@ -12,6 +12,7 @@ #include "config.hpp" #include "protocol.h" +#include "Generator.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" @@ -20,6 +21,8 @@ #include "../../toolkit/rapidjson/stringbuffer.h" #include "Time.hpp" +#include "ASession.fwd.hpp" + #define DARWIN_SESSION_BUFFER_SIZE 2048 #define DARWIN_DEFAULT_THRESHOLD 80 #define DARWIN_ERROR_RETURN 101 @@ -28,30 +31,22 @@ namespace darwin { class Manager; - class Session : public std::enable_shared_from_this { + class ASession : public std::enable_shared_from_this { public: - Session(std::string name, - boost::asio::local::stream_protocol::socket& socket, + ASession(boost::asio::local::stream_protocol::socket& socket, Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex); + Generator& generator); - virtual ~Session() = default; + virtual ~ASession() = default; // Make the manager non copyable & non movable - Session(Session const&) = delete; - - Session(Session const&&) = delete; + ASession(ASession const&) = delete; - Session& operator=(Session const&) = delete; + ASession(ASession const&&) = delete; - Session& operator=(Session const&&) = delete; + ASession& operator=(ASession const&) = delete; - public: - /// Entry point of the execution. - /// MUST be overloaded. - /// WARNING This method will be executed by a thread. - virtual void operator()() = 0; + ASession& operator=(ASession const&&) = delete; public: /// Start the session and the async read of the incoming packet. @@ -75,64 +70,37 @@ namespace darwin { /// \param name the string that represent the output type virtual void SetOutputType(std::string const& output) final; - protected: - /// Return filter code. - virtual long GetFilterCode() noexcept = 0; - - /// Save the result to cache - virtual void SaveToCache(const xxh::hash64_t &hash, unsigned int certitude) const; - - /// Get the result from the cache - virtual bool GetCacheResult(const xxh::hash64_t &hash, unsigned int &certitude); - - /// Generate the hash - virtual xxh::hash64_t GenerateHash(); - - /// Set starting time - void SetStartingTime(); - - /// Get total time elapsed since the starting time - double GetDurationMs(); - /// Get the filter's output type /// /// \return The filter's output type config::output_type GetOutputType(); - /// Get the data to send to the next filter - /// according to the filter's output type - /// - /// \param size data's size - /// \param data data to send - std::string GetDataToSendToFilter(); - /// Get the filter's result in a log form /// /// \return The filter's log virtual std::string GetLogs(); + /// Get the filter's threshold + /// + /// \return The filter's threshold + size_t GetThreshold(); /// Transform the evt id in the header into a string /// /// \return evt_di as string std::string Evt_idToString(); - /// Get the name of the filter - std::string GetFilterName(); - - /// Get the string representation of a rapidjson document - std::string JsonStringify(rapidjson::Document &json); + protected: + - /// Parse the body received. - /// This is the default function, trying to get a JSON array from the _raw_body, - /// if you wan't to recover something else (full/complex JSON, custom data), - /// override the function in the child class. - virtual bool ParseBody(); + /// Get the data to send to the next filter + /// according to the filter's output type + /// + /// \param size data's size + /// \param data data to send + std::string GetDataToSendToFilter(); - /// Parse a line in the body. - /// This function should be implemented in each child, - /// and should be called between every entry to check validity (no early parsing). - virtual bool ParseLine(rapidjson::Value &line) = 0; + /// Send virtual void SendNext() final; @@ -163,6 +131,11 @@ namespace darwin { SendToFilterCallback(const boost::system::error_code& e, std::size_t size); + + std::string JsonStringify(rapidjson::Document &json); + + bool PreParseBody(); + private: /// Set the async read for the header. /// @@ -193,6 +166,7 @@ namespace darwin { /// \param code The error code to send virtual void SendErrorResponse(const std::string& message, const unsigned int code) final; + friend std::shared_ptr Generator::CreateTask(std::shared_ptr s) noexcept; // Not accessible by children private: std::string _filter_name; //!< name of the filter @@ -201,30 +175,20 @@ namespace darwin { config::output_type _output; //!< The filter's output. std::array _buffer; //!< Reading buffer for the body. + std::size_t _threshold = DARWIN_DEFAULT_THRESHOLD; // Accessible by children protected: boost::asio::local::stream_protocol::socket _socket; //!< Session's socket. boost::asio::local::stream_protocol::socket _filter_socket; //!< Filter's socket. Manager& _manager; //!< The associated connection manager. + Generator& _generator; //!< The Task Generator. darwin_filter_packet_t _header; //!< Header received from the session. rapidjson::Document _body; //!< Body received from session (if any). std::string _raw_body; //!< Body received from session (if any), that will not be parsed. std::string _logs; //!< Represents data given in the logs by the Session - std::chrono::time_point _starting_time; - std::vector _certitudes; //!< The Darwin results obtained. - //!< Cache received from the Generator - std::shared_ptr> _cache; - std::mutex& _cache_mutex; - bool _is_cache = false; - std::size_t _threshold = DARWIN_DEFAULT_THRESHOLD; //! _certitudes; }; - - /// Definition of a session's self-managing pointer. - /// - /// \typedef session_ptr_t - typedef std::shared_ptr session_ptr_t; - } diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp new file mode 100755 index 0000000..2897091 --- /dev/null +++ b/samples/base/ATask.cpp @@ -0,0 +1,83 @@ +#include +#include +#include + +#include "Logger.hpp" +#include "Manager.hpp" +#include "ASession.hpp" +#include "ATask.hpp" +#include "errors.hpp" + +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" + +namespace darwin { + + ATask::ATask(std::string name, + std::shared_ptr> cache, + std::mutex& cache_mutex, session_ptr_t s, + darwin_filter_packet_t& header, + rapidjson::Document& body, + std::string& raw_body, + std::string& logs, + std::string& response_body, + std::vector& certitudes) + : _filter_name(name), _cache{cache}, _cache_mutex{cache_mutex}, _s{s}, + _header{header}, _body{body}, _raw_body{raw_body}, _logs{logs}, + _response_body{response_body}, _certitudes{certitudes}, _threshold{_s->GetThreshold()} + { + ; + } + + void ATask::SetStartingTime() { + _starting_time = std::chrono::high_resolution_clock::now(); + } + + double ATask::GetDurationMs() { + return std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - _starting_time + ).count() / ((double)1000); + } + + xxh::hash64_t ATask::GenerateHash() { + // could be easily overridden in the children + return xxh::xxhash<64>(_raw_body); + } + + void ATask::SaveToCache(const xxh::hash64_t &hash, const unsigned int certitude) const { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("SaveToCache:: Saving certitude " + std::to_string(certitude) + " to cache"); + std::unique_lock lck{_cache_mutex}; + _cache->insert(hash, certitude); + } + + bool ATask::GetCacheResult(const xxh::hash64_t &hash, unsigned int& certitude) { + DARWIN_LOGGER; + boost::optional cached_certitude; + + { + std::unique_lock lck{_cache_mutex}; + cached_certitude = _cache->get(hash); + } + + if (cached_certitude != boost::none) { + certitude = cached_certitude.get(); + DARWIN_LOG_DEBUG("GetCacheResult:: Already processed request. Cached certitude is " + + std::to_string(certitude)); + + return true; + } + + return false; + } + + std::string ATask::GetFilterName() { + return _filter_name; + } + + bool ATask::ParseBody() { + return false; + } + +} \ No newline at end of file diff --git a/samples/base/ATask.hpp b/samples/base/ATask.hpp new file mode 100755 index 0000000..5c4a5e5 --- /dev/null +++ b/samples/base/ATask.hpp @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" +#include "../../toolkit/rapidjson/document.h" +#include "../../toolkit/rapidjson/writer.h" +#include "../../toolkit/rapidjson/stringbuffer.h" +#include "ASession.fwd.hpp" +#include "config.hpp" +#include "protocol.h" +#include "Time.hpp" + +namespace darwin { + + class ATask : public std::enable_shared_from_this { + public: + ATask(std::string name, + std::shared_ptr> cache, + std::mutex& cache_mutex, session_ptr_t session, + darwin_filter_packet_t& _header, + rapidjson::Document& _body, + std::string& _raw_body, + std::string& _logs, + std::string& _response_body, + std::vector& _certitudes); + + virtual ~ATask() = default; + + // Make the ATask non copyable & non movable + ATask(ATask const&) = delete; + + ATask(ATask const&&) = delete; + + ATask& operator=(ATask const&) = delete; + + ATask& operator=(ATask const&&) = delete; + + /// Entry point of the execution. + /// MUST be overloaded. + /// WARNING This method will be executed by a thread. + virtual void operator()() = 0; + + protected: + /// Return filter code. + virtual long GetFilterCode() noexcept = 0; + + /// Generate the hash + virtual xxh::hash64_t GenerateHash(); + + /// Set starting time + void SetStartingTime(); + + /// Get total time elapsed since the starting time + double GetDurationMs(); + + /// Save the result to cache + virtual void SaveToCache(const xxh::hash64_t &hash, unsigned int certitude) const; + + /// Get the result from the cache + virtual bool GetCacheResult(const xxh::hash64_t &hash, unsigned int &certitude); + + /// Get the name of the filter + std::string GetFilterName(); + + /// Parse a line in the body. + /// This function should be implemented in each child, + /// and should be called between every entry to check validity (no early parsing). + virtual bool ParseLine(rapidjson::Value &line) = 0; + + /// Parse the body received. + /// This is the default function, trying to get a JSON array from the _raw_body, + /// if you wan't to recover something else (full/complex JSON, custom data), + /// override the function in the child class. + virtual bool ParseBody() ; + + private: + std::string _filter_name; //!< name of the filter + + protected: + //!< Cache received from the Generator + std::shared_ptr> _cache; + std::mutex& _cache_mutex; + + std::chrono::time_point _starting_time; + + bool _is_cache = false; + + session_ptr_t _s; //!< associated session + + darwin_filter_packet_t& _header; //!< Ref to the Header received from the session. + rapidjson::Document& _body; //!< Ref to the Body received from session (if any). + std::string& _raw_body; //!< Ref to the Body received from session (if any), that will not be parsed. + std::string& _logs; //!< Ref to the data given in the logs by the Session + std::string& _response_body; //!< Ref to the body to send back to the client + std::vector& _certitudes; //!< Ref to the Darwin results obtained. + + std::size_t _threshold; //!< Threshold, overriden at creation by the session + + }; + +} \ No newline at end of file diff --git a/samples/base/Manager.hpp b/samples/base/Manager.hpp index 93e0e09..c42fa8e 100644 --- a/samples/base/Manager.hpp +++ b/samples/base/Manager.hpp @@ -9,7 +9,7 @@ #include #include -#include "Session.hpp" +#include "ASession.hpp" namespace darwin { class Manager { diff --git a/samples/base/Server.cpp b/samples/base/Server.cpp deleted file mode 100644 index 4039deb..0000000 --- a/samples/base/Server.cpp +++ /dev/null @@ -1,119 +0,0 @@ -/// \file Server.cpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 02/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - -#include -#include -#include -#include -#include "Server.hpp" -#include "Logger.hpp" -#include "Stats.hpp" - -namespace darwin { - - Server::Server(std::string const& socket_path, - std::string const& output, - std::string const& next_filter_socket, - std::size_t threshold, - Generator& generator) - : _socket_path{socket_path}, _socket_next{next_filter_socket}, _output{output}, - _io_context{}, _threshold{threshold}, _signals{_io_context}, - _acceptor{_io_context, - boost::asio::local::stream_protocol::endpoint( - socket_path)}, - _new_connection{_io_context}, _generator{generator} { - - // Setting the stopping signals for the service - _signals.add(SIGINT); - _signals.add(SIGTERM); -#ifdef SIGQUIT - _signals.add(SIGQUIT); -#endif // !SIGQUIT - - // Start the waiting for a stopping signal - AwaitStop(); - - // Start accepting connections - Accept(); - } - - boost::asio::io_context &Server::GetIOContext() { - return this->_io_context; - } - - void Server::Run() { - // The io_context::run() call will block until all asynchronous operations - // have finished. While the server is running, there is always at least one - // asynchronous operation outstanding: the asynchronous accept call waiting - // for new incoming connections. - - // By using multiple threads, a program can call run() multiple times. - // Once an asynchronous operation is complete, - // the I/O service object will execute the handler in one of these threads. - // If a second operation is completed shortly after the first one, - // the I/O service object can execute the handler in a different thread. - // Now, not only can operations outside of a process be executed concurrently, - // but handlers within the process can be executed concurrently, too. - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Server::Run:: Running..."); - _io_context.run(); - } - - void Server::Clean() { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); - _manager.StopAll(); - unlink(_socket_path.c_str()); - } - - void Server::AwaitStop() { - _signals.async_wait( - boost::bind(&Server::HandleStop, this, - boost::asio::placeholders::error, - boost::asio::placeholders::signal_number)); - } - - void Server::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { - // The server is stopped by cancelling all outstanding asynchronous - // operations. Once all operations have finished the io_context::run() - // call will exit. - DARWIN_LOGGER; - - SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); - DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); - _acceptor.close(); - _io_context.stop(); - } - - void Server::Accept() { - _acceptor.async_accept(_new_connection, - boost::bind(&Server::HandleAccept, this, - boost::asio::placeholders::error)); - } - - void Server::HandleAccept(boost::system::error_code const& e) { - DARWIN_LOGGER; - - if (!_acceptor.is_open()) { - DARWIN_LOG_INFO("Server::HandleAccept:: Acceptor closed, closing server..."); - return; - } - - if (!e) { - DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); - auto task = _generator.CreateTask(_new_connection, _manager); - task->SetNextFilterSocketPath(_socket_next); - task->SetOutputType(_output); - task->SetThreshold(_threshold); - _manager.Start(task); - Accept(); - } else { - DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); - } - } - -} \ No newline at end of file diff --git a/samples/base/Server.hpp b/samples/base/Server.hpp deleted file mode 100644 index 595c5a6..0000000 --- a/samples/base/Server.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/// \file Server.hpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 02/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - -#pragma once - -#include -#include -#include -#include "Session.hpp" -#include "Manager.hpp" -#include "Generator.hpp" - -namespace darwin { - - class Server { - public: - /// Create an async UNIX stream socket server. - /// The server runs on nb_threads thread. - /// - /// \param socket_path Path of the UNIX socket to listen on. - /// \param output Filters' output type - /// \param next_filter_socket Path of the UNIX socket of the filter to send data to. - /// \param threshold Threshold at which the filter will raise a log. - Server(std::string const& socket_path, - std::string const& output, - std::string const& next_filter_socket, - std::size_t threshold, - Generator& generator); - - ~Server() = default; - - // Make the server non copyable & non movable - Server(Server const&) = delete; - - Server(Server const&&) = delete; - - Server& operator=(Server const&) = delete; - - Server& operator=(Server const&&) = delete; - - public: - /// Start the server and the threads. - void Run(); - - /// Clean the server's ressources (sessions, socket) - void Clean(); - - boost::asio::io_context &GetIOContext(); - - private: - /// Start async waiting for the stopping signals. - void AwaitStop(); - - /// Handler called when signal is received by the async wait. - /// - /// \param sig The received signal. - void HandleStop(boost::system::error_code const& error, int sig); - - /// Start an async connection acceptation on _new_session's socket. - void Accept(); - - /// Handler called on async accept trigger. - void HandleAccept(boost::system::error_code const& e); - - private: - std::string _socket_path; //!< Path to the UNIX socket to listen on. - std::string _socket_next; //!< Path to the next filter's UNIX socket. - std::string _output; //!< Filter's output type - boost::asio::io_context _io_context; //!< The async io context. - std::size_t _threshold; //!< Filter's threshold - boost::asio::signal_set _signals; //!< Set of the stopping signals. - boost::asio::local::stream_protocol::acceptor _acceptor; //!< Acceptor for the incoming connections. - boost::asio::local::stream_protocol::socket _new_connection; //!< Socket used to accept a new connection. - Generator& _generator; //!< Generator used to create new Sessions. - Manager _manager; //!< Server's session manager. - }; -} \ No newline at end of file diff --git a/samples/base/TcpServer.cpp b/samples/base/TcpServer.cpp new file mode 100755 index 0000000..46e5687 --- /dev/null +++ b/samples/base/TcpServer.cpp @@ -0,0 +1,82 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include +#include "TcpServer.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + TcpServer::TcpServer(std::string const& socket_path, + std::string const& output, + std::string const& next_filter_socket, + std::size_t threshold, + Generator& generator) + : AServer(output, threshold, generator), + _socket_path{socket_path}, _socket_next{next_filter_socket}, + _acceptor{_io_context, boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 8000)}, + _new_connection{_io_context} { + + this->InitSignalsAndStart(); + } + + + void TcpServer::Clean() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); + _manager.StopAll(); + //todo: can't work, to be changed + unlink(_socket_path.c_str()); + } + + void TcpServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { + // The server is stopped by cancelling all outstanding asynchronous + // operations. Once all operations have finished the io_context::run() + // call will exit. + DARWIN_LOGGER; + + SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); + DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); + _acceptor.close(); + _io_context.stop(); + } + + void TcpServer::Accept() { + _acceptor.async_accept(_new_connection, + boost::bind(&TcpServer::HandleAccept, this, + boost::asio::placeholders::error)); + } + + void TcpServer::HandleAccept(boost::system::error_code const& e) { + DARWIN_LOGGER; + + if (!_acceptor.is_open()) { + DARWIN_LOG_INFO("Server::HandleAccept:: Acceptor closed, closing server..."); + return; + } + + if (!e) { + DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); + //todo to be modified + + // auto task = _generator.CreateTask(_new_connection, _manager); + // task->SetNextFilterSocketPath(_socket_next); + // task->SetOutputType(_output); + // task->SetThreshold(_threshold); + // _manager.Start(task); + Accept(); + } else { + DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); + } + } + +} \ No newline at end of file diff --git a/samples/base/TcpServer.hpp b/samples/base/TcpServer.hpp new file mode 100755 index 0000000..3e8fbcd --- /dev/null +++ b/samples/base/TcpServer.hpp @@ -0,0 +1,61 @@ +/// \file Server.hpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#pragma once + +#include +#include +#include +#include +#include "ASession.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +#include "AServer.hpp" + +namespace darwin { + + class TcpServer : public AServer { + public: + /// Create an async Tcp stream socket server. + /// The server runs on nb_threads thread. + /// + /// \param socket_path Path of the Tcp socket to listen on. + /// \param output Filters' output type + /// \param next_filter_socket Path of the Tcp socket of the filter to send data to. + /// \param threshold Threshold at which the filter will raise a log. + TcpServer(std::string const& socket_path, + std::string const& output, + std::string const& next_filter_socket, + std::size_t threshold, + Generator& generator); + + ~TcpServer() = default; + + // Make the TcpServer non copyable & non movable + TcpServer(TcpServer const&) = delete; + + TcpServer(TcpServer const&&) = delete; + + TcpServer& operator=(TcpServer const&) = delete; + + TcpServer& operator=(TcpServer const&&) = delete; + + void Clean(); + + void HandleStop(boost::system::error_code const& error, int sig); + + void Accept(); + + void HandleAccept(boost::system::error_code const& e); + + private: + std::string _socket_path; //!< Path to the Tcp socket to listen on. + std::string _socket_next; //!< Path to the next filter's Tcp socket. + boost::asio::ip::tcp::acceptor _acceptor; //!< Acceptor for the incoming connections. + boost::asio::ip::tcp::socket _new_connection; //!< Socket used to accept a new connection. + }; +} \ No newline at end of file diff --git a/samples/base/UnixServer.cpp b/samples/base/UnixServer.cpp index d9bc5d7..b69c4c8 100644 --- a/samples/base/UnixServer.cpp +++ b/samples/base/UnixServer.cpp @@ -68,11 +68,11 @@ namespace darwin { if (!e) { DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); - auto task = _generator.CreateTask(_new_connection, _manager); - task->SetNextFilterSocketPath(_socket_next); - task->SetOutputType(_output); - task->SetThreshold(_threshold); - _manager.Start(task); + auto sess = std::make_shared(_new_connection, _manager, _generator); + sess->SetNextFilterSocketPath(_socket_next); + sess->SetOutputType(_output); + sess->SetThreshold(_threshold); + _manager.Start(sess); Accept(); } else { DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); diff --git a/samples/base/UnixServer.hpp b/samples/base/UnixServer.hpp index f8849c5..ba3dad5 100644 --- a/samples/base/UnixServer.hpp +++ b/samples/base/UnixServer.hpp @@ -10,7 +10,7 @@ #include #include #include -#include "Session.hpp" +#include "ASession.hpp" #include "Manager.hpp" #include "Generator.hpp" #include "AServer.hpp" @@ -43,13 +43,13 @@ namespace darwin { UnixServer& operator=(UnixServer const&&) = delete; - void Clean(); + void Clean() override; - void HandleStop(boost::system::error_code const& error, int sig); + void HandleStop(boost::system::error_code const& error, int sig) override; - void Accept(); + void Accept() override; - void HandleAccept(boost::system::error_code const& e); + void HandleAccept(boost::system::error_code const& e) override; private: std::string _socket_path; //!< Path to the UNIX socket to listen on. diff --git a/samples/fdga/DGATask.cpp b/samples/fdga/DGATask.cpp index 6cf8af3..60d6371 100644 --- a/samples/fdga/DGATask.cpp +++ b/samples/fdga/DGATask.cpp @@ -19,21 +19,27 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "DGATask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" #include "protocol.h" #include "tensorflow/core/framework/tensor.h" #include "AlertManager.hpp" -DGATask::DGATask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +DGATask::DGATask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin_filter_packet_t& header, + rapidjson::Document& body, + std::string& raw_body, + std::string& logs, + std::string& response_body, + std::vector& certitudes, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens) - : Session{"dga", socket, manager, cache, cache_mutex}, _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, header, body, raw_body, logs, response_body, certitudes), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, _faup_options(faup_options) { _is_cache = _cache != nullptr; } @@ -48,11 +54,10 @@ long DGATask::GetFilterCode() noexcept { void DGATask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; // Should not fail, as the Session body parser MUST check for validity ! rapidjson::GenericArray array = _body.GetArray(); - for (rapidjson::Value &value : array) { STAT_INPUT_INC; SetStartingTime(); @@ -71,9 +76,9 @@ void DGATask::operator()() { if (GetCacheResult(hash, certitude)) { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_domain, certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _s->Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; _logs += alert_log + '\n'; } @@ -88,9 +93,9 @@ void DGATask::operator()() { certitude = Predict(); if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_domain, certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _s->Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; _logs += alert_log + '\n'; } diff --git a/samples/fdga/DGATask.hpp b/samples/fdga/DGATask.hpp index dba6df2..8436a9f 100644 --- a/samples/fdga/DGATask.hpp +++ b/samples/fdga/DGATask.hpp @@ -15,7 +15,8 @@ #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" #include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "ASession.fwd.hpp" #include "tensorflow/core/public/session.h" #define DARWIN_FILTER_DGA 0x64676164 @@ -27,12 +28,17 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class DGATask : public darwin::Session { +class DGATask : public darwin::ATask { public: - explicit DGATask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit DGATask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin_filter_packet_t& header, + rapidjson::Document& body, + std::string& raw_body, + std::string& logs, + std::string& response_body, + std::vector& certitudes, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens = 50); diff --git a/samples/fdga/Generator.cpp b/samples/fdga/Generator.cpp index ed0e492..9e7790f 100644 --- a/samples/fdga/Generator.cpp +++ b/samples/fdga/Generator.cpp @@ -14,6 +14,7 @@ #include "base/Logger.hpp" #include "DGATask.hpp" #include "Generator.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" #include "tensorflow/core/framework/graph.pb.h" @@ -168,12 +169,19 @@ bool Generator::LoadFaupOptions() { } return true; } +// Method is friend with ASession +std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, + s->_header, s->_body, s->_raw_body, s->_logs, + s->_response_body, s->_certitudes, + _session, _faup_options, _token_map, _max_tokens + ) + ); +} -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _session, _faup_options, _token_map, _max_tokens)); +long Generator::GetFilterCode() const { + return DARWIN_FILTER_DGA; } Generator::~Generator() { diff --git a/samples/fdga/Generator.hpp b/samples/fdga/Generator.hpp index 08a4530..27b089e 100644 --- a/samples/fdga/Generator.hpp +++ b/samples/fdga/Generator.hpp @@ -12,7 +12,7 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "tensorflow/core/public/session.h" @@ -24,9 +24,9 @@ class Generator: public AGenerator { public: static constexpr int DEFAULT_MAX_TOKENS = 75; - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; From 9da70fd21996fe8e98c71cc984cabad1347f41ef Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 14 Jun 2021 14:10:21 +0200 Subject: [PATCH 03/54] Fixed potential corruption : If more than 1 certitude is sent back to client AND a body is set, the body is copied on the memory location of the certitudes It is fixed by using the correct offset when copying the body data. The other function using similar logic has been modified to keep it coherent. --- samples/base/ASession.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 324f5d5..e06784e 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -231,7 +231,7 @@ namespace darwin { * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size */ - std::size_t packet_size = 0; + std::size_t packet_size = 0, packet_size_wo_body = 0; if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { packet_size = sizeof(darwin_filter_packet_t) + (certitude_size - DEFAULT_CERTITUDE_LIST_SIZE) * sizeof(unsigned int); @@ -239,14 +239,14 @@ namespace darwin { packet_size = sizeof(darwin_filter_packet_t); } + packet_size_wo_body = packet_size; + if (not this->_response_body.empty()) { packet_size += this->_response_body.size(); } DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(packet_size)); - // TODO: to be corrected - // si certitudes.size() > default ET body not empty : on écrit sur les dernieres certitudes darwin_filter_packet_t* packet = reinterpret_cast(malloc(packet_size)); if (packet == nullptr) { @@ -270,8 +270,8 @@ namespace darwin { packet->filter_code = _generator.GetFilterCode(); packet->body_size = this->_response_body.size(); memcpy(packet->evt_id, _header.evt_id, 16); - // TODO: potentiellement corruption (effacement de certitudes) - memcpy((char*)(packet) + sizeof(darwin_filter_packet_t), _response_body.c_str(), _response_body.length()); + + memcpy((char*)(packet) + packet_size_wo_body, _response_body.c_str(), _response_body.size()); boost::asio::async_write(_socket, boost::asio::buffer(packet, packet_size), @@ -319,7 +319,7 @@ namespace darwin { * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size */ - std::size_t packet_size = 0; + std::size_t packet_size = 0, packet_size_wo_data = 0; if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { packet_size = sizeof(darwin_filter_packet_t) + (certitude_size - DEFAULT_CERTITUDE_LIST_SIZE) * sizeof(unsigned int); @@ -327,6 +327,8 @@ namespace darwin { packet_size = sizeof(darwin_filter_packet_t); } + packet_size_wo_data = packet_size; + packet_size += data.size(); DARWIN_LOG_DEBUG("ASession::SendToFilter:: Computed packet size: " + std::to_string(packet_size)); @@ -352,7 +354,7 @@ namespace darwin { if(data.size() != 0) { // TODO: set a proper pointer in protocol.h for the body // Yes We Hack... - memcpy(&packet->certitude_list[certitude_size+1], data.c_str(), data.size()); + memcpy((char *)(packet) + packet_size_wo_data, data.c_str(), data.size()); } packet->type = DARWIN_PACKET_FILTER; From 0149aeea622eb21f3065494a857a0e767642158c Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 15 Jun 2021 18:34:40 +0200 Subject: [PATCH 04/54] Separated Sessions into UnixSession and TcpSession UnixSessions are spawn by UnixServer and TcpSessions by TcpServer Included a modification when passing arguments from sessions to task (move instead of ref) --- CMakeLists.txt | 2 + samples/base/ASession.cpp | 74 +++---------------------- samples/base/ASession.hpp | 40 ++++++-------- samples/base/ATask.cpp | 7 ++- samples/base/ATask.hpp | 15 +++--- samples/base/Manager.cpp | 1 + samples/base/Manager.hpp | 2 +- samples/base/TcpServer.cpp | 24 ++++----- samples/base/TcpServer.hpp | 14 ++--- samples/base/TcpSession.cpp | 98 +++++++++++++++++++++++++++++++++ samples/base/TcpSession.hpp | 55 +++++++++++++++++++ samples/base/UnixServer.cpp | 5 +- samples/base/UnixServer.hpp | 2 +- samples/base/UnixSession.cpp | 102 +++++++++++++++++++++++++++++++++++ samples/base/UnixSession.hpp | 57 ++++++++++++++++++++ samples/fdga/DGATask.cpp | 3 +- samples/fdga/DGATask.hpp | 1 - samples/fdga/Generator.cpp | 2 +- 18 files changed, 374 insertions(+), 130 deletions(-) create mode 100755 samples/base/TcpSession.cpp create mode 100755 samples/base/TcpSession.hpp create mode 100755 samples/base/UnixSession.cpp create mode 100755 samples/base/UnixSession.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cfc470d..e6cb170 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,6 +101,8 @@ set( samples/base/TcpServer.cpp samples/base/TcpServer.hpp samples/base/Manager.cpp samples/base/Manager.hpp samples/base/ASession.cpp samples/base/ASession.hpp + samples/base/UnixSession.cpp samples/base/UnixSession.hpp + samples/base/TcpSession.cpp samples/base/TcpSession.hpp samples/base/ATask.cpp samples/base/ATask.hpp toolkit/Network.cpp toolkit/Network.hpp diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index e06784e..226f76a 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -23,12 +23,9 @@ #include "../../toolkit/xxhash.hpp" namespace darwin { - ASession::ASession(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, + ASession::ASession(darwin::Manager& manager, Generator& generator) - : _connected{false}, _socket{std::move(socket)}, - _filter_socket{socket.get_executor()}, - _manager{manager}, _generator{generator} {} + : _manager{manager}, _generator{generator}, _has_next_filter{false} {} @@ -38,17 +35,6 @@ namespace darwin { ReadHeader(); } - void ASession::Stop() { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("ASession::Stop::"); - _socket.close(); - if(_connected) _filter_socket.close(); - } - - void ASession::SetNextFilterSocketPath(std::string const& path){ - _next_filter_path = path; - } - void ASession::SetOutputType(std::string const& output) { _output = config::convert_output_string(output); } @@ -75,17 +61,6 @@ namespace darwin { return ""; } - void ASession::ReadHeader() { - DARWIN_LOGGER; - - DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); - boost::asio::async_read(_socket, - boost::asio::buffer(&_header, sizeof(_header)), - boost::bind(&ASession::ReadHeaderCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - } - void ASession::ReadHeaderCallback(const boost::system::error_code& e, std::size_t size) { DARWIN_LOGGER; @@ -118,17 +93,6 @@ namespace darwin { _manager.Stop(shared_from_this()); } - void ASession::ReadBody(std::size_t size) { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); - - _socket.async_read_some(boost::asio::buffer(_buffer, - size), - boost::bind(&ASession::ReadBodyCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - } - void ASession::ReadBodyCallback(const boost::system::error_code& e, std::size_t size) { DARWIN_LOGGER; @@ -273,11 +237,7 @@ namespace darwin { memcpy((char*)(packet) + packet_size_wo_body, _response_body.c_str(), _response_body.size()); - boost::asio::async_write(_socket, - boost::asio::buffer(packet, packet_size), - boost::bind(&ASession::SendToClientCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); + this->WriteToClient(packet, packet_size); free(packet); this->_response_body.clear(); @@ -287,25 +247,13 @@ namespace darwin { bool ASession::SendToFilter() noexcept { DARWIN_LOGGER; - if (!_next_filter_path.compare("no")) { + if (!_has_next_filter) { DARWIN_LOG_NOTICE("ASession::SendToFilter:: No next filter provided. Ignoring..."); return false; } - if (!_connected) { - DARWIN_LOG_DEBUG("ASession::SendToFilter:: Trying to connect to: " + - _next_filter_path); - try { - _filter_socket.connect( - boost::asio::local::stream_protocol::endpoint( - _next_filter_path.c_str())); - _connected = true; - } catch (std::exception const& e) { - DARWIN_LOG_ERROR(std::string("ASession::SendToFilter:: " - "Unable to connect to next filter: ") + - e.what()); - return false; - } + if( ! ConnectToNextFilter()) { + return false; } std::string data = GetDataToSendToFilter(); @@ -365,12 +313,7 @@ namespace darwin { memcpy(packet->evt_id, _header.evt_id, 16); DARWIN_LOG_DEBUG("ASession:: SendToFilter:: Sending header + data"); - boost::asio::async_write(_filter_socket, - boost::asio::buffer(packet, packet_size), - boost::bind(&ASession::SendToFilterCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - + this->WriteToFilter(packet, packet_size); free(packet); return true; } @@ -395,8 +338,7 @@ namespace darwin { if (e) { DARWIN_LOG_ERROR("ASession::SendToFilterCallback:: " + e.message()); - _filter_socket.close(); - _connected = false; + CloseFilterConnection(); } if(_header.response == DARWIN_RESPONSE_SEND_BOTH) { diff --git a/samples/base/ASession.hpp b/samples/base/ASession.hpp index fbf13e9..9778ff4 100755 --- a/samples/base/ASession.hpp +++ b/samples/base/ASession.hpp @@ -8,11 +8,10 @@ #pragma once #include -#include - #include "config.hpp" #include "protocol.h" #include "Generator.hpp" +#include "Manager.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" @@ -29,17 +28,14 @@ namespace darwin { - class Manager; - class ASession : public std::enable_shared_from_this { public: - ASession(boost::asio::local::stream_protocol::socket& socket, - Manager& manager, + ASession(Manager& manager, Generator& generator); virtual ~ASession() = default; - // Make the manager non copyable & non movable + // Make the Session non copyable & non movable ASession(ASession const&) = delete; ASession(ASession const&&) = delete; @@ -50,21 +46,16 @@ namespace darwin { public: /// Start the session and the async read of the incoming packet. - virtual void Start() final; + virtual void Start(); /// Stop the session and close the socket. - virtual void Stop() final; + virtual void Stop() = 0; /// Set the filter's threshold /// /// \param threshold The threshold wanted. virtual void SetThreshold(std::size_t const& threshold) final; - /// Set the path to the associated decision module UNIX socket - /// - /// \param path Path to the UNIX socket. - virtual void SetNextFilterSocketPath(std::string const& path) final; - /// Set the output's type of the filter /// /// \param name the string that represent the output type @@ -100,7 +91,14 @@ namespace darwin { /// \param data data to send std::string GetDataToSendToFilter(); - + virtual void WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) = 0; + + virtual bool ConnectToNextFilter() = 0; + + virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) = 0; + + virtual void CloseFilterConnection() = 0; + /// Send virtual void SendNext() final; @@ -136,11 +134,10 @@ namespace darwin { bool PreParseBody(); -private: /// Set the async read for the header. /// /// \return -1 on error, 0 on socket closed & sizeof(header) on success. - virtual void ReadHeader() final; + virtual void ReadHeader() = 0; /// Callback of async read for the header. /// Terminate the session on failure. @@ -149,7 +146,7 @@ namespace darwin { /// Set the async read for the body. /// /// \return -1 on error, 0 on socket closed & sizeof(header) on success. - virtual void ReadBody(std::size_t size) final; + virtual void ReadBody(std::size_t size) = 0; /// Callback of async read for the body. /// Terminate the session on failure. @@ -170,17 +167,13 @@ namespace darwin { // Not accessible by children private: std::string _filter_name; //!< name of the filter - bool _connected; //!< True if the socket to the next filter is connected. - std::string _next_filter_path; //!< The socket path to the next filter. config::output_type _output; //!< The filter's output. - std::array _buffer; //!< Reading buffer for the body. std::size_t _threshold = DARWIN_DEFAULT_THRESHOLD; // Accessible by children protected: - boost::asio::local::stream_protocol::socket _socket; //!< Session's socket. - boost::asio::local::stream_protocol::socket _filter_socket; //!< Filter's socket. + std::array _buffer; //!< Reading buffer for the body. Manager& _manager; //!< The associated connection manager. Generator& _generator; //!< The Task Generator. darwin_filter_packet_t _header; //!< Header received from the session. @@ -189,6 +182,7 @@ namespace darwin { std::string _logs; //!< Represents data given in the logs by the Session std::string _response_body; //!< The body to send back to the client std::vector _certitudes; + bool _has_next_filter; }; } diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp index 2897091..f451069 100755 --- a/samples/base/ATask.cpp +++ b/samples/base/ATask.cpp @@ -21,11 +21,10 @@ namespace darwin { rapidjson::Document& body, std::string& raw_body, std::string& logs, - std::string& response_body, - std::vector& certitudes) + std::string& response_body) : _filter_name(name), _cache{cache}, _cache_mutex{cache_mutex}, _s{s}, - _header{header}, _body{body}, _raw_body{raw_body}, _logs{logs}, - _response_body{response_body}, _certitudes{certitudes}, _threshold{_s->GetThreshold()} + _header{std::move(header)}, _body{std::move(body)}, _raw_body{std::move(raw_body)}, _logs{std::move(logs)}, + _response_body{std::move(response_body)}, _threshold{_s->GetThreshold()} { ; } diff --git a/samples/base/ATask.hpp b/samples/base/ATask.hpp index 5c4a5e5..562bed9 100755 --- a/samples/base/ATask.hpp +++ b/samples/base/ATask.hpp @@ -24,8 +24,7 @@ namespace darwin { rapidjson::Document& _body, std::string& _raw_body, std::string& _logs, - std::string& _response_body, - std::vector& _certitudes); + std::string& _response_body); virtual ~ATask() = default; @@ -90,12 +89,12 @@ namespace darwin { session_ptr_t _s; //!< associated session - darwin_filter_packet_t& _header; //!< Ref to the Header received from the session. - rapidjson::Document& _body; //!< Ref to the Body received from session (if any). - std::string& _raw_body; //!< Ref to the Body received from session (if any), that will not be parsed. - std::string& _logs; //!< Ref to the data given in the logs by the Session - std::string& _response_body; //!< Ref to the body to send back to the client - std::vector& _certitudes; //!< Ref to the Darwin results obtained. + darwin_filter_packet_t _header; //!< Ref to the Header received from the session. + rapidjson::Document _body; //!< Ref to the Body received from session (if any). + std::string _raw_body; //!< Ref to the Body received from session (if any), that will not be parsed. + std::string _logs; //!< Ref to the data given in the logs by the Session + std::string _response_body; //!< Ref to the body to send back to the client + std::vector _certitudes; //!< Ref to the Darwin results obtained. std::size_t _threshold; //!< Threshold, overriden at creation by the session diff --git a/samples/base/Manager.cpp b/samples/base/Manager.cpp index 56de739..13b0537 100644 --- a/samples/base/Manager.cpp +++ b/samples/base/Manager.cpp @@ -8,6 +8,7 @@ #include "Logger.hpp" #include "Stats.hpp" #include "Manager.hpp" +#include "ASession.hpp" namespace darwin { diff --git a/samples/base/Manager.hpp b/samples/base/Manager.hpp index c42fa8e..8d17ae0 100644 --- a/samples/base/Manager.hpp +++ b/samples/base/Manager.hpp @@ -9,7 +9,7 @@ #include #include -#include "ASession.hpp" +#include "ASession.fwd.hpp" namespace darwin { class Manager { diff --git a/samples/base/TcpServer.cpp b/samples/base/TcpServer.cpp index 46e5687..49d75ff 100755 --- a/samples/base/TcpServer.cpp +++ b/samples/base/TcpServer.cpp @@ -11,19 +11,20 @@ #include #include #include "TcpServer.hpp" +#include "TcpSession.hpp" #include "Logger.hpp" #include "Stats.hpp" namespace darwin { - TcpServer::TcpServer(std::string const& socket_path, + TcpServer::TcpServer(int port_nb, + int next_filter_port, std::string const& output, - std::string const& next_filter_socket, std::size_t threshold, Generator& generator) : AServer(output, threshold, generator), - _socket_path{socket_path}, _socket_next{next_filter_socket}, - _acceptor{_io_context, boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 8000)}, + _port_nb{port_nb}, _port_nb_next{next_filter_port}, + _acceptor{_io_context, boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), port_nb)}, _new_connection{_io_context} { this->InitSignalsAndStart(); @@ -35,7 +36,7 @@ namespace darwin { DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); _manager.StopAll(); //todo: can't work, to be changed - unlink(_socket_path.c_str()); + // unlink? what? why? } void TcpServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { @@ -66,13 +67,12 @@ namespace darwin { if (!e) { DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); - //todo to be modified - - // auto task = _generator.CreateTask(_new_connection, _manager); - // task->SetNextFilterSocketPath(_socket_next); - // task->SetOutputType(_output); - // task->SetThreshold(_threshold); - // _manager.Start(task); + auto sess = std::make_shared(_new_connection, _manager, _generator); + sess->SetNextFilterPort(_port_nb_next); + sess->SetOutputType(_output); + sess->SetThreshold(_threshold); + _manager.Start(sess); + Accept(); Accept(); } else { DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); diff --git a/samples/base/TcpServer.hpp b/samples/base/TcpServer.hpp index 3e8fbcd..7e3be83 100755 --- a/samples/base/TcpServer.hpp +++ b/samples/base/TcpServer.hpp @@ -27,11 +27,11 @@ namespace darwin { /// \param output Filters' output type /// \param next_filter_socket Path of the Tcp socket of the filter to send data to. /// \param threshold Threshold at which the filter will raise a log. - TcpServer(std::string const& socket_path, - std::string const& output, - std::string const& next_filter_socket, - std::size_t threshold, - Generator& generator); + TcpServer(int port_nb, + int next_filter_port, + std::string const& output, + std::size_t threshold, + Generator& generator); ~TcpServer() = default; @@ -53,8 +53,8 @@ namespace darwin { void HandleAccept(boost::system::error_code const& e); private: - std::string _socket_path; //!< Path to the Tcp socket to listen on. - std::string _socket_next; //!< Path to the next filter's Tcp socket. + int _port_nb; //!< Path to the Tcp socket to listen on. + int _port_nb_next; //!< Path to the next filter's Tcp socket. boost::asio::ip::tcp::acceptor _acceptor; //!< Acceptor for the incoming connections. boost::asio::ip::tcp::socket _new_connection; //!< Socket used to accept a new connection. }; diff --git a/samples/base/TcpSession.cpp b/samples/base/TcpSession.cpp new file mode 100755 index 0000000..2cef591 --- /dev/null +++ b/samples/base/TcpSession.cpp @@ -0,0 +1,98 @@ +#include +#include +#include + +#include "Logger.hpp" +#include "TcpSession.hpp" + +namespace darwin { + TcpSession::TcpSession( + boost::asio::ip::tcp::socket& socket, + Manager& manager, Generator& generator) + : ASession(manager, generator), _connected{false}, + _socket{std::move(socket)}, + _filter_socket{socket.get_executor()}, _next_filter_port{0} + { + ; + } + + void TcpSession::Stop() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("TcpSession::Stop::"); + _socket.close(); + CloseFilterConnection(); + } + + void TcpSession::CloseFilterConnection() { + if(_connected){ + _filter_socket.close(); + _connected = false; + } + } + + void TcpSession::ReadHeader() { + DARWIN_LOGGER; + + DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); + boost::asio::async_read(_socket, + boost::asio::buffer(&_header, sizeof(_header)), + boost::bind(&TcpSession::ReadHeaderCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void TcpSession::ReadBody(std::size_t size) { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); + + _socket.async_read_some(boost::asio::buffer(_buffer, + size), + boost::bind(&TcpSession::ReadBodyCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void TcpSession::WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) { + boost::asio::async_write(_socket, + boost::asio::buffer(packet, packet_size), + boost::bind(&TcpSession::SendToClientCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void TcpSession::WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) { + boost::asio::async_write(_filter_socket, + boost::asio::buffer(packet, packet_size), + boost::bind(&TcpSession::SendToFilterCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + + } + + void TcpSession::SetNextFilterPort(int port){ + if(port > 0 && port < 65536) { + _next_filter_port = port; + _has_next_filter = true; + } + } + + bool TcpSession::ConnectToNextFilter() { + DARWIN_LOGGER; + if (!_connected) { + DARWIN_LOG_DEBUG("TcpSession::SendToFilter:: Trying to connect to: " + + std::to_string(_next_filter_port)); + try { + _filter_socket.connect( + boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), _next_filter_port)); + _connected = true; + } catch (std::exception const& e) { + DARWIN_LOG_ERROR(std::string("TcpSession::SendToFilter:: " + "Unable to connect to next filter: ") + + e.what()); + _connected = false; + } + } + return _connected; + } +} + diff --git a/samples/base/TcpSession.hpp b/samples/base/TcpSession.hpp new file mode 100755 index 0000000..ca8730f --- /dev/null +++ b/samples/base/TcpSession.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include "ASession.hpp" +#include "Generator.hpp" + +namespace darwin { + + class TcpSession: public ASession { + public: + TcpSession(boost::asio::ip::tcp::socket& socket, + Manager& manager, + Generator& generator); + + ~TcpSession() = default; + + void Stop() override final; + + void SetNextFilterPort(int port); + + protected: + + /// Set the async read for the header. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadHeader() override final; + + /// Set the async read for the body. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadBody(std::size_t size) override final; + + /// + virtual void WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) override final; + + /// + virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) override final; + + + /// + virtual bool ConnectToNextFilter() override final; + + virtual void CloseFilterConnection() override final; + + private: + + bool _connected; //!< True if the socket to the next filter is connected. + + boost::asio::ip::tcp::socket _socket; //!< Session's socket. + boost::asio::ip::tcp::socket _filter_socket; //!< Filter's socket. + + int _next_filter_port; + + }; + +} \ No newline at end of file diff --git a/samples/base/UnixServer.cpp b/samples/base/UnixServer.cpp index b69c4c8..42ebc8a 100644 --- a/samples/base/UnixServer.cpp +++ b/samples/base/UnixServer.cpp @@ -15,9 +15,6 @@ namespace darwin { - -// tcp: boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 8000) - UnixServer::UnixServer(std::string const& socket_path, std::string const& output, std::string const& next_filter_socket, @@ -68,7 +65,7 @@ namespace darwin { if (!e) { DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); - auto sess = std::make_shared(_new_connection, _manager, _generator); + auto sess = std::make_shared(_new_connection, _manager, _generator); sess->SetNextFilterSocketPath(_socket_next); sess->SetOutputType(_output); sess->SetThreshold(_threshold); diff --git a/samples/base/UnixServer.hpp b/samples/base/UnixServer.hpp index ba3dad5..86a69cb 100644 --- a/samples/base/UnixServer.hpp +++ b/samples/base/UnixServer.hpp @@ -10,7 +10,7 @@ #include #include #include -#include "ASession.hpp" +#include "UnixSession.hpp" #include "Manager.hpp" #include "Generator.hpp" #include "AServer.hpp" diff --git a/samples/base/UnixSession.cpp b/samples/base/UnixSession.cpp new file mode 100755 index 0000000..6f53dee --- /dev/null +++ b/samples/base/UnixSession.cpp @@ -0,0 +1,102 @@ +#include +#include + +#include "Logger.hpp" +#include "UnixSession.hpp" + +namespace darwin { + + UnixSession::UnixSession( + boost::asio::local::stream_protocol::socket& socket, + Manager& manager, Generator& generator) + : ASession(manager, generator), _connected{false}, + _socket{std::move(socket)}, + _filter_socket{socket.get_executor()} + { + ; + } + + void UnixSession::SetNextFilterSocketPath(std::string const& path){ + if(path.compare("no") != 0) { + _next_filter_path = path; + _has_next_filter = true; + } + } + + void UnixSession::Stop() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("UnixSession::Stop::"); + _socket.close(); + if(_connected) { + _filter_socket.close(); + _connected = false; + } + } + + void UnixSession::CloseFilterConnection() { + if(_connected){ + _filter_socket.close(); + _connected = false; + } + } + + void UnixSession::ReadHeader() { + DARWIN_LOGGER; + + DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); + boost::asio::async_read(_socket, + boost::asio::buffer(&_header, sizeof(_header)), + boost::bind(&UnixSession::ReadHeaderCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void UnixSession::ReadBody(std::size_t size) { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); + + _socket.async_read_some(boost::asio::buffer(_buffer, + size), + boost::bind(&UnixSession::ReadBodyCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void UnixSession::WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) { + boost::asio::async_write(_socket, + boost::asio::buffer(packet, packet_size), + boost::bind(&UnixSession::SendToClientCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void UnixSession::WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) { + boost::asio::async_write(_filter_socket, + boost::asio::buffer(packet, packet_size), + boost::bind(&UnixSession::SendToFilterCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + + } + + bool UnixSession::ConnectToNextFilter() { + DARWIN_LOGGER; + if (!_connected) { + DARWIN_LOG_DEBUG("UnixSession::SendToFilter:: Trying to connect to: " + + _next_filter_path); + try { + _filter_socket.connect( + boost::asio::local::stream_protocol::endpoint( + _next_filter_path.c_str())); + _connected = true; + } catch (std::exception const& e) { + DARWIN_LOG_ERROR(std::string("UnixSession::SendToFilter:: " + "Unable to connect to next filter: ") + + e.what()); + _connected = false; + } + } + return _connected; + } +} + diff --git a/samples/base/UnixSession.hpp b/samples/base/UnixSession.hpp new file mode 100755 index 0000000..4f5e552 --- /dev/null +++ b/samples/base/UnixSession.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include "ASession.hpp" + +namespace darwin { + + class UnixSession: public ASession { + public: + UnixSession(boost::asio::local::stream_protocol::socket& socket, + Manager& manager, + Generator& generator); + + ~UnixSession() = default; + + void Stop() override final; + + /// Set the path to the associated decision module UNIX socket + /// + /// \param path Path to the UNIX socket. + void SetNextFilterSocketPath(std::string const& path); + + + protected: + + /// Set the async read for the header. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadHeader() override final; + + /// Set the async read for the body. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadBody(std::size_t size) override final; + + /// + virtual void WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) override final; + + /// + virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) override final; + + + /// + virtual bool ConnectToNextFilter() override final; + + virtual void CloseFilterConnection() override final; + + private: + + bool _connected; //!< True if the socket to the next filter is connected. + + boost::asio::local::stream_protocol::socket _socket; //!< Session's socket. + boost::asio::local::stream_protocol::socket _filter_socket; //!< Filter's socket. + std::string _next_filter_path; //!< The socket path to the next filter. + + }; + +} \ No newline at end of file diff --git a/samples/fdga/DGATask.cpp b/samples/fdga/DGATask.cpp index 60d6371..c85eb3f 100644 --- a/samples/fdga/DGATask.cpp +++ b/samples/fdga/DGATask.cpp @@ -34,12 +34,11 @@ DGATask::DGATask(std::shared_ptr& certitudes, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens) - : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, header, body, raw_body, logs, response_body, certitudes), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, header, body, raw_body, logs, response_body), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, _faup_options(faup_options) { _is_cache = _cache != nullptr; } diff --git a/samples/fdga/DGATask.hpp b/samples/fdga/DGATask.hpp index 8436a9f..d5e3428 100644 --- a/samples/fdga/DGATask.hpp +++ b/samples/fdga/DGATask.hpp @@ -38,7 +38,6 @@ class DGATask : public darwin::ATask { std::string& raw_body, std::string& logs, std::string& response_body, - std::vector& certitudes, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens = 50); diff --git a/samples/fdga/Generator.cpp b/samples/fdga/Generator.cpp index 9e7790f..dec912b 100644 --- a/samples/fdga/Generator.cpp +++ b/samples/fdga/Generator.cpp @@ -174,7 +174,7 @@ std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) no return std::static_pointer_cast( std::make_shared(_cache, _cache_mutex, s, s->_header, s->_body, s->_raw_body, s->_logs, - s->_response_body, s->_certitudes, + s->_response_body, _session, _faup_options, _token_map, _max_tokens ) ); From 60f0efc06fb9c8b32a32703afb56d5b46f5989aa Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 18 Jun 2021 09:39:15 +0200 Subject: [PATCH 05/54] WIP : First draft of DarwinPacket struct - Used to pass needed data from Session to Task --- samples/base/AGenerator.cpp | 11 ++++ samples/base/AGenerator.hpp | 9 +++- samples/base/ASession.cpp | 80 +++++++++++++--------------- samples/base/ASession.hpp | 15 +++--- samples/base/ATask.cpp | 36 +++++++++---- samples/base/ATask.hpp | 13 +++-- samples/base/DarwinPacket.cpp | 98 +++++++++++++++++++++++++++++++++++ samples/base/DarwinPacket.hpp | 32 ++++++++++++ samples/base/TcpServer.cpp | 1 - samples/base/TcpSession.cpp | 4 +- samples/base/TcpSession.hpp | 2 +- samples/base/UnixSession.cpp | 4 +- samples/base/UnixSession.hpp | 2 +- samples/fdga/DGATask.cpp | 7 +-- samples/fdga/DGATask.hpp | 6 +-- samples/fdga/Generator.cpp | 3 +- 16 files changed, 236 insertions(+), 87 deletions(-) create mode 100644 samples/base/DarwinPacket.cpp create mode 100644 samples/base/DarwinPacket.hpp diff --git a/samples/base/AGenerator.cpp b/samples/base/AGenerator.cpp index 57c8a6f..a88374c 100644 --- a/samples/base/AGenerator.cpp +++ b/samples/base/AGenerator.cpp @@ -13,6 +13,13 @@ #include "AGenerator.hpp" #include "AlertManager.hpp" +AGenerator::AGenerator() +{ + auto opts = tp::ThreadPoolOptions(); + opts.setThreadCount(2); + _threadPool = tp::ThreadPool(opts); +} + bool AGenerator::Configure(std::string const& configFile, const std::size_t cache_size) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("AGenerator:: Configuring..."); @@ -116,3 +123,7 @@ bool AGenerator::ReadConfig(const std::string &configuration_file_path) { conf_file_stream.close(); return true; } + +tp::ThreadPool& AGenerator::GetTaskThreadPool() { + return this->_threadPool; +} diff --git a/samples/base/AGenerator.hpp b/samples/base/AGenerator.hpp index 43199a3..8cbeeaf 100644 --- a/samples/base/AGenerator.hpp +++ b/samples/base/AGenerator.hpp @@ -11,15 +11,17 @@ #include #include #include +#include #include "ATask.hpp" #include "../toolkit/rapidjson/document.h" class AGenerator { public: - AGenerator() = default; + AGenerator(); virtual ~AGenerator() = default; + // Methods to be implemented by the child public: /// Create a new task. @@ -60,6 +62,8 @@ class AGenerator { virtual long GetFilterCode() const = 0; + virtual tp::ThreadPool& GetTaskThreadPool() final; + private: /// Open and read the configuration file. /// Try to load the json format of the configuration. @@ -80,4 +84,7 @@ class AGenerator { protected: std::shared_ptr> _cache; //!< The cache for already processed request std::mutex _cache_mutex; + + tp::ThreadPool _threadPool; + }; diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 226f76a..1d0781d 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -66,7 +66,6 @@ namespace darwin { DARWIN_LOGGER; _raw_body.clear(); - _certitudes.clear(); _logs.clear(); DARWIN_LOG_DEBUG("ASession::ReadHeaderCallback:: Reading header"); @@ -122,11 +121,12 @@ namespace darwin { return; } + /* TODO pushed to Task::ParseBody, check if parsing and error handling is done somewhere if (!PreParseBody()) { DARWIN_LOG_DEBUG("ASession::ReadBodyCallback Something went wrong while parsing the body"); this->SendErrorResponse("Error receiving body: Something went wrong while parsing the body", DARWIN_RESPONSE_CODE_REQUEST_ERROR); return; - } + }*/ ExecuteFilter(); } @@ -140,13 +140,16 @@ namespace darwin { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ASession::ExecuteFilter::"); auto t = _generator.CreateTask(shared_from_this()); //shared pt Task - // TODO ThreadPool.Start(t); - (*t)(); - this->SendNext(); + + _generator.GetTaskThreadPool().post(std::bind(&ATask::run, t)); } - void ASession::SendNext() { - switch(_header.response) { + void ASession::SendNext(DarwinPacket& packet) { + + if(! this->SendToClient(packet)) + Start(); + // Ignoring Sending to filter for now + /*switch(_header.response) { case DARWIN_RESPONSE_SEND_BOTH: if(this->SendToFilter()) break; case DARWIN_RESPONSE_SEND_BACK: @@ -157,25 +160,9 @@ namespace darwin { break; default: Start(); - } + }*/ } - bool ASession::PreParseBody() { - DARWIN_LOGGER; - try { - _body.Parse(_raw_body.c_str()); - - if (!_body.IsArray()) { - DARWIN_LOG_ERROR("ASession:: ParseBody: You must provide a list"); - return false; - } - } catch (...) { - DARWIN_LOG_CRITICAL("ASession:: ParseBody: Could not parse raw body"); - return false; - } - // DARWIN_LOG_DEBUG("ASession:: ParseBody:: parsed body : " + _raw_body); - return true; - } std::string ASession::JsonStringify(rapidjson::Document &json) { rapidjson::StringBuffer buffer; @@ -186,15 +173,18 @@ namespace darwin { return std::string(buffer.GetString()); } - bool ASession::SendToClient() noexcept { + bool ASession::SendToClient(DarwinPacket& packet) noexcept { DARWIN_LOGGER; + auto vec = packet.Serialize(); + this->WriteToClient(vec); + /* OLD CODE const std::size_t certitude_size = _certitudes.size(); /* * Allocate the header + * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size - */ + * / std::size_t packet_size = 0, packet_size_wo_body = 0; if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { packet_size = sizeof(darwin_filter_packet_t) + @@ -205,8 +195,8 @@ namespace darwin { packet_size_wo_body = packet_size; - if (not this->_response_body.empty()) { - packet_size += this->_response_body.size(); + if (not _response_body.empty()) { + packet_size += _response_body.size(); } DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(packet_size)); @@ -221,7 +211,7 @@ namespace darwin { /* * Initialisation of the structure for the padding bytes because of * missing __attribute__((packed)) in the protocol structure. - */ + * / memset(packet, 0, packet_size); for (std::size_t index = 0; index < certitude_size; ++index) { @@ -229,18 +219,17 @@ namespace darwin { } packet->type = DARWIN_PACKET_FILTER; - packet->response = _header.response; + packet->response = _send_header.response; packet->certitude_size = certitude_size; packet->filter_code = _generator.GetFilterCode(); - packet->body_size = this->_response_body.size(); - memcpy(packet->evt_id, _header.evt_id, 16); + packet->body_size = _response_body.size(); + memcpy(packet->evt_id, _send_header.evt_id, 16); memcpy((char*)(packet) + packet_size_wo_body, _response_body.c_str(), _response_body.size()); this->WriteToClient(packet, packet_size); - free(packet); - this->_response_body.clear(); + free(packet);*/ return true; } @@ -255,7 +244,7 @@ namespace darwin { if( ! ConnectToNextFilter()) { return false; } - + /* OLD CODE std::string data = GetDataToSendToFilter(); DARWIN_LOG_DEBUG("ASession::SendToFilter:: data to send: " + data); DARWIN_LOG_DEBUG("ASession::SendToFilter:: data size: " + std::to_string(data.size())); @@ -266,7 +255,7 @@ namespace darwin { * Allocate the header + * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size - */ + * / std::size_t packet_size = 0, packet_size_wo_data = 0; if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { packet_size = sizeof(darwin_filter_packet_t) + @@ -292,7 +281,7 @@ namespace darwin { /* * Initialisation of the structure for the padding bytes because of * missing __attribute__((packed)) in the protocol structure. - */ + * / memset(packet, 0, packet_size); for (std::size_t index = 0; index < certitude_size; ++index) { @@ -306,15 +295,15 @@ namespace darwin { } packet->type = DARWIN_PACKET_FILTER; - packet->response = _header.response == DARWIN_RESPONSE_SEND_BOTH ? DARWIN_RESPONSE_SEND_DARWIN : _header.response; + packet->response = _send_header.response == DARWIN_RESPONSE_SEND_BOTH ? DARWIN_RESPONSE_SEND_DARWIN : _header.response; packet->certitude_size = certitude_size; packet->filter_code = _generator.GetFilterCode(); packet->body_size = data.size(); - memcpy(packet->evt_id, _header.evt_id, 16); + memcpy(packet->evt_id, _send_header.evt_id, 16); DARWIN_LOG_DEBUG("ASession:: SendToFilter:: Sending header + data"); this->WriteToFilter(packet, packet_size); - free(packet); + free(packet);*/ return true; } @@ -342,7 +331,8 @@ namespace darwin { } if(_header.response == DARWIN_RESPONSE_SEND_BOTH) { - this->SendToClient(); + // TODO re handle this after re enabling sendtofilter + // this->SendToClient(); } else { Start(); @@ -372,9 +362,11 @@ namespace darwin { void ASession::SendErrorResponse(const std::string& message, const unsigned int code) { if (this->_header.response != DARWIN_RESPONSE_SEND_BACK && this->_header.response != DARWIN_RESPONSE_SEND_BOTH) return; - this->_response_body.clear(); - this->_response_body += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; - this->SendToClient(); + + // this->_response_body.clear(); + // this->_response_body += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; + DarwinPacket p; //todo better + this->SendToClient(p); } void ASession::SetThreshold(std::size_t const& threshold) { diff --git a/samples/base/ASession.hpp b/samples/base/ASession.hpp index 9778ff4..3732c1a 100755 --- a/samples/base/ASession.hpp +++ b/samples/base/ASession.hpp @@ -12,6 +12,7 @@ #include "protocol.h" #include "Generator.hpp" #include "Manager.hpp" +#include "DarwinPacket.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" @@ -81,6 +82,9 @@ namespace darwin { /// \return evt_di as string std::string Evt_idToString(); + /// Send + virtual void SendNext(DarwinPacket& packet) final; + protected: @@ -91,7 +95,7 @@ namespace darwin { /// \param data data to send std::string GetDataToSendToFilter(); - virtual void WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) = 0; + virtual void WriteToClient(std::vector& packet) = 0; virtual bool ConnectToNextFilter() = 0; @@ -99,14 +103,10 @@ namespace darwin { virtual void CloseFilterConnection() = 0; - - /// Send - virtual void SendNext() final; - /// Send result to the client. /// /// \return false if the function could not send the data, true otherwise. - virtual bool SendToClient() noexcept; + virtual bool SendToClient(DarwinPacket& packet) noexcept; /// Send result to next filter. /// @@ -176,12 +176,11 @@ namespace darwin { std::array _buffer; //!< Reading buffer for the body. Manager& _manager; //!< The associated connection manager. Generator& _generator; //!< The Task Generator. + DarwinPacket _packet; darwin_filter_packet_t _header; //!< Header received from the session. rapidjson::Document _body; //!< Body received from session (if any). std::string _raw_body; //!< Body received from session (if any), that will not be parsed. std::string _logs; //!< Represents data given in the logs by the Session - std::string _response_body; //!< The body to send back to the client - std::vector _certitudes; bool _has_next_filter; }; } diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp index f451069..7254729 100755 --- a/samples/base/ATask.cpp +++ b/samples/base/ATask.cpp @@ -7,6 +7,7 @@ #include "ASession.hpp" #include "ATask.hpp" #include "errors.hpp" +#include "DarwinPacket.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" @@ -17,14 +18,10 @@ namespace darwin { ATask::ATask(std::string name, std::shared_ptr> cache, std::mutex& cache_mutex, session_ptr_t s, - darwin_filter_packet_t& header, - rapidjson::Document& body, - std::string& raw_body, - std::string& logs, - std::string& response_body) + DarwinPacket& packet, + std::string& logs) : _filter_name(name), _cache{cache}, _cache_mutex{cache_mutex}, _s{s}, - _header{std::move(header)}, _body{std::move(body)}, _raw_body{std::move(raw_body)}, _logs{std::move(logs)}, - _response_body{std::move(response_body)}, _threshold{_s->GetThreshold()} + _packet{std::move(packet)}, _logs{logs}, _threshold{_s->GetThreshold()} { ; } @@ -41,7 +38,7 @@ namespace darwin { xxh::hash64_t ATask::GenerateHash() { // could be easily overridden in the children - return xxh::xxhash<64>(_raw_body); + return xxh::xxhash<64>(_packet.body); } void ATask::SaveToCache(const xxh::hash64_t &hash, const unsigned int certitude) const { @@ -76,7 +73,28 @@ namespace darwin { } bool ATask::ParseBody() { - return false; + DARWIN_LOGGER; + try { + _body.Parse(_packet.body.c_str()); + + if (!_body.IsArray()) { + DARWIN_LOG_ERROR("ATask:: ParseBody: You must provide a list"); + return false; + } + } catch (...) { + DARWIN_LOG_CRITICAL("ATask:: ParseBody: Could not parse raw body"); + return false; + } + // DARWIN_LOG_DEBUG("ATask:: ParseBody:: parsed body : " + _raw_body); + return true; + } + + void ATask::run() { + (*this)(); + DarwinPacket p; + + p.type = _packet.type; + _s->SendNext(p); } } \ No newline at end of file diff --git a/samples/base/ATask.hpp b/samples/base/ATask.hpp index 562bed9..9002207 100755 --- a/samples/base/ATask.hpp +++ b/samples/base/ATask.hpp @@ -9,6 +9,7 @@ #include "../../toolkit/rapidjson/writer.h" #include "../../toolkit/rapidjson/stringbuffer.h" #include "ASession.fwd.hpp" +#include "DarwinPacket.hpp" #include "config.hpp" #include "protocol.h" #include "Time.hpp" @@ -20,11 +21,8 @@ namespace darwin { ATask(std::string name, std::shared_ptr> cache, std::mutex& cache_mutex, session_ptr_t session, - darwin_filter_packet_t& _header, - rapidjson::Document& _body, - std::string& _raw_body, - std::string& _logs, - std::string& _response_body); + DarwinPacket& packet, + std::string& logs); virtual ~ATask() = default; @@ -41,6 +39,8 @@ namespace darwin { /// MUST be overloaded. /// WARNING This method will be executed by a thread. virtual void operator()() = 0; + + void run(); protected: /// Return filter code. @@ -89,9 +89,8 @@ namespace darwin { session_ptr_t _s; //!< associated session - darwin_filter_packet_t _header; //!< Ref to the Header received from the session. + DarwinPacket _packet; //!< Ref to the Header received from the session. rapidjson::Document _body; //!< Ref to the Body received from session (if any). - std::string _raw_body; //!< Ref to the Body received from session (if any), that will not be parsed. std::string _logs; //!< Ref to the data given in the logs by the Session std::string _response_body; //!< Ref to the body to send back to the client std::vector _certitudes; //!< Ref to the Darwin results obtained. diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp new file mode 100644 index 0000000..2104130 --- /dev/null +++ b/samples/base/DarwinPacket.cpp @@ -0,0 +1,98 @@ +#include "DarwinPacket.hpp" +#include + + +namespace darwin { + + std::vector DarwinPacket::Serialize() const { + size_t body_size = body.size(), certitude_size = certitude_list.size(); + + size_t size = sizeof(type) + sizeof(response) + sizeof(filter_code) + + sizeof(body_size) + sizeof(evt_id) + sizeof(certitude_size) + certitude_list.size() + + body.size(); + + std::vector ret(size, 0); + + auto pt = ret.data(); + + std::memcpy(pt,(void*)(&type), sizeof(type)); + pt += sizeof(type); + + std::memcpy(pt,(void*)(&response), sizeof(response)); + pt += sizeof(response); + + std::memcpy(pt,(void*)(&filter_code), sizeof(filter_code)); + pt += sizeof(filter_code); + + std::memcpy(pt,(void*)(&body_size), sizeof(body_size)); + pt += sizeof(body_size); + + std::memcpy(pt,(void*)(&evt_id), sizeof(evt_id)); + pt += sizeof(evt_id); + + std::memcpy(pt,(void*)(&certitude_size), sizeof(certitude_size)); + pt += sizeof(certitude_size); + + //certitudes + for(auto certitude: certitude_list) { + std::memcpy(pt,(void*)(&certitude), sizeof(certitude)); + pt += sizeof(certitude); + } + + //body + std::memcpy(pt, body.data(), body.length()); + return ret; + } + + DarwinPacket DarwinPacket::Parse(std::vector& input) { + + size_t minimum_size = sizeof(type) + sizeof(response) + sizeof(filter_code) + + sizeof(size_t /* body size */) + sizeof(evt_id) + sizeof(size_t /* certitude list size */); + + if(input.size() <= minimum_size){ + // error + } + + DarwinPacket packet; + + size_t body_size = 0, certitude_size = 0; + + auto pt = input.data(); + + std::memcpy((void*)(&packet.type), pt, sizeof(type)); + pt += sizeof(type); + + std::memcpy((void*)(&packet.response), pt, sizeof(response)); + pt += sizeof(response); + + std::memcpy((void*)(&packet.filter_code), pt, sizeof(filter_code)); + pt += sizeof(filter_code); + + std::memcpy((void*)(&body_size), pt, sizeof(body_size)); + pt += sizeof(body_size); + + std::memcpy((void*)(&packet.evt_id), pt, sizeof(evt_id)); + pt += sizeof(evt_id); + + std::memcpy((void*)(&certitude_size), pt, sizeof(certitude_size)); + pt += sizeof(certitude_size); + + if (input.size() != minimum_size + body_size + certitude_size) { + // error + } + + packet.certitude_list.reserve(certitude_size); + + for(size_t i=0; i < certitude_size; i++) { + unsigned int cert = 0; + std::memcpy((void*)(&cert), pt, sizeof(cert)); + pt += sizeof(cert); + + packet.certitude_list.push_back(cert); + } + + packet.body = std::string((char*)pt, body_size); + + return packet; + } +} \ No newline at end of file diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp new file mode 100644 index 0000000..f9447d1 --- /dev/null +++ b/samples/base/DarwinPacket.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include "../protocol.h" + +namespace darwin { + struct DarwinPacket { + public: + DarwinPacket() = default; // todo oé oé + ~DarwinPacket() = default; + + enum darwin_packet_type type; //!< The type of information sent. + enum darwin_filter_response_type response; //!< Whom the response will be sent to. + long filter_code; //!< The unique identifier code of a filter. + unsigned char evt_id[16]; //!< An array containing the event ID + std::vector certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. + std::string body; + + static DarwinPacket Parse(std::vector& input); // Unsure + + std::vector Serialize() const; + + + protected: + + + private: + + + }; +} \ No newline at end of file diff --git a/samples/base/TcpServer.cpp b/samples/base/TcpServer.cpp index 49d75ff..67b679b 100755 --- a/samples/base/TcpServer.cpp +++ b/samples/base/TcpServer.cpp @@ -73,7 +73,6 @@ namespace darwin { sess->SetThreshold(_threshold); _manager.Start(sess); Accept(); - Accept(); } else { DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); } diff --git a/samples/base/TcpSession.cpp b/samples/base/TcpSession.cpp index 2cef591..5dbee21 100755 --- a/samples/base/TcpSession.cpp +++ b/samples/base/TcpSession.cpp @@ -52,9 +52,9 @@ namespace darwin { boost::asio::placeholders::bytes_transferred)); } - void TcpSession::WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) { + void TcpSession::WriteToClient(std::vector& packet) { boost::asio::async_write(_socket, - boost::asio::buffer(packet, packet_size), + boost::asio::buffer(packet), boost::bind(&TcpSession::SendToClientCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); diff --git a/samples/base/TcpSession.hpp b/samples/base/TcpSession.hpp index ca8730f..2708274 100755 --- a/samples/base/TcpSession.hpp +++ b/samples/base/TcpSession.hpp @@ -30,7 +30,7 @@ namespace darwin { virtual void ReadBody(std::size_t size) override final; /// - virtual void WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) override final; + virtual void WriteToClient(std::vector& packet) override final; /// virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) override final; diff --git a/samples/base/UnixSession.cpp b/samples/base/UnixSession.cpp index 6f53dee..d841a95 100755 --- a/samples/base/UnixSession.cpp +++ b/samples/base/UnixSession.cpp @@ -62,9 +62,9 @@ namespace darwin { boost::asio::placeholders::bytes_transferred)); } - void UnixSession::WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) { + void UnixSession::WriteToClient(std::vector& packet) { boost::asio::async_write(_socket, - boost::asio::buffer(packet, packet_size), + boost::asio::buffer(packet, packet.size()), boost::bind(&UnixSession::SendToClientCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); diff --git a/samples/base/UnixSession.hpp b/samples/base/UnixSession.hpp index 4f5e552..1d3c31e 100755 --- a/samples/base/UnixSession.hpp +++ b/samples/base/UnixSession.hpp @@ -33,7 +33,7 @@ namespace darwin { virtual void ReadBody(std::size_t size) override final; /// - virtual void WriteToClient(darwin_filter_packet_t* packet, size_t packet_size) override final; + virtual void WriteToClient(std::vector& packet) override final; /// virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) override final; diff --git a/samples/fdga/DGATask.cpp b/samples/fdga/DGATask.cpp index c85eb3f..6601455 100644 --- a/samples/fdga/DGATask.cpp +++ b/samples/fdga/DGATask.cpp @@ -29,16 +29,13 @@ DGATask::DGATask(std::shared_ptr> cache, std::mutex& cache_mutex, darwin::session_ptr_t s, - darwin_filter_packet_t& header, - rapidjson::Document& body, - std::string& raw_body, + darwin::DarwinPacket& packet, std::string& logs, - std::string& response_body, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens) - : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, header, body, raw_body, logs, response_body), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet, logs), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, _faup_options(faup_options) { _is_cache = _cache != nullptr; } diff --git a/samples/fdga/DGATask.hpp b/samples/fdga/DGATask.hpp index d5e3428..002b587 100644 --- a/samples/fdga/DGATask.hpp +++ b/samples/fdga/DGATask.hpp @@ -16,6 +16,7 @@ #include "../../toolkit/xxhash.hpp" #include "protocol.h" #include "ATask.hpp" +#include "DarwinPacket.hpp" #include "ASession.fwd.hpp" #include "tensorflow/core/public/session.h" @@ -33,11 +34,8 @@ class DGATask : public darwin::ATask { explicit DGATask(std::shared_ptr> cache, std::mutex& cache_mutex, darwin::session_ptr_t s, - darwin_filter_packet_t& header, - rapidjson::Document& body, - std::string& raw_body, + darwin::DarwinPacket& packet, std::string& logs, - std::string& response_body, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens = 50); diff --git a/samples/fdga/Generator.cpp b/samples/fdga/Generator.cpp index dec912b..d7c6d2c 100644 --- a/samples/fdga/Generator.cpp +++ b/samples/fdga/Generator.cpp @@ -173,8 +173,7 @@ bool Generator::LoadFaupOptions() { std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) noexcept { return std::static_pointer_cast( std::make_shared(_cache, _cache_mutex, s, - s->_header, s->_body, s->_raw_body, s->_logs, - s->_response_body, + s->_packet, s->_logs, _session, _faup_options, _token_map, _max_tokens ) ); From 551209b3a0c5d1b92166954037515cdc60c9baff Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 18 Jun 2021 15:30:27 +0200 Subject: [PATCH 06/54] Finished integration of DarwinPacket in session (some work is still need for the tasks) --- CMakeLists.txt | 3 ++ samples/base/ASession.cpp | 56 +++++++++++++++---------------- samples/base/ASession.hpp | 8 ++--- samples/base/DarwinPacket.cpp | 63 ++++++++++++++++++++++++++--------- samples/base/DarwinPacket.hpp | 23 +++++++++++-- samples/base/TcpSession.cpp | 4 +-- samples/base/UnixSession.cpp | 4 +-- 7 files changed, 105 insertions(+), 56 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e6cb170..0ab8af6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ set( include_directories( toolkit + toolkit/thread-pool-cpp/include/ samples/base samples/ ${HIREDIS_INCLUDE_DIRS} @@ -96,6 +97,8 @@ set( samples/base/ThreadGroup.cpp samples/base/ThreadGroup.hpp samples/base/AlertManager.cpp samples/base/AlertManager.hpp + samples/base/DarwinPacket.cpp samples/base/DarwinPacket.hpp + samples/base/AServer.cpp samples/base/AServer.hpp samples/base/UnixServer.cpp samples/base/UnixServer.hpp samples/base/TcpServer.cpp samples/base/TcpServer.hpp diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 1d0781d..6e4eb24 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -50,9 +50,9 @@ namespace darwin { std::string ASession::GetDataToSendToFilter(){ switch (GetOutputType()){ case config::output_type::RAW: - return _raw_body; + return _packet.body; case config::output_type::PARSED: - return JsonStringify(_body); + return JsonStringify(_packet.JsonBody()); case config::output_type::NONE: return ""; case config::output_type::LOG: @@ -65,20 +65,21 @@ namespace darwin { std::size_t size) { DARWIN_LOGGER; - _raw_body.clear(); + _packet.clear(); _logs.clear(); DARWIN_LOG_DEBUG("ASession::ReadHeaderCallback:: Reading header"); if (!e) { - if (size != sizeof(_header)) { + if (size != DarwinPacket::getMinimalSize()) { DARWIN_LOG_ERROR("ASession::ReadHeaderCallback:: Mismatching header size"); goto header_callback_stop_session; } - if (_header.body_size == 0) { + _packet = DarwinPacket::ParseHeader(_header_buffer); + if (_packet.parsed_body_size == 0) { ExecuteFilter(); return; } // Else the ReadBodyCallback will call ExecuteFilter - ReadBody(_header.body_size); + ReadBody(_packet.parsed_body_size); return; } @@ -97,23 +98,23 @@ namespace darwin { DARWIN_LOGGER; if (!e) { - _raw_body.append(_buffer.data(), size); + _packet.body.append(_body_buffer.data(), size); DARWIN_LOG_DEBUG("ASession::ReadBodyCallback:: Body len (" + - std::to_string(_raw_body.length()) + + std::to_string(_packet.body.length()) + ") - Header body size (" + - std::to_string(_header.body_size) + + std::to_string(_packet.parsed_body_size) + ")"); - unsigned int bodyLength = _raw_body.length(); - unsigned int totalBodyLength = _header.body_size; + size_t bodyLength = _packet.body.length(); + size_t totalBodyLength = _packet.parsed_body_size; if (bodyLength < totalBodyLength) { ReadBody(totalBodyLength - bodyLength); } else { - if (_raw_body.empty()) { + if (_packet.body.empty()) { DARWIN_LOG_WARNING("ASession::ReadBodyCallback Empty body retrieved"); this->SendErrorResponse("Error receiving body: Empty body retrieved", DARWIN_RESPONSE_CODE_REQUEST_ERROR); return; } - if (_header.body_size <= 0) { + if (_packet.parsed_body_size <= 0) { //TODO useless branch DARWIN_LOG_ERROR( "ASession::ReadBodyCallback Body is not empty, but the header appears to be invalid" ); @@ -176,11 +177,12 @@ namespace darwin { bool ASession::SendToClient(DarwinPacket& packet) noexcept { DARWIN_LOGGER; auto vec = packet.Serialize(); + DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(vec.size())); this->WriteToClient(vec); /* OLD CODE const std::size_t certitude_size = _certitudes.size(); - /* + / * * Allocate the header + * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size @@ -208,7 +210,7 @@ namespace darwin { return false; } - /* + / * * Initialisation of the structure for the padding bytes because of * missing __attribute__((packed)) in the protocol structure. * / @@ -251,7 +253,7 @@ namespace darwin { const std::size_t certitude_size = _certitudes.size(); - /* + / * * Allocate the header + * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size @@ -278,7 +280,7 @@ namespace darwin { return false; } - /* + / * * Initialisation of the structure for the padding bytes because of * missing __attribute__((packed)) in the protocol structure. * / @@ -330,7 +332,7 @@ namespace darwin { CloseFilterConnection(); } - if(_header.response == DARWIN_RESPONSE_SEND_BOTH) { + if(_packet.response == DARWIN_RESPONSE_SEND_BOTH) { // TODO re handle this after re enabling sendtofilter // this->SendToClient(); } @@ -347,10 +349,10 @@ namespace darwin { snprintf(str, 37, "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", - _header.evt_id[0], _header.evt_id[1], _header.evt_id[2], _header.evt_id[3], - _header.evt_id[4], _header.evt_id[5], _header.evt_id[6], _header.evt_id[7], - _header.evt_id[8], _header.evt_id[9], _header.evt_id[10], _header.evt_id[11], - _header.evt_id[12], _header.evt_id[13], _header.evt_id[14], _header.evt_id[15] + _packet.evt_id[0], _packet.evt_id[1], _packet.evt_id[2], _packet.evt_id[3], + _packet.evt_id[4], _packet.evt_id[5], _packet.evt_id[6], _packet.evt_id[7], + _packet.evt_id[8], _packet.evt_id[9], _packet.evt_id[10], _packet.evt_id[11], + _packet.evt_id[12], _packet.evt_id[13], _packet.evt_id[14], _packet.evt_id[15] ); std::string res(str); DARWIN_LOG_DEBUG(std::string("ASession::Evt_idToString:: UUID - ") + res); @@ -360,13 +362,11 @@ namespace darwin { void ASession::SendErrorResponse(const std::string& message, const unsigned int code) { - if (this->_header.response != DARWIN_RESPONSE_SEND_BACK && this->_header.response != DARWIN_RESPONSE_SEND_BOTH) + if (this->_packet.response != DARWIN_RESPONSE_SEND_BACK && this->_packet.response != DARWIN_RESPONSE_SEND_BOTH) return; - - // this->_response_body.clear(); - // this->_response_body += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; - DarwinPacket p; //todo better - this->SendToClient(p); + _packet.body.clear(); + _packet.body += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; + this->SendToClient(_packet); } void ASession::SetThreshold(std::size_t const& threshold) { diff --git a/samples/base/ASession.hpp b/samples/base/ASession.hpp index 3732c1a..acaf705 100755 --- a/samples/base/ASession.hpp +++ b/samples/base/ASession.hpp @@ -132,8 +132,6 @@ namespace darwin { std::string JsonStringify(rapidjson::Document &json); - bool PreParseBody(); - /// Set the async read for the header. /// /// \return -1 on error, 0 on socket closed & sizeof(header) on success. @@ -173,13 +171,11 @@ namespace darwin { // Accessible by children protected: - std::array _buffer; //!< Reading buffer for the body. + std::vector _header_buffer; //!< Reading buffer for the body. + std::array _body_buffer; //!< Reading buffer for the body. Manager& _manager; //!< The associated connection manager. Generator& _generator; //!< The Task Generator. DarwinPacket _packet; - darwin_filter_packet_t _header; //!< Header received from the session. - rapidjson::Document _body; //!< Body received from session (if any). - std::string _raw_body; //!< Body received from session (if any), that will not be parsed. std::string _logs; //!< Represents data given in the logs by the Session bool _has_next_filter; }; diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 2104130..750d450 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -1,6 +1,7 @@ #include "DarwinPacket.hpp" #include - +#include "../../toolkit/rapidjson/writer.h" +#include "../../toolkit/rapidjson/stringbuffer.h" namespace darwin { @@ -44,19 +45,14 @@ namespace darwin { return ret; } - DarwinPacket DarwinPacket::Parse(std::vector& input) { - - size_t minimum_size = sizeof(type) + sizeof(response) + sizeof(filter_code) - + sizeof(size_t /* body size */) + sizeof(evt_id) + sizeof(size_t /* certitude list size */); - + DarwinPacket DarwinPacket::ParseHeader(std::vector& input) { + size_t minimum_size = getMinimalSize(); if(input.size() <= minimum_size){ // error } DarwinPacket packet; - size_t body_size = 0, certitude_size = 0; - auto pt = input.data(); std::memcpy((void*)(&packet.type), pt, sizeof(type)); @@ -68,22 +64,34 @@ namespace darwin { std::memcpy((void*)(&packet.filter_code), pt, sizeof(filter_code)); pt += sizeof(filter_code); - std::memcpy((void*)(&body_size), pt, sizeof(body_size)); - pt += sizeof(body_size); + std::memcpy((void*)(&packet.parsed_body_size), pt, sizeof(parsed_body_size)); + pt += sizeof(parsed_body_size); std::memcpy((void*)(&packet.evt_id), pt, sizeof(evt_id)); pt += sizeof(evt_id); - std::memcpy((void*)(&certitude_size), pt, sizeof(certitude_size)); - pt += sizeof(certitude_size); + std::memcpy((void*)(&packet.parsed_certitude_size), pt, sizeof(parsed_certitude_size)); + pt += sizeof(parsed_certitude_size); + + packet.certitude_list.reserve(packet.parsed_certitude_size); + + packet.body.reserve(packet.parsed_body_size); + + return packet; + } - if (input.size() != minimum_size + body_size + certitude_size) { + + DarwinPacket DarwinPacket::Parse(std::vector& input) { + DarwinPacket packet = ParseHeader(input); + + if (input.size() != getMinimalSize() + packet.parsed_body_size + packet.parsed_certitude_size) { // error } - packet.certitude_list.reserve(certitude_size); + auto pt = input.data(); + pt += getMinimalSize(); - for(size_t i=0; i < certitude_size; i++) { + for(size_t i=0; i < packet.parsed_certitude_size; i++) { unsigned int cert = 0; std::memcpy((void*)(&cert), pt, sizeof(cert)); pt += sizeof(cert); @@ -91,8 +99,31 @@ namespace darwin { packet.certitude_list.push_back(cert); } - packet.body = std::string((char*)pt, body_size); + packet.body = std::string((char*)pt, packet.parsed_body_size); return packet; } + + void DarwinPacket::clear() { + this->type = DARWIN_PACKET_OTHER; + this->response = DARWIN_RESPONSE_SEND_NO; + this->filter_code = 0; + this->parsed_body_size = 0; + std::memset(this->evt_id, 0, sizeof(evt_id)); + this->parsed_certitude_size = 0; + certitude_list.clear(); + body.clear(); + } + + rapidjson::Document& DarwinPacket::JsonBody() { + if(this->_parsed_body == nullptr) { + this->_parsed_body = new rapidjson::Document(); + } + this->_parsed_body->Parse(this->body.c_str()); + return *(this->_parsed_body); + } + + DarwinPacket::~DarwinPacket(){ + delete this->_parsed_body; + } } \ No newline at end of file diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index f9447d1..957881b 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -2,31 +2,50 @@ #include #include +#include +#include #include "../protocol.h" +#include "../../toolkit/rapidjson/document.h" namespace darwin { struct DarwinPacket { public: DarwinPacket() = default; // todo oé oé - ~DarwinPacket() = default; + ~DarwinPacket(); enum darwin_packet_type type; //!< The type of information sent. enum darwin_filter_response_type response; //!< Whom the response will be sent to. long filter_code; //!< The unique identifier code of a filter. + size_t parsed_body_size; unsigned char evt_id[16]; //!< An array containing the event ID + size_t parsed_certitude_size; std::vector certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. std::string body; - static DarwinPacket Parse(std::vector& input); // Unsure + + static DarwinPacket Parse(std::vector& input); // Unsure std::vector Serialize() const; + + constexpr static size_t getMinimalSize() { + return sizeof(type) + sizeof(response) + sizeof(filter_code) + + sizeof(size_t /* body size */) + sizeof(evt_id) + sizeof(size_t /* certitude list size */); + }; + + static DarwinPacket ParseHeader(std::vector& input); + + void clear(); + rapidjson::Document& JsonBody(); protected: private: + // TODO replace with a better solution + // It is added as a ptr because Document, and smart pointers can't be moved but we use move semantics + rapidjson::Document* _parsed_body = nullptr; }; } \ No newline at end of file diff --git a/samples/base/TcpSession.cpp b/samples/base/TcpSession.cpp index 5dbee21..8693eed 100755 --- a/samples/base/TcpSession.cpp +++ b/samples/base/TcpSession.cpp @@ -35,7 +35,7 @@ namespace darwin { DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); boost::asio::async_read(_socket, - boost::asio::buffer(&_header, sizeof(_header)), + boost::asio::buffer(_header_buffer, DarwinPacket::getMinimalSize()), boost::bind(&TcpSession::ReadHeaderCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); @@ -45,7 +45,7 @@ namespace darwin { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); - _socket.async_read_some(boost::asio::buffer(_buffer, + _socket.async_read_some(boost::asio::buffer(_body_buffer, size), boost::bind(&TcpSession::ReadBodyCallback, this, boost::asio::placeholders::error, diff --git a/samples/base/UnixSession.cpp b/samples/base/UnixSession.cpp index d841a95..ca5e50c 100755 --- a/samples/base/UnixSession.cpp +++ b/samples/base/UnixSession.cpp @@ -45,7 +45,7 @@ namespace darwin { DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); boost::asio::async_read(_socket, - boost::asio::buffer(&_header, sizeof(_header)), + boost::asio::buffer(_header_buffer, DarwinPacket::getMinimalSize()), boost::bind(&UnixSession::ReadHeaderCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); @@ -55,7 +55,7 @@ namespace darwin { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ASession::ReadBody:: Starting to read incoming body..."); - _socket.async_read_some(boost::asio::buffer(_buffer, + _socket.async_read_some(boost::asio::buffer(_body_buffer, size), boost::bind(&UnixSession::ReadBodyCallback, this, boost::asio::placeholders::error, From 3c0be86ced7fecdb2e19626cafaad0baf8a19bb1 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 21 Jun 2021 15:49:32 +0200 Subject: [PATCH 07/54] Modified DarwinPacket to be more OOP-like State accessible by const getters only State mutable with only two methods : add Certitude and mutateBody --- samples/base/ASession.cpp | 40 +++++----- samples/base/ASession.hpp | 3 +- samples/base/ATask.cpp | 6 +- samples/base/ATask.hpp | 1 - samples/base/DarwinPacket.cpp | 141 +++++++++++++++++----------------- samples/base/DarwinPacket.hpp | 89 +++++++++++++++++---- samples/base/TcpSession.cpp | 2 +- samples/base/UnixSession.cpp | 4 +- samples/protocol.h | 10 +-- 9 files changed, 175 insertions(+), 121 deletions(-) diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 6e4eb24..1131c47 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -50,7 +50,7 @@ namespace darwin { std::string ASession::GetDataToSendToFilter(){ switch (GetOutputType()){ case config::output_type::RAW: - return _packet.body; + return _packet.GetBody(); case config::output_type::PARSED: return JsonStringify(_packet.JsonBody()); case config::output_type::NONE: @@ -74,12 +74,12 @@ namespace darwin { DARWIN_LOG_ERROR("ASession::ReadHeaderCallback:: Mismatching header size"); goto header_callback_stop_session; } - _packet = DarwinPacket::ParseHeader(_header_buffer); - if (_packet.parsed_body_size == 0) { + _packet = DarwinPacket::ParseHeader(_header); + if (_packet.GetParsedBodySize() == 0) { ExecuteFilter(); return; } // Else the ReadBodyCallback will call ExecuteFilter - ReadBody(_packet.parsed_body_size); + ReadBody(_packet.GetParsedBodySize()); return; } @@ -98,23 +98,23 @@ namespace darwin { DARWIN_LOGGER; if (!e) { - _packet.body.append(_body_buffer.data(), size); + _packet.GetMutableBody().append(_body_buffer.data(), size); DARWIN_LOG_DEBUG("ASession::ReadBodyCallback:: Body len (" + - std::to_string(_packet.body.length()) + + std::to_string(_packet.GetBody().length()) + ") - Header body size (" + - std::to_string(_packet.parsed_body_size) + + std::to_string(_packet.GetParsedBodySize()) + ")"); - size_t bodyLength = _packet.body.length(); - size_t totalBodyLength = _packet.parsed_body_size; + size_t bodyLength = _packet.GetBody().length(); + size_t totalBodyLength = _packet.GetParsedBodySize(); if (bodyLength < totalBodyLength) { ReadBody(totalBodyLength - bodyLength); } else { - if (_packet.body.empty()) { + if (_packet.GetBody().empty()) { DARWIN_LOG_WARNING("ASession::ReadBodyCallback Empty body retrieved"); this->SendErrorResponse("Error receiving body: Empty body retrieved", DARWIN_RESPONSE_CODE_REQUEST_ERROR); return; } - if (_packet.parsed_body_size <= 0) { //TODO useless branch + if (_packet.GetParsedBodySize() <= 0) { //TODO useless branch DARWIN_LOG_ERROR( "ASession::ReadBodyCallback Body is not empty, but the header appears to be invalid" ); @@ -332,7 +332,7 @@ namespace darwin { CloseFilterConnection(); } - if(_packet.response == DARWIN_RESPONSE_SEND_BOTH) { + if(_packet.GetResponse() == DARWIN_RESPONSE_SEND_BOTH) { // TODO re handle this after re enabling sendtofilter // this->SendToClient(); } @@ -345,14 +345,16 @@ namespace darwin { std::string ASession::Evt_idToString() { DARWIN_LOGGER; + + const unsigned char * evt_id = _packet.GetEventId(); char str[37] = {}; snprintf(str, 37, "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", - _packet.evt_id[0], _packet.evt_id[1], _packet.evt_id[2], _packet.evt_id[3], - _packet.evt_id[4], _packet.evt_id[5], _packet.evt_id[6], _packet.evt_id[7], - _packet.evt_id[8], _packet.evt_id[9], _packet.evt_id[10], _packet.evt_id[11], - _packet.evt_id[12], _packet.evt_id[13], _packet.evt_id[14], _packet.evt_id[15] + evt_id[0], evt_id[1], evt_id[2], evt_id[3], + evt_id[4], evt_id[5], evt_id[6], evt_id[7], + evt_id[8], evt_id[9], evt_id[10], evt_id[11], + evt_id[12], evt_id[13], evt_id[14], evt_id[15] ); std::string res(str); DARWIN_LOG_DEBUG(std::string("ASession::Evt_idToString:: UUID - ") + res); @@ -362,10 +364,10 @@ namespace darwin { void ASession::SendErrorResponse(const std::string& message, const unsigned int code) { - if (this->_packet.response != DARWIN_RESPONSE_SEND_BACK && this->_packet.response != DARWIN_RESPONSE_SEND_BOTH) + if (this->_packet.GetResponse() != DARWIN_RESPONSE_SEND_BACK && this->_packet.GetResponse() != DARWIN_RESPONSE_SEND_BOTH) return; - _packet.body.clear(); - _packet.body += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; + _packet.GetMutableBody().clear(); + _packet.GetMutableBody() += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; this->SendToClient(_packet); } diff --git a/samples/base/ASession.hpp b/samples/base/ASession.hpp index acaf705..848cbae 100755 --- a/samples/base/ASession.hpp +++ b/samples/base/ASession.hpp @@ -9,7 +9,6 @@ #include #include "config.hpp" -#include "protocol.h" #include "Generator.hpp" #include "Manager.hpp" #include "DarwinPacket.hpp" @@ -171,7 +170,7 @@ namespace darwin { // Accessible by children protected: - std::vector _header_buffer; //!< Reading buffer for the body. + darwin_filter_packet_t _header; //!< Reading Header std::array _body_buffer; //!< Reading buffer for the body. Manager& _manager; //!< The associated connection manager. Generator& _generator; //!< The Task Generator. diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp index 7254729..ff4dda2 100755 --- a/samples/base/ATask.cpp +++ b/samples/base/ATask.cpp @@ -38,7 +38,7 @@ namespace darwin { xxh::hash64_t ATask::GenerateHash() { // could be easily overridden in the children - return xxh::xxhash<64>(_packet.body); + return xxh::xxhash<64>(_packet.GetBody()); } void ATask::SaveToCache(const xxh::hash64_t &hash, const unsigned int certitude) const { @@ -75,7 +75,7 @@ namespace darwin { bool ATask::ParseBody() { DARWIN_LOGGER; try { - _body.Parse(_packet.body.c_str()); + _body.Parse(_packet.GetBody().c_str()); if (!_body.IsArray()) { DARWIN_LOG_ERROR("ATask:: ParseBody: You must provide a list"); @@ -93,7 +93,7 @@ namespace darwin { (*this)(); DarwinPacket p; - p.type = _packet.type; + //TODO to re-wire _s->SendNext(p); } diff --git a/samples/base/ATask.hpp b/samples/base/ATask.hpp index 9002207..2dd1e74 100755 --- a/samples/base/ATask.hpp +++ b/samples/base/ATask.hpp @@ -11,7 +11,6 @@ #include "ASession.fwd.hpp" #include "DarwinPacket.hpp" #include "config.hpp" -#include "protocol.h" #include "Time.hpp" namespace darwin { diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 750d450..7716f3b 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -6,6 +6,18 @@ namespace darwin { std::vector DarwinPacket::Serialize() const { + darwin_filter_packet_t header { + this->type, + this->response, + this->filter_code, + this->body.size(), + {0}, + this->certitude_list.size(), + 0 + }; + + std::copy(&this->evt_id[0], &this->evt_id[15], &header.evt_id[0]); + size_t body_size = body.size(), certitude_size = certitude_list.size(); size_t size = sizeof(type) + sizeof(response) + sizeof(filter_code) @@ -16,23 +28,8 @@ namespace darwin { auto pt = ret.data(); - std::memcpy(pt,(void*)(&type), sizeof(type)); - pt += sizeof(type); - - std::memcpy(pt,(void*)(&response), sizeof(response)); - pt += sizeof(response); - - std::memcpy(pt,(void*)(&filter_code), sizeof(filter_code)); - pt += sizeof(filter_code); - - std::memcpy(pt,(void*)(&body_size), sizeof(body_size)); - pt += sizeof(body_size); - - std::memcpy(pt,(void*)(&evt_id), sizeof(evt_id)); - pt += sizeof(evt_id); - - std::memcpy(pt,(void*)(&certitude_size), sizeof(certitude_size)); - pt += sizeof(certitude_size); + std::memcpy(pt, &header, sizeof(header) - sizeof(unsigned int)); + pt += sizeof(header) - sizeof(unsigned int); //certitudes for(auto certitude: certitude_list) { @@ -45,62 +42,20 @@ namespace darwin { return ret; } - DarwinPacket DarwinPacket::ParseHeader(std::vector& input) { - size_t minimum_size = getMinimalSize(); - if(input.size() <= minimum_size){ - // error - } - + DarwinPacket DarwinPacket::ParseHeader(darwin_filter_packet_t& input) { + DarwinPacket packet; - - auto pt = input.data(); - - std::memcpy((void*)(&packet.type), pt, sizeof(type)); - pt += sizeof(type); - - std::memcpy((void*)(&packet.response), pt, sizeof(response)); - pt += sizeof(response); - - std::memcpy((void*)(&packet.filter_code), pt, sizeof(filter_code)); - pt += sizeof(filter_code); - - std::memcpy((void*)(&packet.parsed_body_size), pt, sizeof(parsed_body_size)); - pt += sizeof(parsed_body_size); - - std::memcpy((void*)(&packet.evt_id), pt, sizeof(evt_id)); - pt += sizeof(evt_id); - - std::memcpy((void*)(&packet.parsed_certitude_size), pt, sizeof(parsed_certitude_size)); - pt += sizeof(parsed_certitude_size); - - packet.certitude_list.reserve(packet.parsed_certitude_size); - - packet.body.reserve(packet.parsed_body_size); - return packet; - } - - - DarwinPacket DarwinPacket::Parse(std::vector& input) { - DarwinPacket packet = ParseHeader(input); - - if (input.size() != getMinimalSize() + packet.parsed_body_size + packet.parsed_certitude_size) { - // error - } - - auto pt = input.data(); - pt += getMinimalSize(); - - for(size_t i=0; i < packet.parsed_certitude_size; i++) { - unsigned int cert = 0; - std::memcpy((void*)(&cert), pt, sizeof(cert)); - pt += sizeof(cert); - - packet.certitude_list.push_back(cert); - } - - packet.body = std::string((char*)pt, packet.parsed_body_size); - + packet.type = input.type; + packet.response = input.response; + packet.filter_code = input.filter_code; + packet.parsed_body_size = input.body_size; + std::memcpy(packet.evt_id, input.evt_id, sizeof(packet.evt_id)); + packet.parsed_certitude_size = input.certitude_size; + + if(packet.parsed_certitude_size > 0) + packet.AddCertitude(input.certitude_list[0]); + return packet; } @@ -126,4 +81,48 @@ namespace darwin { DarwinPacket::~DarwinPacket(){ delete this->_parsed_body; } + + enum darwin_packet_type DarwinPacket::GetType() const { + return this->type; + } + + enum darwin_filter_response_type DarwinPacket::GetResponse() const { + return this->response; + } + + long DarwinPacket::GetFilterCode() const { + return this->filter_code; + } + + const unsigned char * DarwinPacket::GetEventId() const { + return this->evt_id; + } + + size_t DarwinPacket::GetEventIdSize() const { + return sizeof(this->evt_id); + } + + size_t DarwinPacket::GetParsedCertitudeSize() const { + return this->parsed_certitude_size; + } + + size_t DarwinPacket::GetParsedBodySize() const { + return this->parsed_body_size; + } + + const std::string& DarwinPacket::GetBody() const { + return this->body; + } + + std::string& DarwinPacket::GetMutableBody() { + return this->body; + } + + const std::vector& DarwinPacket::GetCertitudeList() const { + return this->certitude_list; + } + + void DarwinPacket::AddCertitude(unsigned int certitude) { + this->certitude_list.push_back(certitude); + } } \ No newline at end of file diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 957881b..6fdd714 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -4,40 +4,84 @@ #include #include #include -#include "../protocol.h" #include "../../toolkit/rapidjson/document.h" -namespace darwin { - struct DarwinPacket { - public: - DarwinPacket() = default; // todo oé oé - ~DarwinPacket(); +#define DEFAULT_CERTITUDE_LIST_SIZE 1 - enum darwin_packet_type type; //!< The type of information sent. - enum darwin_filter_response_type response; //!< Whom the response will be sent to. - long filter_code; //!< The unique identifier code of a filter. - size_t parsed_body_size; - unsigned char evt_id[16]; //!< An array containing the event ID - size_t parsed_certitude_size; - std::vector certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. - std::string body; +/// Represent the receiver of the results. +/// +/// \enum darwin_response_type +enum darwin_filter_response_type { + DARWIN_RESPONSE_SEND_NO = 0,//!< Don't send results to anybody. + DARWIN_RESPONSE_SEND_BACK, //!< Send results back to caller. + DARWIN_RESPONSE_SEND_DARWIN, //!< Send results to the next filter. + DARWIN_RESPONSE_SEND_BOTH, //!< Send results to both caller and the next filter. +}; + +/// Represent the type of information sent. +/// +/// \enum darwin_packet_type +enum darwin_packet_type { + DARWIN_PACKET_OTHER = 0, //!< Information sent by something else. + DARWIN_PACKET_FILTER, //!< Information sent by another filter. +}; +/// First packet to be sent to a filter. +/// +/// \struct darwin_filter_packet_t +typedef struct { + enum darwin_packet_type type; //!< The type of information sent. + enum darwin_filter_response_type response; //!< Whom the response will be sent to. + long filter_code; //!< The unique identifier code of a filter. + size_t body_size; //!< The complete size of the the parameters to be sent (if needed). + unsigned char evt_id[16]; //!< An array containing the event ID + size_t certitude_size; //!< The size of the list containing the certitudes. + unsigned int certitude_list[DEFAULT_CERTITUDE_LIST_SIZE]; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. +} darwin_filter_packet_t; - static DarwinPacket Parse(std::vector& input); // Unsure +namespace darwin { + + class DarwinPacket { + public: + DarwinPacket() = default; // todo oé oé + virtual ~DarwinPacket(); std::vector Serialize() const; constexpr static size_t getMinimalSize() { return sizeof(type) + sizeof(response) + sizeof(filter_code) - + sizeof(size_t /* body size */) + sizeof(evt_id) + sizeof(size_t /* certitude list size */); + + sizeof(size_t /* body size */) + sizeof(evt_id) + sizeof(size_t /* certitude list size */) + + DEFAULT_CERTITUDE_LIST_SIZE * sizeof(unsigned int); // Protocol has at least DEFAULT (one) certitude }; - static DarwinPacket ParseHeader(std::vector& input); + static DarwinPacket ParseHeader(darwin_filter_packet_t& input); void clear(); rapidjson::Document& JsonBody(); + enum darwin_packet_type GetType() const; + + enum darwin_filter_response_type GetResponse() const; + + long GetFilterCode() const; + + const unsigned char * GetEventId() const; + + size_t GetEventIdSize() const; + + size_t GetParsedCertitudeSize() const; + + size_t GetParsedBodySize() const; + + const std::string& GetBody() const; // we may need a mutable also + + std::string& GetMutableBody(); // we may need a mutable also + + const std::vector& GetCertitudeList() const; // we may need a mutable also + + inline void AddCertitude(unsigned int certitude); + protected: @@ -47,5 +91,16 @@ namespace darwin { // It is added as a ptr because Document, and smart pointers can't be moved but we use move semantics rapidjson::Document* _parsed_body = nullptr; + enum darwin_packet_type type; //!< The type of information sent. + enum darwin_filter_response_type response; //!< Whom the response will be sent to. + long filter_code; //!< The unique identifier code of a filter. + size_t parsed_body_size; + unsigned char evt_id[16]; //!< An array containing the event ID + size_t parsed_certitude_size; + std::vector certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. + std::string body; + }; + + } \ No newline at end of file diff --git a/samples/base/TcpSession.cpp b/samples/base/TcpSession.cpp index 8693eed..f74f28d 100755 --- a/samples/base/TcpSession.cpp +++ b/samples/base/TcpSession.cpp @@ -35,7 +35,7 @@ namespace darwin { DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); boost::asio::async_read(_socket, - boost::asio::buffer(_header_buffer, DarwinPacket::getMinimalSize()), + boost::asio::buffer(&_header, sizeof(_header)), boost::bind(&TcpSession::ReadHeaderCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); diff --git a/samples/base/UnixSession.cpp b/samples/base/UnixSession.cpp index ca5e50c..426650e 100755 --- a/samples/base/UnixSession.cpp +++ b/samples/base/UnixSession.cpp @@ -45,7 +45,7 @@ namespace darwin { DARWIN_LOG_DEBUG("ASession::ReadHeader:: Starting to read incoming header..."); boost::asio::async_read(_socket, - boost::asio::buffer(_header_buffer, DarwinPacket::getMinimalSize()), + boost::asio::buffer(&_header, sizeof(_header)), boost::bind(&UnixSession::ReadHeaderCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); @@ -64,7 +64,7 @@ namespace darwin { void UnixSession::WriteToClient(std::vector& packet) { boost::asio::async_write(_socket, - boost::asio::buffer(packet, packet.size()), + boost::asio::buffer(packet), boost::bind(&UnixSession::SendToClientCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); diff --git a/samples/protocol.h b/samples/protocol.h index c0ce485..679a687 100644 --- a/samples/protocol.h +++ b/samples/protocol.h @@ -29,7 +29,7 @@ extern "C" { /// Represent the receiver of the results. /// /// \enum darwin_response_type -enum darwin_filter_response_type { +enum darwin_filter_response_type_old { DARWIN_RESPONSE_SEND_NO = 0,//!< Don't send results to anybody. DARWIN_RESPONSE_SEND_BACK, //!< Send results back to caller. DARWIN_RESPONSE_SEND_DARWIN, //!< Send results to the next filter. @@ -39,7 +39,7 @@ enum darwin_filter_response_type { /// Represent the type of information sent. /// /// \enum darwin_packet_type -enum darwin_packet_type { +enum darwin_packet_type_old { DARWIN_PACKET_OTHER = 0, //!< Information sent by something else. DARWIN_PACKET_FILTER, //!< Information sent by another filter. }; @@ -48,14 +48,14 @@ enum darwin_packet_type { /// /// \struct darwin_filter_packet_t typedef struct { - enum darwin_packet_type type; //!< The type of information sent. - enum darwin_filter_response_type response; //!< Whom the response will be sent to. + enum darwin_packet_type_old type; //!< The type of information sent. + enum darwin_filter_response_type_old response; //!< Whom the response will be sent to. long filter_code; //!< The unique identifier code of a filter. size_t body_size; //!< The complete size of the the parameters to be sent (if needed). unsigned char evt_id[16]; //!< An array containing the event ID size_t certitude_size; //!< The size of the list containing the certitudes. unsigned int certitude_list[DEFAULT_CERTITUDE_LIST_SIZE]; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. -} darwin_filter_packet_t; +} darwin_filter_packet_t_old; #ifdef __cplusplus }; From e8426bff71323ccd161b6f7d78fc6d20348c2a60 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 21 Jun 2021 23:49:34 +0200 Subject: [PATCH 08/54] Re-wired some things in the tasks Added a few methods for DarwinPacket --- samples/base/ASession.cpp | 2 +- samples/base/ATask.cpp | 13 ++--- samples/base/ATask.hpp | 5 +- samples/base/DarwinPacket.cpp | 106 ++++++++++++++++++++-------------- samples/base/DarwinPacket.hpp | 51 +++++++++++----- samples/fdga/DGATask.cpp | 16 +++-- samples/fdga/DGATask.hpp | 2 - samples/fdga/Generator.cpp | 2 +- 8 files changed, 114 insertions(+), 83 deletions(-) diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 1131c47..0167c78 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -56,7 +56,7 @@ namespace darwin { case config::output_type::NONE: return ""; case config::output_type::LOG: - return GetLogs(); + return _packet.GetLogs(); } return ""; } diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp index ff4dda2..b569aa4 100755 --- a/samples/base/ATask.cpp +++ b/samples/base/ATask.cpp @@ -18,10 +18,9 @@ namespace darwin { ATask::ATask(std::string name, std::shared_ptr> cache, std::mutex& cache_mutex, session_ptr_t s, - DarwinPacket& packet, - std::string& logs) + DarwinPacket& packet) : _filter_name(name), _cache{cache}, _cache_mutex{cache_mutex}, _s{s}, - _packet{std::move(packet)}, _logs{logs}, _threshold{_s->GetThreshold()} + _packet{std::move(packet)}, _threshold{_s->GetThreshold()} { ; } @@ -91,10 +90,10 @@ namespace darwin { void ATask::run() { (*this)(); - DarwinPacket p; - - //TODO to re-wire - _s->SendNext(p); + auto body = _packet.GetMutableBody(); + body.clear(); + body.append(_response_body); + _s->SendNext(_packet); } } \ No newline at end of file diff --git a/samples/base/ATask.hpp b/samples/base/ATask.hpp index 2dd1e74..ce7fb62 100755 --- a/samples/base/ATask.hpp +++ b/samples/base/ATask.hpp @@ -20,8 +20,7 @@ namespace darwin { ATask(std::string name, std::shared_ptr> cache, std::mutex& cache_mutex, session_ptr_t session, - DarwinPacket& packet, - std::string& logs); + DarwinPacket& packet); virtual ~ATask() = default; @@ -90,9 +89,7 @@ namespace darwin { DarwinPacket _packet; //!< Ref to the Header received from the session. rapidjson::Document _body; //!< Ref to the Body received from session (if any). - std::string _logs; //!< Ref to the data given in the logs by the Session std::string _response_body; //!< Ref to the body to send back to the client - std::vector _certitudes; //!< Ref to the Darwin results obtained. std::size_t _threshold; //!< Threshold, overriden at creation by the session diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 7716f3b..22c9700 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -5,24 +5,38 @@ namespace darwin { + DarwinPacket::DarwinPacket( + darwin_packet_type type, + darwin_filter_response_type response, + long filter_code, + unsigned char event_id[16], + size_t certitude_size, + size_t body_size) + : _type{type}, _response{response}, _filter_code{filter_code}, + _parsed_certitude_size{certitude_size}, _parsed_body_size{body_size} + { + std::copy(&event_id[0], &event_id[15], &this->_evt_id[0]); + + } + std::vector DarwinPacket::Serialize() const { darwin_filter_packet_t header { - this->type, - this->response, - this->filter_code, - this->body.size(), + this->_type, + this->_response, + this->_filter_code, + this->_body.size(), {0}, - this->certitude_list.size(), + this->_certitude_list.size(), 0 }; - std::copy(&this->evt_id[0], &this->evt_id[15], &header.evt_id[0]); + std::copy(&this->_evt_id[0], &this->_evt_id[15], &header.evt_id[0]); - size_t body_size = body.size(), certitude_size = certitude_list.size(); + size_t body_size = _body.size(), certitude_size = _certitude_list.size(); - size_t size = sizeof(type) + sizeof(response) + sizeof(filter_code) - + sizeof(body_size) + sizeof(evt_id) + sizeof(certitude_size) + certitude_list.size() - + body.size(); + size_t size = sizeof(_type) + sizeof(_response) + sizeof(_filter_code) + + sizeof(body_size) + sizeof(_evt_id) + sizeof(certitude_size) + _certitude_list.size() + + _body.size(); std::vector ret(size, 0); @@ -32,49 +46,47 @@ namespace darwin { pt += sizeof(header) - sizeof(unsigned int); //certitudes - for(auto certitude: certitude_list) { + for(auto certitude: _certitude_list) { std::memcpy(pt,(void*)(&certitude), sizeof(certitude)); pt += sizeof(certitude); } //body - std::memcpy(pt, body.data(), body.length()); + std::memcpy(pt, _body.data(), _body.length()); return ret; } DarwinPacket DarwinPacket::ParseHeader(darwin_filter_packet_t& input) { - DarwinPacket packet; - - packet.type = input.type; - packet.response = input.response; - packet.filter_code = input.filter_code; - packet.parsed_body_size = input.body_size; - std::memcpy(packet.evt_id, input.evt_id, sizeof(packet.evt_id)); - packet.parsed_certitude_size = input.certitude_size; - - if(packet.parsed_certitude_size > 0) + DarwinPacket packet{ + input.type, input.response, + input.filter_code, + input.evt_id, + input.certitude_size, input.body_size + }; + + if(packet._parsed_certitude_size > 0) packet.AddCertitude(input.certitude_list[0]); return packet; } void DarwinPacket::clear() { - this->type = DARWIN_PACKET_OTHER; - this->response = DARWIN_RESPONSE_SEND_NO; - this->filter_code = 0; - this->parsed_body_size = 0; - std::memset(this->evt_id, 0, sizeof(evt_id)); - this->parsed_certitude_size = 0; - certitude_list.clear(); - body.clear(); + this->_type = DARWIN_PACKET_OTHER; + this->_response = DARWIN_RESPONSE_SEND_NO; + this->_filter_code = 0; + this->_parsed_body_size = 0; + std::memset(this->_evt_id, 0, sizeof(_evt_id)); + this->_parsed_certitude_size = 0; + _certitude_list.clear(); + _body.clear(); } rapidjson::Document& DarwinPacket::JsonBody() { if(this->_parsed_body == nullptr) { this->_parsed_body = new rapidjson::Document(); } - this->_parsed_body->Parse(this->body.c_str()); + this->_parsed_body->Parse(this->_body.c_str()); return *(this->_parsed_body); } @@ -83,46 +95,54 @@ namespace darwin { } enum darwin_packet_type DarwinPacket::GetType() const { - return this->type; + return this->_type; } enum darwin_filter_response_type DarwinPacket::GetResponse() const { - return this->response; + return this->_response; } long DarwinPacket::GetFilterCode() const { - return this->filter_code; + return this->_filter_code; } const unsigned char * DarwinPacket::GetEventId() const { - return this->evt_id; + return this->_evt_id; } size_t DarwinPacket::GetEventIdSize() const { - return sizeof(this->evt_id); + return sizeof(this->_evt_id); } size_t DarwinPacket::GetParsedCertitudeSize() const { - return this->parsed_certitude_size; + return this->_parsed_certitude_size; } size_t DarwinPacket::GetParsedBodySize() const { - return this->parsed_body_size; + return this->_parsed_body_size; } const std::string& DarwinPacket::GetBody() const { - return this->body; + return this->_body; } std::string& DarwinPacket::GetMutableBody() { - return this->body; + return this->_body; } const std::vector& DarwinPacket::GetCertitudeList() const { - return this->certitude_list; + return this->_certitude_list; + } + + const std::string& DarwinPacket::GetLogs() const { + return this->_logs; } - void DarwinPacket::AddCertitude(unsigned int certitude) { - this->certitude_list.push_back(certitude); + std::string& DarwinPacket::GetMutableLogs() { + return this->_logs; + } + + void DarwinPacket::SetLogs(std::string& logs) { + this->_logs = logs; } } \ No newline at end of file diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 6fdd714..e59b170 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -43,14 +43,24 @@ namespace darwin { class DarwinPacket { public: - DarwinPacket() = default; // todo oé oé + DarwinPacket( + enum darwin_packet_type type, + enum darwin_filter_response_type response, + long filter_code, + unsigned char event_id[16], + size_t certitude_size, + size_t body_size); + + DarwinPacket() = default; + + virtual ~DarwinPacket(); std::vector Serialize() const; constexpr static size_t getMinimalSize() { - return sizeof(type) + sizeof(response) + sizeof(filter_code) - + sizeof(size_t /* body size */) + sizeof(evt_id) + sizeof(size_t /* certitude list size */) + return sizeof(_type) + sizeof(_response) + sizeof(_filter_code) + + sizeof(_parsed_body_size) + sizeof(_evt_id) + sizeof(_parsed_certitude_size) + DEFAULT_CERTITUDE_LIST_SIZE * sizeof(unsigned int); // Protocol has at least DEFAULT (one) certitude }; @@ -74,16 +84,23 @@ namespace darwin { size_t GetParsedBodySize() const; - const std::string& GetBody() const; // we may need a mutable also + const std::string& GetBody() const; - std::string& GetMutableBody(); // we may need a mutable also + std::string& GetMutableBody(); - const std::vector& GetCertitudeList() const; // we may need a mutable also + const std::vector& GetCertitudeList() const; - inline void AddCertitude(unsigned int certitude); + inline void AddCertitude(unsigned int certitude) { + this->_certitude_list.push_back(certitude); + }; - protected: + const std::string& GetLogs() const; + std::string& GetMutableLogs(); + + void SetLogs(std::string& logs); + + protected: private: @@ -91,14 +108,16 @@ namespace darwin { // It is added as a ptr because Document, and smart pointers can't be moved but we use move semantics rapidjson::Document* _parsed_body = nullptr; - enum darwin_packet_type type; //!< The type of information sent. - enum darwin_filter_response_type response; //!< Whom the response will be sent to. - long filter_code; //!< The unique identifier code of a filter. - size_t parsed_body_size; - unsigned char evt_id[16]; //!< An array containing the event ID - size_t parsed_certitude_size; - std::vector certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. - std::string body; + darwin_packet_type _type; //!< The type of information sent. + darwin_filter_response_type _response; //!< Whom the response will be sent to. + long _filter_code; //!< The unique identifier code of a filter. + unsigned char _evt_id[16]; //!< An array containing the event ID + size_t _parsed_certitude_size; + size_t _parsed_body_size; + std::vector _certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. + std::string _body; + + std::string _logs; }; diff --git a/samples/fdga/DGATask.cpp b/samples/fdga/DGATask.cpp index 6601455..5967ca3 100644 --- a/samples/fdga/DGATask.cpp +++ b/samples/fdga/DGATask.cpp @@ -22,7 +22,6 @@ #include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "tensorflow/core/framework/tensor.h" #include "AlertManager.hpp" @@ -30,12 +29,11 @@ DGATask::DGATask(std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens) - : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet, logs), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _max_tokens{max_tokens}, _session{session}, _token_map{token_map}, _faup_options(faup_options) { _is_cache = _cache != nullptr; } @@ -51,7 +49,7 @@ long DGATask::GetFilterCode() noexcept { void DGATask::operator()() { DARWIN_LOGGER; bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; - + auto logs = _packet.GetMutableLogs(); // Should not fail, as the Session body parser MUST check for validity ! rapidjson::GenericArray array = _body.GetArray(); for (rapidjson::Value &value : array) { @@ -76,10 +74,10 @@ void DGATask::operator()() { if (is_log) { std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("DGATask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -93,10 +91,10 @@ void DGATask::operator()() { if (is_log) { std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) { SaveToCache(hash, certitude); } @@ -105,7 +103,7 @@ void DGATask::operator()() { } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } diff --git a/samples/fdga/DGATask.hpp b/samples/fdga/DGATask.hpp index 002b587..e66ca5d 100644 --- a/samples/fdga/DGATask.hpp +++ b/samples/fdga/DGATask.hpp @@ -14,7 +14,6 @@ #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" -#include "protocol.h" #include "ATask.hpp" #include "DarwinPacket.hpp" #include "ASession.fwd.hpp" @@ -35,7 +34,6 @@ class DGATask : public darwin::ATask { std::mutex& cache_mutex, darwin::session_ptr_t s, darwin::DarwinPacket& packet, - std::string& logs, std::shared_ptr &session, faup_options_t *faup_options, std::map &token_map, const unsigned int max_tokens = 50); diff --git a/samples/fdga/Generator.cpp b/samples/fdga/Generator.cpp index d7c6d2c..7f0c2a3 100644 --- a/samples/fdga/Generator.cpp +++ b/samples/fdga/Generator.cpp @@ -173,7 +173,7 @@ bool Generator::LoadFaupOptions() { std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) noexcept { return std::static_pointer_cast( std::make_shared(_cache, _cache_mutex, s, - s->_packet, s->_logs, + s->_packet, _session, _faup_options, _token_map, _max_tokens ) ); From 6072b7ff40cd06bac557ab1db250d244ced27eda Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 22 Jun 2021 11:31:40 +0200 Subject: [PATCH 09/54] Put back structs in c header Added ParseBody in ATask::run with error handling --- samples/base/ASession.cpp | 53 ----------------------------------- samples/base/ASession.hpp | 3 ++ samples/base/ATask.cpp | 6 ++++ samples/base/DarwinPacket.hpp | 32 +-------------------- samples/protocol.h | 10 +++---- 5 files changed, 15 insertions(+), 89 deletions(-) diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 0167c78..c2e72c2 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -179,59 +179,6 @@ namespace darwin { auto vec = packet.Serialize(); DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(vec.size())); this->WriteToClient(vec); - /* OLD CODE - const std::size_t certitude_size = _certitudes.size(); - - / * - * Allocate the header + - * the size of the certitude - - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size - * / - std::size_t packet_size = 0, packet_size_wo_body = 0; - if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { - packet_size = sizeof(darwin_filter_packet_t) + - (certitude_size - DEFAULT_CERTITUDE_LIST_SIZE) * sizeof(unsigned int); - } else { - packet_size = sizeof(darwin_filter_packet_t); - } - - packet_size_wo_body = packet_size; - - if (not _response_body.empty()) { - packet_size += _response_body.size(); - } - - DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(packet_size)); - - darwin_filter_packet_t* packet = reinterpret_cast(malloc(packet_size)); - - if (packet == nullptr) { - DARWIN_LOG_CRITICAL("ASession::SendToClient: Could not create a Darwin packet"); - return false; - } - - / * - * Initialisation of the structure for the padding bytes because of - * missing __attribute__((packed)) in the protocol structure. - * / - memset(packet, 0, packet_size); - - for (std::size_t index = 0; index < certitude_size; ++index) { - packet->certitude_list[index] = _certitudes[index]; - } - - packet->type = DARWIN_PACKET_FILTER; - packet->response = _send_header.response; - packet->certitude_size = certitude_size; - packet->filter_code = _generator.GetFilterCode(); - packet->body_size = _response_body.size(); - memcpy(packet->evt_id, _send_header.evt_id, 16); - - memcpy((char*)(packet) + packet_size_wo_body, _response_body.c_str(), _response_body.size()); - - this->WriteToClient(packet, packet_size); - - free(packet);*/ return true; } diff --git a/samples/base/ASession.hpp b/samples/base/ASession.hpp index 848cbae..287a4f6 100755 --- a/samples/base/ASession.hpp +++ b/samples/base/ASession.hpp @@ -161,6 +161,9 @@ namespace darwin { virtual void SendErrorResponse(const std::string& message, const unsigned int code) final; friend std::shared_ptr Generator::CreateTask(std::shared_ptr s) noexcept; + + friend void ATask::run(); + // Not accessible by children private: std::string _filter_name; //!< name of the filter diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp index b569aa4..b21af13 100755 --- a/samples/base/ATask.cpp +++ b/samples/base/ATask.cpp @@ -89,6 +89,12 @@ namespace darwin { } void ATask::run() { + DARWIN_LOGGER; + if( ! this->ParseBody()) { + DARWIN_LOG_DEBUG("ASession::ReadBodyCallback Something went wrong while parsing the body"); + _s->SendErrorResponse("Error receiving body: Something went wrong while parsing the body", DARWIN_RESPONSE_CODE_REQUEST_ERROR); + return; + } (*this)(); auto body = _packet.GetMutableBody(); body.clear(); diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index e59b170..79bf31c 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -4,41 +4,11 @@ #include #include #include +#include "protocol.h" #include "../../toolkit/rapidjson/document.h" #define DEFAULT_CERTITUDE_LIST_SIZE 1 -/// Represent the receiver of the results. -/// -/// \enum darwin_response_type -enum darwin_filter_response_type { - DARWIN_RESPONSE_SEND_NO = 0,//!< Don't send results to anybody. - DARWIN_RESPONSE_SEND_BACK, //!< Send results back to caller. - DARWIN_RESPONSE_SEND_DARWIN, //!< Send results to the next filter. - DARWIN_RESPONSE_SEND_BOTH, //!< Send results to both caller and the next filter. -}; - -/// Represent the type of information sent. -/// -/// \enum darwin_packet_type -enum darwin_packet_type { - DARWIN_PACKET_OTHER = 0, //!< Information sent by something else. - DARWIN_PACKET_FILTER, //!< Information sent by another filter. -}; - -/// First packet to be sent to a filter. -/// -/// \struct darwin_filter_packet_t -typedef struct { - enum darwin_packet_type type; //!< The type of information sent. - enum darwin_filter_response_type response; //!< Whom the response will be sent to. - long filter_code; //!< The unique identifier code of a filter. - size_t body_size; //!< The complete size of the the parameters to be sent (if needed). - unsigned char evt_id[16]; //!< An array containing the event ID - size_t certitude_size; //!< The size of the list containing the certitudes. - unsigned int certitude_list[DEFAULT_CERTITUDE_LIST_SIZE]; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. -} darwin_filter_packet_t; - namespace darwin { class DarwinPacket { diff --git a/samples/protocol.h b/samples/protocol.h index 679a687..c0ce485 100644 --- a/samples/protocol.h +++ b/samples/protocol.h @@ -29,7 +29,7 @@ extern "C" { /// Represent the receiver of the results. /// /// \enum darwin_response_type -enum darwin_filter_response_type_old { +enum darwin_filter_response_type { DARWIN_RESPONSE_SEND_NO = 0,//!< Don't send results to anybody. DARWIN_RESPONSE_SEND_BACK, //!< Send results back to caller. DARWIN_RESPONSE_SEND_DARWIN, //!< Send results to the next filter. @@ -39,7 +39,7 @@ enum darwin_filter_response_type_old { /// Represent the type of information sent. /// /// \enum darwin_packet_type -enum darwin_packet_type_old { +enum darwin_packet_type { DARWIN_PACKET_OTHER = 0, //!< Information sent by something else. DARWIN_PACKET_FILTER, //!< Information sent by another filter. }; @@ -48,14 +48,14 @@ enum darwin_packet_type_old { /// /// \struct darwin_filter_packet_t typedef struct { - enum darwin_packet_type_old type; //!< The type of information sent. - enum darwin_filter_response_type_old response; //!< Whom the response will be sent to. + enum darwin_packet_type type; //!< The type of information sent. + enum darwin_filter_response_type response; //!< Whom the response will be sent to. long filter_code; //!< The unique identifier code of a filter. size_t body_size; //!< The complete size of the the parameters to be sent (if needed). unsigned char evt_id[16]; //!< An array containing the event ID size_t certitude_size; //!< The size of the list containing the certitudes. unsigned int certitude_list[DEFAULT_CERTITUDE_LIST_SIZE]; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. -} darwin_filter_packet_t_old; +} darwin_filter_packet_t; #ifdef __cplusplus }; From f9b8b0f96155a1a3275c5989a56fd4afbd2a8de3 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 23 Jun 2021 18:16:44 +0200 Subject: [PATCH 10/54] Adapted all compiled filters Fixed issues with threadpool instance and size issues in the ASession --- samples/base/AGenerator.cpp | 10 +++-- samples/base/AGenerator.hpp | 4 +- samples/base/ASession.cpp | 11 +---- samples/base/Core.cpp | 16 ++------ samples/base/Core.hpp | 2 - samples/base/DarwinPacket.cpp | 40 ++++++++----------- samples/base/DarwinPacket.hpp | 2 + samples/fanomaly/AnomalyTask.cpp | 28 ++++++------- samples/fanomaly/AnomalyTask.hpp | 15 +++---- samples/fanomaly/Generator.cpp | 21 +++++++--- samples/fanomaly/Generator.hpp | 11 ++--- samples/fbuffer/BufferTask.cpp | 16 ++++---- samples/fbuffer/BufferTask.hpp | 16 ++++---- samples/fbuffer/Generator.cpp | 25 ++++++++---- samples/fbuffer/Generator.hpp | 11 ++--- .../fconnection/ConnectionSupervisionTask.cpp | 27 +++++++------ .../fconnection/ConnectionSupervisionTask.hpp | 17 ++++---- samples/fconnection/Generator.cpp | 24 ++++++++--- samples/fconnection/Generator.hpp | 11 ++--- samples/fdga/Generator.cpp | 6 +++ samples/fdga/Generator.hpp | 2 +- samples/fhostlookup/Generator.cpp | 22 +++++++--- samples/fhostlookup/Generator.hpp | 12 +++--- samples/fhostlookup/HostLookupTask.cpp | 32 +++++++-------- samples/fhostlookup/HostLookupTask.hpp | 13 +++--- samples/finspection/ContentInspectionTask.cpp | 39 +++++++++--------- samples/finspection/ContentInspectionTask.hpp | 17 ++++---- samples/finspection/Generator.cpp | 23 ++++++++--- samples/finspection/Generator.hpp | 11 ++--- samples/fsession/Generator.cpp | 22 +++++++--- samples/fsession/Generator.hpp | 12 +++--- samples/fsession/SessionTask.cpp | 16 ++++---- samples/fsession/SessionTask.hpp | 15 +++---- samples/ftanomaly/Generator.cpp | 26 ++++++++---- samples/ftanomaly/Generator.hpp | 12 +++--- samples/ftanomaly/TAnomalyTask.cpp | 14 +++---- samples/ftanomaly/TAnomalyTask.hpp | 19 ++++----- samples/ftest/Generator.cpp | 23 ++++++++--- samples/ftest/Generator.hpp | 11 ++--- samples/ftest/TestTask.cpp | 24 +++++------ samples/ftest/TestTask.hpp | 13 +++--- samples/fuseragent/Generator.cpp | 23 ++++++++--- samples/fuseragent/Generator.hpp | 11 ++--- samples/fuseragent/UserAgentTask.cpp | 25 ++++++------ samples/fuseragent/UserAgentTask.hpp | 13 +++--- samples/fyara/Generator.cpp | 23 ++++++++--- samples/fyara/Generator.hpp | 11 ++--- samples/fyara/YaraTask.cpp | 34 ++++++++-------- samples/fyara/YaraTask.hpp | 12 +++--- 49 files changed, 489 insertions(+), 354 deletions(-) diff --git a/samples/base/AGenerator.cpp b/samples/base/AGenerator.cpp index a88374c..773ab3e 100644 --- a/samples/base/AGenerator.cpp +++ b/samples/base/AGenerator.cpp @@ -13,11 +13,15 @@ #include "AGenerator.hpp" #include "AlertManager.hpp" -AGenerator::AGenerator() +AGenerator::AGenerator(size_t nb_task_threads): _threadPool{GetThreadPoolOptions(nb_task_threads)} { + ; +} + +tp::ThreadPoolOptions AGenerator::GetThreadPoolOptions(size_t nb_task_threads){ auto opts = tp::ThreadPoolOptions(); - opts.setThreadCount(2); - _threadPool = tp::ThreadPool(opts); + opts.setThreadCount(nb_task_threads); + return opts; } bool AGenerator::Configure(std::string const& configFile, const std::size_t cache_size) { diff --git a/samples/base/AGenerator.hpp b/samples/base/AGenerator.hpp index 8cbeeaf..6c27b81 100644 --- a/samples/base/AGenerator.hpp +++ b/samples/base/AGenerator.hpp @@ -18,7 +18,7 @@ class AGenerator { public: - AGenerator(); + AGenerator(size_t nb_task_threads); virtual ~AGenerator() = default; @@ -81,6 +81,8 @@ class AGenerator { virtual bool ExtractCustomAlertingTags(const rapidjson::Document &configuration, std::string& tags); + static tp::ThreadPoolOptions GetThreadPoolOptions(size_t nb_task_threads); + protected: std::shared_ptr> _cache; //!< The cache for already processed request std::mutex _cache_mutex; diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index c2e72c2..010d43e 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -70,11 +70,11 @@ namespace darwin { DARWIN_LOG_DEBUG("ASession::ReadHeaderCallback:: Reading header"); if (!e) { - if (size != DarwinPacket::getMinimalSize()) { + if (size != sizeof(darwin_filter_packet_t)) { DARWIN_LOG_ERROR("ASession::ReadHeaderCallback:: Mismatching header size"); goto header_callback_stop_session; } - _packet = DarwinPacket::ParseHeader(_header); + _packet = DarwinPacket(_header); if (_packet.GetParsedBodySize() == 0) { ExecuteFilter(); return; @@ -122,13 +122,6 @@ namespace darwin { return; } - /* TODO pushed to Task::ParseBody, check if parsing and error handling is done somewhere - if (!PreParseBody()) { - DARWIN_LOG_DEBUG("ASession::ReadBodyCallback Something went wrong while parsing the body"); - this->SendErrorResponse("Error receiving body: Something went wrong while parsing the body", DARWIN_RESPONSE_CODE_REQUEST_ERROR); - return; - }*/ - ExecuteFilter(); } return; diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index 1a6a431..8a956f6 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -23,8 +23,7 @@ namespace darwin { Core::Core() : _name{}, _socketPath{}, _modConfigPath{}, _monSocketPath{}, - _pidPath{}, _nbThread{0}, - _threadpool{}, daemon{true} {} + _pidPath{}, _nbThread{0}, daemon{true} {} int Core::run() { DARWIN_LOGGER; @@ -37,7 +36,7 @@ namespace darwin { std::thread t{std::bind(&Monitor::Run, std::ref(monitor))}; SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::configuring); - Generator gen{}; + Generator gen{_nbThread}; if (not gen.Configure(_modConfigPath, _cacheSize)) { DARWIN_LOG_CRITICAL("Core:: Run:: Unable to configure the filter"); raise(SIGTERM); @@ -55,17 +54,10 @@ namespace darwin { } SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::running); - DARWIN_LOG_DEBUG("Core::run:: Creating threads..."); - // Starting from 1 because the current process will run - // the io_context too. - for (std::size_t i = 1; i < _nbThread; ++i) { - _threadpool.CreateThread( - std::bind(&AServer::Run, std::ref(server)) - ); - } + DARWIN_LOG_DEBUG("Core::run:: Launching server..."); server.Run(); DARWIN_LOG_DEBUG("Core::run:: Joining threads..."); - _threadpool.JoinAll(); + server.Clean(); } catch (const std::exception& e) { DARWIN_LOG_CRITICAL(std::string("Core::run:: Cannot open unix socket: ") + e.what()); diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 57f6aa7..9f94f19 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -89,8 +89,6 @@ namespace darwin { std::size_t _nbThread; std::size_t _cacheSize; std::size_t _threshold; - ThreadGroup _threadpool; - public: // TODO Maybe a getter is a better idea... diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 22c9700..cf0da99 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -19,6 +19,16 @@ namespace darwin { } + DarwinPacket::DarwinPacket(darwin_filter_packet_t& input) + : _type{input.type}, _response{input.response}, _filter_code{input.filter_code}, + _parsed_certitude_size{input.certitude_size}, _parsed_body_size{input.body_size} + + { + std::copy(&input.evt_id[0], &input.evt_id[15], &this->_evt_id[0]); + if(input.certitude_size > 0) + AddCertitude(input.certitude_list[0]); + } + std::vector DarwinPacket::Serialize() const { darwin_filter_packet_t header { this->_type, @@ -27,23 +37,20 @@ namespace darwin { this->_body.size(), {0}, this->_certitude_list.size(), - 0 + {0} }; - - std::copy(&this->_evt_id[0], &this->_evt_id[15], &header.evt_id[0]); - size_t body_size = _body.size(), certitude_size = _certitude_list.size(); + std::copy(&this->_evt_id[0], &this->_evt_id[15], &header.evt_id[0]); - size_t size = sizeof(_type) + sizeof(_response) + sizeof(_filter_code) - + sizeof(body_size) + sizeof(_evt_id) + sizeof(certitude_size) + _certitude_list.size() + size_t size = sizeof(header) + _certitude_list.size() + _body.size(); std::vector ret(size, 0); auto pt = ret.data(); - std::memcpy(pt, &header, sizeof(header) - sizeof(unsigned int)); - pt += sizeof(header) - sizeof(unsigned int); + std::memcpy(pt, &header, sizeof(header)); + pt += sizeof(header); //certitudes for(auto certitude: _certitude_list) { @@ -55,22 +62,7 @@ namespace darwin { std::memcpy(pt, _body.data(), _body.length()); return ret; } - - DarwinPacket DarwinPacket::ParseHeader(darwin_filter_packet_t& input) { - - DarwinPacket packet{ - input.type, input.response, - input.filter_code, - input.evt_id, - input.certitude_size, input.body_size - }; - - if(packet._parsed_certitude_size > 0) - packet.AddCertitude(input.certitude_list[0]); - - return packet; - } - + void DarwinPacket::clear() { this->_type = DARWIN_PACKET_OTHER; this->_response = DARWIN_RESPONSE_SEND_NO; diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 79bf31c..9ec6c23 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -20,6 +20,8 @@ namespace darwin { unsigned char event_id[16], size_t certitude_size, size_t body_size); + + DarwinPacket(darwin_filter_packet_t& input); DarwinPacket() = default; diff --git a/samples/fanomaly/AnomalyTask.cpp b/samples/fanomaly/AnomalyTask.cpp index c2213a2..f2e603f 100644 --- a/samples/fanomaly/AnomalyTask.cpp +++ b/samples/fanomaly/AnomalyTask.cpp @@ -14,14 +14,14 @@ #include "AnomalyTask.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" +#include "ASession.hpp" #include "AlertManager.hpp" -AnomalyTask::AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex) - : Session{"anomaly", socket, manager, cache, cache_mutex}{} +AnomalyTask::AnomalyTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet) + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet){} void AnomalyTask::operator()() { DARWIN_LOGGER; @@ -40,7 +40,7 @@ void AnomalyTask::operator()() { } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } @@ -77,7 +77,7 @@ bool AnomalyTask::Detection(){ index_anomalies = arma::find(assignments == SIZE_MAX); // index of noises find by the clustering in the _matrix if (index_anomalies.is_empty()){ DARWIN_LOG_DEBUG("AnomalyTask::Detection:: No anomalies found"); - _certitudes.push_back(0); + _packet.AddCertitude(0); return false; } no_anomalies.shed_cols(index_anomalies); // suppress noisy columns, keep only normals data @@ -94,10 +94,10 @@ bool AnomalyTask::Detection(){ alerts.insert_rows(DISTANCE, distances); STAT_MATCH_INC; - _certitudes.push_back(100); + _packet.AddCertitude(100); GenerateAlerts(_ips, index_anomalies, alerts); - if(GetOutputType() == darwin::config::output_type::LOG){ + if(_s->GetOutputType() == darwin::config::output_type::LOG){ GenerateLogs(_ips, index_anomalies, alerts); } return true; @@ -114,15 +114,15 @@ void AnomalyTask::GenerateAlerts(std::vector ips, arma::uvec index_ details += R"("icmp_nb_host": )" + std::to_string(alerts(ICMP_NB_HOST, i)) + ","; details += R"("distance": )" + std::to_string(alerts(DISTANCE, i)); details += "}"; - DARWIN_ALERT_MANAGER.Alert(ips[index_anomalies(i)], 100, Evt_idToString(), details); + DARWIN_ALERT_MANAGER.Alert(ips[index_anomalies(i)], 100, _s->Evt_idToString(), details); } } void AnomalyTask::GenerateLogs(std::vector ips, arma::uvec index_anomalies, arma::mat alerts){ - + auto logs = _packet.GetMutableLogs(); for(unsigned int i=0; iEvt_idToString(); alert_log += R"(", "time": ")" + darwin::time_utils::GetTime(); alert_log += R"(", "filter": ")" + GetFilterName(); alert_log += R"(", "anomaly": {)"; @@ -134,7 +134,7 @@ void AnomalyTask::GenerateLogs(std::vector ips, arma::uvec index_an alert_log += R"("icmp_nb_host": )" + std::to_string(alerts(ICMP_NB_HOST, i)) + ","; alert_log += R"("distance": )" + std::to_string(alerts(DISTANCE, i)); alert_log += "}}"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } }; diff --git a/samples/fanomaly/AnomalyTask.hpp b/samples/fanomaly/AnomalyTask.hpp index 177c91c..de64e9d 100644 --- a/samples/fanomaly/AnomalyTask.hpp +++ b/samples/fanomaly/AnomalyTask.hpp @@ -13,8 +13,9 @@ #include #include -#include "protocol.h" -#include "Session.hpp" +#include "DarwinPacket.hpp" +#include "ATask.hpp" +#include "ASession.fwd.hpp" #include "../../toolkit/lru_cache.hpp" @@ -27,12 +28,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class AnomalyTask: public darwin::Session { +class AnomalyTask: public darwin::ATask { public: - explicit AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex); + explicit AnomalyTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet); ~AnomalyTask() override = default; public: diff --git a/samples/fanomaly/Generator.cpp b/samples/fanomaly/Generator.cpp index 4c2dcf1..ece2ca5 100644 --- a/samples/fanomaly/Generator.cpp +++ b/samples/fanomaly/Generator.cpp @@ -12,8 +12,15 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "AnomalyTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -35,9 +42,13 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration __attribute_ return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_ANOMALY; } \ No newline at end of file diff --git a/samples/fanomaly/Generator.hpp b/samples/fanomaly/Generator.hpp index c165c47..1a6aa74 100644 --- a/samples/fanomaly/Generator.hpp +++ b/samples/fanomaly/Generator.hpp @@ -12,19 +12,20 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; + virtual long GetFilterCode() const override final; + public: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; virtual bool ConfigureAlerting(const std::string& tags) override final; - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; }; \ No newline at end of file diff --git a/samples/fbuffer/BufferTask.cpp b/samples/fbuffer/BufferTask.cpp index f3d237e..013c542 100644 --- a/samples/fbuffer/BufferTask.cpp +++ b/samples/fbuffer/BufferTask.cpp @@ -12,18 +12,18 @@ #include "../../toolkit/Validators.hpp" #include "../toolkit/rapidjson/document.h" #include "BufferTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "AlertManager.hpp" -BufferTask::BufferTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, +BufferTask::BufferTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::vector> &inputs, std::vector> &connectors) - : Session{"buffer", socket, manager, cache, cache_mutex}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _inputs_format(inputs), _connectors(connectors) { } @@ -41,11 +41,11 @@ void BufferTask::operator()() { for (rapidjson::Value &line : array) { STAT_INPUT_INC; if (ParseLine(line)) { - this->_certitudes.push_back(0); + this->_packet.AddCertitude(0); this->AddEntries(); } else { STAT_PARSE_ERROR_INC; - this->_certitudes.push_back(DARWIN_ERROR_RETURN); + this->_packet.AddCertitude(DARWIN_ERROR_RETURN); } DARWIN_LOG_DEBUG("BufferTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms"); } diff --git a/samples/fbuffer/BufferTask.hpp b/samples/fbuffer/BufferTask.hpp index 7f63b0e..3fd5400 100644 --- a/samples/fbuffer/BufferTask.hpp +++ b/samples/fbuffer/BufferTask.hpp @@ -12,12 +12,14 @@ #include "../../toolkit/RedisManager.hpp" #include "BufferThreadManager.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #define DARWIN_FILTER_BUFFER 0x62756672 // Made from: bufr +#define DARWIN_FILTER_NAME "buffer" -class BufferTask final : public darwin::Session { +class BufferTask final : public darwin::ATask { /// This class inherits from Session (see Session.hpp) /// This class handles Tasks for Buffer Filter. /// It parses the body of incoming messages and splits data into several REDIS lists depending on @@ -34,10 +36,10 @@ class BufferTask final : public darwin::Session { ///\param cache_mutex Transfered to Session constructor ///\param inputs This vector holds the name and types of data in input (in the correct order) ///\param connectors This vector holds all the Connectors needed depending on the output Filters in config file. - BufferTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, + BufferTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::vector> &inputs, std::vector> &connectors); diff --git a/samples/fbuffer/Generator.cpp b/samples/fbuffer/Generator.cpp index e86692d..2e0f584 100644 --- a/samples/fbuffer/Generator.cpp +++ b/samples/fbuffer/Generator.cpp @@ -9,9 +9,16 @@ #include "../../toolkit/lru_cache.hpp" #include "base/Logger.hpp" #include "BufferTask.hpp" +#include "ASession.hpp" #include "Generator.hpp" #include "Connectors.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags __attribute__((unused))) { return true; } @@ -246,11 +253,15 @@ std::vector> Generator::FormatRedisListVecto return vector; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Generator::CreateTask:: Creating a new task"); - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, this->_inputs, this->_outputs)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + this->_inputs, this->_outputs + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_BUFFER; } \ No newline at end of file diff --git a/samples/fbuffer/Generator.hpp b/samples/fbuffer/Generator.hpp index 2287ef4..e816001 100644 --- a/samples/fbuffer/Generator.hpp +++ b/samples/fbuffer/Generator.hpp @@ -10,7 +10,7 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "BufferThreadManager.hpp" #include "AConnector.hpp" @@ -28,7 +28,7 @@ class Generator: public AGenerator { ///\brief Unique default constructor /// DOES NOT fill the attributes members. /// They are filled by ConfigureNetworkObject LoadConfig, LoadInputs and LoadOutputs - Generator() = default; + Generator(size_t nb_task_threads); ///\brief default destructor virtual ~Generator() = default; @@ -40,9 +40,10 @@ class Generator: public AGenerator { ///\param manager Transfered to BufferTask constructor /// ///\return A shared ptr on the newly created BufferTask (in the form of a Session) - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; ///\brief Creates the Connectors depending on the _output_configs vector. /// diff --git a/samples/fconnection/ConnectionSupervisionTask.cpp b/samples/fconnection/ConnectionSupervisionTask.cpp index 78765fb..7fccbc9 100644 --- a/samples/fconnection/ConnectionSupervisionTask.cpp +++ b/samples/fconnection/ConnectionSupervisionTask.cpp @@ -13,22 +13,22 @@ #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "AlertManager.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" #include "../../toolkit/lru_cache.hpp" #include "ConnectionSupervisionTask.hpp" +#include "ASession.hpp" #include "../../toolkit/RedisManager.hpp" #include "../toolkit/rapidjson/document.h" -ConnectionSupervisionTask::ConnectionSupervisionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, - unsigned int expire) - : Session{"connection", socket, manager, cache, cache_mutex}, +ConnectionSupervisionTask::ConnectionSupervisionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + unsigned int expire) + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _redis_expire{expire}{} long ConnectionSupervisionTask::GetFilterCode() noexcept { @@ -37,7 +37,8 @@ long ConnectionSupervisionTask::GetFilterCode() noexcept { void ConnectionSupervisionTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); unsigned int certitude; // Should not fail, as the Session body parser MUST check for validity ! @@ -52,21 +53,21 @@ void ConnectionSupervisionTask::operator()() { if(certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_connection, certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_connection, certitude, _s->Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + + std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "connection": ")" + _connection + R"(", "certitude": )" + std::to_string(certitude) + "}"; - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("ConnectionSupervisionTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } diff --git a/samples/fconnection/ConnectionSupervisionTask.hpp b/samples/fconnection/ConnectionSupervisionTask.hpp index dbd545c..0337850 100644 --- a/samples/fconnection/ConnectionSupervisionTask.hpp +++ b/samples/fconnection/ConnectionSupervisionTask.hpp @@ -8,8 +8,9 @@ #pragma once #include -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/RedisManager.hpp" @@ -24,13 +25,13 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class ConnectionSupervisionTask : public darwin::Session { +class ConnectionSupervisionTask : public darwin::ATask { public: - explicit ConnectionSupervisionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, - unsigned int expire); + explicit ConnectionSupervisionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + unsigned int expire); ~ConnectionSupervisionTask() override = default; public: diff --git a/samples/fconnection/Generator.cpp b/samples/fconnection/Generator.cpp index a3e7bd6..9c6e931 100644 --- a/samples/fconnection/Generator.cpp +++ b/samples/fconnection/Generator.cpp @@ -14,8 +14,15 @@ #include "Generator.hpp" #include "base/Logger.hpp" #include "ConnectionSupervisionTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -139,11 +146,18 @@ bool Generator::ConfigRedis(const std::string &redis_socket_path, const std::str return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _redis_expire)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _redis_expire + ) + ); } +long Generator::GetFilterCode() const { + return DARWIN_FILTER_CONNECTION; +} + + Generator::~Generator() = default; diff --git a/samples/fconnection/Generator.hpp b/samples/fconnection/Generator.hpp index 7a5762a..6eba565 100644 --- a/samples/fconnection/Generator.hpp +++ b/samples/fconnection/Generator.hpp @@ -13,18 +13,19 @@ #include "../toolkit/rapidjson/document.h" #include "../../toolkit/RedisManager.hpp" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator(); public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fdga/Generator.cpp b/samples/fdga/Generator.cpp index 7f0c2a3..1c68380 100644 --- a/samples/fdga/Generator.cpp +++ b/samples/fdga/Generator.cpp @@ -18,6 +18,12 @@ #include "AlertManager.hpp" #include "tensorflow/core/framework/graph.pb.h" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; diff --git a/samples/fdga/Generator.hpp b/samples/fdga/Generator.hpp index 27b089e..e4cd74a 100644 --- a/samples/fdga/Generator.hpp +++ b/samples/fdga/Generator.hpp @@ -18,7 +18,7 @@ class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator(); public: diff --git a/samples/fhostlookup/Generator.cpp b/samples/fhostlookup/Generator.cpp index 16a3d0a..e9e3892 100644 --- a/samples/fhostlookup/Generator.cpp +++ b/samples/fhostlookup/Generator.cpp @@ -12,12 +12,18 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "HostLookupTask.hpp" +#include "ASession.hpp" #include "tsl/hopscotch_map.h" #include "tsl/hopscotch_set.h" #include "../toolkit/rapidjson/document.h" #include "../toolkit/rapidjson/istreamwrapper.h" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -224,9 +230,15 @@ bool Generator::LoadRsyslogEntry(const rapidjson::Value& entry) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _database, _feed_name)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _database, _feed_name + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_HOSTLOOKUP; } diff --git a/samples/fhostlookup/Generator.hpp b/samples/fhostlookup/Generator.hpp index a4bc1ad..d4a58bf 100644 --- a/samples/fhostlookup/Generator.hpp +++ b/samples/fhostlookup/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "HostLookupTask.hpp" #include "tsl/hopscotch_map.h" @@ -21,13 +21,13 @@ class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; -public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; protected: enum class db_type { diff --git a/samples/fhostlookup/HostLookupTask.cpp b/samples/fhostlookup/HostLookupTask.cpp index a7b0b0d..d1b6ece 100644 --- a/samples/fhostlookup/HostLookupTask.cpp +++ b/samples/fhostlookup/HostLookupTask.cpp @@ -14,18 +14,18 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "HostLookupTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" #include "AlertManager.hpp" -HostLookupTask::HostLookupTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +HostLookupTask::HostLookupTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, tsl::hopscotch_map>& db, const std::string& feed_name) - : Session{"hostlookup", socket, manager, cache, cache_mutex}, _database{db}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _database{db}, _feed_name{feed_name} { _is_cache = _cache != nullptr; } @@ -40,8 +40,8 @@ long HostLookupTask::GetFilterCode() noexcept { void HostLookupTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; - + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); // Should not fail, as the Session body parser MUST check for validity ! auto array = _body.GetArray(); @@ -58,13 +58,13 @@ void HostLookupTask::operator()() { if (GetCacheResult(hash, certitude)) { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN) { STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_host, certitude, Evt_idToString(), this->AlertDetails()); + DARWIN_ALERT_MANAGER.Alert(_host, certitude, _s->Evt_idToString(), this->AlertDetails()); if (is_log) { std::string alert_log = this->BuildAlert(_host, certitude); - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("HostLookupTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -75,24 +75,24 @@ void HostLookupTask::operator()() { certitude = DBLookup(description); if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN) { STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_host, certitude, Evt_idToString(), this->AlertDetails(description)); + DARWIN_ALERT_MANAGER.Alert(_host, certitude, _s->Evt_idToString(), this->AlertDetails(description)); if (is_log){ std::string alert_log = this->BuildAlert(_host, certitude); - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) SaveToCache(hash, certitude); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } DARWIN_LOG_DEBUG("HostLookupTask:: processed entry in " - + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_certitudes.back())); + + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_packet.GetCertitudeList().back())); } } @@ -110,7 +110,7 @@ const std::string HostLookupTask::AlertDetails(std::string const& description) { const std::string HostLookupTask::BuildAlert(const std::string& host, unsigned int certitude) { std::string alert_log = - R"({"evt_id": ")" + Evt_idToString() + + R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "entry": ")" + host + diff --git a/samples/fhostlookup/HostLookupTask.hpp b/samples/fhostlookup/HostLookupTask.hpp index 9b4a84a..d0ad572 100644 --- a/samples/fhostlookup/HostLookupTask.hpp +++ b/samples/fhostlookup/HostLookupTask.hpp @@ -10,8 +10,9 @@ #include #include "../../toolkit/lru_cache.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "tsl/hopscotch_map.h" #include "tsl/hopscotch_set.h" @@ -24,12 +25,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class HostLookupTask : public darwin::Session { +class HostLookupTask : public darwin::ATask { public: - explicit HostLookupTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit HostLookupTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, tsl::hopscotch_map>& db, const std::string& feed_name); diff --git a/samples/finspection/ContentInspectionTask.cpp b/samples/finspection/ContentInspectionTask.cpp index bfd5efa..fb7bc0f 100644 --- a/samples/finspection/ContentInspectionTask.cpp +++ b/samples/finspection/ContentInspectionTask.cpp @@ -13,15 +13,15 @@ #include "ContentInspectionTask.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" +#include "ASession.hpp" #include "AlertManager.hpp" -ContentInspectionTask::ContentInspectionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +ContentInspectionTask::ContentInspectionTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, Configurations& configurations) - : Session{"content_inspection", socket, manager, cache, cache_mutex} { + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet) { _is_cache = _cache != nullptr; _configurations = configurations; } @@ -33,7 +33,8 @@ long ContentInspectionTask::GetFilterCode() noexcept { void ContentInspectionTask::operator()() { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ContentInspectionTask:: started task"); - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); for(Packet *pkt : _packetList) { unsigned int certitude = 0; @@ -89,19 +90,19 @@ void ContentInspectionTask::operator()() { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; DARWIN_ALERT_MANAGER.SetTags(tagListJson); - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, Evt_idToString(), details); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _s->Evt_idToString(), details); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + R"(, "rules": )" + ruleListJson + R"(, "tags": )" + tagListJson + "}"; - _logs += alert_log + "\n"; + logs += alert_log + "\n"; } } } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_INFO("ContentInspectionTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); freePacket(pkt); @@ -113,36 +114,38 @@ void ContentInspectionTask::operator()() { bool ContentInspectionTask::ParseBody() { DARWIN_LOGGER; _packetList.clear(); - DARWIN_LOG_DEBUG("ContentInspectionTask:: ParseBody: _raw_body: " + _raw_body); + auto raw_body = _packet.GetBody(); + DARWIN_LOG_DEBUG("ContentInspectionTask:: ParseBody: _raw_body: " + raw_body); try { - _logs.clear(); + auto logs = _packet.GetMutableLogs(); + logs.clear(); std::size_t packetMeta = 0, packetMetaEnd; std::size_t packetData, packetDataEnd = 0; std::size_t openingBracket; do { - packetMeta = _raw_body.find("\"{", packetDataEnd + 1); + packetMeta = raw_body.find("\"{", packetDataEnd + 1); if(packetMeta == std::string::npos) { break; } STAT_INPUT_INC; - packetMetaEnd = _raw_body.find("}\",", packetMeta); + packetMetaEnd = raw_body.find("}\",", packetMeta); if(packetMetaEnd == std::string::npos) { DARWIN_LOG_WARNING("ContentInspectionTask:: parse fail 1"); STAT_PARSE_ERROR_INC; break; } - packetData = _raw_body.find("\"{", packetMetaEnd); + packetData = raw_body.find("\"{", packetMetaEnd); if(packetData == std::string::npos) { DARWIN_LOG_WARNING("ContentInspectionTask:: parse fail 2"); STAT_PARSE_ERROR_INC; break; } - packetDataEnd = _raw_body.find("}\"", packetData); + packetDataEnd = raw_body.find("}\"", packetData); if(packetDataEnd == std::string::npos) { DARWIN_LOG_WARNING("ContentInspectionTask:: parse fail 3"); STAT_PARSE_ERROR_INC; @@ -150,8 +153,8 @@ bool ContentInspectionTask::ParseBody() { } _packetList.push_back(getImpcapData( - _raw_body.substr(packetMeta + 1, packetMetaEnd - packetMeta), - _raw_body.substr(packetData + 1, packetDataEnd - packetData) + raw_body.substr(packetMeta + 1, packetMetaEnd - packetMeta), + raw_body.substr(packetData + 1, packetDataEnd - packetData) )); } while(true); diff --git a/samples/finspection/ContentInspectionTask.hpp b/samples/finspection/ContentInspectionTask.hpp index e2f636c..a95f6e8 100644 --- a/samples/finspection/ContentInspectionTask.hpp +++ b/samples/finspection/ContentInspectionTask.hpp @@ -13,8 +13,9 @@ #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/rapidjson/stringbuffer.h" #include "../../toolkit/rapidjson/writer.h" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "tcp_sessions.hpp" #include "stream_buffer.hpp" @@ -37,12 +38,12 @@ typedef struct Configurations_t { // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class ContentInspectionTask : public darwin::Session { +class ContentInspectionTask : public darwin::ATask { public: - explicit ContentInspectionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& _cache_mutex, + explicit ContentInspectionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, Configurations& configurations); ~ContentInspectionTask() override = default; @@ -67,7 +68,7 @@ class ContentInspectionTask : public darwin::Session { std::string GetJsonListFromSet(std::set &input); // Implemented but not used. - bool ParseLine(rapidjson::Value& line __attribute__((unsused))) final {return true;} + bool ParseLine(rapidjson::Value& line __attribute__((unused))) final {return true;} private: Configurations _configurations; diff --git a/samples/finspection/Generator.cpp b/samples/finspection/Generator.cpp index 55db860..bfaf57c 100644 --- a/samples/finspection/Generator.cpp +++ b/samples/finspection/Generator.cpp @@ -12,9 +12,13 @@ #include "../../toolkit/rapidjson/document.h" #include "base/Logger.hpp" #include "Generator.hpp" +#include "ContentInspectionTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" -Generator::Generator() { +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ poolStorage = initPoolStorage(); _configurations.flowCnf = (FlowCnf *)calloc(1, sizeof(FlowCnf)); _configurations.streamsCnf = (StreamsCnf *)calloc(1, sizeof(StreamsCnf)); @@ -168,13 +172,20 @@ bool Generator::LoadConfig(const rapidjson::Document &config) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _configurations)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _configurations + ) + ); } +long Generator::GetFilterCode() const { + return DARWIN_FILTER_CONTENT_INSPECTION; +} + + Generator::~Generator() { destroyTCPPools(); yaraDeleteConfig(_configurations.yaraCnf); diff --git a/samples/finspection/Generator.hpp b/samples/finspection/Generator.hpp index 1eed042..aa0b948 100644 --- a/samples/finspection/Generator.hpp +++ b/samples/finspection/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "data_pool.hpp" #include "AGenerator.hpp" #include "ContentInspectionTask.hpp" @@ -19,13 +19,14 @@ class Generator: public AGenerator { public: - Generator(); + Generator(size_t nb_task_threads); ~Generator(); public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fsession/Generator.cpp b/samples/fsession/Generator.cpp index 61d23e4..e9159e8 100644 --- a/samples/fsession/Generator.cpp +++ b/samples/fsession/Generator.cpp @@ -13,8 +13,15 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "SessionTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -53,9 +60,14 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { return redis.FindAndConnect(); } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet + ) + ); } + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_SESSION; +} \ No newline at end of file diff --git a/samples/fsession/Generator.hpp b/samples/fsession/Generator.hpp index d1c1697..0fee48b 100644 --- a/samples/fsession/Generator.hpp +++ b/samples/fsession/Generator.hpp @@ -15,20 +15,20 @@ extern "C" { #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "../../toolkit/RedisManager.hpp" #include "../toolkit/rapidjson/document.h" #include "AGenerator.hpp" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; -public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fsession/SessionTask.cpp b/samples/fsession/SessionTask.cpp index 5370ca2..a4ed9d8 100644 --- a/samples/fsession/SessionTask.cpp +++ b/samples/fsession/SessionTask.cpp @@ -23,14 +23,14 @@ extern "C" { #include "Logger.hpp" #include "Stats.hpp" #include "SessionTask.hpp" -#include "protocol.h" +#include "ASession.hpp" -SessionTask::SessionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex) - : Session{"session", socket, manager, cache, cache_mutex}{ +SessionTask::SessionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet) + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet){ } long SessionTask::GetFilterCode() noexcept { @@ -52,14 +52,14 @@ void SessionTask::operator()() { certitude = ReadFromSession(_token, _repo_ids); if(certitude) STAT_MATCH_INC; - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("SessionTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } diff --git a/samples/fsession/SessionTask.hpp b/samples/fsession/SessionTask.hpp index 0b8f20b..820590d 100644 --- a/samples/fsession/SessionTask.hpp +++ b/samples/fsession/SessionTask.hpp @@ -19,8 +19,9 @@ extern "C" { #include #include -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/RedisManager.hpp" @@ -34,12 +35,12 @@ extern "C" { // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class SessionTask : public darwin::Session { +class SessionTask : public darwin::ATask { public: - explicit SessionTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex); + explicit SessionTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet); ~SessionTask() override = default; diff --git a/samples/ftanomaly/Generator.cpp b/samples/ftanomaly/Generator.cpp index 1f1da82..441be2f 100644 --- a/samples/ftanomaly/Generator.cpp +++ b/samples/ftanomaly/Generator.cpp @@ -13,9 +13,16 @@ #include "base/Core.hpp" #include "Generator.hpp" #include "TAnomalyTask.hpp" +#include "ASession.hpp" #include "TAnomalyThreadManager.hpp" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -92,10 +99,15 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, - _anomaly_thread_manager, _redis_internal)); -} \ No newline at end of file +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _anomaly_thread_manager, _redis_internal + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_TANOMALY; +} diff --git a/samples/ftanomaly/Generator.hpp b/samples/ftanomaly/Generator.hpp index 088c891..bf4fed4 100644 --- a/samples/ftanomaly/Generator.hpp +++ b/samples/ftanomaly/Generator.hpp @@ -13,7 +13,7 @@ #include "../toolkit/rapidjson/document.h" #include "../toolkit/FileManager.hpp" -#include "Session.hpp" +#include "ATask.hpp" #include "TAnomalyThreadManager.hpp" #include "AGenerator.hpp" @@ -21,13 +21,13 @@ class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; -public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/ftanomaly/TAnomalyTask.cpp b/samples/ftanomaly/TAnomalyTask.cpp index 42bd2d2..d8f9087 100644 --- a/samples/ftanomaly/TAnomalyTask.cpp +++ b/samples/ftanomaly/TAnomalyTask.cpp @@ -13,17 +13,17 @@ #include "../../toolkit/RedisManager.hpp" #include "../../toolkit/lru_cache.hpp" #include "TAnomalyTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" #include "Stats.hpp" -#include "protocol.h" -AnomalyTask::AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +AnomalyTask::AnomalyTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr vat, std::string redis_list_name) - : Session{"tanomaly", socket, manager, cache, cache_mutex}, _redis_list_name{std::move(redis_list_name)}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _redis_list_name{std::move(redis_list_name)}, _anomaly_thread_manager{std::move(vat)} { } @@ -43,11 +43,11 @@ void AnomalyTask::operator()() { STAT_INPUT_INC; if(ParseLine(line)) { REDISAddEntry(); - _certitudes.push_back(0); + _packet.AddCertitude(0); } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } diff --git a/samples/ftanomaly/TAnomalyTask.hpp b/samples/ftanomaly/TAnomalyTask.hpp index e257a70..a5b16f4 100644 --- a/samples/ftanomaly/TAnomalyTask.hpp +++ b/samples/ftanomaly/TAnomalyTask.hpp @@ -8,8 +8,9 @@ #pragma once #include -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "TAnomalyThreadManager.hpp" #include "../../toolkit/RedisManager.hpp" @@ -25,14 +26,14 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class AnomalyTask: public darwin::Session { +class AnomalyTask: public darwin::ATask { public: - explicit AnomalyTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, - std::shared_ptr vat, - std::string redis_list_name); + explicit AnomalyTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + std::shared_ptr vat, + std::string redis_list_name); ~AnomalyTask() override = default; public: diff --git a/samples/ftest/Generator.cpp b/samples/ftest/Generator.cpp index 191b949..171e360 100644 --- a/samples/ftest/Generator.cpp +++ b/samples/ftest/Generator.cpp @@ -12,10 +12,17 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "TestTask.hpp" +#include "ASession.hpp" #include "tsl/hopscotch_map.h" #include "tsl/hopscotch_set.h" #include "AlertManager.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::ConfigureAlerting(const std::string& tags) { DARWIN_LOGGER; @@ -79,9 +86,15 @@ bool Generator::ConfigRedis(std::string redis_socket_path) { return redis.FindAndConnect(); } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _redis_list_name, _redis_channel_name)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _redis_list_name, _redis_channel_name + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_TEST; } diff --git a/samples/ftest/Generator.hpp b/samples/ftest/Generator.hpp index 46dd6f6..8b58fa6 100644 --- a/samples/ftest/Generator.hpp +++ b/samples/ftest/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "../toolkit/Files.hpp" #include "../toolkit/rapidjson/document.h" @@ -19,13 +19,14 @@ class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/ftest/TestTask.cpp b/samples/ftest/TestTask.cpp index ee128d8..83f1799 100644 --- a/samples/ftest/TestTask.cpp +++ b/samples/ftest/TestTask.cpp @@ -13,17 +13,17 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "TestTask.hpp" +#include "ASession.hpp" #include "Logger.hpp" -#include "protocol.h" #include "AlertManager.hpp" -TestTask::TestTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +TestTask::TestTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::string& redis_list, std::string& redis_channel) - : Session{"test", socket, manager, cache, cache_mutex}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _redis_list{redis_list}, _redis_channel{redis_channel} { _is_cache = _cache != nullptr; @@ -51,29 +51,29 @@ void TestTask::operator()() { if(_line == "trigger_redis_list") { DARWIN_LOG_DEBUG("TestTask:: triggered redis list action"); if(REDISAddList(_redis_list, _line)) { - _certitudes.push_back(0); + _packet.AddCertitude(0); } else { - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } else if(_line == "trigger_redis_channel") { DARWIN_LOG_DEBUG("TestTask:: triggered redis channel action"); if(REDISPublishChannel(_redis_channel, _line)) { - _certitudes.push_back(0); + _packet.AddCertitude(0); } else { - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } else { DARWIN_LOG_DEBUG("TestTask:: not triggered specific action, generating alert by default"); - DARWIN_ALERT_MANAGER.Alert(_line, 100, Evt_idToString()); - _certitudes.push_back(0); + DARWIN_ALERT_MANAGER.Alert(_line, 100, _s->Evt_idToString()); + _packet.AddCertitude(0); } } else { - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } } } diff --git a/samples/ftest/TestTask.hpp b/samples/ftest/TestTask.hpp index aa8a140..62c80a9 100644 --- a/samples/ftest/TestTask.hpp +++ b/samples/ftest/TestTask.hpp @@ -10,8 +10,9 @@ #include #include "../../toolkit/lru_cache.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #define DARWIN_FILTER_TEST 0x74657374 #define DARWIN_FILTER_NAME "test" @@ -22,12 +23,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class TestTask : public darwin::Session { +class TestTask : public darwin::ATask { public: - explicit TestTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit TestTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::string& list, std::string& channel); diff --git a/samples/fuseragent/Generator.cpp b/samples/fuseragent/Generator.cpp index 266b83c..3580613 100644 --- a/samples/fuseragent/Generator.cpp +++ b/samples/fuseragent/Generator.cpp @@ -13,6 +13,13 @@ #include "Generator.hpp" #include "tensorflow/core/framework/graph.pb.h" #include "UserAgentTask.hpp" +#include "ASession.hpp" + +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} bool Generator::LoadConfig(const rapidjson::Document &configuration) { DARWIN_LOGGER; @@ -112,11 +119,17 @@ bool Generator::LoadModel(const std::string &model_path) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _session, _token_map, _max_tokens)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _session, _token_map, _max_tokens + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_USER_AGENT; } Generator::~Generator() { diff --git a/samples/fuseragent/Generator.hpp b/samples/fuseragent/Generator.hpp index 70a8732..0f9335c 100644 --- a/samples/fuseragent/Generator.hpp +++ b/samples/fuseragent/Generator.hpp @@ -11,21 +11,22 @@ #include #include "../toolkit/rapidjson/document.h" -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "tensorflow/core/public/session.h" class Generator: public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator(); public: static constexpr int DEFAULT_MAX_TOKENS = 50; - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fuseragent/UserAgentTask.cpp b/samples/fuseragent/UserAgentTask.cpp index 4595cc9..260055e 100644 --- a/samples/fuseragent/UserAgentTask.cpp +++ b/samples/fuseragent/UserAgentTask.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -16,20 +15,20 @@ #include "../../toolkit/xxhash.hpp" #include "../toolkit/rapidjson/document.h" #include "Logger.hpp" -#include "protocol.h" -#include "tensorflow/core/framework/tensor.h" #include "UserAgentTask.hpp" +#include "ASession.hpp" +#include "tensorflow/core/framework/tensor.h" const std::vector UserAgentTask::USER_AGENT_CLASSES({"Desktop", "Tool", "Libraries", "Good bot", "Bad bot", "Mail", "IOT", "Mobile"}); -UserAgentTask::UserAgentTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +UserAgentTask::UserAgentTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr &session, std::map &token_map, const unsigned int max_tokens) - : Session{"user_agent", socket, manager, cache, cache_mutex}, _session{session}, _max_tokens{max_tokens}, _token_map{token_map} { + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _session{session}, _max_tokens{max_tokens}, _token_map{token_map} { _is_cache = _cache != nullptr; } @@ -44,7 +43,7 @@ long UserAgentTask::GetFilterCode() noexcept { void UserAgentTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; for (const std::string &user_agent : _user_agents) { SetStartingTime(); @@ -61,10 +60,10 @@ void UserAgentTask::operator()() { if (GetCacheResult(hash, certitude)) { if (is_log && (certitude>=_threshold)){ - _logs += R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + _logs += R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"user_agent\": \"" + user_agent + "\", \"ua_classification\": " + std::to_string(certitude) + "}\n"; } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("UserAgentTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -73,10 +72,10 @@ void UserAgentTask::operator()() { certitude = Predict(user_agent); if (is_log && (certitude>=_threshold)){ - _logs += R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + _logs += R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"user_agent\": \"" + user_agent + "\", \"ua_classification\": " + std::to_string(certitude) + "}\n"; } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) { SaveToCache(hash, certitude); @@ -183,7 +182,7 @@ bool UserAgentTask::ParseBody() { try { rapidjson::Document document; - document.Parse(body.c_str()); + document.Parse(_packet.GetBody().c_str()); if (!document.IsArray()) { DARWIN_LOG_ERROR("UserAgentTask:: ParseBody: You must provide a list"); diff --git a/samples/fuseragent/UserAgentTask.hpp b/samples/fuseragent/UserAgentTask.hpp index 69434ef..a0ba424 100644 --- a/samples/fuseragent/UserAgentTask.hpp +++ b/samples/fuseragent/UserAgentTask.hpp @@ -13,8 +13,9 @@ #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "tensorflow/core/public/session.h" #define DARWIN_FILTER_USER_AGENT 0x75736572 @@ -23,12 +24,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class UserAgentTask : public darwin::Session { +class UserAgentTask : public darwin::ATask { public: - explicit UserAgentTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit UserAgentTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr &session, std::map &token_map, const unsigned int max_tokens = 50); ~UserAgentTask() override; diff --git a/samples/fyara/Generator.cpp b/samples/fyara/Generator.cpp index a80f047..9c1f5b8 100644 --- a/samples/fyara/Generator.cpp +++ b/samples/fyara/Generator.cpp @@ -11,8 +11,15 @@ #include "../toolkit/lru_cache.hpp" #include "base/Logger.hpp" #include "YaraTask.hpp" +#include "ASession.hpp" #include "Generator.hpp" +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads) +{ + +} + bool Generator::LoadConfig(const rapidjson::Document &configuration) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("Yara:: Generator:: Loading configuration..."); @@ -114,9 +121,15 @@ bool Generator::ConfigureAlerting(const std::string& tags) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, _yaraCompiler->GetEngine(_fastmode, _timeout))); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + _yaraCompiler->GetEngine(_fastmode, _timeout) + ) + ); +} + +long Generator::GetFilterCode() const { + return DARWIN_FILTER_YARA_SCAN; } diff --git a/samples/fyara/Generator.hpp b/samples/fyara/Generator.hpp index 8ede14b..4a934bc 100644 --- a/samples/fyara/Generator.hpp +++ b/samples/fyara/Generator.hpp @@ -11,7 +11,7 @@ #include #include -#include "Session.hpp" +#include "ATask.hpp" #include "AGenerator.hpp" #include "AlertManager.hpp" #include "../../toolkit/rapidjson/document.h" @@ -19,13 +19,14 @@ class Generator : public AGenerator { public: - Generator() = default; + Generator(size_t nb_task_threads); ~Generator() = default; public: - darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const override final; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fyara/YaraTask.cpp b/samples/fyara/YaraTask.cpp index c3b22c2..d0d992c 100644 --- a/samples/fyara/YaraTask.cpp +++ b/samples/fyara/YaraTask.cpp @@ -11,17 +11,18 @@ #include #include "YaraTask.hpp" +#include "ASession.hpp" #include "Stats.hpp" #include "Logger.hpp" #include "AlertManager.hpp" -YaraTask::YaraTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, +YaraTask::YaraTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr yaraEngine) - : Session{"yara", socket, manager, cache, cache_mutex}, + : ATask(DARWIN_FILTER_NAME, cache, cache_mutex, s, packet), _yaraEngine{yaraEngine} { _is_cache = _cache != nullptr; } @@ -36,7 +37,8 @@ long YaraTask::GetFilterCode() noexcept { void YaraTask::operator()() { DARWIN_LOGGER; - bool is_log = GetOutputType() == darwin::config::output_type::LOG; + bool is_log = _s->GetOutputType() == darwin::config::output_type::LOG; + auto logs = _packet.GetMutableLogs(); // Should not fail, as the Session body parser MUST check for validity ! auto array = _body.GetArray(); @@ -57,16 +59,16 @@ void YaraTask::operator()() { if (certitude>=_threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _s->Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + "}\n"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); DARWIN_LOG_DEBUG("YaraTask:: processed entry in " + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(certitude)); continue; @@ -78,7 +80,7 @@ void YaraTask::operator()() { if(_yaraEngine->ScanData(data, certitude) == -1) { DARWIN_LOG_WARNING("YaraTask:: error while scanning, ignoring chunk"); - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); continue; } @@ -90,16 +92,16 @@ void YaraTask::operator()() { std::string tagListJson = YaraTask::GetJsonListFromSet(results.tags); std::string details = "{\"rules\": " + ruleListJson + "}"; - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, Evt_idToString(), details, tagListJson); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _s->Evt_idToString(), details, tagListJson); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + R"(, "rules": )" + ruleListJson + R"(, "tags": )" + tagListJson + "}\n"; - _logs += alert_log + '\n'; + logs += alert_log + '\n'; } } - _certitudes.push_back(certitude); + _packet.AddCertitude(certitude); if (_is_cache) { SaveToCache(hash, certitude); @@ -107,11 +109,11 @@ void YaraTask::operator()() { } else { STAT_PARSE_ERROR_INC; - _certitudes.push_back(DARWIN_ERROR_RETURN); + _packet.AddCertitude(DARWIN_ERROR_RETURN); } DARWIN_LOG_DEBUG("YaraTask:: processed entry in " - + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_certitudes.back())); + + std::to_string(GetDurationMs()) + "ms, certitude: " + std::to_string(_packet.GetCertitudeList().back())); } } diff --git a/samples/fyara/YaraTask.hpp b/samples/fyara/YaraTask.hpp index 9ccdae7..0898312 100644 --- a/samples/fyara/YaraTask.hpp +++ b/samples/fyara/YaraTask.hpp @@ -18,7 +18,9 @@ #include "Encoders.h" #include "Yara.hpp" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #define DARWIN_FILTER_YARA_SCAN 0x79617261 #define DARWIN_FILTER_NAME "yara" @@ -29,12 +31,12 @@ // The code bellow show all what's necessary to have a working task. // For more information about Tasks, please refer to the class definition. -class YaraTask : public darwin::Session { +class YaraTask : public darwin::ATask { public: - explicit YaraTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, + explicit YaraTask(std::shared_ptr> cache, std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, std::shared_ptr yaraEngine); ~YaraTask() override = default; From de6f710014a3e8ff5fe56fac7fa6e4823cf97dab Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 25 Jun 2021 10:10:47 +0200 Subject: [PATCH 11/54] Added parsing of tcp/udp Fixed issues in the serialization Added support for ipv6 in Tcp sessions --- samples/base/Core.cpp | 68 ++++++++++++++++++++++++++++++++--- samples/base/Core.hpp | 13 +++++++ samples/base/DarwinPacket.cpp | 13 ++++--- samples/base/TcpServer.cpp | 2 +- samples/base/TcpSession.cpp | 2 +- toolkit/StringUtils.cpp | 22 ++++++++++++ toolkit/StringUtils.hpp | 11 ++++++ 7 files changed, 120 insertions(+), 11 deletions(-) diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index 8a956f6..33dcc2b 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -16,8 +16,10 @@ #include "Generator.hpp" #include "Logger.hpp" #include "UnixServer.hpp" +#include "TcpServer.hpp" #include "Core.hpp" #include "Stats.hpp" +#include "StringUtils.hpp" namespace darwin { @@ -31,6 +33,8 @@ namespace darwin { SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::starting); + std::unique_ptr server; + try { Monitor monitor{_monSocketPath}; std::thread t{std::bind(&Monitor::Run, std::ref(monitor))}; @@ -45,9 +49,27 @@ namespace darwin { } DARWIN_LOG_DEBUG("Core::run:: Configured generator"); + switch(_net_type) { + case NetworkSocketType::Unix: + DARWIN_LOG_DEBUG("Core::run:: Unix socket configured on path " + _socketPath); + server = std::make_unique(_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen); + break; + case NetworkSocketType::Tcp: + DARWIN_LOG_DEBUG("Core::run:: TCP configured on port " + std::to_string(_net_port)); + server = std::make_unique(_net_port, 0 /* Next filter port */, _nextFilterUnixSocketPath, _threshold, gen); + break; + case NetworkSocketType::Udp: + //not yet implemented + DARWIN_LOG_CRITICAL("Core:: Run:: UDP Not implemented"); + default: + DARWIN_LOG_CRITICAL("Core:: Run:: Unable to configure the filter"); + raise(SIGTERM); + t.join(); + return 1; + } + try { - UnixServer server{_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen}; - if (not gen.ConfigureNetworkObject(server.GetIOContext())) { + if (not gen.ConfigureNetworkObject(server->GetIOContext())) { raise(SIGTERM); t.join(); return 1; @@ -55,10 +77,10 @@ namespace darwin { SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::running); DARWIN_LOG_DEBUG("Core::run:: Launching server..."); - server.Run(); + server->Run(); DARWIN_LOG_DEBUG("Core::run:: Joining threads..."); - server.Clean(); + server->Clean(); } catch (const std::exception& e) { DARWIN_LOG_CRITICAL(std::string("Core::run:: Cannot open unix socket: ") + e.what()); ret = 1; @@ -135,7 +157,8 @@ namespace darwin { // MANDATORY ARGUMENTS log.setName(av[optind]); _name = av[optind]; - _socketPath = av[optind + 1]; + if (! this->ParseSocketAddress(std::string(av[optind + 1]))) + return false; _modConfigPath = av[optind + 2]; _monSocketPath = av[optind + 3]; _pidPath = av[optind + 4]; @@ -239,4 +262,39 @@ namespace darwin { return this->_name; } + bool Core::ParsePort(const char* pathOrAddress) { + DARWIN_LOGGER; + + long port = 0; + if(!darwin::strings::StrToLong(pathOrAddress, port)){ + DARWIN_LOG_ERROR("Core::ParseSocketAddress:: Error while parsing the port number, unrecognized number: '" + std::string(pathOrAddress) + "'"); + return false; + } + + if (port < 0 || port > 65353) { + DARWIN_LOG_ERROR("Core::ParseSocketAddress:: Error while parsing the port number : out of bounds [0; 65353]: '" + std::string(pathOrAddress) + "'"); + return false; + } + + _net_port = static_cast(port); + + return true; + } + + bool Core::ParseSocketAddress(const std::string& pathOrAddress) { + if(pathOrAddress.find("tcp:", 0) == 0){ + _net_type = NetworkSocketType::Tcp; + return ParsePort(&pathOrAddress.c_str()[4]); + + } else if (pathOrAddress.find("udp:", 0) == 0) { + _net_type = NetworkSocketType::Udp; + return ParsePort(&pathOrAddress.c_str()[4]); + + } else { + _net_type = NetworkSocketType::Unix; + _socketPath = pathOrAddress; + return true; + } + } + } diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 9f94f19..67f6011 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -19,6 +19,12 @@ /// \namespace darwin namespace darwin { + enum NetworkSocketType { + Unix, + Tcp, + Udp + }; + /// Singleton. The main class of the program. /// /// \class Core @@ -26,6 +32,10 @@ namespace darwin { private: Core(); + bool ParseSocketAddress(const std::string& pathOrAddress); + + bool ParsePort(const char* pathOrAddress); + public: ~Core() = default; @@ -90,6 +100,9 @@ namespace darwin { std::size_t _cacheSize; std::size_t _threshold; + enum NetworkSocketType _net_type; + int _net_port; + public: // TODO Maybe a getter is a better idea... bool daemon; diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index cf0da99..94488d2 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -42,15 +42,20 @@ namespace darwin { std::copy(&this->_evt_id[0], &this->_evt_id[15], &header.evt_id[0]); - size_t size = sizeof(header) + _certitude_list.size() - + _body.size(); + size_t size = sizeof(header) + _body.size(); + + if( ! _certitude_list.empty()) { + // From the packet format, at least one certitude is always allocated + size += (_certitude_list.size() - 1) * sizeof(unsigned int); + } + std::vector ret(size, 0); auto pt = ret.data(); - std::memcpy(pt, &header, sizeof(header)); - pt += sizeof(header); + std::memcpy(pt, &header, sizeof(header) - sizeof(unsigned int)); + pt += sizeof(header) - sizeof(unsigned int); //certitudes for(auto certitude: _certitude_list) { diff --git a/samples/base/TcpServer.cpp b/samples/base/TcpServer.cpp index 67b679b..7b29af3 100755 --- a/samples/base/TcpServer.cpp +++ b/samples/base/TcpServer.cpp @@ -24,7 +24,7 @@ namespace darwin { Generator& generator) : AServer(output, threshold, generator), _port_nb{port_nb}, _port_nb_next{next_filter_port}, - _acceptor{_io_context, boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), port_nb)}, + _acceptor{_io_context, boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v6(), port_nb)}, _new_connection{_io_context} { this->InitSignalsAndStart(); diff --git a/samples/base/TcpSession.cpp b/samples/base/TcpSession.cpp index f74f28d..3b808df 100755 --- a/samples/base/TcpSession.cpp +++ b/samples/base/TcpSession.cpp @@ -83,7 +83,7 @@ namespace darwin { std::to_string(_next_filter_port)); try { _filter_socket.connect( - boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), _next_filter_port)); + boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v6(), _next_filter_port)); _connected = true; } catch (std::exception const& e) { DARWIN_LOG_ERROR(std::string("TcpSession::SendToFilter:: " diff --git a/toolkit/StringUtils.cpp b/toolkit/StringUtils.cpp index 6f8b4e2..4bf947d 100644 --- a/toolkit/StringUtils.cpp +++ b/toolkit/StringUtils.cpp @@ -6,6 +6,8 @@ /// \brief Copyright (c) 2020 Advens. All rights reserved. #include +#include +#include #include "StringUtils.hpp" @@ -19,3 +21,23 @@ std::vector darwin::strings::SplitString(const std::string& source, res.emplace_back(str); return res; } + +bool darwin::strings::StrToLong(const char * str, long& out) { + char *end; + long l; + errno = 0; + l = strtol(str, &end, 10); + + if ( + ((errno == ERANGE && l == LONG_MAX) || l > INT_MAX) + || ((errno == ERANGE && l == LONG_MIN) || l < INT_MIN) + || (*str == '\0' || *end != '\0') + ) + { + // Overflow, underflow or Error while parsing a number + return false; + } + + out = l; + return true; +} diff --git a/toolkit/StringUtils.hpp b/toolkit/StringUtils.hpp index 93b2fa3..8e615a7 100644 --- a/toolkit/StringUtils.hpp +++ b/toolkit/StringUtils.hpp @@ -19,5 +19,16 @@ namespace darwin { /// ///\return The newly created vector of strings std::vector SplitString(const std::string& source, char delim); + + ///\brief Util to parse a 'long' from a char array, it returns true if it works, + /// false if it couldn't parse the number, 'str' must be null terminated just after the number : + /// -> "70\0" -> true + /// -> "70Hi\0" -> false, 'out' is not modified + /// + ///\param str source to parse (must be null terminated, it will not be checked) + ///\param out a reference to a 'long' which will be set by the function + /// + ///\return true if the parsing works, false otherwise + bool StrToLong(const char * str, long& out); } // namespace strings } // namespace darwin From d957e0100b6ab697d6d0e933d004a83ed7745abf Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 25 Jun 2021 18:14:51 +0200 Subject: [PATCH 12/54] Adapted TcpServer and Config for ip addresses Adapted python manager to handle tcp sockets --- manager/Services.py | 14 +++++---- manager/config.py | 16 +++++++++++ samples/base/Core.cpp | 58 ++++++++++++++++++++++++++++---------- samples/base/Core.hpp | 11 +++++--- samples/base/TcpServer.cpp | 8 ++++-- samples/base/TcpServer.hpp | 4 ++- samples/base/main.cpp | 2 +- 7 files changed, 84 insertions(+), 29 deletions(-) diff --git a/manager/Services.py b/manager/Services.py index 1fe01dc..516e0f4 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -50,9 +50,10 @@ def start_all(self): self.stop_one(filter, no_lock=True) self.clean_one(filter, no_lock=True) else: - logger.debug("Linking UNIX sockets...") - filter['status'] = psutil.STATUS_RUNNING - call(['ln', '-s', filter['socket'], filter['socket_link']]) + if filter['network']['socket_type'] == 'UNIX': + logger.debug("Linking UNIX sockets...") + filter['status'] = psutil.STATUS_RUNNING + call(['ln', '-s', filter['socket'], filter['socket_link']]) def rotate_logs_all(self): """ @@ -99,10 +100,13 @@ def _build_cmd(filt): cmd.append(filt['log_level']) except KeyError: pass - + + if filt['network']['socket_type'] == 'UDP': + cmd.append('-u') + cmd += [ filt['name'], - filt['socket'], + filt['network']['address_path'], filt['config_file'], filt['monitoring'], filt['pid_file'], diff --git a/manager/config.py b/manager/config.py index 405f6cd..a29c324 100644 --- a/manager/config.py +++ b/manager/config.py @@ -193,6 +193,17 @@ def set_defaults(validator, properties, instance, schema): "threshold": { "type": "integer", "default": 100 + }, + "network": { + "type": "object", + "properties": { + "socket_type":{ + "type":"string", + "enum": ["UNIX", "TCP", "UDP"], + "default":"UNIX" + }, + "address_path": {"type": "string"} + } } }, "required": ["name", "exec_path", "config_file"], @@ -280,6 +291,11 @@ def complete_filters_conf(prefix, suffix): filter['socket'] = '{prefix}/sockets{suffix}/{filter}{extension}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension']) filter['socket_link'] = '{prefix}/sockets{suffix}/{filter}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name']) + if 'network' not in filter: + filter['network'] = { "socket_type":"UNIX", "address_path":filter['socket'] } + elif filter['network']['socket_type'] == "UNIX": + filter['network']['address_path'] = filter['socket'] + filter['monitoring'] = '{prefix}/sockets{suffix}/{filter}_mon{extension}.sock'.format( prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension'] diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index 33dcc2b..bfd47e8 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -21,11 +21,13 @@ #include "Stats.hpp" #include "StringUtils.hpp" +#include + namespace darwin { Core::Core() - : _name{}, _socketPath{}, _modConfigPath{}, _monSocketPath{}, - _pidPath{}, _nbThread{0}, daemon{true} {} + : _name{}, _modConfigPath{}, _monSocketPath{}, + _pidPath{}, _nbThread{0}, _socketPath{}, daemon{true} {} int Core::run() { DARWIN_LOGGER; @@ -55,14 +57,16 @@ namespace darwin { server = std::make_unique(_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen); break; case NetworkSocketType::Tcp: - DARWIN_LOG_DEBUG("Core::run:: TCP configured on port " + std::to_string(_net_port)); - server = std::make_unique(_net_port, 0 /* Next filter port */, _nextFilterUnixSocketPath, _threshold, gen); + DARWIN_LOG_DEBUG("Core::run:: TCP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); + server = std::make_unique(_net_address, _net_port, 0 /* Next filter port */, _nextFilterUnixSocketPath, _threshold, gen); break; case NetworkSocketType::Udp: //not yet implemented + DARWIN_LOG_DEBUG("Core::run:: UDP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); DARWIN_LOG_CRITICAL("Core:: Run:: UDP Not implemented"); + __attribute__((fallthrough)); default: - DARWIN_LOG_CRITICAL("Core:: Run:: Unable to configure the filter"); + DARWIN_LOG_CRITICAL("Core:: Run:: Network Configuration problem"); raise(SIGTERM); t.join(); return 1; @@ -115,10 +119,12 @@ namespace darwin { int opt; std::string log_level; + bool is_udp = false; + // OPTIONS log.setLevel(logger::Warning); // Log level by default opt = -1; - while((opt = getopt(ac, av, ":l:h")) != -1) + while((opt = getopt(ac, av, ":l:hu")) != -1) { DARWIN_LOG_DEBUG("OPT : " + std::to_string(opt)); DARWIN_LOG_DEBUG("OPTIND : " + std::to_string(optind)); @@ -144,6 +150,9 @@ namespace darwin { DARWIN_LOG_ERROR("Core:: Program Arguments:: Unknown option"); Core::Usage(); return false; + case 'u': + is_udp = true; + break; } } @@ -157,7 +166,7 @@ namespace darwin { // MANDATORY ARGUMENTS log.setName(av[optind]); _name = av[optind]; - if (! this->ParseSocketAddress(std::string(av[optind + 1]))) + if (! this->ParseSocketAddress(std::string(av[optind + 1]), is_udp)) return false; _modConfigPath = av[optind + 2]; _monSocketPath = av[optind + 3]; @@ -281,20 +290,39 @@ namespace darwin { return true; } - bool Core::ParseSocketAddress(const std::string& pathOrAddress) { - if(pathOrAddress.find("tcp:", 0) == 0){ - _net_type = NetworkSocketType::Tcp; - return ParsePort(&pathOrAddress.c_str()[4]); + bool Core::ParseSocketAddress(const std::string& pathOrAddress, bool isUdp) { + DARWIN_LOGGER; + size_t colon = pathOrAddress.rfind(':'); + if (colon != std::string::npos) { + boost::system::error_code e; + auto addr = pathOrAddress.substr(0, colon); + if(addr.find('[') != std::string::npos && addr.rfind(']') != std::string::npos) { + addr.pop_back(); + addr.erase(0, 1); + } + auto port = pathOrAddress.substr(colon+1, pathOrAddress.length()-1); + bool portRes = ParsePort(port.c_str()); - } else if (pathOrAddress.find("udp:", 0) == 0) { - _net_type = NetworkSocketType::Udp; - return ParsePort(&pathOrAddress.c_str()[4]); + _net_address = boost::asio::ip::make_address(addr, e); + if( ! portRes || e.failed()) { + DARWIN_LOG_CRITICAL("Error while parsing the ip address: " + pathOrAddress); + return false; + } + if(isUdp) { + _net_type = NetworkSocketType::Udp; + } else { + _net_type = NetworkSocketType::Tcp; + } } else { _net_type = NetworkSocketType::Unix; _socketPath = pathOrAddress; - return true; } + return true; + } + + bool Core::IsDaemon() const { + return this->daemon; } } diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 67f6011..a1a49a4 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -12,6 +12,8 @@ #include "Monitor.hpp" #include "ThreadGroup.hpp" +#include + #ifndef PID_PATH # define PID_PATH "/var/run/darwin/" #endif // !PID_PATH @@ -32,7 +34,7 @@ namespace darwin { private: Core(); - bool ParseSocketAddress(const std::string& pathOrAddress); + bool ParseSocketAddress(const std::string& pathOrAddress, bool isUdp); bool ParsePort(const char* pathOrAddress); @@ -88,9 +90,10 @@ namespace darwin { /// \return true on success, false otherwise. bool SetLogLevel(std::string level); + bool IsDaemon() const; + private: std::string _name; - std::string _socketPath; std::string _modConfigPath; std::string _monSocketPath; std::string _pidPath; @@ -101,10 +104,10 @@ namespace darwin { std::size_t _threshold; enum NetworkSocketType _net_type; + std::string _socketPath; + boost::asio::ip::address _net_address; int _net_port; - public: - // TODO Maybe a getter is a better idea... bool daemon; }; } diff --git a/samples/base/TcpServer.cpp b/samples/base/TcpServer.cpp index 7b29af3..d1482eb 100755 --- a/samples/base/TcpServer.cpp +++ b/samples/base/TcpServer.cpp @@ -17,14 +17,16 @@ namespace darwin { - TcpServer::TcpServer(int port_nb, + TcpServer::TcpServer(boost::asio::ip::address const& address, + int port_nb, int next_filter_port, std::string const& output, std::size_t threshold, Generator& generator) : AServer(output, threshold, generator), - _port_nb{port_nb}, _port_nb_next{next_filter_port}, - _acceptor{_io_context, boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v6(), port_nb)}, + _address{address}, _port_nb{port_nb}, + _port_nb_next{next_filter_port}, + _acceptor{_io_context, boost::asio::ip::tcp::endpoint(_address, port_nb)}, _new_connection{_io_context} { this->InitSignalsAndStart(); diff --git a/samples/base/TcpServer.hpp b/samples/base/TcpServer.hpp index 7e3be83..fe708ea 100755 --- a/samples/base/TcpServer.hpp +++ b/samples/base/TcpServer.hpp @@ -27,7 +27,8 @@ namespace darwin { /// \param output Filters' output type /// \param next_filter_socket Path of the Tcp socket of the filter to send data to. /// \param threshold Threshold at which the filter will raise a log. - TcpServer(int port_nb, + TcpServer(boost::asio::ip::address const& address, + int port, int next_filter_port, std::string const& output, std::size_t threshold, @@ -53,6 +54,7 @@ namespace darwin { void HandleAccept(boost::system::error_code const& e); private: + boost::asio::ip::address _address; int _port_nb; //!< Path to the Tcp socket to listen on. int _port_nb_next; //!< Path to the next filter's Tcp socket. boost::asio::ip::tcp::acceptor _acceptor; //!< Acceptor for the incoming connections. diff --git a/samples/base/main.cpp b/samples/base/main.cpp index 85cf59d..32e548f 100644 --- a/samples/base/main.cpp +++ b/samples/base/main.cpp @@ -31,7 +31,7 @@ int main(int ac, char**av) { return 1; DARWIN_LOG_INFO("Configured"); - if (core.daemon) { + if (core.IsDaemon()) { daemon(1, 0); } From 711dc9d4e7ae47e0f33cc350d4a7cb0f6539fe91 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 29 Jun 2021 23:23:02 +0200 Subject: [PATCH 13/54] DarwinPacket: fixed issues with the serialization The packet was illformed near the certitudes NextFilterConnector: Working WIP --- CMakeLists.txt | 1 + samples/base/ASession.cpp | 102 +++------------------------ samples/base/ASession.hpp | 15 +--- samples/base/ATask.cpp | 2 +- samples/base/Core.cpp | 67 ++++-------------- samples/base/Core.hpp | 18 ++--- samples/base/DarwinPacket.cpp | 12 ++-- samples/base/NextFilterConnector.cpp | 92 ++++++++++++++++++++++++ samples/base/NextFilterConnector.hpp | 45 ++++++++++++ samples/base/TcpSession.cpp | 28 -------- samples/base/TcpSession.hpp | 7 -- samples/base/UnixSession.cpp | 29 -------- samples/base/UnixSession.hpp | 7 -- toolkit/Network.cpp | 55 +++++++++++++++ toolkit/Network.hpp | 12 ++++ 15 files changed, 247 insertions(+), 245 deletions(-) create mode 100644 samples/base/NextFilterConnector.cpp create mode 100644 samples/base/NextFilterConnector.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ab8af6..a89c9dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,7 @@ set( samples/base/AlertManager.cpp samples/base/AlertManager.hpp samples/base/DarwinPacket.cpp samples/base/DarwinPacket.hpp + samples/base/NextFilterConnector.cpp samples/base/NextFilterConnector.hpp samples/base/AServer.cpp samples/base/AServer.hpp samples/base/UnixServer.cpp samples/base/UnixServer.hpp diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 010d43e..7abdbab 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -17,6 +17,7 @@ #include "Generator.hpp" #include "ASession.hpp" #include "errors.hpp" +#include "Core.hpp" #include "../../toolkit/lru_cache.hpp" #include "../../toolkit/xxhash.h" @@ -139,22 +140,19 @@ namespace darwin { } void ASession::SendNext(DarwinPacket& packet) { - - if(! this->SendToClient(packet)) - Start(); - // Ignoring Sending to filter for now - /*switch(_header.response) { + // TODO : Unsure of the logic behind the if's and and the Start calls + switch(packet.GetResponse()) { case DARWIN_RESPONSE_SEND_BOTH: - if(this->SendToFilter()) break; + this->SendToFilter(packet); case DARWIN_RESPONSE_SEND_BACK: - if(not this->SendToClient()) Start(); + if(not this->SendToClient(packet)) Start(); break; case DARWIN_RESPONSE_SEND_DARWIN: - if(not this->SendToFilter()) Start(); + if(not this->SendToFilter(packet)) Start(); break; default: Start(); - }*/ + } } @@ -175,7 +173,7 @@ namespace darwin { return true; } - bool ASession::SendToFilter() noexcept { + bool ASession::SendToFilter(DarwinPacket& packet) noexcept { DARWIN_LOGGER; if (!_has_next_filter) { @@ -183,69 +181,9 @@ namespace darwin { return false; } - if( ! ConnectToNextFilter()) { - return false; - } - /* OLD CODE - std::string data = GetDataToSendToFilter(); - DARWIN_LOG_DEBUG("ASession::SendToFilter:: data to send: " + data); - DARWIN_LOG_DEBUG("ASession::SendToFilter:: data size: " + std::to_string(data.size())); - - const std::size_t certitude_size = _certitudes.size(); - - / * - * Allocate the header + - * the size of the certitude - - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size - * / - std::size_t packet_size = 0, packet_size_wo_data = 0; - if (certitude_size > DEFAULT_CERTITUDE_LIST_SIZE) { - packet_size = sizeof(darwin_filter_packet_t) + - (certitude_size - DEFAULT_CERTITUDE_LIST_SIZE) * sizeof(unsigned int); - } else { - packet_size = sizeof(darwin_filter_packet_t); - } - - packet_size_wo_data = packet_size; - - packet_size += data.size(); - - DARWIN_LOG_DEBUG("ASession::SendToFilter:: Computed packet size: " + std::to_string(packet_size)); - - darwin_filter_packet_t* packet; - packet = (darwin_filter_packet_t *) malloc(packet_size); - - if (!packet) { - DARWIN_LOG_CRITICAL("ASession:: SendToFilter:: Could not create a Darwin packet"); - return false; - } - - / * - * Initialisation of the structure for the padding bytes because of - * missing __attribute__((packed)) in the protocol structure. - * / - memset(packet, 0, packet_size); - - for (std::size_t index = 0; index < certitude_size; ++index) { - packet->certitude_list[index] = _certitudes[index]; - } + NextFilterConnector& c = Core::instance().GetNextFilterconnector(); - if(data.size() != 0) { - // TODO: set a proper pointer in protocol.h for the body - // Yes We Hack... - memcpy((char *)(packet) + packet_size_wo_data, data.c_str(), data.size()); - } - - packet->type = DARWIN_PACKET_FILTER; - packet->response = _send_header.response == DARWIN_RESPONSE_SEND_BOTH ? DARWIN_RESPONSE_SEND_DARWIN : _header.response; - packet->certitude_size = certitude_size; - packet->filter_code = _generator.GetFilterCode(); - packet->body_size = data.size(); - memcpy(packet->evt_id, _send_header.evt_id, 16); - - DARWIN_LOG_DEBUG("ASession:: SendToFilter:: Sending header + data"); - this->WriteToFilter(packet, packet_size); - free(packet);*/ + c.Send(packet); return true; } @@ -263,26 +201,6 @@ namespace darwin { Start(); } - void ASession::SendToFilterCallback(const boost::system::error_code& e, - std::size_t size __attribute__((unused))) { - DARWIN_LOGGER; - - if (e) { - DARWIN_LOG_ERROR("ASession::SendToFilterCallback:: " + e.message()); - CloseFilterConnection(); - } - - if(_packet.GetResponse() == DARWIN_RESPONSE_SEND_BOTH) { - // TODO re handle this after re enabling sendtofilter - // this->SendToClient(); - } - else { - Start(); - } - } - - - std::string ASession::Evt_idToString() { DARWIN_LOGGER; diff --git a/samples/base/ASession.hpp b/samples/base/ASession.hpp index 287a4f6..2d15b2e 100755 --- a/samples/base/ASession.hpp +++ b/samples/base/ASession.hpp @@ -96,10 +96,6 @@ namespace darwin { virtual void WriteToClient(std::vector& packet) = 0; - virtual bool ConnectToNextFilter() = 0; - - virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) = 0; - virtual void CloseFilterConnection() = 0; /// Send result to the client. @@ -110,7 +106,7 @@ namespace darwin { /// Send result to next filter. /// /// \return false if the function could not send the data, true otherwise. - virtual bool SendToFilter() noexcept; + virtual bool SendToFilter(DarwinPacket& packet) noexcept; /// Called when data is sent using Send() method. /// Terminate the session on failure. @@ -120,15 +116,6 @@ namespace darwin { SendToClientCallback(const boost::system::error_code& e, std::size_t size); - /// Called when data is sent using SendToFilter() method. - /// Terminate the filter session on failure. - /// - /// \param size The number of byte sent. - virtual void - SendToFilterCallback(const boost::system::error_code& e, - std::size_t size); - - std::string JsonStringify(rapidjson::Document &json); /// Set the async read for the header. diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp index b21af13..33d5507 100755 --- a/samples/base/ATask.cpp +++ b/samples/base/ATask.cpp @@ -96,7 +96,7 @@ namespace darwin { return; } (*this)(); - auto body = _packet.GetMutableBody(); + auto& body = _packet.GetMutableBody(); body.clear(); body.append(_response_body); _s->SendNext(_packet); diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index bfd47e8..13de8b3 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -40,27 +40,29 @@ namespace darwin { try { Monitor monitor{_monSocketPath}; std::thread t{std::bind(&Monitor::Run, std::ref(monitor))}; + std::thread t_nextfilter{std::bind(&NextFilterConnector::Run, std::ref(GetNextFilterconnector()))}; SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::configuring); Generator gen{_nbThread}; if (not gen.Configure(_modConfigPath, _cacheSize)) { DARWIN_LOG_CRITICAL("Core:: Run:: Unable to configure the filter"); raise(SIGTERM); + t_nextfilter.join(); t.join(); return 1; } DARWIN_LOG_DEBUG("Core::run:: Configured generator"); switch(_net_type) { - case NetworkSocketType::Unix: + case network::NetworkSocketType::Unix: DARWIN_LOG_DEBUG("Core::run:: Unix socket configured on path " + _socketPath); server = std::make_unique(_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen); break; - case NetworkSocketType::Tcp: + case network::NetworkSocketType::Tcp: DARWIN_LOG_DEBUG("Core::run:: TCP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); server = std::make_unique(_net_address, _net_port, 0 /* Next filter port */, _nextFilterUnixSocketPath, _threshold, gen); break; - case NetworkSocketType::Udp: + case network::NetworkSocketType::Udp: //not yet implemented DARWIN_LOG_DEBUG("Core::run:: UDP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); DARWIN_LOG_CRITICAL("Core:: Run:: UDP Not implemented"); @@ -68,6 +70,7 @@ namespace darwin { default: DARWIN_LOG_CRITICAL("Core:: Run:: Network Configuration problem"); raise(SIGTERM); + t_nextfilter.join(); t.join(); return 1; } @@ -75,6 +78,7 @@ namespace darwin { try { if (not gen.ConfigureNetworkObject(server->GetIOContext())) { raise(SIGTERM); + t_nextfilter.join(); t.join(); return 1; } @@ -91,6 +95,7 @@ namespace darwin { raise(SIGTERM); } DARWIN_LOG_DEBUG("Core::run:: Joining monitoring thread..."); + t_nextfilter.join(); if (t.joinable()) t.join(); } catch (const std::exception& e) { @@ -166,7 +171,7 @@ namespace darwin { // MANDATORY ARGUMENTS log.setName(av[optind]); _name = av[optind]; - if (! this->ParseSocketAddress(std::string(av[optind + 1]), is_udp)) + if (! network::ParseSocketAddress(std::string(av[optind + 1]), is_udp, _net_type, _net_address, _net_port, _socketPath)) return false; _modConfigPath = av[optind + 2]; _monSocketPath = av[optind + 3]; @@ -271,58 +276,16 @@ namespace darwin { return this->_name; } - bool Core::ParsePort(const char* pathOrAddress) { - DARWIN_LOGGER; - - long port = 0; - if(!darwin::strings::StrToLong(pathOrAddress, port)){ - DARWIN_LOG_ERROR("Core::ParseSocketAddress:: Error while parsing the port number, unrecognized number: '" + std::string(pathOrAddress) + "'"); - return false; - } - - if (port < 0 || port > 65353) { - DARWIN_LOG_ERROR("Core::ParseSocketAddress:: Error while parsing the port number : out of bounds [0; 65353]: '" + std::string(pathOrAddress) + "'"); - return false; - } - - _net_port = static_cast(port); - - return true; + bool Core::IsDaemon() const { + return this->daemon; } - bool Core::ParseSocketAddress(const std::string& pathOrAddress, bool isUdp) { - DARWIN_LOGGER; - size_t colon = pathOrAddress.rfind(':'); - if (colon != std::string::npos) { - boost::system::error_code e; - auto addr = pathOrAddress.substr(0, colon); - if(addr.find('[') != std::string::npos && addr.rfind(']') != std::string::npos) { - addr.pop_back(); - addr.erase(0, 1); - } - auto port = pathOrAddress.substr(colon+1, pathOrAddress.length()-1); - bool portRes = ParsePort(port.c_str()); - - _net_address = boost::asio::ip::make_address(addr, e); - if( ! portRes || e.failed()) { - DARWIN_LOG_CRITICAL("Error while parsing the ip address: " + pathOrAddress); - return false; - } - - if(isUdp) { - _net_type = NetworkSocketType::Udp; - } else { - _net_type = NetworkSocketType::Tcp; - } - } else { - _net_type = NetworkSocketType::Unix; - _socketPath = pathOrAddress; + NextFilterConnector& Core::GetNextFilterconnector() noexcept { + if(!_next_filter_connector){ + _next_filter_connector.emplace(_nextFilterUnixSocketPath, false); } - return true; - } - bool Core::IsDaemon() const { - return this->daemon; + return *_next_filter_connector; } } diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index a1a49a4..92d41c1 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -11,6 +11,8 @@ #include #include "Monitor.hpp" #include "ThreadGroup.hpp" +#include "Network.hpp" +#include "NextFilterConnector.hpp" #include @@ -21,12 +23,6 @@ /// \namespace darwin namespace darwin { - enum NetworkSocketType { - Unix, - Tcp, - Udp - }; - /// Singleton. The main class of the program. /// /// \class Core @@ -34,10 +30,6 @@ namespace darwin { private: Core(); - bool ParseSocketAddress(const std::string& pathOrAddress, bool isUdp); - - bool ParsePort(const char* pathOrAddress); - public: ~Core() = default; @@ -92,6 +84,8 @@ namespace darwin { bool IsDaemon() const; + NextFilterConnector& GetNextFilterconnector() noexcept; + private: std::string _name; std::string _modConfigPath; @@ -103,7 +97,9 @@ namespace darwin { std::size_t _cacheSize; std::size_t _threshold; - enum NetworkSocketType _net_type; + std::optional _next_filter_connector; + + network::NetworkSocketType _net_type; std::string _socketPath; boost::asio::ip::address _net_address; int _net_port; diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 94488d2..8f57507 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -1,5 +1,7 @@ #include "DarwinPacket.hpp" #include +#include +#include #include "../../toolkit/rapidjson/writer.h" #include "../../toolkit/rapidjson/stringbuffer.h" @@ -47,18 +49,20 @@ namespace darwin { if( ! _certitude_list.empty()) { // From the packet format, at least one certitude is always allocated size += (_certitude_list.size() - 1) * sizeof(unsigned int); + header.certitude_list[0] = _certitude_list[0]; } std::vector ret(size, 0); - auto pt = ret.data(); + unsigned char * pt = ret.data(); - std::memcpy(pt, &header, sizeof(header) - sizeof(unsigned int)); - pt += sizeof(header) - sizeof(unsigned int); + std::memcpy(pt, &header, sizeof(header)); + pt += sizeof(header) - sizeof(int); //certitudes - for(auto certitude: _certitude_list) { + for(size_t i=1; i < _certitude_list.size(); i++) { + auto certitude = _certitude_list[i]; std::memcpy(pt,(void*)(&certitude), sizeof(certitude)); pt += sizeof(certitude); } diff --git a/samples/base/NextFilterConnector.cpp b/samples/base/NextFilterConnector.cpp new file mode 100644 index 0000000..a837784 --- /dev/null +++ b/samples/base/NextFilterConnector.cpp @@ -0,0 +1,92 @@ +#include "NextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + NextFilterConnector::NextFilterConnector(std::string const& path_address, bool is_udp) + : _io_context{}, _tcp_socket{_io_context}, _unix_socket{_io_context}, + _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)} + { + _address_path_parsing = network::ParseSocketAddress(path_address, is_udp, _net_type, _net_address, _net_port, _socketPath); + } + + bool NextFilterConnector::Run() { + if (! _address_path_parsing) { + return false; + } + _io_context.run(); + return true; + } + + bool NextFilterConnector::Connect() { + DARWIN_LOGGER; + boost::system::error_code ec; + switch(_net_type) { + case network::NetworkSocketType::Unix: + _unix_socket.connect(boost::asio::local::stream_protocol::endpoint(_socketPath), ec); + break; + case network::NetworkSocketType::Tcp: + _tcp_socket.connect(boost::asio::ip::tcp::endpoint(_net_address, _net_port), ec); + break; + case network::NetworkSocketType::Udp: + DARWIN_LOG_CRITICAL("NextFilterConnector::Connect:: UDP Not implemented"); + return false; + default: + DARWIN_LOG_CRITICAL("NextFilterConnector::Connect:: Unhandled case"); + return false; + } + + if(ec) { + DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + return false; + } + return true; + } + + void NextFilterConnector::Send(DarwinPacket& packet) { + auto b = _buffer_list.insert(_buffer_list.end(), std::move(boost::asio::buffer(packet.Serialize()))); + this->Send(*b); + } + + void NextFilterConnector::Send(boost::asio::const_buffer const& packet) { + DARWIN_LOGGER; + if(_nb_attempts > _max_attempts){ + DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); + return; + } + while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { + std::this_thread::sleep_for(_attempts_delay_ms); + } + + switch(_net_type) { + case network::NetworkSocketType::Unix: + boost::asio::async_write(_unix_socket, packet, boost::bind(&NextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + break; + case network::NetworkSocketType::Tcp: + boost::asio::async_write(_tcp_socket, packet, boost::bind(&NextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + break; + case network::NetworkSocketType::Udp: + + break; + } + } + + void NextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, boost::asio::const_buffer const& buffer) { + if(ec) { + Send(buffer); + return; + } + + //TODO _buffer_list.remove(buffer); (remove buffer) + _nb_attempts = 0; + } +} \ No newline at end of file diff --git a/samples/base/NextFilterConnector.hpp b/samples/base/NextFilterConnector.hpp new file mode 100644 index 0000000..6ea5033 --- /dev/null +++ b/samples/base/NextFilterConnector.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include + +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class NextFilterConnector { + public: + NextFilterConnector(std::string const& path_address, bool is_udp); + ~NextFilterConnector() = default; + + bool Run(); + + bool Connect(); + + void Send(DarwinPacket& packet); + + private: + + void Send(boost::asio::const_buffer const& packet); + + void SendCallback(const boost::system::error_code& e, size_t size, boost::asio::const_buffer const& buffer); + + network::NetworkSocketType _net_type; + std::string _socketPath; + boost::asio::ip::address _net_address; + int _net_port; + + boost::asio::io_context _io_context; + boost::asio::ip::tcp::socket _tcp_socket; + boost::asio::local::stream_protocol::socket _unix_socket; + std::list _buffer_list; + + size_t _max_attempts; + size_t _nb_attempts; + std::chrono::milliseconds _attempts_delay_ms; + bool _address_path_parsing; + }; +} \ No newline at end of file diff --git a/samples/base/TcpSession.cpp b/samples/base/TcpSession.cpp index 3b808df..16a828b 100755 --- a/samples/base/TcpSession.cpp +++ b/samples/base/TcpSession.cpp @@ -60,39 +60,11 @@ namespace darwin { boost::asio::placeholders::bytes_transferred)); } - void TcpSession::WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) { - boost::asio::async_write(_filter_socket, - boost::asio::buffer(packet, packet_size), - boost::bind(&TcpSession::SendToFilterCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - - } - void TcpSession::SetNextFilterPort(int port){ if(port > 0 && port < 65536) { _next_filter_port = port; _has_next_filter = true; } } - - bool TcpSession::ConnectToNextFilter() { - DARWIN_LOGGER; - if (!_connected) { - DARWIN_LOG_DEBUG("TcpSession::SendToFilter:: Trying to connect to: " + - std::to_string(_next_filter_port)); - try { - _filter_socket.connect( - boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v6(), _next_filter_port)); - _connected = true; - } catch (std::exception const& e) { - DARWIN_LOG_ERROR(std::string("TcpSession::SendToFilter:: " - "Unable to connect to next filter: ") + - e.what()); - _connected = false; - } - } - return _connected; - } } diff --git a/samples/base/TcpSession.hpp b/samples/base/TcpSession.hpp index 2708274..d5517db 100755 --- a/samples/base/TcpSession.hpp +++ b/samples/base/TcpSession.hpp @@ -32,13 +32,6 @@ namespace darwin { /// virtual void WriteToClient(std::vector& packet) override final; - /// - virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) override final; - - - /// - virtual bool ConnectToNextFilter() override final; - virtual void CloseFilterConnection() override final; private: diff --git a/samples/base/UnixSession.cpp b/samples/base/UnixSession.cpp index 426650e..f0b4042 100755 --- a/samples/base/UnixSession.cpp +++ b/samples/base/UnixSession.cpp @@ -69,34 +69,5 @@ namespace darwin { boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); } - - void UnixSession::WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) { - boost::asio::async_write(_filter_socket, - boost::asio::buffer(packet, packet_size), - boost::bind(&UnixSession::SendToFilterCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred)); - - } - - bool UnixSession::ConnectToNextFilter() { - DARWIN_LOGGER; - if (!_connected) { - DARWIN_LOG_DEBUG("UnixSession::SendToFilter:: Trying to connect to: " + - _next_filter_path); - try { - _filter_socket.connect( - boost::asio::local::stream_protocol::endpoint( - _next_filter_path.c_str())); - _connected = true; - } catch (std::exception const& e) { - DARWIN_LOG_ERROR(std::string("UnixSession::SendToFilter:: " - "Unable to connect to next filter: ") + - e.what()); - _connected = false; - } - } - return _connected; - } } diff --git a/samples/base/UnixSession.hpp b/samples/base/UnixSession.hpp index 1d3c31e..e29911b 100755 --- a/samples/base/UnixSession.hpp +++ b/samples/base/UnixSession.hpp @@ -35,13 +35,6 @@ namespace darwin { /// virtual void WriteToClient(std::vector& packet) override final; - /// - virtual void WriteToFilter(darwin_filter_packet_t* packet, size_t packet_size) override final; - - - /// - virtual bool ConnectToNextFilter() override final; - virtual void CloseFilterConnection() override final; private: diff --git a/toolkit/Network.cpp b/toolkit/Network.cpp index c6131c6..5e3ad43 100644 --- a/toolkit/Network.cpp +++ b/toolkit/Network.cpp @@ -20,7 +20,9 @@ extern "C" { #include #include #include +#include #include "Logger.hpp" +#include "StringUtils.hpp" #include "Network.hpp" @@ -91,5 +93,58 @@ namespace darwin { return str; } + + bool ParsePort(const char* path_address, int& out_port) { + DARWIN_LOGGER; + + long port = 0; + if(!darwin::strings::StrToLong(path_address, port)){ + DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number, unrecognized number: '" + std::string(path_address) + "'"); + return false; + } + + if (port < 0 || port > 65353) { + DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number : out of bounds [0; 65353]: '" + std::string(path_address) + "'"); + return false; + } + + out_port = static_cast(port); + + return true; + } + + bool ParseSocketAddress(const std::string& path_address, bool is_udp, + NetworkSocketType& out_net_type, boost::asio::ip::address& out_net_address, int& out_port, std::string& out_unix_path) + { + DARWIN_LOGGER; + size_t colon = path_address.rfind(':'); + if (colon != std::string::npos) { + boost::system::error_code e; + auto addr = path_address.substr(0, colon); + if(addr.find('[') != std::string::npos && addr.rfind(']') != std::string::npos) { + addr.pop_back(); + addr.erase(0, 1); + } + auto port = path_address.substr(colon+1, path_address.length()-1); + bool portRes = ParsePort(port.c_str(), out_port); + + out_net_address = boost::asio::ip::make_address(addr, e); + if( ! portRes || e.failed()) { + DARWIN_LOG_CRITICAL("Network::ParseSocketAddress::Error while parsing the ip address: " + path_address); + return false; + } + + if(is_udp) { + out_net_type = NetworkSocketType::Udp; + } else { + out_net_type = NetworkSocketType::Tcp; + } + } else { + out_net_type = NetworkSocketType::Unix; + out_unix_path = path_address; + } + return true; + } + } } diff --git a/toolkit/Network.hpp b/toolkit/Network.hpp index eb3ea01..6e1ad5c 100644 --- a/toolkit/Network.hpp +++ b/toolkit/Network.hpp @@ -18,6 +18,13 @@ extern "C" { namespace darwin { /// \namespace network namespace network { + + enum NetworkSocketType { + Unix, + Tcp, + Udp + }; + /// Get the IP address type. /// /// \param ip_address_string The IP address to get the type from. @@ -46,5 +53,10 @@ namespace darwin { /// /// \param sa The in6_addr structure to be stringified. std::string GetStringAddrFromSockAddrIn6(const in6_addr &addr); + + bool ParseSocketAddress(const std::string& path_address, bool is_udp, + NetworkSocketType& out_net_type, boost::asio::ip::address& out_net_address, int& out_port, std::string& out_unix_path); + + bool ParsePort(const char* path_address, int& out_port); } } From 3a46fe0179d26e4ac600600ffb5f20cd0293c799 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 30 Jun 2021 15:33:24 +0200 Subject: [PATCH 14/54] Separated NextFilterConnector in : Abstract/Tcp/Unix, fixed configuration for nextFilter --- CMakeLists.txt | 4 +- samples/base/ANextFilterConnector.cpp | 32 +++++++++ samples/base/ANextFilterConnector.hpp | 38 ++++++++++ samples/base/ASession.cpp | 2 +- samples/base/Core.cpp | 47 +++++++++--- samples/base/Core.hpp | 9 +-- samples/base/NextFilterConnector.cpp | 92 ------------------------ samples/base/NextFilterConnector.hpp | 45 ------------ samples/base/TcpNextFilterConnector.cpp | 45 ++++++++++++ samples/base/TcpNextFilterConnector.hpp | 29 ++++++++ samples/base/TcpServer.cpp | 3 - samples/base/TcpServer.hpp | 2 - samples/base/UnixNextFilterConnector.cpp | 44 ++++++++++++ samples/base/UnixNextFilterConnector.hpp | 28 ++++++++ samples/base/UnixServer.cpp | 4 +- samples/base/UnixServer.hpp | 2 - 16 files changed, 265 insertions(+), 161 deletions(-) create mode 100644 samples/base/ANextFilterConnector.cpp create mode 100644 samples/base/ANextFilterConnector.hpp delete mode 100644 samples/base/NextFilterConnector.cpp delete mode 100644 samples/base/NextFilterConnector.hpp create mode 100644 samples/base/TcpNextFilterConnector.cpp create mode 100644 samples/base/TcpNextFilterConnector.hpp create mode 100644 samples/base/UnixNextFilterConnector.cpp create mode 100644 samples/base/UnixNextFilterConnector.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a89c9dc..018824b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,7 +98,9 @@ set( samples/base/AlertManager.cpp samples/base/AlertManager.hpp samples/base/DarwinPacket.cpp samples/base/DarwinPacket.hpp - samples/base/NextFilterConnector.cpp samples/base/NextFilterConnector.hpp + samples/base/ANextFilterConnector.cpp samples/base/ANextFilterConnector.hpp + samples/base/UnixNextFilterConnector.cpp samples/base/UnixNextFilterConnector.hpp + samples/base/TcpNextFilterConnector.cpp samples/base/TcpNextFilterConnector.hpp samples/base/AServer.cpp samples/base/AServer.hpp samples/base/UnixServer.cpp samples/base/UnixServer.hpp diff --git a/samples/base/ANextFilterConnector.cpp b/samples/base/ANextFilterConnector.cpp new file mode 100644 index 0000000..e2baba8 --- /dev/null +++ b/samples/base/ANextFilterConnector.cpp @@ -0,0 +1,32 @@ +#include "ANextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + ANextFilterConnector::ANextFilterConnector() + : _io_context{}, _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)} + { + } + + void ANextFilterConnector::Run() { + _io_context.run(); + } + + void ANextFilterConnector::Send(DarwinPacket& packet) { + auto b = _buffer_list.insert(_buffer_list.end(), std::move(boost::asio::buffer(packet.Serialize()))); + this->Send(*b); + } + + void ANextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, boost::asio::const_buffer const& buffer) { + if(ec) { + Send(buffer); + return; + } + + //TODO _buffer_list.remove(buffer); (remove buffer) + _nb_attempts = 0; + } +} \ No newline at end of file diff --git a/samples/base/ANextFilterConnector.hpp b/samples/base/ANextFilterConnector.hpp new file mode 100644 index 0000000..bbaeffa --- /dev/null +++ b/samples/base/ANextFilterConnector.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include + +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class ANextFilterConnector { + public: + ANextFilterConnector(); + virtual ~ANextFilterConnector() = default; + + void Run(); + + virtual bool Connect() = 0; + + void Send(DarwinPacket& packet); + + protected: + + virtual void Send(boost::asio::const_buffer const& packet) = 0; + + virtual /*?*/ void SendCallback(const boost::system::error_code& e, size_t size, boost::asio::const_buffer const& buffer); + + boost::asio::io_context _io_context; + std::list _buffer_list; + + size_t _max_attempts; + size_t _nb_attempts; + std::chrono::milliseconds _attempts_delay_ms; + bool _address_path_parsing; + }; +} \ No newline at end of file diff --git a/samples/base/ASession.cpp b/samples/base/ASession.cpp index 7abdbab..22548b3 100755 --- a/samples/base/ASession.cpp +++ b/samples/base/ASession.cpp @@ -181,7 +181,7 @@ namespace darwin { return false; } - NextFilterConnector& c = Core::instance().GetNextFilterconnector(); + ANextFilterConnector& c = Core::instance().GetNextFilterconnector(); c.Send(packet); return true; diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index 13de8b3..fbda543 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -20,6 +20,8 @@ #include "Core.hpp" #include "Stats.hpp" #include "StringUtils.hpp" +#include "UnixNextFilterConnector.hpp" +#include "TcpNextFilterConnector.hpp" #include @@ -40,7 +42,7 @@ namespace darwin { try { Monitor monitor{_monSocketPath}; std::thread t{std::bind(&Monitor::Run, std::ref(monitor))}; - std::thread t_nextfilter{std::bind(&NextFilterConnector::Run, std::ref(GetNextFilterconnector()))}; + std::thread t_nextfilter{std::bind(&ANextFilterConnector::Run, std::ref(GetNextFilterconnector()))}; SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::configuring); Generator gen{_nbThread}; @@ -56,11 +58,11 @@ namespace darwin { switch(_net_type) { case network::NetworkSocketType::Unix: DARWIN_LOG_DEBUG("Core::run:: Unix socket configured on path " + _socketPath); - server = std::make_unique(_socketPath, _output, _nextFilterUnixSocketPath, _threshold, gen); + server = std::make_unique(_socketPath, _output, _threshold, gen); break; case network::NetworkSocketType::Tcp: DARWIN_LOG_DEBUG("Core::run:: TCP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); - server = std::make_unique(_net_address, _net_port, 0 /* Next filter port */, _nextFilterUnixSocketPath, _threshold, gen); + server = std::make_unique(_net_address, _net_port, _output, _threshold, gen); break; case network::NetworkSocketType::Udp: //not yet implemented @@ -177,7 +179,9 @@ namespace darwin { _monSocketPath = av[optind + 3]; _pidPath = av[optind + 4]; _output = av[optind + 5]; - _nextFilterUnixSocketPath = av[optind + 6]; + + if (!SetNextFilterConnector(std::string(av[optind + 6]), is_udp)) + return false; if (!GetULArg(_nbThread, av[optind + 7])) return false; if (!GetULArg(_cacheSize, av[optind + 8])) @@ -188,6 +192,32 @@ namespace darwin { return true; } + bool Core::SetNextFilterConnector(std::string const& path_address, bool is_udp) { + DARWIN_LOGGER; + network::NetworkSocketType net_type; + std::string path; + int port; + boost::asio::ip::address addr; + if (! network::ParseSocketAddress(path_address, is_udp, net_type, addr, port, path)) + return false; + + switch(net_type) { + case network::NetworkSocketType::Unix: + _next_filter_connector = std::make_unique(path); + return true; + case network::NetworkSocketType::Tcp: + _next_filter_connector = std::make_unique(addr, port); + return true; + case network::NetworkSocketType::Udp: + DARWIN_LOG_CRITICAL("Core:: SetNextFilterConnector:: UDP Not implemented"); + return false; + default: + DARWIN_LOG_CRITICAL("Core:: SetNextFilterConnector:: Next Filter Configuration error"); + return false; + } + return false; + } + void Core::Usage() { std::cout << "Usage: ./darwin filter_name socket_path config_file" << " monitoring_socket_path pid_file output next_filter_socket_path nb_thread " @@ -280,11 +310,12 @@ namespace darwin { return this->daemon; } - NextFilterConnector& Core::GetNextFilterconnector() noexcept { - if(!_next_filter_connector){ - _next_filter_connector.emplace(_nextFilterUnixSocketPath, false); + ANextFilterConnector& Core::GetNextFilterconnector() { + if(!_next_filter_connector) { + DARWIN_LOGGER; + DARWIN_LOG_CRITICAL("Core::GetNextFilterConnector :: Configure must be called before GetNextFilterConnector"); + throw std::logic_error("Core:: Configure must be called before GetNextFilterConnector()"); } - return *_next_filter_connector; } diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 92d41c1..0b08634 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -12,7 +12,7 @@ #include "Monitor.hpp" #include "ThreadGroup.hpp" #include "Network.hpp" -#include "NextFilterConnector.hpp" +#include "ANextFilterConnector.hpp" #include @@ -84,20 +84,21 @@ namespace darwin { bool IsDaemon() const; - NextFilterConnector& GetNextFilterconnector() noexcept; + bool SetNextFilterConnector(std::string const& path_address, bool is_udp); + + ANextFilterConnector& GetNextFilterconnector(); private: std::string _name; std::string _modConfigPath; std::string _monSocketPath; std::string _pidPath; - std::string _nextFilterUnixSocketPath; std::string _output; std::size_t _nbThread; std::size_t _cacheSize; std::size_t _threshold; - std::optional _next_filter_connector; + std::unique_ptr _next_filter_connector; network::NetworkSocketType _net_type; std::string _socketPath; diff --git a/samples/base/NextFilterConnector.cpp b/samples/base/NextFilterConnector.cpp deleted file mode 100644 index a837784..0000000 --- a/samples/base/NextFilterConnector.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include "NextFilterConnector.hpp" - -#include - -#include "Logger.hpp" - -namespace darwin { - - NextFilterConnector::NextFilterConnector(std::string const& path_address, bool is_udp) - : _io_context{}, _tcp_socket{_io_context}, _unix_socket{_io_context}, - _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)} - { - _address_path_parsing = network::ParseSocketAddress(path_address, is_udp, _net_type, _net_address, _net_port, _socketPath); - } - - bool NextFilterConnector::Run() { - if (! _address_path_parsing) { - return false; - } - _io_context.run(); - return true; - } - - bool NextFilterConnector::Connect() { - DARWIN_LOGGER; - boost::system::error_code ec; - switch(_net_type) { - case network::NetworkSocketType::Unix: - _unix_socket.connect(boost::asio::local::stream_protocol::endpoint(_socketPath), ec); - break; - case network::NetworkSocketType::Tcp: - _tcp_socket.connect(boost::asio::ip::tcp::endpoint(_net_address, _net_port), ec); - break; - case network::NetworkSocketType::Udp: - DARWIN_LOG_CRITICAL("NextFilterConnector::Connect:: UDP Not implemented"); - return false; - default: - DARWIN_LOG_CRITICAL("NextFilterConnector::Connect:: Unhandled case"); - return false; - } - - if(ec) { - DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); - return false; - } - return true; - } - - void NextFilterConnector::Send(DarwinPacket& packet) { - auto b = _buffer_list.insert(_buffer_list.end(), std::move(boost::asio::buffer(packet.Serialize()))); - this->Send(*b); - } - - void NextFilterConnector::Send(boost::asio::const_buffer const& packet) { - DARWIN_LOGGER; - if(_nb_attempts > _max_attempts){ - DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); - return; - } - while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { - std::this_thread::sleep_for(_attempts_delay_ms); - } - - switch(_net_type) { - case network::NetworkSocketType::Unix: - boost::asio::async_write(_unix_socket, packet, boost::bind(&NextFilterConnector::SendCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred, - packet)); - break; - case network::NetworkSocketType::Tcp: - boost::asio::async_write(_tcp_socket, packet, boost::bind(&NextFilterConnector::SendCallback, this, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred, - packet)); - break; - case network::NetworkSocketType::Udp: - - break; - } - } - - void NextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, boost::asio::const_buffer const& buffer) { - if(ec) { - Send(buffer); - return; - } - - //TODO _buffer_list.remove(buffer); (remove buffer) - _nb_attempts = 0; - } -} \ No newline at end of file diff --git a/samples/base/NextFilterConnector.hpp b/samples/base/NextFilterConnector.hpp deleted file mode 100644 index 6ea5033..0000000 --- a/samples/base/NextFilterConnector.hpp +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "DarwinPacket.hpp" -#include "Network.hpp" - -namespace darwin { - - class NextFilterConnector { - public: - NextFilterConnector(std::string const& path_address, bool is_udp); - ~NextFilterConnector() = default; - - bool Run(); - - bool Connect(); - - void Send(DarwinPacket& packet); - - private: - - void Send(boost::asio::const_buffer const& packet); - - void SendCallback(const boost::system::error_code& e, size_t size, boost::asio::const_buffer const& buffer); - - network::NetworkSocketType _net_type; - std::string _socketPath; - boost::asio::ip::address _net_address; - int _net_port; - - boost::asio::io_context _io_context; - boost::asio::ip::tcp::socket _tcp_socket; - boost::asio::local::stream_protocol::socket _unix_socket; - std::list _buffer_list; - - size_t _max_attempts; - size_t _nb_attempts; - std::chrono::milliseconds _attempts_delay_ms; - bool _address_path_parsing; - }; -} \ No newline at end of file diff --git a/samples/base/TcpNextFilterConnector.cpp b/samples/base/TcpNextFilterConnector.cpp new file mode 100644 index 0000000..a53af1e --- /dev/null +++ b/samples/base/TcpNextFilterConnector.cpp @@ -0,0 +1,45 @@ +#include "TcpNextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + TcpNextFilterConnector::TcpNextFilterConnector(boost::asio::ip::address const& net_address, int port) + : ANextFilterConnector{}, _net_address{net_address}, _net_port{port}, _socket{_io_context} + { + + } + + + bool TcpNextFilterConnector::Connect() { + DARWIN_LOGGER; + boost::system::error_code ec; + + _socket.connect(boost::asio::ip::tcp::endpoint(_net_address, _net_port), ec); + + if(ec) { + DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + return false; + } + return true; + } + + void TcpNextFilterConnector::Send(boost::asio::const_buffer const& packet) { + DARWIN_LOGGER; + if(_nb_attempts > _max_attempts){ + DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); + return; + } + while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { + std::this_thread::sleep_for(_attempts_delay_ms); + } + + boost::asio::async_write(_socket, packet, boost::bind(&TcpNextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + + } +} \ No newline at end of file diff --git a/samples/base/TcpNextFilterConnector.hpp b/samples/base/TcpNextFilterConnector.hpp new file mode 100644 index 0000000..2d2ff20 --- /dev/null +++ b/samples/base/TcpNextFilterConnector.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include + +#include "ANextFilterConnector.hpp" +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class TcpNextFilterConnector: public ANextFilterConnector { + public: + TcpNextFilterConnector(boost::asio::ip::address const& net_address, int port); + virtual ~TcpNextFilterConnector() = default; + + virtual bool Connect(); + + private: + + virtual void Send(boost::asio::const_buffer const& packet); + + boost::asio::ip::address _net_address; + int _net_port; + boost::asio::ip::tcp::socket _socket; + }; +} \ No newline at end of file diff --git a/samples/base/TcpServer.cpp b/samples/base/TcpServer.cpp index d1482eb..17bd24a 100755 --- a/samples/base/TcpServer.cpp +++ b/samples/base/TcpServer.cpp @@ -19,13 +19,11 @@ namespace darwin { TcpServer::TcpServer(boost::asio::ip::address const& address, int port_nb, - int next_filter_port, std::string const& output, std::size_t threshold, Generator& generator) : AServer(output, threshold, generator), _address{address}, _port_nb{port_nb}, - _port_nb_next{next_filter_port}, _acceptor{_io_context, boost::asio::ip::tcp::endpoint(_address, port_nb)}, _new_connection{_io_context} { @@ -70,7 +68,6 @@ namespace darwin { if (!e) { DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); auto sess = std::make_shared(_new_connection, _manager, _generator); - sess->SetNextFilterPort(_port_nb_next); sess->SetOutputType(_output); sess->SetThreshold(_threshold); _manager.Start(sess); diff --git a/samples/base/TcpServer.hpp b/samples/base/TcpServer.hpp index fe708ea..72fa315 100755 --- a/samples/base/TcpServer.hpp +++ b/samples/base/TcpServer.hpp @@ -29,7 +29,6 @@ namespace darwin { /// \param threshold Threshold at which the filter will raise a log. TcpServer(boost::asio::ip::address const& address, int port, - int next_filter_port, std::string const& output, std::size_t threshold, Generator& generator); @@ -56,7 +55,6 @@ namespace darwin { private: boost::asio::ip::address _address; int _port_nb; //!< Path to the Tcp socket to listen on. - int _port_nb_next; //!< Path to the next filter's Tcp socket. boost::asio::ip::tcp::acceptor _acceptor; //!< Acceptor for the incoming connections. boost::asio::ip::tcp::socket _new_connection; //!< Socket used to accept a new connection. }; diff --git a/samples/base/UnixNextFilterConnector.cpp b/samples/base/UnixNextFilterConnector.cpp new file mode 100644 index 0000000..c822e60 --- /dev/null +++ b/samples/base/UnixNextFilterConnector.cpp @@ -0,0 +1,44 @@ +#include "UnixNextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + UnixNextFilterConnector::UnixNextFilterConnector(std::string const& path) + : ANextFilterConnector{}, _socket_path{path}, _socket{_io_context} + { + } + + + bool UnixNextFilterConnector::Connect() { + DARWIN_LOGGER; + boost::system::error_code ec; + + _socket.connect(boost::asio::local::stream_protocol::endpoint(_socket_path), ec); + + if(ec) { + DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + return false; + } + return true; + } + + void UnixNextFilterConnector::Send(boost::asio::const_buffer const& packet) { + DARWIN_LOGGER; + if(_nb_attempts > _max_attempts){ + DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); + return; + } + while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { + std::this_thread::sleep_for(_attempts_delay_ms); + } + + boost::asio::async_write(_socket, packet, boost::bind(&UnixNextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + + } +} \ No newline at end of file diff --git a/samples/base/UnixNextFilterConnector.hpp b/samples/base/UnixNextFilterConnector.hpp new file mode 100644 index 0000000..1d0fb79 --- /dev/null +++ b/samples/base/UnixNextFilterConnector.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include + +#include "ANextFilterConnector.hpp" +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class UnixNextFilterConnector: public ANextFilterConnector { + public: + UnixNextFilterConnector(std::string const& path); + virtual ~UnixNextFilterConnector() = default; + + virtual bool Connect(); + + private: + + virtual void Send(boost::asio::const_buffer const& packet); + + std::string _socket_path; + boost::asio::local::stream_protocol::socket _socket; + }; +} \ No newline at end of file diff --git a/samples/base/UnixServer.cpp b/samples/base/UnixServer.cpp index 42ebc8a..a88a428 100644 --- a/samples/base/UnixServer.cpp +++ b/samples/base/UnixServer.cpp @@ -17,11 +17,10 @@ namespace darwin { UnixServer::UnixServer(std::string const& socket_path, std::string const& output, - std::string const& next_filter_socket, std::size_t threshold, Generator& generator) : AServer(output, threshold, generator), - _socket_path{socket_path}, _socket_next{next_filter_socket}, + _socket_path{socket_path}, _acceptor{_io_context, boost::asio::local::stream_protocol::endpoint( socket_path)}, _new_connection{_io_context} { @@ -66,7 +65,6 @@ namespace darwin { if (!e) { DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); auto sess = std::make_shared(_new_connection, _manager, _generator); - sess->SetNextFilterSocketPath(_socket_next); sess->SetOutputType(_output); sess->SetThreshold(_threshold); _manager.Start(sess); diff --git a/samples/base/UnixServer.hpp b/samples/base/UnixServer.hpp index 86a69cb..41422d4 100644 --- a/samples/base/UnixServer.hpp +++ b/samples/base/UnixServer.hpp @@ -28,7 +28,6 @@ namespace darwin { /// \param threshold Threshold at which the filter will raise a log. UnixServer(std::string const& socket_path, std::string const& output, - std::string const& next_filter_socket, std::size_t threshold, Generator& generator); @@ -53,7 +52,6 @@ namespace darwin { private: std::string _socket_path; //!< Path to the UNIX socket to listen on. - std::string _socket_next; //!< Path to the next filter's UNIX socket. boost::asio::local::stream_protocol::acceptor _acceptor; //!< Acceptor for the incoming connections. boost::asio::local::stream_protocol::socket _new_connection; //!< Socket used to accept a new connection. }; From 070660b1d910b2556ec0736cd8590343177da85a Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 30 Jun 2021 15:40:01 +0200 Subject: [PATCH 15/54] Moved all network related objects to network folder --- CMakeLists.txt | 19 ++++++++++--------- .../{ => network}/ANextFilterConnector.cpp | 0 .../{ => network}/ANextFilterConnector.hpp | 0 samples/base/{ => network}/AServer.cpp | 0 samples/base/{ => network}/AServer.hpp | 0 samples/base/{ => network}/ASession.cpp | 0 samples/base/{ => network}/ASession.fwd.hpp | 0 samples/base/{ => network}/ASession.hpp | 0 .../{ => network}/TcpNextFilterConnector.cpp | 0 .../{ => network}/TcpNextFilterConnector.hpp | 0 samples/base/{ => network}/TcpServer.cpp | 0 samples/base/{ => network}/TcpServer.hpp | 0 samples/base/{ => network}/TcpSession.cpp | 0 samples/base/{ => network}/TcpSession.hpp | 0 .../{ => network}/UnixNextFilterConnector.cpp | 0 .../{ => network}/UnixNextFilterConnector.hpp | 0 samples/base/{ => network}/UnixServer.cpp | 0 samples/base/{ => network}/UnixServer.hpp | 0 samples/base/{ => network}/UnixSession.cpp | 0 samples/base/{ => network}/UnixSession.hpp | 0 20 files changed, 10 insertions(+), 9 deletions(-) rename samples/base/{ => network}/ANextFilterConnector.cpp (100%) rename samples/base/{ => network}/ANextFilterConnector.hpp (100%) rename samples/base/{ => network}/AServer.cpp (100%) rename samples/base/{ => network}/AServer.hpp (100%) rename samples/base/{ => network}/ASession.cpp (100%) rename samples/base/{ => network}/ASession.fwd.hpp (100%) rename samples/base/{ => network}/ASession.hpp (100%) rename samples/base/{ => network}/TcpNextFilterConnector.cpp (100%) rename samples/base/{ => network}/TcpNextFilterConnector.hpp (100%) rename samples/base/{ => network}/TcpServer.cpp (100%) rename samples/base/{ => network}/TcpServer.hpp (100%) rename samples/base/{ => network}/TcpSession.cpp (100%) rename samples/base/{ => network}/TcpSession.hpp (100%) rename samples/base/{ => network}/UnixNextFilterConnector.cpp (100%) rename samples/base/{ => network}/UnixNextFilterConnector.hpp (100%) rename samples/base/{ => network}/UnixServer.cpp (100%) rename samples/base/{ => network}/UnixServer.hpp (100%) rename samples/base/{ => network}/UnixSession.cpp (100%) rename samples/base/{ => network}/UnixSession.hpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 018824b..f8c1f31 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ include_directories( toolkit toolkit/thread-pool-cpp/include/ samples/base + samples/base/network samples/ ${HIREDIS_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} @@ -98,17 +99,17 @@ set( samples/base/AlertManager.cpp samples/base/AlertManager.hpp samples/base/DarwinPacket.cpp samples/base/DarwinPacket.hpp - samples/base/ANextFilterConnector.cpp samples/base/ANextFilterConnector.hpp - samples/base/UnixNextFilterConnector.cpp samples/base/UnixNextFilterConnector.hpp - samples/base/TcpNextFilterConnector.cpp samples/base/TcpNextFilterConnector.hpp + samples/base/network/ANextFilterConnector.cpp samples/base/network/ANextFilterConnector.hpp + samples/base/network/UnixNextFilterConnector.cpp samples/base/network/UnixNextFilterConnector.hpp + samples/base/network/TcpNextFilterConnector.cpp samples/base/network/TcpNextFilterConnector.hpp - samples/base/AServer.cpp samples/base/AServer.hpp - samples/base/UnixServer.cpp samples/base/UnixServer.hpp - samples/base/TcpServer.cpp samples/base/TcpServer.hpp + samples/base/network/AServer.cpp samples/base/network/AServer.hpp + samples/base/network/UnixServer.cpp samples/base/network/UnixServer.hpp + samples/base/network/TcpServer.cpp samples/base/network/TcpServer.hpp samples/base/Manager.cpp samples/base/Manager.hpp - samples/base/ASession.cpp samples/base/ASession.hpp - samples/base/UnixSession.cpp samples/base/UnixSession.hpp - samples/base/TcpSession.cpp samples/base/TcpSession.hpp + samples/base/network/ASession.cpp samples/base/network/ASession.hpp + samples/base/network/UnixSession.cpp samples/base/network/UnixSession.hpp + samples/base/network/TcpSession.cpp samples/base/network/TcpSession.hpp samples/base/ATask.cpp samples/base/ATask.hpp toolkit/Network.cpp toolkit/Network.hpp diff --git a/samples/base/ANextFilterConnector.cpp b/samples/base/network/ANextFilterConnector.cpp similarity index 100% rename from samples/base/ANextFilterConnector.cpp rename to samples/base/network/ANextFilterConnector.cpp diff --git a/samples/base/ANextFilterConnector.hpp b/samples/base/network/ANextFilterConnector.hpp similarity index 100% rename from samples/base/ANextFilterConnector.hpp rename to samples/base/network/ANextFilterConnector.hpp diff --git a/samples/base/AServer.cpp b/samples/base/network/AServer.cpp similarity index 100% rename from samples/base/AServer.cpp rename to samples/base/network/AServer.cpp diff --git a/samples/base/AServer.hpp b/samples/base/network/AServer.hpp similarity index 100% rename from samples/base/AServer.hpp rename to samples/base/network/AServer.hpp diff --git a/samples/base/ASession.cpp b/samples/base/network/ASession.cpp similarity index 100% rename from samples/base/ASession.cpp rename to samples/base/network/ASession.cpp diff --git a/samples/base/ASession.fwd.hpp b/samples/base/network/ASession.fwd.hpp similarity index 100% rename from samples/base/ASession.fwd.hpp rename to samples/base/network/ASession.fwd.hpp diff --git a/samples/base/ASession.hpp b/samples/base/network/ASession.hpp similarity index 100% rename from samples/base/ASession.hpp rename to samples/base/network/ASession.hpp diff --git a/samples/base/TcpNextFilterConnector.cpp b/samples/base/network/TcpNextFilterConnector.cpp similarity index 100% rename from samples/base/TcpNextFilterConnector.cpp rename to samples/base/network/TcpNextFilterConnector.cpp diff --git a/samples/base/TcpNextFilterConnector.hpp b/samples/base/network/TcpNextFilterConnector.hpp similarity index 100% rename from samples/base/TcpNextFilterConnector.hpp rename to samples/base/network/TcpNextFilterConnector.hpp diff --git a/samples/base/TcpServer.cpp b/samples/base/network/TcpServer.cpp similarity index 100% rename from samples/base/TcpServer.cpp rename to samples/base/network/TcpServer.cpp diff --git a/samples/base/TcpServer.hpp b/samples/base/network/TcpServer.hpp similarity index 100% rename from samples/base/TcpServer.hpp rename to samples/base/network/TcpServer.hpp diff --git a/samples/base/TcpSession.cpp b/samples/base/network/TcpSession.cpp similarity index 100% rename from samples/base/TcpSession.cpp rename to samples/base/network/TcpSession.cpp diff --git a/samples/base/TcpSession.hpp b/samples/base/network/TcpSession.hpp similarity index 100% rename from samples/base/TcpSession.hpp rename to samples/base/network/TcpSession.hpp diff --git a/samples/base/UnixNextFilterConnector.cpp b/samples/base/network/UnixNextFilterConnector.cpp similarity index 100% rename from samples/base/UnixNextFilterConnector.cpp rename to samples/base/network/UnixNextFilterConnector.cpp diff --git a/samples/base/UnixNextFilterConnector.hpp b/samples/base/network/UnixNextFilterConnector.hpp similarity index 100% rename from samples/base/UnixNextFilterConnector.hpp rename to samples/base/network/UnixNextFilterConnector.hpp diff --git a/samples/base/UnixServer.cpp b/samples/base/network/UnixServer.cpp similarity index 100% rename from samples/base/UnixServer.cpp rename to samples/base/network/UnixServer.cpp diff --git a/samples/base/UnixServer.hpp b/samples/base/network/UnixServer.hpp similarity index 100% rename from samples/base/UnixServer.hpp rename to samples/base/network/UnixServer.hpp diff --git a/samples/base/UnixSession.cpp b/samples/base/network/UnixSession.cpp similarity index 100% rename from samples/base/UnixSession.cpp rename to samples/base/network/UnixSession.cpp diff --git a/samples/base/UnixSession.hpp b/samples/base/network/UnixSession.hpp similarity index 100% rename from samples/base/UnixSession.hpp rename to samples/base/network/UnixSession.hpp From 3be3f485d879d274f93a9609c2e459f8b4531d17 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 30 Jun 2021 16:12:22 +0200 Subject: [PATCH 16/54] Fixed Set of asio buffer in ANextFilterConnector --- samples/base/network/ANextFilterConnector.cpp | 10 ++++++---- samples/base/network/ANextFilterConnector.hpp | 8 ++++---- samples/base/network/TcpNextFilterConnector.cpp | 4 ++-- samples/base/network/TcpNextFilterConnector.hpp | 2 +- samples/base/network/UnixNextFilterConnector.cpp | 4 ++-- samples/base/network/UnixNextFilterConnector.hpp | 2 +- 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/samples/base/network/ANextFilterConnector.cpp b/samples/base/network/ANextFilterConnector.cpp index e2baba8..a2e6c16 100644 --- a/samples/base/network/ANextFilterConnector.cpp +++ b/samples/base/network/ANextFilterConnector.cpp @@ -16,17 +16,19 @@ namespace darwin { } void ANextFilterConnector::Send(DarwinPacket& packet) { - auto b = _buffer_list.insert(_buffer_list.end(), std::move(boost::asio::buffer(packet.Serialize()))); - this->Send(*b); + std::vector v = packet.Serialize(); + std::shared_ptr buf_ptr = std::make_shared(v.data(), v.size()); + _buffer_set.insert(buf_ptr); + this->Send(buf_ptr); } - void ANextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, boost::asio::const_buffer const& buffer) { + void ANextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, std::shared_ptr buffer) { if(ec) { Send(buffer); return; } - //TODO _buffer_list.remove(buffer); (remove buffer) + _buffer_set.erase(buffer); _nb_attempts = 0; } } \ No newline at end of file diff --git a/samples/base/network/ANextFilterConnector.hpp b/samples/base/network/ANextFilterConnector.hpp index bbaeffa..5f9e698 100644 --- a/samples/base/network/ANextFilterConnector.hpp +++ b/samples/base/network/ANextFilterConnector.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include @@ -23,12 +23,12 @@ namespace darwin { protected: - virtual void Send(boost::asio::const_buffer const& packet) = 0; + virtual void Send(std::shared_ptr packet) = 0; - virtual /*?*/ void SendCallback(const boost::system::error_code& e, size_t size, boost::asio::const_buffer const& buffer); + virtual /*?*/ void SendCallback(const boost::system::error_code& e, size_t size, std::shared_ptr buffer); boost::asio::io_context _io_context; - std::list _buffer_list; + std::set> _buffer_set; size_t _max_attempts; size_t _nb_attempts; diff --git a/samples/base/network/TcpNextFilterConnector.cpp b/samples/base/network/TcpNextFilterConnector.cpp index a53af1e..fe9f06f 100644 --- a/samples/base/network/TcpNextFilterConnector.cpp +++ b/samples/base/network/TcpNextFilterConnector.cpp @@ -26,7 +26,7 @@ namespace darwin { return true; } - void TcpNextFilterConnector::Send(boost::asio::const_buffer const& packet) { + void TcpNextFilterConnector::Send(std::shared_ptr packet) { DARWIN_LOGGER; if(_nb_attempts > _max_attempts){ DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); @@ -36,7 +36,7 @@ namespace darwin { std::this_thread::sleep_for(_attempts_delay_ms); } - boost::asio::async_write(_socket, packet, boost::bind(&TcpNextFilterConnector::SendCallback, this, + boost::asio::async_write(_socket, *packet, boost::bind(&TcpNextFilterConnector::SendCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, packet)); diff --git a/samples/base/network/TcpNextFilterConnector.hpp b/samples/base/network/TcpNextFilterConnector.hpp index 2d2ff20..50e2a06 100644 --- a/samples/base/network/TcpNextFilterConnector.hpp +++ b/samples/base/network/TcpNextFilterConnector.hpp @@ -20,7 +20,7 @@ namespace darwin { private: - virtual void Send(boost::asio::const_buffer const& packet); + virtual void Send(std::shared_ptr packet); boost::asio::ip::address _net_address; int _net_port; diff --git a/samples/base/network/UnixNextFilterConnector.cpp b/samples/base/network/UnixNextFilterConnector.cpp index c822e60..0e6dacd 100644 --- a/samples/base/network/UnixNextFilterConnector.cpp +++ b/samples/base/network/UnixNextFilterConnector.cpp @@ -25,7 +25,7 @@ namespace darwin { return true; } - void UnixNextFilterConnector::Send(boost::asio::const_buffer const& packet) { + void UnixNextFilterConnector::Send(std::shared_ptr packet) { DARWIN_LOGGER; if(_nb_attempts > _max_attempts){ DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); @@ -35,7 +35,7 @@ namespace darwin { std::this_thread::sleep_for(_attempts_delay_ms); } - boost::asio::async_write(_socket, packet, boost::bind(&UnixNextFilterConnector::SendCallback, this, + boost::asio::async_write(_socket, *packet, boost::bind(&UnixNextFilterConnector::SendCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, packet)); diff --git a/samples/base/network/UnixNextFilterConnector.hpp b/samples/base/network/UnixNextFilterConnector.hpp index 1d0fb79..4491dce 100644 --- a/samples/base/network/UnixNextFilterConnector.hpp +++ b/samples/base/network/UnixNextFilterConnector.hpp @@ -20,7 +20,7 @@ namespace darwin { private: - virtual void Send(boost::asio::const_buffer const& packet); + virtual void Send(std::shared_ptr packet); std::string _socket_path; boost::asio::local::stream_protocol::socket _socket; From 2436f5de3f46797c432b850df762ab24d0f37463 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 1 Jul 2021 11:19:21 +0200 Subject: [PATCH 17/54] Fixed memory issues: - DarwinPacket copied only the first 15 bytes of event id - Destructor of AServer wasn't virtual leading to memory leak when UnixServer was destroyed --- samples/base/DarwinPacket.cpp | 6 +++--- samples/base/network/AServer.hpp | 2 +- samples/base/network/ASession.cpp | 26 ++++++++++++++------------ samples/base/network/TcpServer.hpp | 2 +- samples/base/network/UnixServer.hpp | 2 +- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 8f57507..0caf1b5 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -17,7 +17,7 @@ namespace darwin { : _type{type}, _response{response}, _filter_code{filter_code}, _parsed_certitude_size{certitude_size}, _parsed_body_size{body_size} { - std::copy(&event_id[0], &event_id[15], &this->_evt_id[0]); + std::memcpy(this->_evt_id, event_id, 16); } @@ -26,7 +26,7 @@ namespace darwin { _parsed_certitude_size{input.certitude_size}, _parsed_body_size{input.body_size} { - std::copy(&input.evt_id[0], &input.evt_id[15], &this->_evt_id[0]); + std::memcpy(this->_evt_id, input.evt_id, 16); if(input.certitude_size > 0) AddCertitude(input.certitude_list[0]); } @@ -42,7 +42,7 @@ namespace darwin { {0} }; - std::copy(&this->_evt_id[0], &this->_evt_id[15], &header.evt_id[0]); + std::memcpy(header.evt_id, this->_evt_id, 16); size_t size = sizeof(header) + _body.size(); diff --git a/samples/base/network/AServer.hpp b/samples/base/network/AServer.hpp index b2bb05c..d84df8e 100644 --- a/samples/base/network/AServer.hpp +++ b/samples/base/network/AServer.hpp @@ -19,7 +19,7 @@ namespace darwin { /// \param threshold Threshold at which the filter will raise a log. // AServer() = default; - ~AServer() = default; + virtual ~AServer() = default; // Make the server non copyable & non movable AServer(AServer const&) = delete; diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index 22548b3..d1cc92f 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -205,18 +206,19 @@ namespace darwin { DARWIN_LOGGER; const unsigned char * evt_id = _packet.GetEventId(); - char str[37] = {}; - snprintf(str, - 37, - "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", - evt_id[0], evt_id[1], evt_id[2], evt_id[3], - evt_id[4], evt_id[5], evt_id[6], evt_id[7], - evt_id[8], evt_id[9], evt_id[10], evt_id[11], - evt_id[12], evt_id[13], evt_id[14], evt_id[15] - ); - std::string res(str); - DARWIN_LOG_DEBUG(std::string("ASession::Evt_idToString:: UUID - ") + res); - return res; + + std::ostringstream oss; + auto default_flags = oss.flags(); + + for(size_t i=0; i<16;i++){ + if(i==4 || i==6 || i==8 || i==10){ + oss.flags(default_flags); + oss << '-'; + } + oss << std::hex << std::setw(2) << static_cast(evt_id[i]); + } + DARWIN_LOG_DEBUG(std::string("ASession::Evt_idToString:: UUID - ") + oss.str()); + return oss.str(); } diff --git a/samples/base/network/TcpServer.hpp b/samples/base/network/TcpServer.hpp index 72fa315..7820f35 100755 --- a/samples/base/network/TcpServer.hpp +++ b/samples/base/network/TcpServer.hpp @@ -33,7 +33,7 @@ namespace darwin { std::size_t threshold, Generator& generator); - ~TcpServer() = default; + virtual ~TcpServer() = default; // Make the TcpServer non copyable & non movable TcpServer(TcpServer const&) = delete; diff --git a/samples/base/network/UnixServer.hpp b/samples/base/network/UnixServer.hpp index 41422d4..ed1944e 100644 --- a/samples/base/network/UnixServer.hpp +++ b/samples/base/network/UnixServer.hpp @@ -31,7 +31,7 @@ namespace darwin { std::size_t threshold, Generator& generator); - ~UnixServer() = default; + virtual ~UnixServer() = default; // Make the UnixServer non copyable & non movable UnixServer(UnixServer const&) = delete; From 80955dbb5d4b8f2b017762c7bb4ce8a21a9726f0 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 1 Jul 2021 11:49:36 +0200 Subject: [PATCH 18/54] Fixed problem with the manager Service.update did not update nettwork field --- manager/Services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manager/Services.py b/manager/Services.py index 516e0f4..91f49ec 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -373,6 +373,8 @@ def update(self, names, prefix, suffix): prefix=prefix, suffix=suffix, name=n, extension=new[n]['extension'] ) + + new[n]['network']['address_path'] = new[n]['socket'] new[n]['monitoring'] = '{prefix}/sockets{suffix}/{name}_mon{extension}.sock'.format( prefix=prefix, suffix=suffix, From 55dfa070e90b20741ab170e326b10bceb4473af0 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 1 Jul 2021 11:50:40 +0200 Subject: [PATCH 19/54] Fixed manager (failed previous commit) --- manager/Services.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/manager/Services.py b/manager/Services.py index 91f49ec..62e655a 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -373,8 +373,9 @@ def update(self, names, prefix, suffix): prefix=prefix, suffix=suffix, name=n, extension=new[n]['extension'] ) - - new[n]['network']['address_path'] = new[n]['socket'] + # TODO Handle TCP case? + if new[n]['network']['socket_type'] == 'UNIX': + new[n]['network']['address_path'] = new[n]['socket'] new[n]['monitoring'] = '{prefix}/sockets{suffix}/{name}_mon{extension}.sock'.format( prefix=prefix, suffix=suffix, From 98a303904e31f35bea615e8a486de6b45e5d5e72 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 2 Jul 2021 14:41:22 +0200 Subject: [PATCH 20/54] RedisManager: modified Rate Limit connection logic Added retry attempts, modified tests accordingly --- tests/core/redis.py | 25 +++++++++++++------------ toolkit/RedisManager.cpp | 22 +++++++++++++++++----- toolkit/RedisManager.hpp | 3 +++ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/tests/core/redis.py b/tests/core/redis.py index d624348..40c87fd 100644 --- a/tests/core/redis.py +++ b/tests/core/redis.py @@ -185,16 +185,14 @@ def master_replica_transfer(function_name, healthcheck): return False if return_code != 0: - logging.error("{}: Filter didn't return correct code, " + - "waited for 0 but got {}".format(function_name, return_code)) + logging.error("{}: Filter didn't return correct code, waited for 0 but got {}".format(function_name, return_code)) return False with replica.connect() as new_master_connection: num_entries = new_master_connection.llen(REDIS_LIST_NAME) if num_entries != 2: - logging.error("{}: Wrong number of entries in {}, " + - "expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) + logging.error("{}: Wrong number of entries in {}, expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) return False return True @@ -242,16 +240,14 @@ def master_replica_failover(function_name, healthcheck): return False if return_code != 0: - logging.error("{}: Filter didn't return correct code, " + - "waited for 0 but got {}".format(function_name, return_code)) + logging.error("{}: Filter didn't return correct code, waited for 0 but got {}".format(function_name, return_code)) return False with replica.connect() as new_master_connection: num_entries = new_master_connection.llen(REDIS_LIST_NAME) if num_entries != 2: - logging.error("{}: Wrong number of entries in {}, " + - "expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) + logging.error("{}: Wrong number of entries in {}, expected 2 but got {}".format(function_name, REDIS_LIST_NAME, num_entries)) return False return True @@ -296,8 +292,8 @@ def thread_brute(filter, count_log): number = master.get_number_of_connections() - # 5 threads - if number != 5: + # 6 threads : 5 task threads + the main thread for configuring the redis socket + if number != 6: logging.error("multi_thread_master: wrong number of active connections: expected 5 but got " + str(number)) return False @@ -352,9 +348,14 @@ def thread_brute_time(filter, time_end): # new generated connections generated, minus the one generated by the call to get it new_connections = replica.connect().info()['total_connections_received'] - initial_connections_num - 1 - if new_connections > 10: + # In darwin::RedisManager, there is at most 2 connections attempts per 8 seconds per thread + # We try de post something in a stopped redis and expect a controlled number of connection attempts + # The limit is calculated as : + # 20 connections : 5 threads * 2 attempts * 2 (the 9 seconds span let the discover happen twice) + # We measure generally only 19 because there was usually already one attempt done in the first part of the test + if new_connections > 20: logging.error("master_replica_discovery_rate_limiting: Wrong number of new connection attempts, " + - "was supposed to have 10 new at most, but got ".format(new_connections)) + "was supposed to have 10 new at most, but got {}".format(new_connections)) return False return True \ No newline at end of file diff --git a/toolkit/RedisManager.cpp b/toolkit/RedisManager.cpp index 0b28392..87d55f3 100644 --- a/toolkit/RedisManager.cpp +++ b/toolkit/RedisManager.cpp @@ -171,18 +171,30 @@ namespace darwin { // rate limit if(std::time(nullptr) > threadData->_lastDiscovery + this->_healthCheckInterval) { std::time(&(threadData->_lastDiscovery)); + threadData->_retryAttempts = 1; // this is the first attempt /* try first to connect to current active connection (potentially updated), then search master in case of failure */ - if(this->Connect() and this->IsMaster()) + if ((this->Connect() && this->IsMaster()) || this->FindAndConnect() || this->FindAndConnectWithRateLimiting()) { + threadData->_retryAttempts = 0; return true; - else - return this->FindAndConnect(); + } else { + return false; + } + } + else if(threadData->_retryAttempts < this ->_maxRetryAttempts) { + // less than 8 seconds since last discovery, retrying + threadData->_retryAttempts++; + if ((this->Connect() && this->IsMaster()) || this->FindAndConnect() || this->FindAndConnectWithRateLimiting()) { + threadData->_retryAttempts = 0; + return true; + } else { + return false; + } } else { DARWIN_LOG_DEBUG("RedisManager::FindAndConnectWithRateLimiting:: Rate limiting applied"); + return false; } - - return false; } diff --git a/toolkit/RedisManager.hpp b/toolkit/RedisManager.hpp index dded065..19dd039 100644 --- a/toolkit/RedisManager.hpp +++ b/toolkit/RedisManager.hpp @@ -40,6 +40,7 @@ namespace darwin { redisContext* _redisContext = nullptr; time_t _redisLastUse = 0; time_t _lastDiscovery = 0; + size_t _retryAttempts = 0; }; struct RedisConnectionInfo { @@ -112,6 +113,7 @@ namespace darwin { // default values static constexpr int HEALTH_CHECK_INTERVAL = 8; // more than 8 seconds without calling redis will trigger a health check static constexpr int CONNECTION_TIMEOUT = 5; // 5 seconds for connections' timeout + static constexpr size_t MAX_RETRY_ATTEMPTS = 2; public: RedisManager(const RedisManager&) = delete; @@ -331,6 +333,7 @@ namespace darwin { private: time_t _healthCheckInterval = HEALTH_CHECK_INTERVAL; // will execute a HealthCheck if the connection wasn't used for x seconds time_t _connectTimeout = 0; // timeout when trying to connect + size_t _maxRetryAttempts = MAX_RETRY_ATTEMPTS; std::set> _threadSet; // set containing each thread's own context std::mutex _threadSetMut; // set mutex RedisConnectionInfo _activeConnection; // current active connection From 36e2ae0c798a79d7b4ec40bee84189ca4bce3cea Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 6 Jul 2021 16:37:19 +0200 Subject: [PATCH 21/54] Adapted ATask, Asession and DarwinPacket Proprietary filters needed additional methods for accessing private fields of a packet --- samples/base/AGenerator.hpp | 2 -- samples/base/ATask.cpp | 2 ++ samples/base/DarwinPacket.cpp | 12 ++++++++++++ samples/base/DarwinPacket.hpp | 6 ++++++ samples/base/network/ASession.cpp | 6 +++--- samples/ftest/TestTask.cpp | 9 ++++++++- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/samples/base/AGenerator.hpp b/samples/base/AGenerator.hpp index 6c27b81..4159141 100644 --- a/samples/base/AGenerator.hpp +++ b/samples/base/AGenerator.hpp @@ -60,8 +60,6 @@ class AGenerator { Configure(std::string const& configFile, const std::size_t cache_size) final; - virtual long GetFilterCode() const = 0; - virtual tp::ThreadPool& GetTaskThreadPool() final; private: diff --git a/samples/base/ATask.cpp b/samples/base/ATask.cpp index 33d5507..7cf1c72 100755 --- a/samples/base/ATask.cpp +++ b/samples/base/ATask.cpp @@ -96,6 +96,8 @@ namespace darwin { return; } (*this)(); + + _packet.SetFilterCode(this->GetFilterCode()); auto& body = _packet.GetMutableBody(); body.clear(); body.append(_response_body); diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 0caf1b5..86012d3 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -103,10 +103,18 @@ namespace darwin { return this->_response; } + void DarwinPacket::SetResponse(enum darwin_filter_response_type response) { + this->_response = response; + } + long DarwinPacket::GetFilterCode() const { return this->_filter_code; } + void DarwinPacket::SetFilterCode(long filter_code) { + this->_filter_code = filter_code; + } + const unsigned char * DarwinPacket::GetEventId() const { return this->_evt_id; } @@ -135,6 +143,10 @@ namespace darwin { return this->_certitude_list; } + std::vector& DarwinPacket::GetMutableCertitudeList() { + return this->_certitude_list; + } + const std::string& DarwinPacket::GetLogs() const { return this->_logs; } diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 9ec6c23..4877018 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -46,8 +46,12 @@ namespace darwin { enum darwin_filter_response_type GetResponse() const; + void SetResponse(enum darwin_filter_response_type); + long GetFilterCode() const; + void SetFilterCode(long filter_code); + const unsigned char * GetEventId() const; size_t GetEventIdSize() const; @@ -62,6 +66,8 @@ namespace darwin { const std::vector& GetCertitudeList() const; + std::vector& GetMutableCertitudeList(); + inline void AddCertitude(unsigned int certitude) { this->_certitude_list.push_back(certitude); }; diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index d1cc92f..75e1b24 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -168,9 +168,9 @@ namespace darwin { bool ASession::SendToClient(DarwinPacket& packet) noexcept { DARWIN_LOGGER; - auto vec = packet.Serialize(); - DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(vec.size())); - this->WriteToClient(vec); + std::vector serialized_packet = std::move(packet.Serialize()); + DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(serialized_packet.size())); + this->WriteToClient(serialized_packet); return true; } diff --git a/samples/ftest/TestTask.cpp b/samples/ftest/TestTask.cpp index 83f1799..1e74cc2 100644 --- a/samples/ftest/TestTask.cpp +++ b/samples/ftest/TestTask.cpp @@ -66,7 +66,14 @@ void TestTask::operator()() { _packet.AddCertitude(DARWIN_ERROR_RETURN); } } - else { + else if (_line == "load_test") { + DARWIN_LOG_DEBUG("TestTask:: load_test triggered, doing work for some times (2 000 000 mults)"); + int n = _line.size(); + for(int i=0; i< 2000000;i++){ + n *=i; + } + _packet.AddCertitude(n & 1); + } else { DARWIN_LOG_DEBUG("TestTask:: not triggered specific action, generating alert by default"); DARWIN_ALERT_MANAGER.Alert(_line, 100, _s->Evt_idToString()); _packet.AddCertitude(0); From b8cbf638009dbdaaa7d51d1d4de5a4c598df6798 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 6 Jul 2021 18:10:50 +0200 Subject: [PATCH 22/54] Adapted tests to TCP : Added possibility to run all tests in TCP mode --- samples/fanomaly/Generator.hpp | 2 +- samples/fbuffer/Generator.hpp | 2 +- samples/fconnection/Generator.hpp | 2 +- samples/fdga/Generator.hpp | 2 +- samples/fhostlookup/Generator.hpp | 2 +- samples/finspection/Generator.hpp | 2 +- samples/fsession/Generator.hpp | 2 +- samples/ftanomaly/Generator.hpp | 2 +- samples/ftest/Generator.hpp | 2 +- samples/fuseragent/Generator.hpp | 2 +- samples/fyara/Generator.hpp | 2 +- tests/conf.py | 3 +++ tests/filters/fanomaly.py | 3 +-- tests/filters/fbuffer.py | 18 +++++++----------- tests/filters/fconnection.py | 9 +++------ tests/filters/fhostlookup.py | 3 +-- tests/filters/ftanomaly.py | 16 +++++----------- tests/filters/fyara.py | 3 +-- tests/tools/filter.py | 23 +++++++++++++++++++---- 19 files changed, 51 insertions(+), 49 deletions(-) diff --git a/samples/fanomaly/Generator.hpp b/samples/fanomaly/Generator.hpp index 1a6aa74..2b0e184 100644 --- a/samples/fanomaly/Generator.hpp +++ b/samples/fanomaly/Generator.hpp @@ -20,7 +20,7 @@ class Generator: public AGenerator { Generator(size_t nb_task_threads); ~Generator() = default; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; public: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fbuffer/Generator.hpp b/samples/fbuffer/Generator.hpp index e816001..192b2e5 100644 --- a/samples/fbuffer/Generator.hpp +++ b/samples/fbuffer/Generator.hpp @@ -43,7 +43,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; ///\brief Creates the Connectors depending on the _output_configs vector. /// diff --git a/samples/fconnection/Generator.hpp b/samples/fconnection/Generator.hpp index 6eba565..cf30a21 100644 --- a/samples/fconnection/Generator.hpp +++ b/samples/fconnection/Generator.hpp @@ -25,7 +25,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fdga/Generator.hpp b/samples/fdga/Generator.hpp index e4cd74a..947e11e 100644 --- a/samples/fdga/Generator.hpp +++ b/samples/fdga/Generator.hpp @@ -26,7 +26,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fhostlookup/Generator.hpp b/samples/fhostlookup/Generator.hpp index d4a58bf..04973e7 100644 --- a/samples/fhostlookup/Generator.hpp +++ b/samples/fhostlookup/Generator.hpp @@ -27,7 +27,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; protected: enum class db_type { diff --git a/samples/finspection/Generator.hpp b/samples/finspection/Generator.hpp index aa0b948..8069304 100644 --- a/samples/finspection/Generator.hpp +++ b/samples/finspection/Generator.hpp @@ -26,7 +26,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fsession/Generator.hpp b/samples/fsession/Generator.hpp index 0fee48b..78e868f 100644 --- a/samples/fsession/Generator.hpp +++ b/samples/fsession/Generator.hpp @@ -28,7 +28,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/ftanomaly/Generator.hpp b/samples/ftanomaly/Generator.hpp index bf4fed4..9990353 100644 --- a/samples/ftanomaly/Generator.hpp +++ b/samples/ftanomaly/Generator.hpp @@ -27,7 +27,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/ftest/Generator.hpp b/samples/ftest/Generator.hpp index 8b58fa6..364d42c 100644 --- a/samples/ftest/Generator.hpp +++ b/samples/ftest/Generator.hpp @@ -26,7 +26,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; protected: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fuseragent/Generator.hpp b/samples/fuseragent/Generator.hpp index 0f9335c..3377609 100644 --- a/samples/fuseragent/Generator.hpp +++ b/samples/fuseragent/Generator.hpp @@ -26,7 +26,7 @@ class Generator: public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fyara/Generator.hpp b/samples/fyara/Generator.hpp index 4a934bc..251b7e0 100644 --- a/samples/fyara/Generator.hpp +++ b/samples/fyara/Generator.hpp @@ -26,7 +26,7 @@ class Generator : public AGenerator { virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept override final; - virtual long GetFilterCode() const override final; + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/tests/conf.py b/tests/conf.py index 18f1956..73e8a27 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -8,6 +8,9 @@ DEFAULT_MANAGER_PATH = '/home/darwin/manager/manager.py' DEFAULT_FILTER_PATH = '/home/darwin/filters/' +DEFAULT_PROTOCOL = 'tcp' +DEFAULT_ADDRESS = '127.0.0.1:12121' + # TEST CONFIG VALGRIND_MEMCHECK = False \ No newline at end of file diff --git a/tests/filters/fanomaly.py b/tests/filters/fanomaly.py index 58206f3..2728837 100644 --- a/tests/filters/fanomaly.py +++ b/tests/filters/fanomaly.py @@ -45,8 +45,7 @@ def test(test_name, data, expected_certitudes, expected_certitudes_size): return False # SEND TEST - darwin_api = DarwinApi(socket_path=anomaly_filter.socket, - socket_type="unix", ) + darwin_api = anomaly_filter.get_darwin_api() results = darwin_api.bulk_call( data, diff --git a/tests/filters/fbuffer.py b/tests/filters/fbuffer.py index 43771bb..664902c 100644 --- a/tests/filters/fbuffer.py +++ b/tests/filters/fbuffer.py @@ -132,8 +132,7 @@ def data_to_bytes(data): return False # SEND TEST - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix", ) + darwin_api = buffer_filter.get_darwin_api() if bulk: darwin_api.bulk_call( @@ -413,7 +412,7 @@ def thread_working_test(): buffer_filter = Buffer() buffer_filter.configure(config_buffer) - test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock") + test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock", socket_type='unix') test_filter.configure(config_test) @@ -426,8 +425,7 @@ def thread_working_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix", ) + darwin_api = buffer_filter.get_darwin_api() data = buffer_filter.get_test_data() @@ -539,7 +537,7 @@ def fanomaly_connector_and_send_test(): buffer_filter = Buffer() buffer_filter.configure(config_buffer) - test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock") + test_filter = Filter(filter_name="anomaly", socket_path="/tmp/anomaly.sock", socket_type='unix') test_filter.configure(config_test) # START FILTER @@ -554,8 +552,7 @@ def fanomaly_connector_and_send_test(): # SEND TEST data = buffer_filter.get_test_data() - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix") + darwin_api = buffer_filter.get_darwin_api() darwin_api.bulk_call( data, @@ -654,7 +651,7 @@ def sum_tests(test_name, values=[], required_log_lines=0, expected_alert=1, init buffer_filter = Buffer() buffer_filter.configure(config_buffer) - test_filter = Filter(filter_name="test", socket_path="/tmp/test.sock") + test_filter = Filter(filter_name="test", socket_path="/tmp/test.sock", socket_type='unix') test_filter.configure(config_test) # Potentially add init data to redis @@ -677,8 +674,7 @@ def sum_tests(test_name, values=[], required_log_lines=0, expected_alert=1, init # SEND values - darwin_api = DarwinApi(socket_path=buffer_filter.socket, - socket_type="unix") + darwin_api = buffer_filter.get_darwin_api() for test_value in values: darwin_api.call( ["", test_value], diff --git a/tests/filters/fconnection.py b/tests/filters/fconnection.py index 47a1e1a..d527b01 100644 --- a/tests/filters/fconnection.py +++ b/tests/filters/fconnection.py @@ -65,8 +65,7 @@ def new_connection_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=connection_filter.socket, - socket_type="unix", ) + darwin_api = connection_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -125,8 +124,7 @@ def known_connection_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=connection_filter.socket, - socket_type="unix", ) + darwin_api = connection_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -184,8 +182,7 @@ def new_connection_to_known_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=connection_filter.socket, - socket_type="unix", ) + darwin_api = connection_filter.get_darwin_api() darwin_api.bulk_call( [ diff --git a/tests/filters/fhostlookup.py b/tests/filters/fhostlookup.py index 8c23456..2f48405 100644 --- a/tests/filters/fhostlookup.py +++ b/tests/filters/fhostlookup.py @@ -92,8 +92,7 @@ def test(test_name, init_data, data, expected_certitudes, db_type="json"): return False # SEND TEST - darwin_api = DarwinApi(socket_path=hostlookup_filter.socket, - socket_type="unix", ) + darwin_api = hostlookup_filter.get_darwin_api() results = darwin_api.bulk_call( data, diff --git a/tests/filters/ftanomaly.py b/tests/filters/ftanomaly.py index 435d83d..7256a9d 100644 --- a/tests/filters/ftanomaly.py +++ b/tests/filters/ftanomaly.py @@ -209,8 +209,7 @@ def data_to_bytes(data): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( data, @@ -353,8 +352,7 @@ def thread_working_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( [ @@ -413,8 +411,7 @@ def alert_in_redis_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() data = tanomaly_filter.get_test_data() darwin_api.bulk_call( @@ -466,9 +463,7 @@ def alert_published_test(): if not tanomaly_filter.valgrind_start(): return False # SEND TEST - - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( tanomaly_filter.get_test_data(), @@ -536,8 +531,7 @@ def alert_in_file_test(): return False # SEND TEST - darwin_api = DarwinApi(socket_path=tanomaly_filter.socket, - socket_type="unix", ) + darwin_api = tanomaly_filter.get_darwin_api() darwin_api.bulk_call( tanomaly_filter.get_test_data(), diff --git a/tests/filters/fyara.py b/tests/filters/fyara.py index 552d73a..fbc7213 100644 --- a/tests/filters/fyara.py +++ b/tests/filters/fyara.py @@ -204,8 +204,7 @@ def input_with_several_rules(data, expected_certitudes=[0]): return False # SEND TEST - darwin_api = DarwinApi(socket_path=yara_filter.socket, - socket_type="unix") + darwin_api = yara_filter.get_darwin_api() diff --git a/tests/tools/filter.py b/tests/tools/filter.py index 90e2ae4..2b490d6 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -5,7 +5,7 @@ import uuid import subprocess -from conf import DEFAULT_FILTER_PATH, VALGRIND_MEMCHECK, TEST_FILES_DIR +from conf import DEFAULT_FILTER_PATH, VALGRIND_MEMCHECK, TEST_FILES_DIR, DEFAULT_PROTOCOL, DEFAULT_ADDRESS from darwin import DarwinApi from time import sleep from tools.redis_utils import RedisServer @@ -17,9 +17,15 @@ class Filter(): - def __init__(self, path=None, config_file=None, filter_name="filter", socket_path=None, monitoring_socket_path=None, pid_file=None, output="NONE", next_filter_socket_path="no", nb_threads=1, cache_size=0, threshold=101, log_level="DEVELOPER"): + def __init__(self, path=None, config_file=None, filter_name="filter", socket_path=None, monitoring_socket_path=None, pid_file=None, output="NONE", next_filter_socket_path="no", nb_threads=1, cache_size=0, threshold=101, log_level="DEVELOPER", socket_type=DEFAULT_PROTOCOL): self.filter_name = filter_name - self.socket = socket_path if socket_path else "{}/{}.sock".format(TEST_FILES_DIR, filter_name) + self.socket_type = socket_type + if socket_type == 'unix': + self.socket = socket_path if socket_path else "{}/{}.sock".format(TEST_FILES_DIR, filter_name) + elif socket_type == 'tcp': + self.socket = socket_path if socket_path else DEFAULT_ADDRESS + self.host = self.socket.split(':')[0] + self.port = int(self.socket.split(':')[1]) self.config = config_file if config_file else "{}/{}.conf".format(TEST_FILES_DIR, filter_name) self.path = path if path else "{}darwin_{}".format(DEFAULT_FILTER_PATH, filter_name) self.monitor = monitoring_socket_path if monitoring_socket_path else "{}/{}_mon.sock".format(TEST_FILES_DIR, filter_name) @@ -30,6 +36,15 @@ def __init__(self, path=None, config_file=None, filter_name="filter", socket_pat self.pubsub = None self.prepare_log_file() + def get_darwin_api(self): + return DarwinApi(socket_type=self.socket_type, socket_path=self.socket, socket_host=self.host, socket_port=self.port) + if self.socket_type == 'unix': + api = DarwinApi(socket_type='unix', socket_path=self.socket) + elif self.socket_type == 'tcp': + api = DarwinApi(socket_type='tcp', socket_host=self.socket.split(':')[0], socket_port=int(self.socket.split(':')[1])) + else: + return None + def prepare_log_file(self): # TODO variabilize once path can be changed self.log_file = open("/var/log/darwin/darwin.log", 'a+') @@ -160,7 +175,7 @@ def send_single(self, line): """ Send a single line. """ - api = DarwinApi(socket_type="unix", socket_path=self.socket) + api = self.get_darwin_api() ret = api.call(line, response_type="back") api.close() From 00b207d2081906936df2fcac37c4b4c98a298be2 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 8 Jul 2021 10:13:57 +0200 Subject: [PATCH 23/54] Added tests for TCP connections Assured working with ipv4 and ipv6 addresses Fixed a few log lines in the redis test file --- tests/core/base.py | 120 +++++++++++++++++++++++++++++++++++++----- tests/core/redis.py | 22 ++++---- tests/tools/filter.py | 30 ++++++----- 3 files changed, 134 insertions(+), 38 deletions(-) diff --git a/tests/core/base.py b/tests/core/base.py index 61cc0b3..e6b9bc1 100644 --- a/tests/core/base.py +++ b/tests/core/base.py @@ -14,8 +14,12 @@ def run(): tests = [ check_start_stop, check_pid_file, - check_socket_create_delete, - check_socket_connection, + check_unix_socket_create_delete, + check_unix_socket_connection, + check_tcp_socket_create_delete, + check_tcp_socket_connection, + check_tcp6_socket_create_delete, + check_tcp6_socket_connection, check_socket_monitor_create_delete, check_socket_monitor_connection, check_start_wrong_conf, @@ -78,39 +82,129 @@ def check_pid_file(): return True -def check_socket_create_delete(): - filter = Filter(filter_name="test") - pid = -1 +def check_unix_socket_create_delete(): + filter = Filter(filter_name="test", socket_type='unix') filter.configure(FTEST_CONFIG) filter.valgrind_start() if not access(filter.socket, F_OK): - logging.error("check_socket_create_delete: Socket file not accesible") + logging.error("check_unix_socket_create_delete: Socket file not accesible") return False filter.stop() if access(filter.socket, F_OK): - logging.error("check_socket_create_delete: Socket file not deleted") + logging.error("check_unix_socket_create_delete: Socket file not deleted") return False return True -def check_socket_connection(): - filter = Filter(filter_name="test") - pid = -1 +def check_unix_socket_connection(): + filter = Filter(filter_name="test", socket_type='unix') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("test\n", filter_code=0x74657374, response_type="back") + api.close() + except Exception as e: + logging.error("check_unix_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + +def check_tcp_socket_create_delete(): + filter = Filter(filter_name="test", socket_type='tcp', socket_path='127.0.0.1:12323') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + with socket.socket(socket.AF_INET) as s: + s.settimeout(2) + res = s.connect_ex(('127.0.0.1',12323)) + s.close() + + if res != 0: + logging.error("check_tcp_socket_create_delete: Socket file not accesible") + return False + filter.stop() + + with socket.socket(socket.AF_INET) as s: + s.settimeout(2) + res = s.connect_ex(('127.0.0.1',12323)) + s.close() + + if res == 0: + logging.error("check_tcp_socket_create_delete: Socket file not deleted") + return False + + return True + + +def check_tcp_socket_connection(): #todo + filter = Filter(filter_name="test", socket_type='tcp', socket_path='127.0.0.1:12123') filter.configure(FTEST_CONFIG) filter.valgrind_start() try: - api = DarwinApi(socket_path=filter.socket, socket_type="unix") + api = filter.get_darwin_api() api.call("test\n", filter_code=0x74657374, response_type="back") api.close() except Exception as e: - logging.error("check_socket_connection_back: Error connecting to socket: {}".format(e)) + logging.error("check_tcp_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + +def check_tcp6_socket_create_delete(): + filter = Filter(filter_name="test", socket_type='tcp', socket_path='[::1]:12123') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + with socket.socket(socket.AF_INET6) as s: + s.settimeout(2) + res = s.connect_ex(('::1',12123)) + s.close() + + if res != 0: + logging.error("check_tcp6_socket_create_delete: Socket file not accesible") + return False + + filter.stop() + + with socket.socket(socket.AF_INET6) as s: + s.settimeout(2) + res = s.connect_ex(('::1',12123)) + s.close() + + if res == 0: + logging.error("check_tcp6_socket_create_delete: Socket file not deleted") + return False + + return True + + +def check_tcp6_socket_connection(): + filter = Filter(filter_name="test", socket_type='tcp', socket_path='[::1]:1111') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("test\n", filter_code=0x74657374, response_type="back") + api.close() + except Exception as e: + logging.error("check_tcp6_socket_connection: Error connecting to socket: {}".format(e)) return False filter.stop() @@ -119,7 +213,6 @@ def check_socket_connection(): def check_socket_monitor_create_delete(): filter = Filter(filter_name="test") - pid = -1 filter.configure(FTEST_CONFIG) filter.valgrind_start() @@ -139,7 +232,6 @@ def check_socket_monitor_create_delete(): def check_socket_monitor_connection(): filter = Filter(filter_name="test") - pid = -1 filter.configure(FTEST_CONFIG) filter.valgrind_start() diff --git a/tests/core/redis.py b/tests/core/redis.py index 40c87fd..df71d62 100644 --- a/tests/core/redis.py +++ b/tests/core/redis.py @@ -82,13 +82,13 @@ def master_replica(): message = master.channel_get_message() if message is '': - logging.error("master_replica: expected to get a message in channel {} " + - "but got nothing".format(REDIS_CHANNEL_NAME)) + logging.error(("master_replica: expected to get a message in channel {} " + + "but got nothing").format(REDIS_CHANNEL_NAME)) return False if message != REDIS_CHANNEL_TRIGGER: - logging.error("master_replica: expected to get a message in channel {} saying '{}' " + - "but got '{}' instead".format(REDIS_CHANNEL_NAME, REDIS_CHANNEL_TRIGGER, message)) + logging.error(("master_replica: expected to get a message in channel {} saying '{}' " + + "but got '{}' instead").format(REDIS_CHANNEL_NAME, REDIS_CHANNEL_TRIGGER, message)) return False return True @@ -143,8 +143,8 @@ def master_replica_master_temp_fail(): num_list_entries = master_connection.llen(REDIS_LIST_NAME) if num_list_entries != 1: - logging.error("master_replica_master_temp_fail: wrong number of entries in the redis list {}: " + - "expected 1 but got {}".format(REDIS_LIST_NAME, num_list_entries)) + logging.error(("master_replica_master_temp_fail: wrong number of entries in the redis list {}: " + + "expected 1 but got {}").format(REDIS_LIST_NAME, num_list_entries)) return False return True @@ -271,7 +271,7 @@ def multi_thread_master(): thread_list = [] def thread_brute(filter, count_log): - for count in range(0, count_log): + for _ in range(0, count_log): try: filter.send_single(REDIS_LIST_TRIGGER) except: @@ -279,7 +279,7 @@ def thread_brute(filter, count_log): return True - for num in range(0, 5): + for _ in range(0, 5): thread_list.append(threading.Thread(target=thread_brute, args=(filter, 500))) for thread in thread_list: @@ -294,7 +294,7 @@ def thread_brute(filter, count_log): # 6 threads : 5 task threads + the main thread for configuring the redis socket if number != 6: - logging.error("multi_thread_master: wrong number of active connections: expected 5 but got " + str(number)) + logging.error("multi_thread_master: wrong number of active connections: expected 6 but got {}".format(number)) return False return True @@ -313,7 +313,7 @@ def master_replica_discovery_rate_limiting(): # success filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: - logging.error("master_replica_discovery_rate_limiting: Could not connect to test filter: {}".format(function_name, e)) + logging.error("master_replica_discovery_rate_limiting: Could not connect to test filter: {}".format(e)) return False # master shuts down @@ -333,7 +333,7 @@ def thread_brute_time(filter, time_end): thread_list = [] - for num in range(0, 5): + for _ in range(0, 5): thread_list.append(threading.Thread(target=thread_brute_time, args=(filter, 9))) # ought to crash if command fails diff --git a/tests/tools/filter.py b/tests/tools/filter.py index 2b490d6..f9af7bc 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -19,13 +19,7 @@ class Filter(): def __init__(self, path=None, config_file=None, filter_name="filter", socket_path=None, monitoring_socket_path=None, pid_file=None, output="NONE", next_filter_socket_path="no", nb_threads=1, cache_size=0, threshold=101, log_level="DEVELOPER", socket_type=DEFAULT_PROTOCOL): self.filter_name = filter_name - self.socket_type = socket_type - if socket_type == 'unix': - self.socket = socket_path if socket_path else "{}/{}.sock".format(TEST_FILES_DIR, filter_name) - elif socket_type == 'tcp': - self.socket = socket_path if socket_path else DEFAULT_ADDRESS - self.host = self.socket.split(':')[0] - self.port = int(self.socket.split(':')[1]) + self.set_socket_info(socket_type, socket_path) self.config = config_file if config_file else "{}/{}.conf".format(TEST_FILES_DIR, filter_name) self.path = path if path else "{}darwin_{}".format(DEFAULT_FILTER_PATH, filter_name) self.monitor = monitoring_socket_path if monitoring_socket_path else "{}/{}_mon.sock".format(TEST_FILES_DIR, filter_name) @@ -36,14 +30,24 @@ def __init__(self, path=None, config_file=None, filter_name="filter", socket_pat self.pubsub = None self.prepare_log_file() + def set_socket_info(self, socket_type, socket_path_address): + self.socket_type = socket_type + if socket_type == 'unix': + self.socket = socket_path_address if socket_path_address else "{}/{}.sock".format(TEST_FILES_DIR, self.filter_name) + self.host = None + self.port = -1 + elif socket_type == 'tcp': + self.socket = socket_path_address if socket_path_address else default_address + splitted_address = self.socket.rsplit(':', 1) + if '[' in splitted_address[0]: # ipv6 address + self.socket_type = 'tcp6' + self.host = splitted_address[0][1:-1] + else: + self.host = splitted_address[0] + self.port = int(splitted_address[1]) + def get_darwin_api(self): return DarwinApi(socket_type=self.socket_type, socket_path=self.socket, socket_host=self.host, socket_port=self.port) - if self.socket_type == 'unix': - api = DarwinApi(socket_type='unix', socket_path=self.socket) - elif self.socket_type == 'tcp': - api = DarwinApi(socket_type='tcp', socket_host=self.socket.split(':')[0], socket_port=int(self.socket.split(':')[1])) - else: - return None def prepare_log_file(self): # TODO variabilize once path can be changed From 5139e2a8c2b5d31a361d6d4619514753ee7980ef Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 16 Jul 2021 10:30:16 +0200 Subject: [PATCH 24/54] Added UDP protocol to darwin Fixed a few problems and cleaned some code Added tests for tcp and udp Added possibility to run all test in unix or tcp --- CMakeLists.txt | 5 ++ manager/Services.py | 7 +- manager/config.py | 15 ++++ manager/manager.py | 2 +- samples/base/Core.cpp | 24 +++--- samples/base/network/ANextFilterConnector.cpp | 2 +- samples/base/network/ANextFilterConnector.hpp | 1 - samples/base/network/ASession.fwd.hpp | 3 + samples/base/network/TcpServer.cpp | 12 ++- .../base/network/UdpNextFilterConnector.cpp | 47 +++++++++++ .../base/network/UdpNextFilterConnector.hpp | 30 +++++++ samples/base/network/UdpServer.cpp | 84 +++++++++++++++++++ samples/base/network/UdpServer.hpp | 63 ++++++++++++++ samples/base/network/UdpSession.cpp | 66 +++++++++++++++ samples/base/network/UdpSession.hpp | 42 ++++++++++ tests/core/alert.py | 2 +- tests/core/base.py | 58 ++++++++++++- tests/test.py | 23 +++++ tests/tools/filter.py | 39 +++++---- toolkit/Network.cpp | 13 +-- 20 files changed, 487 insertions(+), 51 deletions(-) create mode 100644 samples/base/network/UdpNextFilterConnector.cpp create mode 100644 samples/base/network/UdpNextFilterConnector.hpp create mode 100755 samples/base/network/UdpServer.cpp create mode 100755 samples/base/network/UdpServer.hpp create mode 100755 samples/base/network/UdpSession.cpp create mode 100755 samples/base/network/UdpSession.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f8c1f31..b3e256f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ if (NOT DEFINED FILTER) CONTENT_INSPECTION BUFFER YARA + TEST ) else (NOT DEFINED FILTER) set( @@ -80,6 +81,7 @@ include_directories( ${Boost_INCLUDE_DIRS} ) +add_compile_options(-Wno-unknown-pragmas) #################### # CORE SOURCES # @@ -102,14 +104,17 @@ set( samples/base/network/ANextFilterConnector.cpp samples/base/network/ANextFilterConnector.hpp samples/base/network/UnixNextFilterConnector.cpp samples/base/network/UnixNextFilterConnector.hpp samples/base/network/TcpNextFilterConnector.cpp samples/base/network/TcpNextFilterConnector.hpp + samples/base/network/UdpNextFilterConnector.cpp samples/base/network/UdpNextFilterConnector.hpp samples/base/network/AServer.cpp samples/base/network/AServer.hpp samples/base/network/UnixServer.cpp samples/base/network/UnixServer.hpp samples/base/network/TcpServer.cpp samples/base/network/TcpServer.hpp + samples/base/network/UdpServer.cpp samples/base/network/UdpServer.hpp samples/base/Manager.cpp samples/base/Manager.hpp samples/base/network/ASession.cpp samples/base/network/ASession.hpp samples/base/network/UnixSession.cpp samples/base/network/UnixSession.hpp samples/base/network/TcpSession.cpp samples/base/network/TcpSession.hpp + samples/base/network/UdpSession.cpp samples/base/network/UdpSession.hpp samples/base/ATask.cpp samples/base/ATask.hpp toolkit/Network.cpp toolkit/Network.hpp diff --git a/manager/Services.py b/manager/Services.py index 62e655a..8765e54 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -103,7 +103,10 @@ def _build_cmd(filt): if filt['network']['socket_type'] == 'UDP': cmd.append('-u') - + + if filt['next_filter_unix_socket']['socket_type'] == 'UDP': + cmd.append('-v') + cmd += [ filt['name'], filt['network']['address_path'], @@ -111,7 +114,7 @@ def _build_cmd(filt): filt['monitoring'], filt['pid_file'], filt['output'], - filt['next_filter_unix_socket'], + filt['next_filter_network']['address_path'], str(filt['nb_thread']), str(filt['cache_size']), str(filt['threshold']), diff --git a/manager/config.py b/manager/config.py index a29c324..b702a44 100644 --- a/manager/config.py +++ b/manager/config.py @@ -190,6 +190,17 @@ def set_defaults(validator, properties, instance, schema): "default": "NONE" }, "next_filter": {"type": "string"}, + "next_filter_network": { + "type": "object", + "properties": { + "socket_type":{ + "type":"string", + "enum": ["UNIX", "TCP", "UDP"], + "default":"UNIX" + }, + "address_path": {"type": "string"} + } + }, "threshold": { "type": "integer", "default": 100 @@ -287,6 +298,10 @@ def complete_filters_conf(prefix, suffix): prefix=prefix, suffix=suffix, next_filter=filter['next_filter'] ) + if 'next_filter_network' not in filter: + filter['next_filter_network'] = { "socket_type":"UNIX", "address_path":filter['next_filter_unix_socket'] } + elif filter['next_filter_network']['socket_type'] == "UNIX": + filter['next_filter_network']['address_path'] = filter['next_filter_unix_socket'] filter['socket'] = '{prefix}/sockets{suffix}/{filter}{extension}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension']) filter['socket_link'] = '{prefix}/sockets{suffix}/{filter}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name']) diff --git a/manager/manager.py b/manager/manager.py index 44270d9..a779808 100644 --- a/manager/manager.py +++ b/manager/manager.py @@ -33,7 +33,7 @@ def create_dirs(dirs, prefix, suffix): for d in dirs: path = '{}/{}{}'.format(prefix, d, suffix) if not os.path.exists(path): - os.mkdir(path) + os.makedirs(path, exist_ok=True) # Argparse parser = argparse.ArgumentParser() diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index fbda543..628da01 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -17,11 +17,13 @@ #include "Logger.hpp" #include "UnixServer.hpp" #include "TcpServer.hpp" +#include "UdpServer.hpp" #include "Core.hpp" #include "Stats.hpp" #include "StringUtils.hpp" #include "UnixNextFilterConnector.hpp" #include "TcpNextFilterConnector.hpp" +#include "UdpNextFilterConnector.hpp" #include @@ -65,10 +67,9 @@ namespace darwin { server = std::make_unique(_net_address, _net_port, _output, _threshold, gen); break; case network::NetworkSocketType::Udp: - //not yet implemented DARWIN_LOG_DEBUG("Core::run:: UDP configured on address " + _net_address.to_string() + ":" + std::to_string(_net_port)); - DARWIN_LOG_CRITICAL("Core:: Run:: UDP Not implemented"); - __attribute__((fallthrough)); + server = std::make_unique(_net_address, _net_port, _output, _threshold, gen); + break; default: DARWIN_LOG_CRITICAL("Core:: Run:: Network Configuration problem"); raise(SIGTERM); @@ -97,7 +98,7 @@ namespace darwin { raise(SIGTERM); } DARWIN_LOG_DEBUG("Core::run:: Joining monitoring thread..."); - t_nextfilter.join(); + t_nextfilter.join(); if (t.joinable()) t.join(); } catch (const std::exception& e) { @@ -127,11 +128,11 @@ namespace darwin { std::string log_level; bool is_udp = false; - + bool is_next_filter_udp = false; // OPTIONS log.setLevel(logger::Warning); // Log level by default opt = -1; - while((opt = getopt(ac, av, ":l:hu")) != -1) + while((opt = getopt(ac, av, ":l:huv")) != -1) { DARWIN_LOG_DEBUG("OPT : " + std::to_string(opt)); DARWIN_LOG_DEBUG("OPTIND : " + std::to_string(optind)); @@ -160,6 +161,9 @@ namespace darwin { case 'u': is_udp = true; break; + case 'v': + is_next_filter_udp = true; + break; } } @@ -180,7 +184,7 @@ namespace darwin { _pidPath = av[optind + 4]; _output = av[optind + 5]; - if (!SetNextFilterConnector(std::string(av[optind + 6]), is_udp)) + if (!SetNextFilterConnector(std::string(av[optind + 6]), is_next_filter_udp)) return false; if (!GetULArg(_nbThread, av[optind + 7])) return false; @@ -209,10 +213,10 @@ namespace darwin { _next_filter_connector = std::make_unique(addr, port); return true; case network::NetworkSocketType::Udp: - DARWIN_LOG_CRITICAL("Core:: SetNextFilterConnector:: UDP Not implemented"); - return false; + _next_filter_connector = std::make_unique(addr, port); + return true; default: - DARWIN_LOG_CRITICAL("Core:: SetNextFilterConnector:: Next Filter Configuration error"); + DARWIN_LOG_CRITICAL("Core:: SetNextFilterConnector:: Next Filter Configuration error : unrecognized type"); return false; } return false; diff --git a/samples/base/network/ANextFilterConnector.cpp b/samples/base/network/ANextFilterConnector.cpp index a2e6c16..a313194 100644 --- a/samples/base/network/ANextFilterConnector.cpp +++ b/samples/base/network/ANextFilterConnector.cpp @@ -23,7 +23,7 @@ namespace darwin { } void ANextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, std::shared_ptr buffer) { - if(ec) { + if(ec || bytes_transferred != buffer->size()) { Send(buffer); return; } diff --git a/samples/base/network/ANextFilterConnector.hpp b/samples/base/network/ANextFilterConnector.hpp index 5f9e698..67ab9dd 100644 --- a/samples/base/network/ANextFilterConnector.hpp +++ b/samples/base/network/ANextFilterConnector.hpp @@ -33,6 +33,5 @@ namespace darwin { size_t _max_attempts; size_t _nb_attempts; std::chrono::milliseconds _attempts_delay_ms; - bool _address_path_parsing; }; } \ No newline at end of file diff --git a/samples/base/network/ASession.fwd.hpp b/samples/base/network/ASession.fwd.hpp index f86d63f..4020842 100644 --- a/samples/base/network/ASession.fwd.hpp +++ b/samples/base/network/ASession.fwd.hpp @@ -1,5 +1,6 @@ #pragma once #include +#include // Forward Declaration mandatory because of the circular dependency between Tasks, Sessions and Generators namespace darwin { @@ -10,4 +11,6 @@ namespace darwin { /// \typedef session_ptr_t // typedef std::shared_ptr session_ptr_t; typedef std::shared_ptr session_ptr_t; + + typedef std::array udp_buffer_t; } \ No newline at end of file diff --git a/samples/base/network/TcpServer.cpp b/samples/base/network/TcpServer.cpp index 17bd24a..018d804 100755 --- a/samples/base/network/TcpServer.cpp +++ b/samples/base/network/TcpServer.cpp @@ -33,10 +33,8 @@ namespace darwin { void TcpServer::Clean() { DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); + DARWIN_LOG_DEBUG("TcpServer::Clean:: Cleaning server..."); _manager.StopAll(); - //todo: can't work, to be changed - // unlink? what? why? } void TcpServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { @@ -46,7 +44,7 @@ namespace darwin { DARWIN_LOGGER; SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); - DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); + DARWIN_LOG_DEBUG("TcpServer::Handle:: Closing acceptor"); _acceptor.close(); _io_context.stop(); } @@ -61,19 +59,19 @@ namespace darwin { DARWIN_LOGGER; if (!_acceptor.is_open()) { - DARWIN_LOG_INFO("Server::HandleAccept:: Acceptor closed, closing server..."); + DARWIN_LOG_INFO("TcpServer::HandleAccept:: Acceptor closed, closing server..."); return; } if (!e) { - DARWIN_LOG_DEBUG("Server::HandleAccept:: New connection accepted"); + DARWIN_LOG_DEBUG("TcpServer::HandleAccept:: New connection accepted"); auto sess = std::make_shared(_new_connection, _manager, _generator); sess->SetOutputType(_output); sess->SetThreshold(_threshold); _manager.Start(sess); Accept(); } else { - DARWIN_LOG_ERROR("Server::HandleAccept:: Error accepting connection, no longer accepting"); + DARWIN_LOG_ERROR("TcpServer::HandleAccept:: Error accepting connection, no longer accepting"); } } diff --git a/samples/base/network/UdpNextFilterConnector.cpp b/samples/base/network/UdpNextFilterConnector.cpp new file mode 100644 index 0000000..ab19dfb --- /dev/null +++ b/samples/base/network/UdpNextFilterConnector.cpp @@ -0,0 +1,47 @@ +#include "UdpNextFilterConnector.hpp" + +#include + +#include "Logger.hpp" + +namespace darwin { + + UdpNextFilterConnector::UdpNextFilterConnector(boost::asio::ip::address const& net_address, int port) + : ANextFilterConnector{}, _net_address{net_address}, _net_port{port}, _endpoint{_net_address, static_cast(_net_port)}, + _socket{_io_context} + { + + } + + + bool UdpNextFilterConnector::Connect() { + DARWIN_LOGGER; + boost::system::error_code ec; + + _socket.connect(_endpoint, ec); + + if(ec) { + DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + return false; + } + return true; + } + + void UdpNextFilterConnector::Send(std::shared_ptr packet) { + DARWIN_LOGGER; + if(_nb_attempts > _max_attempts){ + DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); + return; + } + while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { + std::this_thread::sleep_for(_attempts_delay_ms); + } + + _socket.async_send_to(*packet, + _endpoint, + boost::bind(&UdpNextFilterConnector::SendCallback, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred, + packet)); + } +} \ No newline at end of file diff --git a/samples/base/network/UdpNextFilterConnector.hpp b/samples/base/network/UdpNextFilterConnector.hpp new file mode 100644 index 0000000..53f8863 --- /dev/null +++ b/samples/base/network/UdpNextFilterConnector.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include +#include + +#include "ANextFilterConnector.hpp" +#include "DarwinPacket.hpp" +#include "Network.hpp" + +namespace darwin { + + class UdpNextFilterConnector: public ANextFilterConnector { + public: + UdpNextFilterConnector(boost::asio::ip::address const& net_address, int port); + virtual ~UdpNextFilterConnector() = default; + + virtual bool Connect(); + + private: + + virtual void Send(std::shared_ptr packet); + + boost::asio::ip::address _net_address; + int _net_port; + boost::asio::ip::udp::endpoint _endpoint; + boost::asio::ip::udp::socket _socket; + }; +} \ No newline at end of file diff --git a/samples/base/network/UdpServer.cpp b/samples/base/network/UdpServer.cpp new file mode 100755 index 0000000..0712b69 --- /dev/null +++ b/samples/base/network/UdpServer.cpp @@ -0,0 +1,84 @@ +/// \file Server.cpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#include +#include +#include +#include +#include +#include "UdpServer.hpp" +#include "UdpSession.hpp" +#include "Logger.hpp" +#include "Stats.hpp" + +namespace darwin { + + UdpServer::UdpServer(boost::asio::ip::address const& address, + int port_nb, + std::string const& output, + std::size_t threshold, + Generator& generator) + : AServer(output, threshold, generator), + _address{address}, _port_nb{port_nb}, + _endpoint{_address, static_cast(_port_nb)}, + _new_connection{_io_context,_endpoint} + { + this->InitSignalsAndStart(); + } + + + + + void UdpServer::Clean() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Server::Clean:: Cleaning server..."); + _manager.StopAll(); + } + + void UdpServer::HandleStop(boost::system::error_code const& error __attribute__((unused)), int sig __attribute__((unused))) { + // The server is stopped by cancelling all outstanding asynchronous + // operations. Once all operations have finished the io_context::run() + // call will exit. + DARWIN_LOGGER; + + SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::stopping); + DARWIN_LOG_DEBUG("Server::Handle:: Closing acceptor"); + _io_context.stop(); + } + + void UdpServer::Accept() { + _new_connection.async_receive(boost::asio::buffer(_buffer), + boost::bind(&UdpServer::HandleReceive, this, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred)); + } + + void UdpServer::HandleReceive(boost::system::error_code const& e, size_t bytes_received) { + DARWIN_LOGGER; + + if (!e) { + if(bytes_received >= sizeof(darwin_filter_packet_t)){ + DARWIN_LOG_DEBUG("UdpServer::HandleReceive:: New connection accepted"); + auto sess = std::make_shared(_buffer, _manager, _generator); + sess->SetOutputType(_output); + sess->SetThreshold(_threshold); + sess->ReadBody(bytes_received); + } else { + DARWIN_LOG_ERROR("UdpServer::HandleReceive:: packet too small (in bytes) : " + std::to_string(bytes_received)); + } + } else { + DARWIN_LOG_ERROR("UdpServer::HandleReceive:: Error accepting connection, no longer accepting"); + } + Accept(); + } + + void UdpServer::HandleAccept(boost::system::error_code const& e) { + return; + } + + +} \ No newline at end of file diff --git a/samples/base/network/UdpServer.hpp b/samples/base/network/UdpServer.hpp new file mode 100755 index 0000000..35ae3ca --- /dev/null +++ b/samples/base/network/UdpServer.hpp @@ -0,0 +1,63 @@ +/// \file Server.hpp +/// \authors hsoszynski +/// \version 1.0 +/// \date 02/07/18 +/// \license GPLv3 +/// \brief Copyright (c) 2018 Advens. All rights reserved. + +#pragma once + +#include +#include +#include +#include +#include "ASession.hpp" +#include "Manager.hpp" +#include "Generator.hpp" +#include "AServer.hpp" + +namespace darwin { + + class UdpServer : public AServer { + public: + /// Create an async Udp stream socket server. + /// The server runs on nb_threads thread. + /// + /// \param socket_path Path of the Udp socket to listen on. + /// \param output Filters' output type + /// \param next_filter_socket Path of the Udp socket of the filter to send data to. + /// \param threshold Threshold at which the filter will raise a log. + UdpServer(boost::asio::ip::address const& address, + int port, + std::string const& output, + std::size_t threshold, + Generator& generator); + + virtual ~UdpServer() = default; + + // Make the UdpServer non copyable & non movable + UdpServer(UdpServer const&) = delete; + + UdpServer(UdpServer const&&) = delete; + + UdpServer& operator=(UdpServer const&) = delete; + + UdpServer& operator=(UdpServer const&&) = delete; + + void Clean(); + + void HandleStop(boost::system::error_code const& error, int sig); + + void Accept(); + + void HandleAccept(boost::system::error_code const& e); + + void HandleReceive(boost::system::error_code const& e, size_t bytes_received); + private: + boost::asio::ip::address _address; + int _port_nb; //!< Path to the Udp socket to listen on. + boost::asio::ip::udp::endpoint _endpoint; + boost::asio::ip::udp::socket _new_connection; //!< Socket used to accept a new connection. + udp_buffer_t _buffer; + }; +} \ No newline at end of file diff --git a/samples/base/network/UdpSession.cpp b/samples/base/network/UdpSession.cpp new file mode 100755 index 0000000..74b49ca --- /dev/null +++ b/samples/base/network/UdpSession.cpp @@ -0,0 +1,66 @@ +#include +#include +#include + +#include "Logger.hpp" +#include "UdpSession.hpp" + +namespace darwin { + UdpSession::UdpSession(const udp_buffer_t& buffer, + Manager& manager, Generator& generator) + : ASession(manager, generator), _buffer{buffer} + { + ; + } + + void UdpSession::Stop() { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("UdpSession::Stop::"); + CloseFilterConnection(); + } + + void UdpSession::CloseFilterConnection() { + } + + void UdpSession::ReadHeader() { + //NOP + return; + } + + void UdpSession::ReadBody(std::size_t size) { + DARWIN_LOGGER; + + if(size < sizeof(_header)){ + DARWIN_LOG_ERROR("Error parsing header, buffer is less than expected : " + std::to_string(size)); + return; + } + + std::memcpy(&_header, _buffer.data(), sizeof(_header)); + + if(size != sizeof(_header) + _header.body_size){ + DARWIN_LOG_ERROR("Error parsing header sizes, expected " + std::to_string(sizeof(_header) + _header.body_size) + " but buffer is " + std::to_string(size)); + return; + } + + //TODO Certitudes parsing + + _packet = DarwinPacket(_header); + auto& body = _packet.GetMutableBody(); + body.append(_buffer.begin() + sizeof(_header), _header.body_size); + + this->ExecuteFilter(); + } + + void UdpSession::WriteToClient(std::vector& packet) { + return; + } + + void UdpSession::ExecuteFilter() { + ASession::ExecuteFilter(); + // ASession::Start(); + } + + void UdpSession::SetNextFilterPort(int port){ + } +} + diff --git a/samples/base/network/UdpSession.hpp b/samples/base/network/UdpSession.hpp new file mode 100755 index 0000000..12e54eb --- /dev/null +++ b/samples/base/network/UdpSession.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include "ASession.hpp" +#include "Generator.hpp" + +namespace darwin { + + class UdpSession: public ASession { + public: + UdpSession(const udp_buffer_t& buffer, + Manager& manager, + Generator& generator); + + ~UdpSession() = default; + + void Stop() override final; + + void SetNextFilterPort(int port); + + /// Set the async read for the body. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadBody(std::size_t size) override; + + protected: + + /// Set the async read for the header. + /// + /// \return -1 on error, 0 on socket closed & sizeof(header) on success. + virtual void ReadHeader() override; + + /// + virtual void WriteToClient(std::vector& packet) override; + + virtual void CloseFilterConnection() override; + virtual void ExecuteFilter() override; + private: + + const udp_buffer_t& _buffer; + }; + +} \ No newline at end of file diff --git a/tests/core/alert.py b/tests/core/alert.py index d5bb7e4..b6b114a 100644 --- a/tests/core/alert.py +++ b/tests/core/alert.py @@ -43,7 +43,7 @@ def clean_files(self): pass def send(self, data): - api = DarwinApi(socket_type="unix", socket_path=self.socket) + api = self.get_darwin_api() api.call([data], response_type='no') def get_redis_alerts(self): diff --git a/tests/core/base.py b/tests/core/base.py index e6b9bc1..29432f1 100644 --- a/tests/core/base.py +++ b/tests/core/base.py @@ -5,7 +5,6 @@ from tools.filter import Filter from tools.output import print_result from core.utils import DEFAULT_PATH, FTEST_CONFIG, RESP_MON_STATUS_RUNNING -from darwin import DarwinApi @@ -20,6 +19,8 @@ def run(): check_tcp_socket_connection, check_tcp6_socket_create_delete, check_tcp6_socket_connection, + check_udp_socket_connection, + check_udp6_socket_connection, check_socket_monitor_create_delete, check_socket_monitor_connection, check_start_wrong_conf, @@ -146,7 +147,7 @@ def check_tcp_socket_create_delete(): return True -def check_tcp_socket_connection(): #todo +def check_tcp_socket_connection(): filter = Filter(filter_name="test", socket_type='tcp', socket_path='127.0.0.1:12123') filter.configure(FTEST_CONFIG) @@ -211,6 +212,59 @@ def check_tcp6_socket_connection(): return True +# These tests are not done as there is no reliable way to check if a udp socket is open and listening +# unreliable ways include listening for a icmp packet back but it depends on the +# system and if the packet is blocked by firewalls + +# def check_udp_socket_create_delete() +# def check_udp6_socket_create_delete() + +def check_udp_socket_connection(): + filter = Filter(filter_name="test", socket_type='udp', socket_path='127.0.0.1:12123') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("udp test", filter_code=0x74657374, response_type="no") + # sleep to let the filter process the call + #TODO check alert + api.call("udp test2", filter_code=0x74657374, response_type="no") + api.call("udp test3", filter_code=0x74657374, response_type="no") + api.call("udp test4", filter_code=0x74657374, response_type="no") + api.call("udp test5", filter_code=0x74657374, response_type="no") + sleep(2) + + api.close() + except Exception as e: + logging.error("check_udp_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + +def check_udp6_socket_connection(): + filter = Filter(filter_name="test", socket_type='udp', socket_path='[::1]:1111') + + filter.configure(FTEST_CONFIG) + filter.valgrind_start() + + try: + api = filter.get_darwin_api() + api.call("test\n", filter_code=0x74657374, response_type="no") + sleep(1) #let filter process + # todo check alert + api.close() + except Exception as e: + logging.error("check_udp6_socket_connection: Error connecting to socket: {}".format(e)) + return False + + filter.stop() + return True + + def check_socket_monitor_create_delete(): filter = Filter(filter_name="test") diff --git a/tests/test.py b/tests/test.py index db8214c..b6e984f 100644 --- a/tests/test.py +++ b/tests/test.py @@ -1,9 +1,32 @@ import logging +import argparse from sys import stderr import manager_socket.test as manager_socket import core.test as core import filters.test as filters +# `conf` is imported whole because variables inside +# may be modified by the command line +import conf + +parser = argparse.ArgumentParser() +socket_arg_group = parser.add_mutually_exclusive_group(required=False) +socket_arg_group.add_argument("--tcp", action="store_true") +socket_arg_group.add_argument("--udp", action="store_true") +socket_arg_group.add_argument("--unix", action="store_true") +parser.add_argument("ip_address", nargs='?', + help="(For tcp and udp only) The IP address formatted like this: ipv4: 'x.x.x.x:port', ipv6: '[x:x:x:x:x:x:x]:port'") +args = parser.parse_args() + +if args.tcp: + conf.DEFAULT_PROTOCOL = 'tcp' +elif args.udp: + conf.DEFAULT_PROTOCOL = 'udp' +elif args.unix: + conf.DEFAULT_PROTOCOL = 'unix' + +if args.ip_address != None: + conf.DEFAULT_ADDRESS = args.ip_address if __name__ == "__main__": logging.basicConfig(filename="test_error.log", filemode='w', level=logging.ERROR) diff --git a/tests/tools/filter.py b/tests/tools/filter.py index f9af7bc..e0a0499 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -2,10 +2,11 @@ import logging import os import os.path -import uuid import subprocess -from conf import DEFAULT_FILTER_PATH, VALGRIND_MEMCHECK, TEST_FILES_DIR, DEFAULT_PROTOCOL, DEFAULT_ADDRESS +# `conf` is imported whole because variables inside +# may be modified by the command line +import conf from darwin import DarwinApi from time import sleep from tools.redis_utils import RedisServer @@ -17,34 +18,38 @@ class Filter(): - def __init__(self, path=None, config_file=None, filter_name="filter", socket_path=None, monitoring_socket_path=None, pid_file=None, output="NONE", next_filter_socket_path="no", nb_threads=1, cache_size=0, threshold=101, log_level="DEVELOPER", socket_type=DEFAULT_PROTOCOL): + def __init__(self, path=None, config_file=None, filter_name="filter", socket_path=None, monitoring_socket_path=None, pid_file=None, output="NONE", next_filter_socket_path="no", nb_threads=1, cache_size=0, threshold=101, log_level="DEVELOPER", socket_type=None): self.filter_name = filter_name + self.process = None self.set_socket_info(socket_type, socket_path) - self.config = config_file if config_file else "{}/{}.conf".format(TEST_FILES_DIR, filter_name) - self.path = path if path else "{}darwin_{}".format(DEFAULT_FILTER_PATH, filter_name) - self.monitor = monitoring_socket_path if monitoring_socket_path else "{}/{}_mon.sock".format(TEST_FILES_DIR, filter_name) - self.pid = pid_file if pid_file else "{}/{}.pid".format(TEST_FILES_DIR, filter_name) + self.config = config_file if config_file else "{}/{}.conf".format(conf.TEST_FILES_DIR, filter_name) + self.path = path if path else "{}darwin_{}".format(conf.DEFAULT_FILTER_PATH, filter_name) + self.monitor = monitoring_socket_path if monitoring_socket_path else "{}/{}_mon.sock".format(conf.TEST_FILES_DIR, filter_name) + self.pid = pid_file if pid_file else "{}/{}.pid".format(conf.TEST_FILES_DIR, filter_name) self.cmd = [self.path, "-l", log_level, self.filter_name, self.socket, self.config, self.monitor, self.pid, output, next_filter_socket_path, str(nb_threads), str(cache_size), str(threshold)] - self.process = None + if self.socket_type.startswith('udp'): + self.cmd += ['-u'] self.error_code = 99 # For valgrind testing self.pubsub = None self.prepare_log_file() def set_socket_info(self, socket_type, socket_path_address): - self.socket_type = socket_type - if socket_type == 'unix': - self.socket = socket_path_address if socket_path_address else "{}/{}.sock".format(TEST_FILES_DIR, self.filter_name) + self.socket_type = socket_type if socket_type else conf.DEFAULT_PROTOCOL + if self.socket_type == 'unix': + self.socket = socket_path_address if socket_path_address else "{}/{}.sock".format(conf.TEST_FILES_DIR, self.filter_name) self.host = None self.port = -1 - elif socket_type == 'tcp': - self.socket = socket_path_address if socket_path_address else default_address + elif self.socket_type == 'tcp' or self.socket_type == 'udp': + self.socket = socket_path_address if socket_path_address else conf.DEFAULT_ADDRESS splitted_address = self.socket.rsplit(':', 1) if '[' in splitted_address[0]: # ipv6 address - self.socket_type = 'tcp6' + self.socket_type += '6' self.host = splitted_address[0][1:-1] else: self.host = splitted_address[0] self.port = int(splitted_address[1]) + else: + raise NotImplementedError("Unrecognized protocol : '{}'".format(self.socket_type)) def get_darwin_api(self): return DarwinApi(socket_type=self.socket_type, socket_path=self.socket, socket_host=self.host, socket_port=self.port) @@ -58,7 +63,7 @@ def prepare_log_file(self): def __del__(self): if self.process and self.process.poll() is None: - if VALGRIND_MEMCHECK is False: + if conf.VALGRIND_MEMCHECK is False: self.stop() else: self.valgrind_stop() @@ -108,7 +113,7 @@ def stop(self): return self.check_stop() def valgrind_start(self): - if VALGRIND_MEMCHECK is False: + if conf.VALGRIND_MEMCHECK is False: return self.start() command = ['valgrind', '--tool=memcheck', @@ -123,7 +128,7 @@ def valgrind_start(self): return self.check_start() def valgrind_stop(self): - if VALGRIND_MEMCHECK is False: + if conf.VALGRIND_MEMCHECK is False: return self.stop() self.process.terminate() try: diff --git a/toolkit/Network.cpp b/toolkit/Network.cpp index 5e3ad43..9f3c665 100644 --- a/toolkit/Network.cpp +++ b/toolkit/Network.cpp @@ -120,25 +120,20 @@ namespace darwin { size_t colon = path_address.rfind(':'); if (colon != std::string::npos) { boost::system::error_code e; - auto addr = path_address.substr(0, colon); + std::string addr = path_address.substr(0, colon); if(addr.find('[') != std::string::npos && addr.rfind(']') != std::string::npos) { addr.pop_back(); addr.erase(0, 1); } - auto port = path_address.substr(colon+1, path_address.length()-1); + std::string port = path_address.substr(colon+1, path_address.length()-1); bool portRes = ParsePort(port.c_str(), out_port); - out_net_address = boost::asio::ip::make_address(addr, e); if( ! portRes || e.failed()) { DARWIN_LOG_CRITICAL("Network::ParseSocketAddress::Error while parsing the ip address: " + path_address); return false; } - - if(is_udp) { - out_net_type = NetworkSocketType::Udp; - } else { - out_net_type = NetworkSocketType::Tcp; - } + out_net_address = boost::asio::ip::make_address(addr, e); + out_net_type= is_udp ? NetworkSocketType::Udp : NetworkSocketType::Tcp; } else { out_net_type = NetworkSocketType::Unix; out_unix_path = path_address; From b8d24dbe2cfeb6df8d60894037acb6fa3c636e65 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 20 Jul 2021 09:27:14 +0200 Subject: [PATCH 25/54] fixed few issues with the manager --- manager/Services.py | 2 +- manager/config.py | 5 +++-- samples/base/network/ASession.hpp | 2 +- tests/filters/fsession.py | 16 +++++----------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/manager/Services.py b/manager/Services.py index 8765e54..0999ad5 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -104,7 +104,7 @@ def _build_cmd(filt): if filt['network']['socket_type'] == 'UDP': cmd.append('-u') - if filt['next_filter_unix_socket']['socket_type'] == 'UDP': + if filt['next_filter_network']['socket_type'] == 'UDP': cmd.append('-v') cmd += [ diff --git a/manager/config.py b/manager/config.py index b702a44..a719eef 100644 --- a/manager/config.py +++ b/manager/config.py @@ -195,8 +195,8 @@ def set_defaults(validator, properties, instance, schema): "properties": { "socket_type":{ "type":"string", - "enum": ["UNIX", "TCP", "UDP"], - "default":"UNIX" + "enum": ["NONE", "UNIX", "TCP", "UDP"], + "default":"NONE" }, "address_path": {"type": "string"} } @@ -293,6 +293,7 @@ def complete_filters_conf(prefix, suffix): if not filter['next_filter']: filter['next_filter_unix_socket'] = 'no' + filter['next_filter_network'] = { "socket_type":"NONE", "address_path":filter['next_filter_unix_socket'] } else: filter['next_filter_unix_socket'] = '{prefix}/sockets{suffix}/{next_filter}.sock'.format( prefix=prefix, suffix=suffix, diff --git a/samples/base/network/ASession.hpp b/samples/base/network/ASession.hpp index 2d15b2e..7a1ccac 100755 --- a/samples/base/network/ASession.hpp +++ b/samples/base/network/ASession.hpp @@ -139,7 +139,7 @@ namespace darwin { std::size_t size) final; /// Execute the filter and - virtual void ExecuteFilter() final; + virtual void ExecuteFilter(); /// Sends a response with a body containing an error message /// diff --git a/tests/filters/fsession.py b/tests/filters/fsession.py index a8c9a0d..cd822aa 100644 --- a/tests/filters/fsession.py +++ b/tests/filters/fsession.py @@ -7,7 +7,6 @@ from tools.filter import Filter from tools.output import print_result from conf import TEST_FILES_DIR -from darwin import DarwinApi REDIS_SOCKET = f"{TEST_FILES_DIR}/redis_session.sock" @@ -143,8 +142,7 @@ def _input_tests(test_name, data, expected_certitudes, populate_redis=None, expe return False # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( data, @@ -346,8 +344,7 @@ def no_timeout_no_refresh(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -411,8 +408,7 @@ def timeout_but_no_change_to_ttl(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -477,8 +473,7 @@ def set_new_ttl(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ @@ -543,8 +538,7 @@ def timeout_with_change_to_ttl(): sleep(3) # SEND TEST - darwin_api = DarwinApi(socket_path=session_filter.socket, - socket_type="unix") + darwin_api = session_filter.get_darwin_api() results = darwin_api.bulk_call( [ From 9fabcd93cef2abfb5f7b67f37cc400206fa27720 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 20 Jul 2021 16:49:00 +0200 Subject: [PATCH 26/54] Added comments documentation Removed unused methods --- samples/base/AGenerator.hpp | 15 +- samples/base/ATask.hpp | 19 +++ samples/base/Core.hpp | 18 +++ samples/base/DarwinPacket.hpp | 132 +++++++++++++++++- samples/base/network/ANextFilterConnector.hpp | 48 ++++++- samples/base/network/AServer.hpp | 26 ++-- samples/base/network/ASession.fwd.hpp | 4 +- samples/base/network/ASession.hpp | 26 ++-- .../base/network/TcpNextFilterConnector.hpp | 22 +++ samples/base/network/TcpServer.hpp | 31 ++-- samples/base/network/TcpSession.cpp | 19 +-- samples/base/network/TcpSession.hpp | 32 +++-- .../base/network/UdpNextFilterConnector.hpp | 16 +++ samples/base/network/UdpServer.cpp | 2 +- samples/base/network/UdpServer.hpp | 31 ++-- samples/base/network/UdpSession.cpp | 14 +- samples/base/network/UdpSession.hpp | 22 ++- .../base/network/UnixNextFilterConnector.hpp | 16 ++- samples/base/network/UnixServer.hpp | 15 +- samples/base/network/UnixSession.cpp | 22 +-- samples/base/network/UnixSession.hpp | 25 ++-- toolkit/Network.cpp | 8 +- toolkit/Network.hpp | 20 ++- 23 files changed, 434 insertions(+), 149 deletions(-) diff --git a/samples/base/AGenerator.hpp b/samples/base/AGenerator.hpp index 4159141..657b61e 100644 --- a/samples/base/AGenerator.hpp +++ b/samples/base/AGenerator.hpp @@ -24,11 +24,12 @@ class AGenerator { // Methods to be implemented by the child public: - /// Create a new task. /// - /// \param socket The Session's socket. - /// \param manager The Session manager. - /// \return A pointer to a new session. + /// \brief Create a Task object + /// + /// \param s a shred pointer to the sessions creating the task + /// \return std::shared_ptr a shared pointer to the created task + /// virtual std::shared_ptr CreateTask(darwin::session_ptr_t s) noexcept = 0; virtual bool ConfigureNetworkObject(boost::asio::io_context &context); @@ -79,6 +80,12 @@ class AGenerator { virtual bool ExtractCustomAlertingTags(const rapidjson::Document &configuration, std::string& tags); + /// + /// \brief Get the configuration object of the task thread pool + /// + /// \param nb_task_threads number of workers (threads) to spawn for the tasks + /// \return tp::ThreadPoolOptions + /// static tp::ThreadPoolOptions GetThreadPoolOptions(size_t nb_task_threads); protected: diff --git a/samples/base/ATask.hpp b/samples/base/ATask.hpp index ce7fb62..6e92524 100755 --- a/samples/base/ATask.hpp +++ b/samples/base/ATask.hpp @@ -1,3 +1,13 @@ +/// +/// \file ATask.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Task to be extended by filter's tasks +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -17,6 +27,15 @@ namespace darwin { class ATask : public std::enable_shared_from_this { public: + /// + /// \brief Construct a new ATask object + /// + /// \param name name of the filter + /// \param cache shared pointer to the cache to use + /// \param cache_mutex mutex of the cache + /// \param session shared pointer to the session spawning the task + /// \param packet DarwinPacket parsed from the session, moved to the task + /// ATask(std::string name, std::shared_ptr> cache, std::mutex& cache_mutex, session_ptr_t session, diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 0b08634..36b47e1 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -82,10 +82,28 @@ namespace darwin { /// \return true on success, false otherwise. bool SetLogLevel(std::string level); + /// + /// \brief returns the daemon attribute, it is set ine the Configure method + /// + /// \return true If the filter is configured to be ran as deamon bool IsDaemon() const; + /// + /// \brief Set the Next Filter Connector object + /// + /// \param path_address string to be parsed, can be a socket path, + /// an ipv4 address with the port specified 'x.x.x.x:pppp' or + /// an ipv6 address with the port specified '[xx:xx:xx:xx:xx]:ppp' + /// \param is_udp true if we should use udp, the parameter is unused if path_address is a socket + /// \return true if there was no error setting up the Next FilterConnector + /// bool SetNextFilterConnector(std::string const& path_address, bool is_udp); + /// + /// \brief Get a ref to the NextFilterConnector + /// + /// \return ANextFilterConnector& a ref to the next filter + /// \throws std::logic_error if the NextFilterConnector is not set (happens if SetNextFilterConnector was not called) ANextFilterConnector& GetNextFilterconnector(); private: diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 4877018..2b1e547 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -1,3 +1,13 @@ +/// +/// \file DarwinPacket.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief A Description of a darwin packet with serializing and deserializing capacity +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -13,6 +23,16 @@ namespace darwin { class DarwinPacket { public: + /// + /// \brief Construct a new Darwin Packet object + /// + /// \param type type of the packet + /// \param response response type of the packet + /// \param filter_code filter code + /// \param event_id event id + /// \param certitude_size size of the certitude + /// \param body_size size of the body + /// DarwinPacket( enum darwin_packet_type type, enum darwin_filter_response_type response, @@ -21,61 +41,165 @@ namespace darwin { size_t certitude_size, size_t body_size); + /// + /// \brief Construct a new Darwin Packet object from a darwin_filter_packet_t + /// + /// \param input the native struct to be parsed + /// DarwinPacket(darwin_filter_packet_t& input); DarwinPacket() = default; - virtual ~DarwinPacket(); + /// + /// \brief Serialize the packet + /// + /// \return std::vector a vector holding the serialized packet + /// std::vector Serialize() const; + /// + /// \brief Get the Minimal Size of a packet + /// + /// \return constexpr size_t size of the packet + /// constexpr static size_t getMinimalSize() { return sizeof(_type) + sizeof(_response) + sizeof(_filter_code) + sizeof(_parsed_body_size) + sizeof(_evt_id) + sizeof(_parsed_certitude_size) + DEFAULT_CERTITUDE_LIST_SIZE * sizeof(unsigned int); // Protocol has at least DEFAULT (one) certitude }; - static DarwinPacket ParseHeader(darwin_filter_packet_t& input); - + /// + /// \brief Clears the packet + /// + /// void clear(); + /// TODO check if useful + /// \brief Parse the body as a json document and returns a reference to it + /// + /// \return rapidjson::Document& the reference to the parsed body + /// rapidjson::Document& JsonBody(); + /// + /// \brief Get the Type + /// + /// \return enum darwin_packet_type + /// enum darwin_packet_type GetType() const; + /// + /// \brief Get the Response Type + /// + /// \return enum darwin_filter_response_type + /// enum darwin_filter_response_type GetResponse() const; + /// + /// \brief Set the Response Type + /// + /// void SetResponse(enum darwin_filter_response_type); + /// + /// \brief Get the Filter Code + /// + /// \return long + /// long GetFilterCode() const; + /// + /// \brief Set the Filter Code + /// + /// \param filter_code + /// void SetFilterCode(long filter_code); + /// + /// \brief Get the Event Id pointer + /// + /// \return const unsigned char* pointer of a 16 bytes buffer + /// const unsigned char * GetEventId() const; - + /// + /// \brief Get the Event Id Size + /// + /// \return size_t 16 + /// size_t GetEventIdSize() const; + /// + /// \brief Get the Parsed Certitude Size, it may differs from GetCertitudeList().size() + /// + /// \return size_t Parsed Certitude Size + /// size_t GetParsedCertitudeSize() const; + /// + /// \brief Get the Parsed Body Size, it may differ from GetBody().size() + /// + /// \return size_t Parsed Body size + /// size_t GetParsedBodySize() const; + /// + /// \brief Get a const ref to the Body + /// + /// \return const std::string& body + /// const std::string& GetBody() const; + /// + /// \brief Get a mutable ref to the Body + /// + /// \return std::string& body + /// std::string& GetMutableBody(); + /// + /// \brief Get a const ref to the Certitude List + /// + /// \return const std::vector& certitude list + /// const std::vector& GetCertitudeList() const; + /// + /// \brief Get a mutable ref to the Certitude List + /// + /// \return std::vector& certitude list + /// std::vector& GetMutableCertitudeList(); + /// + /// \brief Add a certitude to the list + /// + /// \param certitude to add + /// inline void AddCertitude(unsigned int certitude) { this->_certitude_list.push_back(certitude); }; + /// + /// \brief Get a const ref to the Logs + /// + /// \return const std::string& logs + /// const std::string& GetLogs() const; + /// + /// \brief Get a mutable ref to the Logs + /// + /// \return std::string& logs + /// std::string& GetMutableLogs(); + /// + /// \brief Set the Logs + /// + /// \param logs + /// void SetLogs(std::string& logs); protected: diff --git a/samples/base/network/ANextFilterConnector.hpp b/samples/base/network/ANextFilterConnector.hpp index 67ab9dd..1d6a011 100644 --- a/samples/base/network/ANextFilterConnector.hpp +++ b/samples/base/network/ANextFilterConnector.hpp @@ -1,3 +1,13 @@ +/// +/// \file ANextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Connector to the Next Filter to be extended for different protocols +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -15,17 +25,49 @@ namespace darwin { ANextFilterConnector(); virtual ~ANextFilterConnector() = default; - void Run(); + /// + /// \brief Runs the io_context from the class + /// + /// + virtual void Run() final; + /// + /// \brief Tries to connect to the next filter + /// + /// \return true if the connection is established + /// \return false else + /// virtual bool Connect() = 0; - void Send(DarwinPacket& packet); + /// + /// \brief Attempt to send a DarwinPacket to the next filter, it will atttempt to connect by calling this->Connect() + /// The Packet will be copied to a boost::asio::const_buffer and kept in a set + /// There will be _max_attempts tries before aborting the packet + /// + /// \param packet packet to be sent + /// + virtual void Send(DarwinPacket& packet); protected: + /// + /// \brief Attempt to send a packet to the next filter, it will atttempt to connect by calling this->Connect() + /// The Packet will be copied to a boost::asio::const_buffer and kept in a set + /// There will be _max_attempts tries before aborting the packet + /// + /// \param packet a shared pointer to a boost asio buffer to be sent + /// virtual void Send(std::shared_ptr packet) = 0; - virtual /*?*/ void SendCallback(const boost::system::error_code& e, size_t size, std::shared_ptr buffer); + /// + /// \brief Callback called by the io_context when the async 'Send' operation has ended + /// This Callback will call 'Send(buffer)' if there is an error + /// + /// \param e error code returned + /// \param size number of bytes sent + /// \param buffer shared pointer to buffer sent + /// + virtual void SendCallback(const boost::system::error_code& e, size_t size, std::shared_ptr buffer); boost::asio::io_context _io_context; std::set> _buffer_set; diff --git a/samples/base/network/AServer.hpp b/samples/base/network/AServer.hpp index d84df8e..38deb70 100644 --- a/samples/base/network/AServer.hpp +++ b/samples/base/network/AServer.hpp @@ -1,3 +1,13 @@ +/// +/// \file AServer.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Server to be extended or different protocols +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -10,15 +20,6 @@ namespace darwin { class AServer { public: - /// Create an async UNIX stream socket server. - /// The server runs on nb_threads thread. - /// - /// \param socket_path Path of the UNIX socket to listen on. - /// \param output Filters' output type - /// \param next_filter_socket Path of the UNIX socket of the filter to send data to. - /// \param threshold Threshold at which the filter will raise a log. - // AServer() = default; - virtual ~AServer() = default; // Make the server non copyable & non movable @@ -41,6 +42,13 @@ namespace darwin { protected: + /// + /// \brief Construct a new AServer object + /// + /// \param output Filter's output type + /// \param threshold Threshold at which the filter will raise a log + /// \param generator Reference to the Task Generator + /// AServer(std::string const& output, std::size_t threshold, Generator& generator); diff --git a/samples/base/network/ASession.fwd.hpp b/samples/base/network/ASession.fwd.hpp index 4020842..cbcfb17 100644 --- a/samples/base/network/ASession.fwd.hpp +++ b/samples/base/network/ASession.fwd.hpp @@ -9,8 +9,10 @@ namespace darwin { /// Definition of a session's self-managing pointer. /// /// \typedef session_ptr_t - // typedef std::shared_ptr session_ptr_t; typedef std::shared_ptr session_ptr_t; + /// Definition of a UDP buffer + /// + /// \typedef udp_buffer_t typedef std::array udp_buffer_t; } \ No newline at end of file diff --git a/samples/base/network/ASession.hpp b/samples/base/network/ASession.hpp index 7a1ccac..73f3d2d 100755 --- a/samples/base/network/ASession.hpp +++ b/samples/base/network/ASession.hpp @@ -1,10 +1,14 @@ -/// \file Session.hpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 05/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - +/// +/// \file ASession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Abstract Session to be extended for different protocols +/// It is created by Servers and managed by a Manager +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -30,6 +34,12 @@ namespace darwin { class ASession : public std::enable_shared_from_this { public: + /// + /// \brief Construct a new ASession object + /// + /// \param manager reference to the manager that will manage this session + /// \param generator referece to the task generator + /// ASession(Manager& manager, Generator& generator); @@ -96,8 +106,6 @@ namespace darwin { virtual void WriteToClient(std::vector& packet) = 0; - virtual void CloseFilterConnection() = 0; - /// Send result to the client. /// /// \return false if the function could not send the data, true otherwise. diff --git a/samples/base/network/TcpNextFilterConnector.hpp b/samples/base/network/TcpNextFilterConnector.hpp index 50e2a06..5c87de9 100644 --- a/samples/base/network/TcpNextFilterConnector.hpp +++ b/samples/base/network/TcpNextFilterConnector.hpp @@ -1,3 +1,13 @@ +/// +/// \file TcpNextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Connection to NExt Filter (TCP Protocol) +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -13,9 +23,21 @@ namespace darwin { class TcpNextFilterConnector: public ANextFilterConnector { public: + /// + /// \brief Construct a new Tcp Next Filter Connector object + /// + /// \param net_address parsed ip address of the next filter + /// \param port port of the next filter + /// TcpNextFilterConnector(boost::asio::ip::address const& net_address, int port); virtual ~TcpNextFilterConnector() = default; + /// + /// \brief Attempts to connect to the next Filter + /// + /// \return true if the attempt is successful + /// \return false else + /// virtual bool Connect(); private: diff --git a/samples/base/network/TcpServer.hpp b/samples/base/network/TcpServer.hpp index 7820f35..1583a00 100755 --- a/samples/base/network/TcpServer.hpp +++ b/samples/base/network/TcpServer.hpp @@ -1,10 +1,13 @@ -/// \file Server.hpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 02/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - +/// +/// \file TcpServer.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Server for TCP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -20,13 +23,15 @@ namespace darwin { class TcpServer : public AServer { public: - /// Create an async Tcp stream socket server. - /// The server runs on nb_threads thread. /// - /// \param socket_path Path of the Tcp socket to listen on. - /// \param output Filters' output type - /// \param next_filter_socket Path of the Tcp socket of the filter to send data to. - /// \param threshold Threshold at which the filter will raise a log. + /// \brief Construct a new Tcp Server object + /// + /// \param address parsed ip address of this filter + /// \param port port of this filter + /// \param output filter's output type + /// \param threshold filter's threshold + /// \param generator ref to the task generator + /// TcpServer(boost::asio::ip::address const& address, int port, std::string const& output, diff --git a/samples/base/network/TcpSession.cpp b/samples/base/network/TcpSession.cpp index 16a828b..eb2bb32 100755 --- a/samples/base/network/TcpSession.cpp +++ b/samples/base/network/TcpSession.cpp @@ -9,9 +9,7 @@ namespace darwin { TcpSession::TcpSession( boost::asio::ip::tcp::socket& socket, Manager& manager, Generator& generator) - : ASession(manager, generator), _connected{false}, - _socket{std::move(socket)}, - _filter_socket{socket.get_executor()}, _next_filter_port{0} + : ASession(manager, generator), _socket{std::move(socket)} { ; } @@ -20,14 +18,6 @@ namespace darwin { DARWIN_LOGGER; DARWIN_LOG_DEBUG("TcpSession::Stop::"); _socket.close(); - CloseFilterConnection(); - } - - void TcpSession::CloseFilterConnection() { - if(_connected){ - _filter_socket.close(); - _connected = false; - } } void TcpSession::ReadHeader() { @@ -59,12 +49,5 @@ namespace darwin { boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)); } - - void TcpSession::SetNextFilterPort(int port){ - if(port > 0 && port < 65536) { - _next_filter_port = port; - _has_next_filter = true; - } - } } diff --git a/samples/base/network/TcpSession.hpp b/samples/base/network/TcpSession.hpp index d5517db..67cd1c4 100755 --- a/samples/base/network/TcpSession.hpp +++ b/samples/base/network/TcpSession.hpp @@ -1,3 +1,13 @@ +/// +/// \file TcpSession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Session for TCP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include "ASession.hpp" @@ -7,6 +17,13 @@ namespace darwin { class TcpSession: public ASession { public: + /// + /// \brief Construct a new Tcp Session object + /// + /// \param socket ref to the socket of a connection, the socket will be moved + /// \param manager ref to the sessions' manager + /// \param generator ref to the task generator + /// TcpSession(boost::asio::ip::tcp::socket& socket, Manager& manager, Generator& generator); @@ -14,35 +31,22 @@ namespace darwin { ~TcpSession() = default; void Stop() override final; - - void SetNextFilterPort(int port); - + protected: /// Set the async read for the header. /// - /// \return -1 on error, 0 on socket closed & sizeof(header) on success. virtual void ReadHeader() override final; /// Set the async read for the body. /// - /// \return -1 on error, 0 on socket closed & sizeof(header) on success. virtual void ReadBody(std::size_t size) override final; /// virtual void WriteToClient(std::vector& packet) override final; - virtual void CloseFilterConnection() override final; - private: - - bool _connected; //!< True if the socket to the next filter is connected. - boost::asio::ip::tcp::socket _socket; //!< Session's socket. - boost::asio::ip::tcp::socket _filter_socket; //!< Filter's socket. - - int _next_filter_port; - }; } \ No newline at end of file diff --git a/samples/base/network/UdpNextFilterConnector.hpp b/samples/base/network/UdpNextFilterConnector.hpp index 53f8863..e981544 100644 --- a/samples/base/network/UdpNextFilterConnector.hpp +++ b/samples/base/network/UdpNextFilterConnector.hpp @@ -1,3 +1,13 @@ +/// +/// \file UdpNextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Connector to Next Filter for UDP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -13,6 +23,12 @@ namespace darwin { class UdpNextFilterConnector: public ANextFilterConnector { public: + /// + /// \brief Construct a new Udp Next Filter Connector object + /// + /// \param net_address parsed IP address of the next filter + /// \param port port of the next filter + /// UdpNextFilterConnector(boost::asio::ip::address const& net_address, int port); virtual ~UdpNextFilterConnector() = default; diff --git a/samples/base/network/UdpServer.cpp b/samples/base/network/UdpServer.cpp index 0712b69..1ea9618 100755 --- a/samples/base/network/UdpServer.cpp +++ b/samples/base/network/UdpServer.cpp @@ -76,7 +76,7 @@ namespace darwin { Accept(); } - void UdpServer::HandleAccept(boost::system::error_code const& e) { + void UdpServer::HandleAccept(boost::system::error_code const& e __attribute__((unused))) { return; } diff --git a/samples/base/network/UdpServer.hpp b/samples/base/network/UdpServer.hpp index 35ae3ca..acce378 100755 --- a/samples/base/network/UdpServer.hpp +++ b/samples/base/network/UdpServer.hpp @@ -1,10 +1,13 @@ -/// \file Server.hpp -/// \authors hsoszynski -/// \version 1.0 -/// \date 02/07/18 -/// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. - +/// +/// \file UdpServer.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Server for TCP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include @@ -20,13 +23,15 @@ namespace darwin { class UdpServer : public AServer { public: - /// Create an async Udp stream socket server. - /// The server runs on nb_threads thread. /// - /// \param socket_path Path of the Udp socket to listen on. - /// \param output Filters' output type - /// \param next_filter_socket Path of the Udp socket of the filter to send data to. - /// \param threshold Threshold at which the filter will raise a log. + /// \brief Construct a new Udp Server object + /// + /// \param address parsed ip address of this filter + /// \param port port of this filter + /// \param output filter's output type + /// \param threshold filter's threshold + /// \param generator ref to the task generator + /// UdpServer(boost::asio::ip::address const& address, int port, std::string const& output, diff --git a/samples/base/network/UdpSession.cpp b/samples/base/network/UdpSession.cpp index 74b49ca..9363d8f 100755 --- a/samples/base/network/UdpSession.cpp +++ b/samples/base/network/UdpSession.cpp @@ -16,10 +16,6 @@ namespace darwin { void UdpSession::Stop() { DARWIN_LOGGER; DARWIN_LOG_DEBUG("UdpSession::Stop::"); - CloseFilterConnection(); - } - - void UdpSession::CloseFilterConnection() { } void UdpSession::ReadHeader() { @@ -51,16 +47,8 @@ namespace darwin { this->ExecuteFilter(); } - void UdpSession::WriteToClient(std::vector& packet) { + void UdpSession::WriteToClient(std::vector& packet __attribute((unused))) { return; } - - void UdpSession::ExecuteFilter() { - ASession::ExecuteFilter(); - // ASession::Start(); - } - - void UdpSession::SetNextFilterPort(int port){ - } } diff --git a/samples/base/network/UdpSession.hpp b/samples/base/network/UdpSession.hpp index 12e54eb..da46977 100755 --- a/samples/base/network/UdpSession.hpp +++ b/samples/base/network/UdpSession.hpp @@ -1,3 +1,13 @@ +/// +/// \file UdpSession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Session for UDP Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include "ASession.hpp" @@ -7,6 +17,13 @@ namespace darwin { class UdpSession: public ASession { public: + /// + /// \brief Construct a new Udp Session object + /// + /// \param buffer ref to a buffer containing a udp datagram + /// \param manager ref to the sessions' manager + /// \param generator ref to the task generator + /// UdpSession(const udp_buffer_t& buffer, Manager& manager, Generator& generator); @@ -19,21 +36,16 @@ namespace darwin { /// Set the async read for the body. /// - /// \return -1 on error, 0 on socket closed & sizeof(header) on success. virtual void ReadBody(std::size_t size) override; protected: /// Set the async read for the header. /// - /// \return -1 on error, 0 on socket closed & sizeof(header) on success. virtual void ReadHeader() override; - /// virtual void WriteToClient(std::vector& packet) override; - virtual void CloseFilterConnection() override; - virtual void ExecuteFilter() override; private: const udp_buffer_t& _buffer; diff --git a/samples/base/network/UnixNextFilterConnector.hpp b/samples/base/network/UnixNextFilterConnector.hpp index 4491dce..af10504 100644 --- a/samples/base/network/UnixNextFilterConnector.hpp +++ b/samples/base/network/UnixNextFilterConnector.hpp @@ -1,4 +1,13 @@ -#pragma once +/// +/// \file UnixNextFilterConnector.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Connector to the NExt filter for Unix socket Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #include #include @@ -13,6 +22,11 @@ namespace darwin { class UnixNextFilterConnector: public ANextFilterConnector { public: + /// + /// \brief Construct a new Unix Next Filter Connector object + /// + /// \param path path of the unix socket of the next filter + /// UnixNextFilterConnector(std::string const& path); virtual ~UnixNextFilterConnector() = default; diff --git a/samples/base/network/UnixServer.hpp b/samples/base/network/UnixServer.hpp index ed1944e..995ae20 100644 --- a/samples/base/network/UnixServer.hpp +++ b/samples/base/network/UnixServer.hpp @@ -1,9 +1,9 @@ -/// \file Server.hpp +/// \file UnixServer.hpp /// \authors hsoszynski -/// \version 1.0 +/// \version 2.0 /// \date 02/07/18 /// \license GPLv3 -/// \brief Copyright (c) 2018 Advens. All rights reserved. +/// \brief Copyright (c) 2018-2021 Advens. All rights reserved. #pragma once @@ -19,13 +19,14 @@ namespace darwin { class UnixServer : public AServer { public: - /// Create an async UNIX stream socket server. - /// The server runs on nb_threads thread. /// - /// \param socket_path Path of the UNIX socket to listen on. + /// \brief Construct a new Unix Server object + /// + /// \param socket_path Path of the UNIX socket to listen on /// \param output Filters' output type - /// \param next_filter_socket Path of the UNIX socket of the filter to send data to. /// \param threshold Threshold at which the filter will raise a log. + /// \param generator ref to the task generator + /// UnixServer(std::string const& socket_path, std::string const& output, std::size_t threshold, diff --git a/samples/base/network/UnixSession.cpp b/samples/base/network/UnixSession.cpp index f0b4042..8a47138 100755 --- a/samples/base/network/UnixSession.cpp +++ b/samples/base/network/UnixSession.cpp @@ -9,35 +9,15 @@ namespace darwin { UnixSession::UnixSession( boost::asio::local::stream_protocol::socket& socket, Manager& manager, Generator& generator) - : ASession(manager, generator), _connected{false}, - _socket{std::move(socket)}, - _filter_socket{socket.get_executor()} + : ASession(manager, generator), _socket{std::move(socket)} { ; } - void UnixSession::SetNextFilterSocketPath(std::string const& path){ - if(path.compare("no") != 0) { - _next_filter_path = path; - _has_next_filter = true; - } - } - void UnixSession::Stop() { DARWIN_LOGGER; DARWIN_LOG_DEBUG("UnixSession::Stop::"); _socket.close(); - if(_connected) { - _filter_socket.close(); - _connected = false; - } - } - - void UnixSession::CloseFilterConnection() { - if(_connected){ - _filter_socket.close(); - _connected = false; - } } void UnixSession::ReadHeader() { diff --git a/samples/base/network/UnixSession.hpp b/samples/base/network/UnixSession.hpp index e29911b..03aede4 100755 --- a/samples/base/network/UnixSession.hpp +++ b/samples/base/network/UnixSession.hpp @@ -1,3 +1,13 @@ +/// +/// \file UnixSession.hpp +/// \author Thibaud Cartegnie (thibaud.cartegnie@advens.fr) +/// \brief Session for Unix Socket Protocol +/// \version 1.0 +/// \date 20-07-2021 +/// +/// @copyright Copyright (c) 2021 +/// +/// #pragma once #include "ASession.hpp" @@ -6,6 +16,13 @@ namespace darwin { class UnixSession: public ASession { public: + /// + /// \brief Construct a new Unix Session object + /// + /// \param socket ref to the unix socket, it will be moved to the object + /// \param manager ref to the sessions' manager + /// \param generator ref to the task generator + /// UnixSession(boost::asio::local::stream_protocol::socket& socket, Manager& manager, Generator& generator); @@ -35,16 +52,8 @@ namespace darwin { /// virtual void WriteToClient(std::vector& packet) override final; - virtual void CloseFilterConnection() override final; - private: - - bool _connected; //!< True if the socket to the next filter is connected. - boost::asio::local::stream_protocol::socket _socket; //!< Session's socket. - boost::asio::local::stream_protocol::socket _filter_socket; //!< Filter's socket. - std::string _next_filter_path; //!< The socket path to the next filter. - }; } \ No newline at end of file diff --git a/toolkit/Network.cpp b/toolkit/Network.cpp index 9f3c665..1067b50 100644 --- a/toolkit/Network.cpp +++ b/toolkit/Network.cpp @@ -94,17 +94,17 @@ namespace darwin { return str; } - bool ParsePort(const char* path_address, int& out_port) { + bool ParsePort(const std::string& str_port, int& out_port) { DARWIN_LOGGER; long port = 0; - if(!darwin::strings::StrToLong(path_address, port)){ - DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number, unrecognized number: '" + std::string(path_address) + "'"); + if(!darwin::strings::StrToLong(str_port.c_str(), port)){ + DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number, unrecognized number: '" + str_port+ "'"); return false; } if (port < 0 || port > 65353) { - DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number : out of bounds [0; 65353]: '" + std::string(path_address) + "'"); + DARWIN_LOG_ERROR("Network::ParsePort:: Error while parsing the port number : out of bounds [0; 65353]: '" + str_port + "'"); return false; } diff --git a/toolkit/Network.hpp b/toolkit/Network.hpp index 6e1ad5c..51afd9f 100644 --- a/toolkit/Network.hpp +++ b/toolkit/Network.hpp @@ -54,9 +54,27 @@ namespace darwin { /// \param sa The in6_addr structure to be stringified. std::string GetStringAddrFromSockAddrIn6(const in6_addr &addr); + /// + /// \brief Parse a string to get the socket path if any or the ipv4/v6 address + /// + /// \param path_address the string to parse + /// \param is_udp true if we should use udp instead of tcp (it is not used) + /// \param out_net_type type of address/path parsed + /// \param out_net_address ip address to fill when parsed + /// \param out_port ip port to fill when parsed + /// \param out_unix_path unix sock path to fill when parsed + /// \return true if the parsing was successful + /// bool ParseSocketAddress(const std::string& path_address, bool is_udp, NetworkSocketType& out_net_type, boost::asio::ip::address& out_net_address, int& out_port, std::string& out_unix_path); - bool ParsePort(const char* path_address, int& out_port); + /// + /// \brief parse an ip port, returns early if there is an error while parsing or if the port is not a valid port number + /// + /// \param str_port port to parse + /// \param out_port port to fill with the port + /// \return true if the parsing was successful + /// + bool ParsePort(const std::string& str_port, int& out_port); } } From 47e58b708604cafbf98ec26bc90de942c4c2d3b1 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 21 Jul 2021 18:17:59 +0200 Subject: [PATCH 27/54] Fixed method names --- samples/base/DarwinPacket.cpp | 4 ++-- samples/base/DarwinPacket.hpp | 4 ++-- samples/base/network/ASession.cpp | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 86012d3..40fa016 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -99,11 +99,11 @@ namespace darwin { return this->_type; } - enum darwin_filter_response_type DarwinPacket::GetResponse() const { + enum darwin_filter_response_type DarwinPacket::GetResponseType() const { return this->_response; } - void DarwinPacket::SetResponse(enum darwin_filter_response_type response) { + void DarwinPacket::SetResponseType(enum darwin_filter_response_type response) { this->_response = response; } diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 2b1e547..6a79579 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -95,13 +95,13 @@ namespace darwin { /// /// \return enum darwin_filter_response_type /// - enum darwin_filter_response_type GetResponse() const; + enum darwin_filter_response_type GetResponseType() const; /// /// \brief Set the Response Type /// /// - void SetResponse(enum darwin_filter_response_type); + void SetResponseType(enum darwin_filter_response_type); /// /// \brief Get the Filter Code diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index 75e1b24..c674f3a 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -116,7 +116,7 @@ namespace darwin { this->SendErrorResponse("Error receiving body: Empty body retrieved", DARWIN_RESPONSE_CODE_REQUEST_ERROR); return; } - if (_packet.GetParsedBodySize() <= 0) { //TODO useless branch + if (_packet.GetParsedBodySize() == 0) { DARWIN_LOG_ERROR( "ASession::ReadBodyCallback Body is not empty, but the header appears to be invalid" ); @@ -135,14 +135,14 @@ namespace darwin { void ASession::ExecuteFilter() { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ASession::ExecuteFilter::"); - auto t = _generator.CreateTask(shared_from_this()); //shared pt Task + auto t = _generator.CreateTask(shared_from_this()); _generator.GetTaskThreadPool().post(std::bind(&ATask::run, t)); } void ASession::SendNext(DarwinPacket& packet) { // TODO : Unsure of the logic behind the if's and and the Start calls - switch(packet.GetResponse()) { + switch(packet.GetResponseType()) { case DARWIN_RESPONSE_SEND_BOTH: this->SendToFilter(packet); case DARWIN_RESPONSE_SEND_BACK: From 3daf60ce3c102a447e0a8968c7df6a395e6dfb10 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 22 Jul 2021 18:04:01 +0200 Subject: [PATCH 28/54] Few modification of NextFilter handling Specification of move semantic for DarwinPacket --- samples/base/Core.cpp | 42 ++++++++++++++++++---------- samples/base/Core.hpp | 8 +++--- samples/base/DarwinPacket.cpp | 37 +++++++++++++++++++----- samples/base/DarwinPacket.hpp | 27 ++++++++++++------ samples/base/network/ASession.cpp | 16 +++++------ samples/base/network/ASession.hpp | 1 - samples/base/network/UdpSession.cpp | 19 +++++++++---- samples/base/network/UnixSession.hpp | 6 ---- 8 files changed, 101 insertions(+), 55 deletions(-) diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index 628da01..aef0b91 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -44,15 +44,20 @@ namespace darwin { try { Monitor monitor{_monSocketPath}; std::thread t{std::bind(&Monitor::Run, std::ref(monitor))}; - std::thread t_nextfilter{std::bind(&ANextFilterConnector::Run, std::ref(GetNextFilterconnector()))}; + std::thread t_nextfilter; + if(_next_filter_connector){ + t_nextfilter = std::thread{std::bind(&ANextFilterConnector::Run, std::ref(*_next_filter_connector))}; + } SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::configuring); Generator gen{_nbThread}; if (not gen.Configure(_modConfigPath, _cacheSize)) { DARWIN_LOG_CRITICAL("Core:: Run:: Unable to configure the filter"); raise(SIGTERM); - t_nextfilter.join(); - t.join(); + if(t_nextfilter.joinable()) + t_nextfilter.join(); + if(t.joinable()) + t.join(); return 1; } DARWIN_LOG_DEBUG("Core::run:: Configured generator"); @@ -73,16 +78,20 @@ namespace darwin { default: DARWIN_LOG_CRITICAL("Core:: Run:: Network Configuration problem"); raise(SIGTERM); - t_nextfilter.join(); - t.join(); + if(t_nextfilter.joinable()) + t_nextfilter.join(); + if (t.joinable()) + t.join(); return 1; } try { if (not gen.ConfigureNetworkObject(server->GetIOContext())) { raise(SIGTERM); - t_nextfilter.join(); - t.join(); + if(t_nextfilter.joinable()) + t_nextfilter.join(); + if (t.joinable()) + t.join(); return 1; } SET_FILTER_STATUS(darwin::stats::FilterStatusEnum::running); @@ -98,7 +107,8 @@ namespace darwin { raise(SIGTERM); } DARWIN_LOG_DEBUG("Core::run:: Joining monitoring thread..."); - t_nextfilter.join(); + if(t_nextfilter.joinable()) + t_nextfilter.join(); if (t.joinable()) t.join(); } catch (const std::exception& e) { @@ -198,6 +208,10 @@ namespace darwin { bool Core::SetNextFilterConnector(std::string const& path_address, bool is_udp) { DARWIN_LOGGER; + if(path_address == "no"){ + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: No Next Filter set"); + return true; + } network::NetworkSocketType net_type; std::string path; int port; @@ -207,12 +221,15 @@ namespace darwin { switch(net_type) { case network::NetworkSocketType::Unix: + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: Next Filter UNIX set on " + path); _next_filter_connector = std::make_unique(path); return true; case network::NetworkSocketType::Tcp: + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: Next Filter TCP set on " + addr.to_string() + ":" + std::to_string(port)); _next_filter_connector = std::make_unique(addr, port); return true; case network::NetworkSocketType::Udp: + DARWIN_LOG_DEBUG("Core::SetNextFilterConnector :: Next Filter UDP set on " + addr.to_string() + ":" + std::to_string(port)); _next_filter_connector = std::make_unique(addr, port); return true; default: @@ -314,13 +331,8 @@ namespace darwin { return this->daemon; } - ANextFilterConnector& Core::GetNextFilterconnector() { - if(!_next_filter_connector) { - DARWIN_LOGGER; - DARWIN_LOG_CRITICAL("Core::GetNextFilterConnector :: Configure must be called before GetNextFilterConnector"); - throw std::logic_error("Core:: Configure must be called before GetNextFilterConnector()"); - } - return *_next_filter_connector; + ANextFilterConnector* Core::GetNextFilterconnector() { + return _next_filter_connector.get(); } } diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 36b47e1..8f6efc1 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -100,11 +100,11 @@ namespace darwin { bool SetNextFilterConnector(std::string const& path_address, bool is_udp); /// - /// \brief Get a ref to the NextFilterConnector + /// \brief Get a pointer to the NextFilterConnector, it may be a nullptr if no NextFilter was set during the configuration + /// If it is not nullptr, it has the lifetime of the darwin::Core instance (most of the duration of the program) /// - /// \return ANextFilterConnector& a ref to the next filter - /// \throws std::logic_error if the NextFilterConnector is not set (happens if SetNextFilterConnector was not called) - ANextFilterConnector& GetNextFilterconnector(); + /// \return ANextFilterConnector* a pointer to the next filter connector or nullptr + ANextFilterConnector* GetNextFilterconnector(); private: std::string _name; diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 40fa016..ae6925d 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -14,13 +14,39 @@ namespace darwin { unsigned char event_id[16], size_t certitude_size, size_t body_size) - : _type{type}, _response{response}, _filter_code{filter_code}, + :_type{type}, _response{response}, _filter_code{filter_code}, _parsed_certitude_size{certitude_size}, _parsed_body_size{body_size} { std::memcpy(this->_evt_id, event_id, 16); } + DarwinPacket::DarwinPacket(DarwinPacket&& other) noexcept + : _type{std::move(other._type)}, _response{std::move(other._response)}, _filter_code{std::move(other._filter_code)}, + _parsed_certitude_size{std::move(other._parsed_certitude_size)}, _parsed_body_size{std::move(other._parsed_body_size)}, + _certitude_list{std::move(other._certitude_list)}, _body{std::move(other._body)}, + _parsed_body{other._parsed_body.release()}, _logs{std::move(other._logs)} + { + std::memcpy(this->_evt_id, other._evt_id, sizeof(this->_evt_id)); + std::memset(other._evt_id, 0, sizeof(other._evt_id)); + } + + DarwinPacket& DarwinPacket::operator=(DarwinPacket&& other) noexcept { + this->clear(); + _type = std::move(other._type); + _response = std::move(other._response); + _filter_code = std::move(other._filter_code); + std::memcpy(this->_evt_id, other._evt_id, sizeof(this->_evt_id)); + std::memset(other._evt_id, 0, sizeof(other._evt_id)); + _parsed_certitude_size = std::move(other._parsed_certitude_size); + _parsed_body_size = std::move(other._parsed_body_size); + _certitude_list = std::move(other._certitude_list); + _body = std::move(other._body); + _parsed_body.reset(other._parsed_body.release()); + _logs = std::move(other._logs); + return *this; + } + DarwinPacket::DarwinPacket(darwin_filter_packet_t& input) : _type{input.type}, _response{input.response}, _filter_code{input.filter_code}, _parsed_certitude_size{input.certitude_size}, _parsed_body_size{input.body_size} @@ -81,20 +107,17 @@ namespace darwin { this->_parsed_certitude_size = 0; _certitude_list.clear(); _body.clear(); + this->_parsed_body.reset(nullptr); } rapidjson::Document& DarwinPacket::JsonBody() { - if(this->_parsed_body == nullptr) { - this->_parsed_body = new rapidjson::Document(); + if(! this->_parsed_body) { + this->_parsed_body = std::make_unique(); } this->_parsed_body->Parse(this->_body.c_str()); return *(this->_parsed_body); } - DarwinPacket::~DarwinPacket(){ - delete this->_parsed_body; - } - enum darwin_packet_type DarwinPacket::GetType() const { return this->_type; } diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 6a79579..ae87180 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -50,7 +50,16 @@ namespace darwin { DarwinPacket() = default; - virtual ~DarwinPacket(); + DarwinPacket(DarwinPacket&& other) noexcept; + + DarwinPacket& operator=(DarwinPacket&& other) noexcept; + + virtual ~DarwinPacket() = default; + + // No copy allowed of a DarwinPacket, only moves + // It may be implemented in the future as a deep copy if this is needed + DarwinPacket(const DarwinPacket&) = delete; + DarwinPacket& operator=(const DarwinPacket&) = delete; /// /// \brief Serialize the packet @@ -76,9 +85,11 @@ namespace darwin { /// void clear(); - /// TODO check if useful + /// /// \brief Parse the body as a json document and returns a reference to it - /// + /// While the parsed Body is kept inside the DarwinPacket, subsequent calls to JsonBody() + /// will re-parse the body as the body may have been modified + /// /// \return rapidjson::Document& the reference to the parsed body /// rapidjson::Document& JsonBody(); @@ -106,7 +117,7 @@ namespace darwin { /// /// \brief Get the Filter Code /// - /// \return long + /// \return long filter code /// long GetFilterCode() const; @@ -206,10 +217,6 @@ namespace darwin { private: - // TODO replace with a better solution - // It is added as a ptr because Document, and smart pointers can't be moved but we use move semantics - rapidjson::Document* _parsed_body = nullptr; - darwin_packet_type _type; //!< The type of information sent. darwin_filter_response_type _response; //!< Whom the response will be sent to. long _filter_code; //!< The unique identifier code of a filter. @@ -219,6 +226,10 @@ namespace darwin { std::vector _certitude_list; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. std::string _body; + // Parsed Body is stored as late-initiliaized pointer because is cannot be moved, nor assigned + // For move semantics, we want to be able to move this, hence the std::unique_ptr + std::unique_ptr _parsed_body; + std::string _logs; }; diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index c674f3a..0218a9b 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -27,7 +27,7 @@ namespace darwin { ASession::ASession(darwin::Manager& manager, Generator& generator) - : _manager{manager}, _generator{generator}, _has_next_filter{false} {} + : _manager{manager}, _generator{generator} {} @@ -76,7 +76,7 @@ namespace darwin { DARWIN_LOG_ERROR("ASession::ReadHeaderCallback:: Mismatching header size"); goto header_callback_stop_session; } - _packet = DarwinPacket(_header); + _packet = std::move(DarwinPacket(_header)); if (_packet.GetParsedBodySize() == 0) { ExecuteFilter(); return; @@ -145,6 +145,7 @@ namespace darwin { switch(packet.GetResponseType()) { case DARWIN_RESPONSE_SEND_BOTH: this->SendToFilter(packet); + __attribute((fallthrough)); case DARWIN_RESPONSE_SEND_BACK: if(not this->SendToClient(packet)) Start(); break; @@ -177,14 +178,13 @@ namespace darwin { bool ASession::SendToFilter(DarwinPacket& packet) noexcept { DARWIN_LOGGER; - if (!_has_next_filter) { + ANextFilterConnector* connector_ptr = Core::instance().GetNextFilterconnector(); + + if (!connector_ptr) { DARWIN_LOG_NOTICE("ASession::SendToFilter:: No next filter provided. Ignoring..."); return false; } - - ANextFilterConnector& c = Core::instance().GetNextFilterconnector(); - - c.Send(packet); + connector_ptr->Send(packet); return true; } @@ -224,7 +224,7 @@ namespace darwin { void ASession::SendErrorResponse(const std::string& message, const unsigned int code) { - if (this->_packet.GetResponse() != DARWIN_RESPONSE_SEND_BACK && this->_packet.GetResponse() != DARWIN_RESPONSE_SEND_BOTH) + if (this->_packet.GetResponseType() != DARWIN_RESPONSE_SEND_BACK && this->_packet.GetResponseType() != DARWIN_RESPONSE_SEND_BOTH) return; _packet.GetMutableBody().clear(); _packet.GetMutableBody() += "{\"error\":\"" + message + "\", \"error_code\":" + std::to_string(code) + "}"; diff --git a/samples/base/network/ASession.hpp b/samples/base/network/ASession.hpp index 73f3d2d..ad3d424 100755 --- a/samples/base/network/ASession.hpp +++ b/samples/base/network/ASession.hpp @@ -174,7 +174,6 @@ namespace darwin { Generator& _generator; //!< The Task Generator. DarwinPacket _packet; std::string _logs; //!< Represents data given in the logs by the Session - bool _has_next_filter; }; } diff --git a/samples/base/network/UdpSession.cpp b/samples/base/network/UdpSession.cpp index 9363d8f..b15a2d3 100755 --- a/samples/base/network/UdpSession.cpp +++ b/samples/base/network/UdpSession.cpp @@ -32,17 +32,24 @@ namespace darwin { } std::memcpy(&_header, _buffer.data(), sizeof(_header)); - - if(size != sizeof(_header) + _header.body_size){ + + //The header can hold one certitude, if there are more, we have to parsed them accordingly + size_t size_cert_above_header = _header.certitude_size > 1 ? (_header.certitude_size-1) * sizeof(unsigned int) : 0; + if(size != sizeof(_header) + _header.body_size + size_cert_above_header){ DARWIN_LOG_ERROR("Error parsing header sizes, expected " + std::to_string(sizeof(_header) + _header.body_size) + " but buffer is " + std::to_string(size)); return; } - //TODO Certitudes parsing + _packet = std::move(DarwinPacket(_header)); + + for(size_t i=1; i < _header.certitude_size; i++){ + unsigned int cert = 0; + std::memcpy(&cert, _buffer.data() + sizeof(_header) + (i-1)*sizeof(unsigned int), sizeof(unsigned int)); + _packet.AddCertitude(cert); + } - _packet = DarwinPacket(_header); - auto& body = _packet.GetMutableBody(); - body.append(_buffer.begin() + sizeof(_header), _header.body_size); + std::string& body = _packet.GetMutableBody(); + body.append(_buffer.begin() + sizeof(_header) + size_cert_above_header, _header.body_size); this->ExecuteFilter(); } diff --git a/samples/base/network/UnixSession.hpp b/samples/base/network/UnixSession.hpp index 03aede4..7c6a3e2 100755 --- a/samples/base/network/UnixSession.hpp +++ b/samples/base/network/UnixSession.hpp @@ -31,12 +31,6 @@ namespace darwin { void Stop() override final; - /// Set the path to the associated decision module UNIX socket - /// - /// \param path Path to the UNIX socket. - void SetNextFilterSocketPath(std::string const& path); - - protected: /// Set the async read for the header. From 06f9cbf6c6d0c924fa8684c10a939b84ca11102a Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Sun, 25 Jul 2021 22:42:08 +0200 Subject: [PATCH 29/54] Few fixes when passing information to next filters --- samples/base/DarwinPacket.cpp | 4 ++-- samples/base/network/ANextFilterConnector.cpp | 2 +- samples/base/network/ANextFilterConnector.hpp | 2 +- samples/base/network/ASession.cpp | 24 +++++++++++++++---- samples/ftest/TestTask.cpp | 1 + 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index ae6925d..3ead041 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -84,11 +84,11 @@ namespace darwin { unsigned char * pt = ret.data(); std::memcpy(pt, &header, sizeof(header)); - pt += sizeof(header) - sizeof(int); + pt += sizeof(header); //certitudes for(size_t i=1; i < _certitude_list.size(); i++) { - auto certitude = _certitude_list[i]; + unsigned int certitude = _certitude_list[i]; std::memcpy(pt,(void*)(&certitude), sizeof(certitude)); pt += sizeof(certitude); } diff --git a/samples/base/network/ANextFilterConnector.cpp b/samples/base/network/ANextFilterConnector.cpp index a313194..d5a0650 100644 --- a/samples/base/network/ANextFilterConnector.cpp +++ b/samples/base/network/ANextFilterConnector.cpp @@ -7,7 +7,7 @@ namespace darwin { ANextFilterConnector::ANextFilterConnector() - : _io_context{}, _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)} + : _io_context{}, _nb_attempts{0}, _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)} { } diff --git a/samples/base/network/ANextFilterConnector.hpp b/samples/base/network/ANextFilterConnector.hpp index 1d6a011..722e79e 100644 --- a/samples/base/network/ANextFilterConnector.hpp +++ b/samples/base/network/ANextFilterConnector.hpp @@ -72,8 +72,8 @@ namespace darwin { boost::asio::io_context _io_context; std::set> _buffer_set; - size_t _max_attempts; size_t _nb_attempts; + size_t _max_attempts; std::chrono::milliseconds _attempts_delay_ms; }; } \ No newline at end of file diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index 0218a9b..6212a19 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -77,11 +77,16 @@ namespace darwin { goto header_callback_stop_session; } _packet = std::move(DarwinPacket(_header)); - if (_packet.GetParsedBodySize() == 0) { + if (_packet.GetParsedBodySize() == 0 && _packet.GetParsedCertitudeSize() <= 1) { ExecuteFilter(); return; - } // Else the ReadBodyCallback will call ExecuteFilter - ReadBody(_packet.GetParsedBodySize()); + } // Else the ReadBodyCallback willcall ExecuteFilter + size_t sizeToRead = _packet.GetParsedBodySize(); + if(_packet.GetParsedCertitudeSize() > 1){ + sizeToRead += _packet.GetParsedCertitudeSize() * sizeof(unsigned int); + } + + ReadBody(sizeToRead); return; } @@ -100,7 +105,18 @@ namespace darwin { DARWIN_LOGGER; if (!e) { - _packet.GetMutableBody().append(_body_buffer.data(), size); + size_t already_parsed_size = 0; + if(_packet.GetParsedCertitudeSize() > 1){ + while(_packet.GetParsedCertitudeSize() != _packet.GetCertitudeList().size() + && already_parsed_size + sizeof(unsigned int) <= size) { + unsigned int cert = 0; + std::memcpy(&cert, _body_buffer.data() + already_parsed_size, sizeof(cert)); + already_parsed_size += sizeof(cert); + _packet.AddCertitude(cert); + } + } + + _packet.GetMutableBody().append(_body_buffer.data() + already_parsed_size, size - already_parsed_size); DARWIN_LOG_DEBUG("ASession::ReadBodyCallback:: Body len (" + std::to_string(_packet.GetBody().length()) + ") - Header body size (" + diff --git a/samples/ftest/TestTask.cpp b/samples/ftest/TestTask.cpp index 1e74cc2..4bdee95 100644 --- a/samples/ftest/TestTask.cpp +++ b/samples/ftest/TestTask.cpp @@ -77,6 +77,7 @@ void TestTask::operator()() { DARWIN_LOG_DEBUG("TestTask:: not triggered specific action, generating alert by default"); DARWIN_ALERT_MANAGER.Alert(_line, 100, _s->Evt_idToString()); _packet.AddCertitude(0); + _response_body.append(_packet.GetBody()); } } else { From 8694a4409dd50c007ab959cdef58b58a7685046b Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 26 Jul 2021 17:04:35 +0200 Subject: [PATCH 30/54] Added a tests for multi filters using the manager Fixed a fex things accordingly --- manager/HeartBeat.py | 28 ++++- manager/Services.py | 8 +- samples/base/Core.cpp | 4 + samples/base/Core.hpp | 2 + samples/ftest/TestTask.cpp | 3 +- tests/manager_socket/multi_filters_test.py | 125 +++++++++++++++++++++ tests/manager_socket/test.py | 2 + 7 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 tests/manager_socket/multi_filters_test.py diff --git a/manager/HeartBeat.py b/manager/HeartBeat.py index f5e7c76..264ab13 100644 --- a/manager/HeartBeat.py +++ b/manager/HeartBeat.py @@ -7,7 +7,7 @@ __doc__ = 'The heartbeat functions and main class' from os import kill, access, F_OK - +import socket class HeartBeat: """ @@ -15,7 +15,7 @@ class HeartBeat: """ @staticmethod - def check_socket(file): + def check_unix_socket(file): try: if access(file, F_OK): return True @@ -23,6 +23,30 @@ def check_socket(file): except Exception: return False + def check_network_socket(filter_network): + if filter_network['socket_type'] == 'UNIX': + return HeartBeat.check_unix_socket(filter_network['address_path']) + + elif filter_network['socket_type'] == 'TCP': + # parse ip address field + splitted_address = filter_network['address_path'].rsplit(':', 1) + if '[' in splitted_address[0]: # ipv6 address + host_address = splitted_address[0][1:-1] + else: + host_address = splitted_address[0] + host_port = int(splitted_address[1]) + #test connection to socket + if ':' in host_address: # ipv6 + s = socket.socket(socket.AF_INET6) + else: + s = socket.socket(socket.AF_INET) + s.settimeout(2) + res = s.connect_ex((host_address,host_port)) + s.close() + return res == 0 + else: #No check for UDP + return True + @staticmethod def check_pid_file(file): """ diff --git a/manager/Services.py b/manager/Services.py index 0999ad5..2bed431 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -541,7 +541,7 @@ def _wait_process_ready(content): status = "Process not running" continue - if not HeartBeat.check_socket(content['monitoring']): + if not HeartBeat.check_unix_socket(content['monitoring']): status = "Monitoring socket not created" continue @@ -560,7 +560,7 @@ def _wait_process_ready(content): continue if "running" in resp: - if not HeartBeat.check_socket(content['socket']): + if not HeartBeat.check_network_socket(content['network']): status = "Main socket not created" continue else: @@ -584,10 +584,10 @@ def hb_one(self, filter): if not HeartBeat.check_process(pid): raise Exception('Process not running') - if not HeartBeat.check_socket(filter['socket']): + if not HeartBeat.check_network_socket(filter['network']): raise Exception('Socket not accessible') - if not HeartBeat.check_socket(filter['monitoring']): + if not HeartBeat.check_unix_socket(filter['monitoring']): raise Exception('Monitoring socket not accessible') except Exception as e: diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index aef0b91..5bdf5c7 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -335,4 +335,8 @@ namespace darwin { return _next_filter_connector.get(); } + const std::string& Core::GetName() const { + return this->_name; + } + } diff --git a/samples/base/Core.hpp b/samples/base/Core.hpp index 8f6efc1..166536a 100644 --- a/samples/base/Core.hpp +++ b/samples/base/Core.hpp @@ -106,6 +106,8 @@ namespace darwin { /// \return ANextFilterConnector* a pointer to the next filter connector or nullptr ANextFilterConnector* GetNextFilterconnector(); + const std::string& GetName() const; + private: std::string _name; std::string _modConfigPath; diff --git a/samples/ftest/TestTask.cpp b/samples/ftest/TestTask.cpp index 4bdee95..910719d 100644 --- a/samples/ftest/TestTask.cpp +++ b/samples/ftest/TestTask.cpp @@ -16,6 +16,7 @@ #include "ASession.hpp" #include "Logger.hpp" #include "AlertManager.hpp" +#include "Core.hpp" TestTask::TestTask(std::shared_ptr> cache, std::mutex& cache_mutex, @@ -75,7 +76,7 @@ void TestTask::operator()() { _packet.AddCertitude(n & 1); } else { DARWIN_LOG_DEBUG("TestTask:: not triggered specific action, generating alert by default"); - DARWIN_ALERT_MANAGER.Alert(_line, 100, _s->Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_line, 100, _s->Evt_idToString(), "{\"filter_name\":\"" + darwin::Core::instance().GetName() + "\"}"); _packet.AddCertitude(0); _response_body.append(_packet.GetBody()); } diff --git a/tests/manager_socket/multi_filters_test.py b/tests/manager_socket/multi_filters_test.py new file mode 100644 index 0000000..3e3a404 --- /dev/null +++ b/tests/manager_socket/multi_filters_test.py @@ -0,0 +1,125 @@ +from manager_socket.utils import CONF_FTEST, PATH_CONF_FTEST +import time +from tools.output import print_result +from tools.darwin_utils import darwin_configure, darwin_stop, darwin_start, darwin_remove_configuration +from tools.filter import Filter +import conf + + +def run(): + tests = [ + alerting_tests, + ] + + for i in tests: + print_result("Monitoring: " + i.__name__, i) + +ONE_FILTER = """ +{{ + "name": "{name}", + "exec_path": "{filter_path}darwin_test", + "config_file": "/tmp/test.conf", + "output": "NONE", + "next_filter": "{next_filter}", + "next_filter_network":{next_filter_network}, + "nb_thread": 1, + "log_level": "DEBUG", + "cache_size": 0, + "network":{network} + }} +""" +UNIX_NETWORK = """{{ + "socket_type":"UNIX" +}}""" + +TCP_NETWORK = """{{ + "socket_type":"TCP", + "address_path":"[::1]:{port}" + }}""" +UDP_NETWORK = """{{ + "socket_type":"UDP", + "address_path":"[::1]:{port}" + }}""" +network_map = { + 'unix': UNIX_NETWORK, + 'tcp': TCP_NETWORK, + 'udp': UDP_NETWORK +} +CONFIG = """ +{{ + "version": 2, + "filters": [ + {filter_1}, + {filter_2}, + {filter_3} + ], + "report_stats": {{ + "file": {{ + "filepath": "/tmp/darwin-stats", + "permissions": 640 + }}, + "interval": 5 + }} +}} +""" + + +def alerting_tests(): + tests = [ + ['unix', 'unix', 'unix'], + ['tcp', 'tcp', 'tcp'], + ['udp', 'udp', 'udp'], + ['udp', 'unix', 'unix'], + ['udp', 'tcp', 'tcp'] + ] + + for test in tests: + filter_1 = ONE_FILTER.format(name='test_1', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='test_2', next_filter_network=network_map[test[1]].format(port=8282), + network=network_map[test[0]].format(port=8181)) + filter_2 = ONE_FILTER.format(name='test_2', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='test_3', next_filter_network=network_map[test[2]].format(port=8383), + network=network_map[test[1]].format(port=8282)) + filter_3 = ONE_FILTER.format(name='test_3', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='', next_filter_network='{}', + network=network_map[test[2]].format(port=8383)) + config = CONFIG.format(filter_1=filter_1, filter_2=filter_2, filter_3=filter_3) + + darwin_configure(config) + darwin_configure(CONF_FTEST, path=PATH_CONF_FTEST) + + # Erasing last alerts + file = open("/tmp/test_test.log","w") + file.truncate(0) + file.close() + + p = darwin_start() + # socket path is given according to how it is currently generated in the manager, + # this may have to change in the future + + path_addr = '{}/sockets/test_1.1.sock'.format(conf.TEST_FILES_DIR) if test[0] == 'unix' else '[::]:8181' + f = Filter(socket_type=test[0], socket_path=path_addr) + api = f.get_darwin_api() + + api.call("Hello", response_type="darwin") + # arbitrary sleep length, should be enough on most systems + # We verify that One alert per filter were written + time.sleep(2) + line1, line2, line3 = False, False, False + file = open("/tmp/test_test.log","r") + for l in file.readlines(): + if 'test_1' in l: + line1 = True + elif 'test_2' in l: + line2 = True + elif 'test_3' in l: + line3 = True + file.close() + darwin_stop(p) + darwin_remove_configuration() + darwin_remove_configuration(path=PATH_CONF_FTEST) + + if not line1 or not line2 or not line3: + print('test failed: ', test) + return False + return True diff --git a/tests/manager_socket/test.py b/tests/manager_socket/test.py index d081557..7a250e7 100644 --- a/tests/manager_socket/test.py +++ b/tests/manager_socket/test.py @@ -2,6 +2,7 @@ import manager_socket.monitor_test as monitor_test import manager_socket.update_test as update_test import manager_socket.reporting_test as reporting_test +import manager_socket.multi_filters_test as multi_filters_test from tools.output import print_results def run(): @@ -10,6 +11,7 @@ def run(): monitor_test.run() update_test.run() reporting_test.run() + multi_filters_test.run() print() print() \ No newline at end of file From 7b61fddbcfc665eca5754576fcdb1f26b368bb05 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 27 Jul 2021 10:55:39 +0200 Subject: [PATCH 31/54] Fixed issue with UUIDs EvtIdToString passed from Session to DarwinPacket --- samples/base/DarwinPacket.cpp | 17 +++++++++++++++ samples/base/DarwinPacket.hpp | 7 +++++++ samples/base/network/ASession.cpp | 21 ------------------- samples/base/network/ASession.hpp | 5 ----- samples/fanomaly/AnomalyTask.cpp | 4 ++-- .../fconnection/ConnectionSupervisionTask.cpp | 4 ++-- samples/fdga/DGATask.cpp | 8 +++---- samples/fhostlookup/HostLookupTask.cpp | 6 +++--- samples/finspection/ContentInspectionTask.cpp | 4 ++-- samples/ftest/TestTask.cpp | 2 +- samples/fuseragent/UserAgentTask.cpp | 4 ++-- samples/fyara/YaraTask.cpp | 8 +++---- tests/manager_socket/multi_filters_test.py | 8 ++++--- 13 files changed, 49 insertions(+), 49 deletions(-) diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 3ead041..657e10f 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -4,6 +4,7 @@ #include #include "../../toolkit/rapidjson/writer.h" #include "../../toolkit/rapidjson/stringbuffer.h" +#include "Logger.hpp" namespace darwin { @@ -118,6 +119,22 @@ namespace darwin { return *(this->_parsed_body); } + std::string DarwinPacket::Evt_idToString() const { + DARWIN_LOGGER; + std::ostringstream oss; + auto default_flags = oss.flags(); + + for(size_t i=0; i(_evt_id[i]) & 0xFF); + } + DARWIN_LOG_DEBUG("DarwinPacket::Evt_idToString:: UUID - " + oss.str()); + return oss.str(); + } + enum darwin_packet_type DarwinPacket::GetType() const { return this->_type; } diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index ae87180..6f094a0 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -141,6 +141,13 @@ namespace darwin { /// size_t GetEventIdSize() const; + /// + /// \brief Serialize the event id as a UUID String + /// + /// \return std::string UUID encoded event id + /// + std::string Evt_idToString() const; + /// /// \brief Get the Parsed Certitude Size, it may differs from GetCertitudeList().size() /// diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index 6212a19..734e960 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -218,27 +218,6 @@ namespace darwin { Start(); } - std::string ASession::Evt_idToString() { - DARWIN_LOGGER; - - const unsigned char * evt_id = _packet.GetEventId(); - - std::ostringstream oss; - auto default_flags = oss.flags(); - - for(size_t i=0; i<16;i++){ - if(i==4 || i==6 || i==8 || i==10){ - oss.flags(default_flags); - oss << '-'; - } - oss << std::hex << std::setw(2) << static_cast(evt_id[i]); - } - DARWIN_LOG_DEBUG(std::string("ASession::Evt_idToString:: UUID - ") + oss.str()); - return oss.str(); - } - - - void ASession::SendErrorResponse(const std::string& message, const unsigned int code) { if (this->_packet.GetResponseType() != DARWIN_RESPONSE_SEND_BACK && this->_packet.GetResponseType() != DARWIN_RESPONSE_SEND_BOTH) return; diff --git a/samples/base/network/ASession.hpp b/samples/base/network/ASession.hpp index ad3d424..617fd3e 100755 --- a/samples/base/network/ASession.hpp +++ b/samples/base/network/ASession.hpp @@ -86,11 +86,6 @@ namespace darwin { /// \return The filter's threshold size_t GetThreshold(); - /// Transform the evt id in the header into a string - /// - /// \return evt_di as string - std::string Evt_idToString(); - /// Send virtual void SendNext(DarwinPacket& packet) final; diff --git a/samples/fanomaly/AnomalyTask.cpp b/samples/fanomaly/AnomalyTask.cpp index f2e603f..c3c05f0 100644 --- a/samples/fanomaly/AnomalyTask.cpp +++ b/samples/fanomaly/AnomalyTask.cpp @@ -114,7 +114,7 @@ void AnomalyTask::GenerateAlerts(std::vector ips, arma::uvec index_ details += R"("icmp_nb_host": )" + std::to_string(alerts(ICMP_NB_HOST, i)) + ","; details += R"("distance": )" + std::to_string(alerts(DISTANCE, i)); details += "}"; - DARWIN_ALERT_MANAGER.Alert(ips[index_anomalies(i)], 100, _s->Evt_idToString(), details); + DARWIN_ALERT_MANAGER.Alert(ips[index_anomalies(i)], 100, _packet.Evt_idToString(), details); } } @@ -122,7 +122,7 @@ void AnomalyTask::GenerateLogs(std::vector ips, arma::uvec index_an auto logs = _packet.GetMutableLogs(); for(unsigned int i=0; iEvt_idToString(); + alert_log = R"({"evt_id": ")" + _packet.Evt_idToString(); alert_log += R"(", "time": ")" + darwin::time_utils::GetTime(); alert_log += R"(", "filter": ")" + GetFilterName(); alert_log += R"(", "anomaly": {)"; diff --git a/samples/fconnection/ConnectionSupervisionTask.cpp b/samples/fconnection/ConnectionSupervisionTask.cpp index 7fccbc9..cefd270 100644 --- a/samples/fconnection/ConnectionSupervisionTask.cpp +++ b/samples/fconnection/ConnectionSupervisionTask.cpp @@ -53,9 +53,9 @@ void ConnectionSupervisionTask::operator()() { if(certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_connection, certitude, _s->Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_connection, certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "connection": ")" + _connection + R"(", "certitude": )" + std::to_string(certitude) + "}"; logs += alert_log + "\n"; } diff --git a/samples/fdga/DGATask.cpp b/samples/fdga/DGATask.cpp index 5967ca3..ea730f6 100644 --- a/samples/fdga/DGATask.cpp +++ b/samples/fdga/DGATask.cpp @@ -70,9 +70,9 @@ void DGATask::operator()() { if (GetCacheResult(hash, certitude)) { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _s->Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; logs += alert_log + '\n'; } @@ -87,9 +87,9 @@ void DGATask::operator()() { certitude = Predict(); if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _s->Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert(_domain, certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"domain\": \""+ _domain + "\", \"dga_prob\": " + std::to_string(certitude) + "}"; logs += alert_log + '\n'; } diff --git a/samples/fhostlookup/HostLookupTask.cpp b/samples/fhostlookup/HostLookupTask.cpp index d1b6ece..6604aa4 100644 --- a/samples/fhostlookup/HostLookupTask.cpp +++ b/samples/fhostlookup/HostLookupTask.cpp @@ -58,7 +58,7 @@ void HostLookupTask::operator()() { if (GetCacheResult(hash, certitude)) { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN) { STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_host, certitude, _s->Evt_idToString(), this->AlertDetails()); + DARWIN_ALERT_MANAGER.Alert(_host, certitude, _packet.Evt_idToString(), this->AlertDetails()); if (is_log) { std::string alert_log = this->BuildAlert(_host, certitude); logs += alert_log + "\n"; @@ -75,7 +75,7 @@ void HostLookupTask::operator()() { certitude = DBLookup(description); if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN) { STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert(_host, certitude, _s->Evt_idToString(), this->AlertDetails(description)); + DARWIN_ALERT_MANAGER.Alert(_host, certitude, _packet.Evt_idToString(), this->AlertDetails(description)); if (is_log){ std::string alert_log = this->BuildAlert(_host, certitude); logs += alert_log + "\n"; @@ -110,7 +110,7 @@ const std::string HostLookupTask::AlertDetails(std::string const& description) { const std::string HostLookupTask::BuildAlert(const std::string& host, unsigned int certitude) { std::string alert_log = - R"({"evt_id": ")" + _s->Evt_idToString() + + R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "entry": ")" + host + diff --git a/samples/finspection/ContentInspectionTask.cpp b/samples/finspection/ContentInspectionTask.cpp index fb7bc0f..35c1837 100644 --- a/samples/finspection/ContentInspectionTask.cpp +++ b/samples/finspection/ContentInspectionTask.cpp @@ -90,9 +90,9 @@ void ContentInspectionTask::operator()() { if (certitude >= _threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; DARWIN_ALERT_MANAGER.SetTags(tagListJson); - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _s->Evt_idToString(), details); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _packet.Evt_idToString(), details); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + R"(, "rules": )" + ruleListJson + R"(, "tags": )" + tagListJson + "}"; diff --git a/samples/ftest/TestTask.cpp b/samples/ftest/TestTask.cpp index 910719d..2baf8ab 100644 --- a/samples/ftest/TestTask.cpp +++ b/samples/ftest/TestTask.cpp @@ -76,7 +76,7 @@ void TestTask::operator()() { _packet.AddCertitude(n & 1); } else { DARWIN_LOG_DEBUG("TestTask:: not triggered specific action, generating alert by default"); - DARWIN_ALERT_MANAGER.Alert(_line, 100, _s->Evt_idToString(), "{\"filter_name\":\"" + darwin::Core::instance().GetName() + "\"}"); + DARWIN_ALERT_MANAGER.Alert(_line, 100, _packet.Evt_idToString(), "{\"filter_name\":\"" + darwin::Core::instance().GetName() + "\"}"); _packet.AddCertitude(0); _response_body.append(_packet.GetBody()); } diff --git a/samples/fuseragent/UserAgentTask.cpp b/samples/fuseragent/UserAgentTask.cpp index 260055e..cccab94 100644 --- a/samples/fuseragent/UserAgentTask.cpp +++ b/samples/fuseragent/UserAgentTask.cpp @@ -60,7 +60,7 @@ void UserAgentTask::operator()() { if (GetCacheResult(hash, certitude)) { if (is_log && (certitude>=_threshold)){ - _logs += R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + _logs += R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"user_agent\": \"" + user_agent + "\", \"ua_classification\": " + std::to_string(certitude) + "}\n"; } _packet.AddCertitude(certitude); @@ -72,7 +72,7 @@ void UserAgentTask::operator()() { certitude = Predict(user_agent); if (is_log && (certitude>=_threshold)){ - _logs += R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + _logs += R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + "\", \"user_agent\": \"" + user_agent + "\", \"ua_classification\": " + std::to_string(certitude) + "}\n"; } _packet.AddCertitude(certitude); diff --git a/samples/fyara/YaraTask.cpp b/samples/fyara/YaraTask.cpp index d0d992c..e156879 100644 --- a/samples/fyara/YaraTask.cpp +++ b/samples/fyara/YaraTask.cpp @@ -59,10 +59,10 @@ void YaraTask::operator()() { if (certitude>=_threshold and certitude < DARWIN_ERROR_RETURN){ STAT_MATCH_INC; - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _s->Evt_idToString()); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _packet.Evt_idToString()); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + "}\n"; logs += alert_log + '\n'; @@ -92,10 +92,10 @@ void YaraTask::operator()() { std::string tagListJson = YaraTask::GetJsonListFromSet(results.tags); std::string details = "{\"rules\": " + ruleListJson + "}"; - DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _s->Evt_idToString(), details, tagListJson); + DARWIN_ALERT_MANAGER.Alert("raw_data", certitude, _packet.Evt_idToString(), details, tagListJson); if (is_log) { - std::string alert_log = R"({"evt_id": ")" + _s->Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + + std::string alert_log = R"({"evt_id": ")" + _packet.Evt_idToString() + R"(", "time": ")" + darwin::time_utils::GetTime() + R"(", "filter": ")" + GetFilterName() + R"(", "certitude": )" + std::to_string(certitude) + R"(, "rules": )" + ruleListJson + R"(, "tags": )" + tagListJson + "}\n"; logs += alert_log + '\n'; diff --git a/tests/manager_socket/multi_filters_test.py b/tests/manager_socket/multi_filters_test.py index 3e3a404..4dbc547 100644 --- a/tests/manager_socket/multi_filters_test.py +++ b/tests/manager_socket/multi_filters_test.py @@ -63,14 +63,16 @@ def run(): }} """ - +# Test different combinations of tcp/udp/unix socket and verifies that everytime, 3 alerts are spawned def alerting_tests(): tests = [ ['unix', 'unix', 'unix'], ['tcp', 'tcp', 'tcp'], ['udp', 'udp', 'udp'], ['udp', 'unix', 'unix'], - ['udp', 'tcp', 'tcp'] + ['udp', 'tcp', 'tcp'], + ['tcp', 'udp', 'unix'], + ['unix', 'tcp', 'udp'], ] for test in tests: @@ -94,9 +96,9 @@ def alerting_tests(): file.close() p = darwin_start() + # socket path is given according to how it is currently generated in the manager, # this may have to change in the future - path_addr = '{}/sockets/test_1.1.sock'.format(conf.TEST_FILES_DIR) if test[0] == 'unix' else '[::]:8181' f = Filter(socket_type=test[0], socket_path=path_addr) api = f.get_darwin_api() From aa1b53ceec2cee644f06f5295f628eeebef5d3df Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 27 Jul 2021 16:42:34 +0200 Subject: [PATCH 32/54] Fixed problems with next filters, cleaned tests Also cleaned ASession a bit --- samples/base/network/ANextFilterConnector.cpp | 10 +- samples/base/network/ANextFilterConnector.hpp | 1 + samples/base/network/ASession.cpp | 38 +++--- samples/base/network/ASession.hpp | 17 ++- .../base/network/TcpNextFilterConnector.cpp | 20 +-- .../base/network/UdpNextFilterConnector.cpp | 14 +-- .../base/network/UnixNextFilterConnector.cpp | 21 ++-- tests/manager_socket/multi_filters_test.py | 117 +++++++++--------- 8 files changed, 130 insertions(+), 108 deletions(-) diff --git a/samples/base/network/ANextFilterConnector.cpp b/samples/base/network/ANextFilterConnector.cpp index d5a0650..5891986 100644 --- a/samples/base/network/ANextFilterConnector.cpp +++ b/samples/base/network/ANextFilterConnector.cpp @@ -7,7 +7,7 @@ namespace darwin { ANextFilterConnector::ANextFilterConnector() - : _io_context{}, _nb_attempts{0}, _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)} + : _io_context{}, _nb_attempts{0}, _max_attempts{3}, _attempts_delay_ms{std::chrono::milliseconds(1000)}, _is_connected{false} { } @@ -23,7 +23,15 @@ namespace darwin { } void ANextFilterConnector::SendCallback(const boost::system::error_code& ec, size_t bytes_transferred, std::shared_ptr buffer) { + DARWIN_LOGGER; if(ec || bytes_transferred != buffer->size()) { + if(ec){ + _is_connected = false; + DARWIN_LOG_NOTICE("ANextFilterConnector::SendCallback Error when sending to Next Filter, retrying, error:" + ec.message()); + } else { + DARWIN_LOG_NOTICE("ANextFilterConnector::SendCallback Error when sending to Next Filter, retrying, error: mismatched size, was :" + + std::to_string(bytes_transferred) + ", expected:" + std::to_string(buffer->size())); + } Send(buffer); return; } diff --git a/samples/base/network/ANextFilterConnector.hpp b/samples/base/network/ANextFilterConnector.hpp index 722e79e..e03cac1 100644 --- a/samples/base/network/ANextFilterConnector.hpp +++ b/samples/base/network/ANextFilterConnector.hpp @@ -75,5 +75,6 @@ namespace darwin { size_t _nb_attempts; size_t _max_attempts; std::chrono::milliseconds _attempts_delay_ms; + bool _is_connected; }; } \ No newline at end of file diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index 734e960..bd2042a 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -41,7 +41,7 @@ namespace darwin { _output = config::convert_output_string(output); } - config::output_type ASession::GetOutputType() { + config::output_type ASession::GetOutputType() const { return _output; } @@ -49,18 +49,21 @@ namespace darwin { return _logs; } - std::string ASession::GetDataToSendToFilter(){ - switch (GetOutputType()){ + void ASession::ModifyDataToSendToFilter(DarwinPacket &packet) const { + std::string& body = packet.GetMutableBody(); + switch (this->_output){ case config::output_type::RAW: - return _packet.GetBody(); + return; case config::output_type::PARSED: - return JsonStringify(_packet.JsonBody()); + body = std::move(JsonStringify(packet.JsonBody())); + return; case config::output_type::NONE: - return ""; + body.clear(); + return; case config::output_type::LOG: - return _packet.GetLogs(); + body = packet.GetLogs(); + return; } - return ""; } void ASession::ReadHeaderCallback(const boost::system::error_code& e, @@ -157,16 +160,18 @@ namespace darwin { } void ASession::SendNext(DarwinPacket& packet) { - // TODO : Unsure of the logic behind the if's and and the Start calls switch(packet.GetResponseType()) { case DARWIN_RESPONSE_SEND_BOTH: + // We try to send to filter, then send back to client anyway this->SendToFilter(packet); __attribute((fallthrough)); case DARWIN_RESPONSE_SEND_BACK: - if(not this->SendToClient(packet)) Start(); + this->SendToClient(packet); + // We don't resume the session by calling Start() because the SendToClientCallback already does break; case DARWIN_RESPONSE_SEND_DARWIN: - if(not this->SendToFilter(packet)) Start(); + this->SendToFilter(packet); + Start(); break; default: Start(); @@ -174,7 +179,7 @@ namespace darwin { } - std::string ASession::JsonStringify(rapidjson::Document &json) { + std::string ASession::JsonStringify(rapidjson::Document &json) const { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -183,25 +188,24 @@ namespace darwin { return std::string(buffer.GetString()); } - bool ASession::SendToClient(DarwinPacket& packet) noexcept { + void ASession::SendToClient(DarwinPacket& packet) noexcept { DARWIN_LOGGER; std::vector serialized_packet = std::move(packet.Serialize()); DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(serialized_packet.size())); this->WriteToClient(serialized_packet); - return true; } - bool ASession::SendToFilter(DarwinPacket& packet) noexcept { + void ASession::SendToFilter(DarwinPacket& packet) noexcept { DARWIN_LOGGER; ANextFilterConnector* connector_ptr = Core::instance().GetNextFilterconnector(); if (!connector_ptr) { DARWIN_LOG_NOTICE("ASession::SendToFilter:: No next filter provided. Ignoring..."); - return false; + return; } + this->ModifyDataToSendToFilter(packet); connector_ptr->Send(packet); - return true; } void ASession::SendToClientCallback(const boost::system::error_code& e, diff --git a/samples/base/network/ASession.hpp b/samples/base/network/ASession.hpp index 617fd3e..f4f5c1a 100755 --- a/samples/base/network/ASession.hpp +++ b/samples/base/network/ASession.hpp @@ -74,7 +74,7 @@ namespace darwin { /// Get the filter's output type /// /// \return The filter's output type - config::output_type GetOutputType(); + config::output_type GetOutputType() const; /// Get the filter's result in a log form /// @@ -95,21 +95,20 @@ namespace darwin { /// Get the data to send to the next filter /// according to the filter's output type /// - /// \param size data's size - /// \param data data to send - std::string GetDataToSendToFilter(); + /// \param packet packet to modify (the field _body may be modified) + void ModifyDataToSendToFilter(DarwinPacket& packet) const; virtual void WriteToClient(std::vector& packet) = 0; /// Send result to the client. + /// Possible failures must be handled in SendToClientCallback /// - /// \return false if the function could not send the data, true otherwise. - virtual bool SendToClient(DarwinPacket& packet) noexcept; + virtual void SendToClient(DarwinPacket& packet) noexcept; /// Send result to next filter. + /// Possible failures must be handled in SendToFilterCallback /// - /// \return false if the function could not send the data, true otherwise. - virtual bool SendToFilter(DarwinPacket& packet) noexcept; + virtual void SendToFilter(DarwinPacket& packet) noexcept; /// Called when data is sent using Send() method. /// Terminate the session on failure. @@ -119,7 +118,7 @@ namespace darwin { SendToClientCallback(const boost::system::error_code& e, std::size_t size); - std::string JsonStringify(rapidjson::Document &json); + std::string JsonStringify(rapidjson::Document &json) const; /// Set the async read for the header. /// diff --git a/samples/base/network/TcpNextFilterConnector.cpp b/samples/base/network/TcpNextFilterConnector.cpp index fe9f06f..5916a34 100644 --- a/samples/base/network/TcpNextFilterConnector.cpp +++ b/samples/base/network/TcpNextFilterConnector.cpp @@ -20,20 +20,26 @@ namespace darwin { _socket.connect(boost::asio::ip::tcp::endpoint(_net_address, _net_port), ec); if(ec) { - DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + _nb_attempts++; + DARWIN_LOG_ERROR("TcpNextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); return false; } + // Reset the counter on success + _nb_attempts = 0; + _is_connected = true; return true; } void TcpNextFilterConnector::Send(std::shared_ptr packet) { DARWIN_LOGGER; - if(_nb_attempts > _max_attempts){ - DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); - return; - } - while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { - std::this_thread::sleep_for(_attempts_delay_ms); + if(! _is_connected){ + while( ! this->Connect()) { + if(_nb_attempts >= _max_attempts){ + DARWIN_LOG_ERROR("TcpNextFilterConnector::Send:: Maximal number of attempts reached, aborting packet"); + return; + } + std::this_thread::sleep_for(_attempts_delay_ms); + } } boost::asio::async_write(_socket, *packet, boost::bind(&TcpNextFilterConnector::SendCallback, this, diff --git a/samples/base/network/UdpNextFilterConnector.cpp b/samples/base/network/UdpNextFilterConnector.cpp index ab19dfb..9a846bd 100644 --- a/samples/base/network/UdpNextFilterConnector.cpp +++ b/samples/base/network/UdpNextFilterConnector.cpp @@ -21,7 +21,8 @@ namespace darwin { _socket.connect(_endpoint, ec); if(ec) { - DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + // We handle the case but, since UDP is stateless, this _socket.connect call does nothing and never fails + DARWIN_LOG_ERROR("UdpNextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); return false; } return true; @@ -29,13 +30,10 @@ namespace darwin { void UdpNextFilterConnector::Send(std::shared_ptr packet) { DARWIN_LOGGER; - if(_nb_attempts > _max_attempts){ - DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); - return; - } - while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { - std::this_thread::sleep_for(_attempts_delay_ms); - } + // The Connect call cannot fail, we don't have to check and we don't have to retry it + this->Connect(); + + DARWIN_LOG_DEBUG("UdpNextFilterConnector::Send : Sending packet to UDP endpoint, no retries"); _socket.async_send_to(*packet, _endpoint, diff --git a/samples/base/network/UnixNextFilterConnector.cpp b/samples/base/network/UnixNextFilterConnector.cpp index 0e6dacd..371e7c0 100644 --- a/samples/base/network/UnixNextFilterConnector.cpp +++ b/samples/base/network/UnixNextFilterConnector.cpp @@ -19,22 +19,27 @@ namespace darwin { _socket.connect(boost::asio::local::stream_protocol::endpoint(_socket_path), ec); if(ec) { - DARWIN_LOG_ERROR("NextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); + _nb_attempts++; + DARWIN_LOG_ERROR("UnixNextFilterConnector::Connect:: Connexion error : " + std::string(ec.message())); return false; } + // Reset the counter on success + _nb_attempts = 0; + _is_connected = true; return true; } void UnixNextFilterConnector::Send(std::shared_ptr packet) { DARWIN_LOGGER; - if(_nb_attempts > _max_attempts){ - DARWIN_LOG_ERROR("NextFilterConnector::Send:: Maximal number of attempts reached"); - return; + if(! _is_connected){ + while( ! this->Connect()) { + if(_nb_attempts >= _max_attempts){ + DARWIN_LOG_ERROR("UnixNextFilterConnector::Send:: Maximal number of attempts reached, aborting packet"); + return; + } + std::this_thread::sleep_for(_attempts_delay_ms); + } } - while( ! this->Connect() && _nb_attempts++ <= _max_attempts) { - std::this_thread::sleep_for(_attempts_delay_ms); - } - boost::asio::async_write(_socket, *packet, boost::bind(&UnixNextFilterConnector::SendCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, diff --git a/tests/manager_socket/multi_filters_test.py b/tests/manager_socket/multi_filters_test.py index 4dbc547..1e8d172 100644 --- a/tests/manager_socket/multi_filters_test.py +++ b/tests/manager_socket/multi_filters_test.py @@ -1,3 +1,4 @@ +import functools from manager_socket.utils import CONF_FTEST, PATH_CONF_FTEST import time from tools.output import print_result @@ -7,19 +8,26 @@ def run(): - tests = [ - alerting_tests, + scenarios = [ + ['unix', 'unix', 'unix'], + ['tcp', 'tcp', 'tcp'], + ['udp', 'udp', 'udp'], + ['udp', 'unix', 'unix'], + ['tcp', 'unix', 'unix'], + ['udp', 'tcp', 'tcp'], + ['tcp', 'udp', 'unix'], + ['unix', 'tcp', 'udp'], ] - for i in tests: - print_result("Monitoring: " + i.__name__, i) + for i in scenarios: + print_result("Multi Filters: Alerting tests : " + ' -> '.join(i), functools.partial(alerting_tests, i)) ONE_FILTER = """ {{ "name": "{name}", "exec_path": "{filter_path}darwin_test", "config_file": "/tmp/test.conf", - "output": "NONE", + "output": "RAW", "next_filter": "{next_filter}", "next_filter_network":{next_filter_network}, "nb_thread": 1, @@ -64,64 +72,57 @@ def run(): """ # Test different combinations of tcp/udp/unix socket and verifies that everytime, 3 alerts are spawned -def alerting_tests(): - tests = [ - ['unix', 'unix', 'unix'], - ['tcp', 'tcp', 'tcp'], - ['udp', 'udp', 'udp'], - ['udp', 'unix', 'unix'], - ['udp', 'tcp', 'tcp'], - ['tcp', 'udp', 'unix'], - ['unix', 'tcp', 'udp'], - ] +def alerting_tests(filters_list): + assert(len(filters_list) == 3) + filter_1 = ONE_FILTER.format(name='test_1', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='test_2', next_filter_network=network_map[filters_list[1]].format(port=8282), + network=network_map[filters_list[0]].format(port=8181)) + filter_2 = ONE_FILTER.format(name='test_2', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='test_3', next_filter_network=network_map[filters_list[2]].format(port=8383), + network=network_map[filters_list[1]].format(port=8282)) + filter_3 = ONE_FILTER.format(name='test_3', filter_path=conf.DEFAULT_FILTER_PATH, + next_filter='', next_filter_network='{}', + network=network_map[filters_list[2]].format(port=8383)) + config = CONFIG.format(filter_1=filter_1, filter_2=filter_2, filter_3=filter_3) - for test in tests: - filter_1 = ONE_FILTER.format(name='test_1', filter_path=conf.DEFAULT_FILTER_PATH, - next_filter='test_2', next_filter_network=network_map[test[1]].format(port=8282), - network=network_map[test[0]].format(port=8181)) - filter_2 = ONE_FILTER.format(name='test_2', filter_path=conf.DEFAULT_FILTER_PATH, - next_filter='test_3', next_filter_network=network_map[test[2]].format(port=8383), - network=network_map[test[1]].format(port=8282)) - filter_3 = ONE_FILTER.format(name='test_3', filter_path=conf.DEFAULT_FILTER_PATH, - next_filter='', next_filter_network='{}', - network=network_map[test[2]].format(port=8383)) - config = CONFIG.format(filter_1=filter_1, filter_2=filter_2, filter_3=filter_3) + darwin_configure(config) + darwin_configure(CONF_FTEST, path=PATH_CONF_FTEST) - darwin_configure(config) - darwin_configure(CONF_FTEST, path=PATH_CONF_FTEST) + # Erasing last alerts + file = open("/tmp/test_test.log","w") + file.truncate(0) + file.close() - # Erasing last alerts - file = open("/tmp/test_test.log","w") - file.truncate(0) - file.close() + p = darwin_start() - p = darwin_start() + # socket path is given according to how it is currently generated in the manager, + # this may have to change in the future + path_addr = '{}/sockets/test_1.1.sock'.format(conf.TEST_FILES_DIR) if filters_list[0] == 'unix' else '[::]:8181' + f = Filter(socket_type=filters_list[0], socket_path=path_addr) + api = f.get_darwin_api() - # socket path is given according to how it is currently generated in the manager, - # this may have to change in the future - path_addr = '{}/sockets/test_1.1.sock'.format(conf.TEST_FILES_DIR) if test[0] == 'unix' else '[::]:8181' - f = Filter(socket_type=test[0], socket_path=path_addr) - api = f.get_darwin_api() + api.call("Hello", response_type="darwin") + time.sleep(0.2) + api.call("There", response_type="darwin") - api.call("Hello", response_type="darwin") - # arbitrary sleep length, should be enough on most systems - # We verify that One alert per filter were written - time.sleep(2) - line1, line2, line3 = False, False, False - file = open("/tmp/test_test.log","r") - for l in file.readlines(): - if 'test_1' in l: - line1 = True - elif 'test_2' in l: - line2 = True - elif 'test_3' in l: - line3 = True - file.close() - darwin_stop(p) - darwin_remove_configuration() - darwin_remove_configuration(path=PATH_CONF_FTEST) + # arbitrary sleep length, should be enough on most systems + # We verify that One alert per filter were written + time.sleep(2) + line1, line2, line3 = 0, 0, 0 + file = open("/tmp/test_test.log","r") + for l in file.readlines(): + if 'test_1' in l: + line1 += 1 + elif 'test_2' in l: + line2 += 1 + elif 'test_3' in l: + line3 += 1 + file.close() + darwin_stop(p) + darwin_remove_configuration() + darwin_remove_configuration(path=PATH_CONF_FTEST) - if not line1 or not line2 or not line3: - print('test failed: ', test) - return False + if line1 != 2 or line2 != 2 or line3 != 2: + print('test failed: ', filters_list) + return False return True From aa05f12f33cb7dca4016851ad3ad3395d29271fc Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 27 Jul 2021 18:47:59 +0200 Subject: [PATCH 33/54] Modified config of next filter for easier configuration --- manager/config.py | 58 ++++++++++++---------- tests/manager_socket/multi_filters_test.py | 9 ++-- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/manager/config.py b/manager/config.py index a719eef..22c0ec1 100644 --- a/manager/config.py +++ b/manager/config.py @@ -189,18 +189,21 @@ def set_defaults(validator, properties, instance, schema): "enum": ["NONE", "RAW", "LOG", "PARSED"], "default": "NONE" }, - "next_filter": {"type": "string"}, - "next_filter_network": { - "type": "object", - "properties": { - "socket_type":{ - "type":"string", - "enum": ["NONE", "UNIX", "TCP", "UDP"], - "default":"NONE" - }, - "address_path": {"type": "string"} + "next_filter": { + "OneOf":[ + {"type": "string"}, + {"type": "object", + "properties": { + "socket_type":{ + "type":"string", + "enum": ["NONE", "UNIX", "TCP", "UDP"], + "default":"NONE" + }, + "address_path": {"type": "string"} + } } - }, + ] + }, "threshold": { "type": "integer", "default": 100 @@ -290,20 +293,7 @@ def complete_filters_conf(prefix, suffix): filter['failures'] = 0 filter['extension'] = '.1' filter['pid_file'] = '{prefix}/run{suffix}/{filter}{extension}.pid'.format(prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension']) - - if not filter['next_filter']: - filter['next_filter_unix_socket'] = 'no' - filter['next_filter_network'] = { "socket_type":"NONE", "address_path":filter['next_filter_unix_socket'] } - else: - filter['next_filter_unix_socket'] = '{prefix}/sockets{suffix}/{next_filter}.sock'.format( - prefix=prefix, suffix=suffix, - next_filter=filter['next_filter'] - ) - if 'next_filter_network' not in filter: - filter['next_filter_network'] = { "socket_type":"UNIX", "address_path":filter['next_filter_unix_socket'] } - elif filter['next_filter_network']['socket_type'] == "UNIX": - filter['next_filter_network']['address_path'] = filter['next_filter_unix_socket'] - + filter['socket'] = '{prefix}/sockets{suffix}/{filter}{extension}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension']) filter['socket_link'] = '{prefix}/sockets{suffix}/{filter}.sock'.format(prefix=prefix, suffix=suffix, filter=filter['name']) @@ -316,3 +306,21 @@ def complete_filters_conf(prefix, suffix): prefix=prefix, suffix=suffix, filter=filter['name'], extension=filter['extension'] ) + + # Next filter is setup with a second loop (we need network information already setup) + for _, filter in filters.items(): + if 'next_filter' not in filter: + filter['next_filter_network'] = { "socket_type":"NONE", "address_path":"no" } + else: + if isinstance(filter['next_filter'], str): + #check other filters + modified = False + for _, other_filter in filters.items(): + if filter['next_filter'] == other_filter['name']: + filter['next_filter_network'] = other_filter['network'] + modified = True + if not modified: + raise ConfParseError("Filter '{}' had next_filter configured to '{}' but it was not found in the configuration" + .format(filter['name'], filter['next_filter'])) + else: + filter['next_filter_network'] = filter['next_filter'] diff --git a/tests/manager_socket/multi_filters_test.py b/tests/manager_socket/multi_filters_test.py index 1e8d172..67f3bad 100644 --- a/tests/manager_socket/multi_filters_test.py +++ b/tests/manager_socket/multi_filters_test.py @@ -28,8 +28,7 @@ def run(): "exec_path": "{filter_path}darwin_test", "config_file": "/tmp/test.conf", "output": "RAW", - "next_filter": "{next_filter}", - "next_filter_network":{next_filter_network}, + {next_filter} "nb_thread": 1, "log_level": "DEBUG", "cache_size": 0, @@ -75,13 +74,13 @@ def run(): def alerting_tests(filters_list): assert(len(filters_list) == 3) filter_1 = ONE_FILTER.format(name='test_1', filter_path=conf.DEFAULT_FILTER_PATH, - next_filter='test_2', next_filter_network=network_map[filters_list[1]].format(port=8282), + next_filter='"next_filter": "test_2",', network=network_map[filters_list[0]].format(port=8181)) filter_2 = ONE_FILTER.format(name='test_2', filter_path=conf.DEFAULT_FILTER_PATH, - next_filter='test_3', next_filter_network=network_map[filters_list[2]].format(port=8383), + next_filter='"next_filter": "test_3",', network=network_map[filters_list[1]].format(port=8282)) filter_3 = ONE_FILTER.format(name='test_3', filter_path=conf.DEFAULT_FILTER_PATH, - next_filter='', next_filter_network='{}', + next_filter='', network=network_map[filters_list[2]].format(port=8383)) config = CONFIG.format(filter_1=filter_1, filter_2=filter_2, filter_3=filter_3) From b0880292a48094631fdde843e6d439a38c6ad443 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 28 Jul 2021 17:09:35 +0200 Subject: [PATCH 34/54] Modified protocol.h added packed attribute to darwin header Removed certitude list default size Modified parsing and serializing accordingly --- manager/config.py | 2 +- samples/base/DarwinPacket.cpp | 19 ++++--------------- samples/base/DarwinPacket.hpp | 4 +--- samples/base/network/ASession.cpp | 10 +++------- samples/base/network/UdpSession.cpp | 12 ++++++------ samples/fbuffer/Connectors/AConnector.cpp | 6 +++--- samples/protocol.h | 6 +++--- tests/tools/filter.py | 4 ++-- 8 files changed, 23 insertions(+), 40 deletions(-) diff --git a/manager/config.py b/manager/config.py index 22c0ec1..158d763 100644 --- a/manager/config.py +++ b/manager/config.py @@ -309,7 +309,7 @@ def complete_filters_conf(prefix, suffix): # Next filter is setup with a second loop (we need network information already setup) for _, filter in filters.items(): - if 'next_filter' not in filter: + if 'next_filter' not in filter or filter['next_filter'] == '': filter['next_filter_network'] = { "socket_type":"NONE", "address_path":"no" } else: if isinstance(filter['next_filter'], str): diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 657e10f..58ce07e 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -51,11 +51,8 @@ namespace darwin { DarwinPacket::DarwinPacket(darwin_filter_packet_t& input) : _type{input.type}, _response{input.response}, _filter_code{input.filter_code}, _parsed_certitude_size{input.certitude_size}, _parsed_body_size{input.body_size} - { std::memcpy(this->_evt_id, input.evt_id, 16); - if(input.certitude_size > 0) - AddCertitude(input.certitude_list[0]); } std::vector DarwinPacket::Serialize() const { @@ -66,29 +63,21 @@ namespace darwin { this->_body.size(), {0}, this->_certitude_list.size(), - {0} + {} }; std::memcpy(header.evt_id, this->_evt_id, 16); - size_t size = sizeof(header) + _body.size(); - - if( ! _certitude_list.empty()) { - // From the packet format, at least one certitude is always allocated - size += (_certitude_list.size() - 1) * sizeof(unsigned int); - header.certitude_list[0] = _certitude_list[0]; - } - - + size_t size = sizeof(header) + _body.size() + _certitude_list.size() * sizeof(unsigned int); + std::vector ret(size, 0); unsigned char * pt = ret.data(); std::memcpy(pt, &header, sizeof(header)); pt += sizeof(header); - //certitudes - for(size_t i=1; i < _certitude_list.size(); i++) { + for(size_t i=0; i < _certitude_list.size(); i++) { unsigned int certitude = _certitude_list[i]; std::memcpy(pt,(void*)(&certitude), sizeof(certitude)); pt += sizeof(certitude); diff --git a/samples/base/DarwinPacket.hpp b/samples/base/DarwinPacket.hpp index 6f094a0..76609f7 100644 --- a/samples/base/DarwinPacket.hpp +++ b/samples/base/DarwinPacket.hpp @@ -17,7 +17,6 @@ #include "protocol.h" #include "../../toolkit/rapidjson/document.h" -#define DEFAULT_CERTITUDE_LIST_SIZE 1 namespace darwin { @@ -75,8 +74,7 @@ namespace darwin { /// constexpr static size_t getMinimalSize() { return sizeof(_type) + sizeof(_response) + sizeof(_filter_code) - + sizeof(_parsed_body_size) + sizeof(_evt_id) + sizeof(_parsed_certitude_size) - + DEFAULT_CERTITUDE_LIST_SIZE * sizeof(unsigned int); // Protocol has at least DEFAULT (one) certitude + + sizeof(_parsed_body_size) + sizeof(_evt_id) + sizeof(_parsed_certitude_size); }; /// diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index bd2042a..177c4e7 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -80,16 +80,12 @@ namespace darwin { goto header_callback_stop_session; } _packet = std::move(DarwinPacket(_header)); - if (_packet.GetParsedBodySize() == 0 && _packet.GetParsedCertitudeSize() <= 1) { + if (_packet.GetParsedBodySize() == 0 && _packet.GetParsedCertitudeSize() == 0) { ExecuteFilter(); return; } // Else the ReadBodyCallback willcall ExecuteFilter - size_t sizeToRead = _packet.GetParsedBodySize(); - if(_packet.GetParsedCertitudeSize() > 1){ - sizeToRead += _packet.GetParsedCertitudeSize() * sizeof(unsigned int); - } - ReadBody(sizeToRead); + ReadBody(_packet.GetParsedBodySize() + _packet.GetParsedCertitudeSize()); return; } @@ -109,7 +105,7 @@ namespace darwin { if (!e) { size_t already_parsed_size = 0; - if(_packet.GetParsedCertitudeSize() > 1){ + if(_packet.GetParsedCertitudeSize() > 0){ while(_packet.GetParsedCertitudeSize() != _packet.GetCertitudeList().size() && already_parsed_size + sizeof(unsigned int) <= size) { unsigned int cert = 0; diff --git a/samples/base/network/UdpSession.cpp b/samples/base/network/UdpSession.cpp index b15a2d3..f4337a6 100755 --- a/samples/base/network/UdpSession.cpp +++ b/samples/base/network/UdpSession.cpp @@ -34,22 +34,22 @@ namespace darwin { std::memcpy(&_header, _buffer.data(), sizeof(_header)); //The header can hold one certitude, if there are more, we have to parsed them accordingly - size_t size_cert_above_header = _header.certitude_size > 1 ? (_header.certitude_size-1) * sizeof(unsigned int) : 0; - if(size != sizeof(_header) + _header.body_size + size_cert_above_header){ - DARWIN_LOG_ERROR("Error parsing header sizes, expected " + std::to_string(sizeof(_header) + _header.body_size) + " but buffer is " + std::to_string(size)); + size_t cert_size = _header.certitude_size * sizeof(unsigned int); + if(size != sizeof(_header) + _header.body_size + cert_size){ + DARWIN_LOG_ERROR("Error parsing header sizes, expected " + std::to_string(sizeof(_header) + _header.body_size + cert_size) + " but buffer is " + std::to_string(size)); return; } _packet = std::move(DarwinPacket(_header)); - for(size_t i=1; i < _header.certitude_size; i++){ + for(size_t i=0; i < _header.certitude_size; i++){ unsigned int cert = 0; - std::memcpy(&cert, _buffer.data() + sizeof(_header) + (i-1)*sizeof(unsigned int), sizeof(unsigned int)); + std::memcpy(&cert, _buffer.data() + sizeof(_header) + i*sizeof(unsigned int), sizeof(unsigned int)); _packet.AddCertitude(cert); } std::string& body = _packet.GetMutableBody(); - body.append(_buffer.begin() + sizeof(_header) + size_cert_above_header, _header.body_size); + body.append(_buffer.begin() + sizeof(_header) + cert_size, _header.body_size); this->ExecuteFilter(); } diff --git a/samples/fbuffer/Connectors/AConnector.cpp b/samples/fbuffer/Connectors/AConnector.cpp index a4c7614..4e8d3e2 100644 --- a/samples/fbuffer/Connectors/AConnector.cpp +++ b/samples/fbuffer/Connectors/AConnector.cpp @@ -215,14 +215,14 @@ bool AConnector::SendToFilter(std::vector &logs) { } // Prepare packet - DARWIN_LOG_DEBUG("AConnector::SendToFilter:: data to send: " + data + ", data size: " + std::to_string(data.size())); + DARWIN_LOG_DEBUG("AConnector::SendToFilter:: data to send: size:" + std::to_string(data.size()) + ", data : " + data); /* * Allocate the header + * the size of the certitude - * DEFAULT_CERTITUDE_LIST_SIZE certitude already in header size */ std::size_t certitude_size = 1; - std::size_t packet_size = sizeof(darwin_filter_packet_t) + data.size(); + std::size_t packet_size = sizeof(darwin_filter_packet_t) + data.size() + certitude_size*sizeof(unsigned int); DARWIN_LOG_DEBUG("AConnector::SendToFilter:: Computed packet size: " + std::to_string(packet_size)); darwin_filter_packet_t* packet; @@ -240,7 +240,7 @@ bool AConnector::SendToFilter(std::vector &logs) { if(data.size() != 0) { // TODO: set a proper pointer in protocol.h for the body // Yes We Hack... - memcpy(&packet->certitude_list[certitude_size + 1], data.c_str(), data.size()); + memcpy(&packet->certitude_list[certitude_size], data.c_str(), data.size()); } packet->type = DARWIN_PACKET_FILTER; packet->response = DARWIN_RESPONSE_SEND_DARWIN; diff --git a/samples/protocol.h b/samples/protocol.h index c0ce485..c02b8d2 100644 --- a/samples/protocol.h +++ b/samples/protocol.h @@ -24,7 +24,6 @@ extern "C" { #define DARWIN_FILTER_CODE_NO 0x00000000 // the default certitude list size, which is 1, to allow FMAs (see flexible array members on C99) for both C and C++ code -#define DEFAULT_CERTITUDE_LIST_SIZE 1 /// Represent the receiver of the results. /// @@ -54,8 +53,9 @@ typedef struct { size_t body_size; //!< The complete size of the the parameters to be sent (if needed). unsigned char evt_id[16]; //!< An array containing the event ID size_t certitude_size; //!< The size of the list containing the certitudes. - unsigned int certitude_list[DEFAULT_CERTITUDE_LIST_SIZE]; //!< The scores or the certitudes of the module. May be used to pass other info in specific cases. -} darwin_filter_packet_t; + unsigned int certitude_list[0]; //!< DEPRECATED, certitudes are no longer passed through this array, + //!< they must be appened after the header, before the body +} __attribute__((packed)) darwin_filter_packet_t; #ifdef __cplusplus }; diff --git a/tests/tools/filter.py b/tests/tools/filter.py index e0a0499..9a5f2e6 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -51,8 +51,8 @@ def set_socket_info(self, socket_type, socket_path_address): else: raise NotImplementedError("Unrecognized protocol : '{}'".format(self.socket_type)) - def get_darwin_api(self): - return DarwinApi(socket_type=self.socket_type, socket_path=self.socket, socket_host=self.host, socket_port=self.port) + def get_darwin_api(self, verbose=False): + return DarwinApi(socket_type=self.socket_type, socket_path=self.socket, socket_host=self.host, socket_port=self.port, verbose=verbose) def prepare_log_file(self): # TODO variabilize once path can be changed From 13f4ab325b07e6343003c7696fce75c6c30f2639 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 29 Jul 2021 11:03:35 +0200 Subject: [PATCH 35/54] Fixed clang/gcc warnings Most case were unused variables or poorly chosen types --- samples/base/network/ASession.cpp | 6 +++--- samples/base/network/TcpServer.cpp | 2 +- samples/base/network/UdpSession.cpp | 2 +- samples/fbuffer/Connectors/SumConnector.hpp | 6 +++--- samples/fconnection/ConnectionSupervisionTask.cpp | 1 - samples/finspection/ContentInspectionTask.cpp | 1 - samples/finspection/data_pool.cpp | 6 ++---- samples/finspection/data_pool.hpp | 2 +- samples/finspection/extract_impcap.cpp | 5 +---- samples/finspection/flow.cpp | 1 + samples/finspection/stream_buffer.cpp | 2 +- samples/finspection/yara_utils.cpp | 2 +- samples/fsession/SessionTask.cpp | 2 +- samples/ftanomaly/TAnomalyThreadManager.cpp | 2 +- samples/fyara/YaraTask.cpp | 1 + 15 files changed, 18 insertions(+), 23 deletions(-) diff --git a/samples/base/network/ASession.cpp b/samples/base/network/ASession.cpp index 177c4e7..08d0184 100755 --- a/samples/base/network/ASession.cpp +++ b/samples/base/network/ASession.cpp @@ -55,7 +55,7 @@ namespace darwin { case config::output_type::RAW: return; case config::output_type::PARSED: - body = std::move(JsonStringify(packet.JsonBody())); + body = JsonStringify(packet.JsonBody()); return; case config::output_type::NONE: body.clear(); @@ -79,7 +79,7 @@ namespace darwin { DARWIN_LOG_ERROR("ASession::ReadHeaderCallback:: Mismatching header size"); goto header_callback_stop_session; } - _packet = std::move(DarwinPacket(_header)); + _packet = DarwinPacket(_header); if (_packet.GetParsedBodySize() == 0 && _packet.GetParsedCertitudeSize() == 0) { ExecuteFilter(); return; @@ -186,7 +186,7 @@ namespace darwin { void ASession::SendToClient(DarwinPacket& packet) noexcept { DARWIN_LOGGER; - std::vector serialized_packet = std::move(packet.Serialize()); + std::vector serialized_packet = packet.Serialize(); DARWIN_LOG_DEBUG("ASession::SendToClient: Computed packet size: " + std::to_string(serialized_packet.size())); this->WriteToClient(serialized_packet); } diff --git a/samples/base/network/TcpServer.cpp b/samples/base/network/TcpServer.cpp index 018d804..c330ba7 100755 --- a/samples/base/network/TcpServer.cpp +++ b/samples/base/network/TcpServer.cpp @@ -24,7 +24,7 @@ namespace darwin { Generator& generator) : AServer(output, threshold, generator), _address{address}, _port_nb{port_nb}, - _acceptor{_io_context, boost::asio::ip::tcp::endpoint(_address, port_nb)}, + _acceptor{_io_context, boost::asio::ip::tcp::endpoint(_address, _port_nb)}, _new_connection{_io_context} { this->InitSignalsAndStart(); diff --git a/samples/base/network/UdpSession.cpp b/samples/base/network/UdpSession.cpp index f4337a6..4d02fa7 100755 --- a/samples/base/network/UdpSession.cpp +++ b/samples/base/network/UdpSession.cpp @@ -40,7 +40,7 @@ namespace darwin { return; } - _packet = std::move(DarwinPacket(_header)); + _packet = DarwinPacket(_header); for(size_t i=0; i < _header.certitude_size; i++){ unsigned int cert = 0; diff --git a/samples/fbuffer/Connectors/SumConnector.hpp b/samples/fbuffer/Connectors/SumConnector.hpp index 622ae24..1192488 100644 --- a/samples/fbuffer/Connectors/SumConnector.hpp +++ b/samples/fbuffer/Connectors/SumConnector.hpp @@ -35,7 +35,7 @@ class SumConnector final : public AConnector { ///\brief test every Redis Key in _redis_lists to ensure keys are of the correct type if they exist in Redis /// ///\return true if the keys are not in redis, or are of the good type, false otherwise. - virtual bool PrepareKeysInRedis(); + virtual bool PrepareKeysInRedis() override; ///\brief This function sends data to the REDIS storage. It overrides default pure virtual one as each filter doesn't need the same data. /// @@ -60,7 +60,7 @@ class SumConnector final : public AConnector { ///\param sum_name unused parameter /// ///\return always true - virtual bool REDISReinsertLogs(std::vector &logs __attribute__((unused)), const std::string &sum_name __attribute__((unused))); + virtual bool REDISReinsertLogs(std::vector &logs __attribute__((unused)), const std::string &sum_name __attribute__((unused))) override; ///\brief Get the sum from Redis /// @@ -85,5 +85,5 @@ class SumConnector final : public AConnector { ///\param format The string to fill with the result /// ///\return True on success (formatting successful), False otherwise. - virtual bool FormatDataToSendToFilter(std::vector &logs, std::string &formatted); + virtual bool FormatDataToSendToFilter(std::vector &logs, std::string &formatted) override; }; \ No newline at end of file diff --git a/samples/fconnection/ConnectionSupervisionTask.cpp b/samples/fconnection/ConnectionSupervisionTask.cpp index cefd270..e9a24a7 100644 --- a/samples/fconnection/ConnectionSupervisionTask.cpp +++ b/samples/fconnection/ConnectionSupervisionTask.cpp @@ -114,7 +114,6 @@ bool ConnectionSupervisionTask::ParseLine(rapidjson::Value& line){ unsigned int ConnectionSupervisionTask::REDISLookup(const std::string& connection) noexcept { DARWIN_LOGGER; DARWIN_LOG_DEBUG("ConnectionSupervisionTask:: Looking up '" + connection + "' in the Redis"); - int redisReplyCode; darwin::toolkit::RedisManager& redis = darwin::toolkit::RedisManager::GetInstance(); long long int result; diff --git a/samples/finspection/ContentInspectionTask.cpp b/samples/finspection/ContentInspectionTask.cpp index 35c1837..f806170 100644 --- a/samples/finspection/ContentInspectionTask.cpp +++ b/samples/finspection/ContentInspectionTask.cpp @@ -122,7 +122,6 @@ bool ContentInspectionTask::ParseBody() { logs.clear(); std::size_t packetMeta = 0, packetMetaEnd; std::size_t packetData, packetDataEnd = 0; - std::size_t openingBracket; do { packetMeta = raw_body.find("\"{", packetDataEnd + 1); diff --git a/samples/finspection/data_pool.cpp b/samples/finspection/data_pool.cpp index 68a5413..872f308 100644 --- a/samples/finspection/data_pool.cpp +++ b/samples/finspection/data_pool.cpp @@ -202,7 +202,7 @@ DataObject *getOrCreateAvailableObject(DataPool *pool) { return object; } -DataPool *createPool(char *poolName, constructor_t objectConstructor, +DataPool *createPool(const char *poolName, constructor_t objectConstructor, destructor_t objectDestructor, resetor_t objectResetor, uint32_t minAvailableElems) { DARWIN_LOGGER; @@ -346,8 +346,6 @@ void *memoryManagerDoWork(void *pData) { DARWIN_LOG_DEBUG("memory manager: cleanup finished, memory freed: " + std::to_string(totalMemFreed) + "," " total memory used: " + std::to_string(poolStorage->totalDataSize)); - uint32_t ratio = (unsigned long int)((float)poolStorage->totalDataSize/(float)poolStorage->maxDataSize*100); - DARWIN_LOG_DEBUG("memory manager: starting TCP session cleanup"); DataObject *sessObject; TcpSession *session; @@ -403,7 +401,7 @@ void startMemoryManager(MemManagerParams *memManagerParams) { pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); if(pthread_create(&(memManagerParams->thread), &attr, memoryManagerDoWork, (void *)memManagerParams) != 0) { - memManagerParams->thread = NULL; + memManagerParams->thread = 0; pthread_attr_destroy(&attr); return; } diff --git a/samples/finspection/data_pool.hpp b/samples/finspection/data_pool.hpp index bfdc404..804d340 100644 --- a/samples/finspection/data_pool.hpp +++ b/samples/finspection/data_pool.hpp @@ -101,7 +101,7 @@ uint32_t deleteDataObjectFromPool(DataObject *, DataPool *); void setObjectAvailable(DataObject *); void updateDataObjectSize(DataObject *, int); DataObject *getOrCreateAvailableObject(DataPool *); -DataPool *createPool(char *, constructor_t, destructor_t, resetor_t, uint32_t); +DataPool *createPool(const char *, constructor_t, destructor_t, resetor_t, uint32_t); void destroyPool(DataPool *); PoolStorage *initPoolStorage(); void deletePoolStorage(PoolStorage *); diff --git a/samples/finspection/extract_impcap.cpp b/samples/finspection/extract_impcap.cpp index 94ab9ea..345edaf 100644 --- a/samples/finspection/extract_impcap.cpp +++ b/samples/finspection/extract_impcap.cpp @@ -31,8 +31,6 @@ #include "../toolkit/rapidjson/document.h" Packet *getImpcapData(std::string impcapMeta, std::string impcapData) { - DARWIN_LOGGER; - int localret; uint32_t contentLength; const char *content; uint16_t ethType; @@ -85,9 +83,8 @@ Packet *getImpcapData(std::string impcapMeta, std::string impcapData) { uint8_t *ImpcapDataDecode(const char *hex, uint32_t length) { uint8_t *retBuf = (uint8_t *)malloc(length/2*sizeof(uint8_t)); - int i; - for(i = 0; i < length; ++i) { + for(uint32_t i = 0; i < length; ++i) { if(i%2) { retBuf[i/2] <<= 4; if(hex[i] >= '0' && hex[i] <= '9') { diff --git a/samples/finspection/flow.cpp b/samples/finspection/flow.cpp index d303dea..9514e96 100644 --- a/samples/finspection/flow.cpp +++ b/samples/finspection/flow.cpp @@ -135,6 +135,7 @@ static inline int addFlowToList(Flow *flow, FlowList *flowList) { } } +__attribute((unused)) static inline int removeFlowFromList(Flow *flow, FlowList *flowList) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("removeFlowFromList"); diff --git a/samples/finspection/stream_buffer.cpp b/samples/finspection/stream_buffer.cpp index afb84cd..1a180d9 100644 --- a/samples/finspection/stream_buffer.cpp +++ b/samples/finspection/stream_buffer.cpp @@ -197,7 +197,7 @@ int streamBufferExtend(StreamBuffer *sb, uint32_t extLength) { return -1; } -static inline void streamBufferShift(StreamBuffer *sb, int amount) { +static inline void streamBufferShift(StreamBuffer *sb, uint32_t amount) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("streamBufferShift, amount=" + std::to_string(amount)); diff --git a/samples/finspection/yara_utils.cpp b/samples/finspection/yara_utils.cpp index 7298118..6f1ede6 100644 --- a/samples/finspection/yara_utils.cpp +++ b/samples/finspection/yara_utils.cpp @@ -287,7 +287,7 @@ YaraResults yaraScan(uint8_t *buffer, uint32_t buffLen, StreamBuffer *sb) { } #if YR_MAJOR_VERSION == 3 -void yaraErrorCallback(int errorLevel, const char *fileName, int lineNumber, const char *message, void *userData) { +void yaraErrorCallback(int errorLevel, const char *fileName, int lineNumber, const char *message, void *userData __attribute((unused))) { DARWIN_LOGGER; char errStr[2048]; if(fileName) { diff --git a/samples/fsession/SessionTask.cpp b/samples/fsession/SessionTask.cpp index 139750c..6a47ef8 100644 --- a/samples/fsession/SessionTask.cpp +++ b/samples/fsession/SessionTask.cpp @@ -91,7 +91,7 @@ std::string SessionTask::JoinRepoIDs(const std::vector &repo_ids) { } -bool SessionTask::REDISResetExpire(const std::string &token, const std::string &repo_id) { +bool SessionTask::REDISResetExpire(const std::string &token, const std::string &repo_id __attribute((unused))) { DARWIN_LOGGER; long long int ttl; darwin::toolkit::RedisManager& redis = darwin::toolkit::RedisManager::GetInstance(); diff --git a/samples/ftanomaly/TAnomalyThreadManager.cpp b/samples/ftanomaly/TAnomalyThreadManager.cpp index f7e5918..e4ea21e 100644 --- a/samples/ftanomaly/TAnomalyThreadManager.cpp +++ b/samples/ftanomaly/TAnomalyThreadManager.cpp @@ -110,7 +110,7 @@ void AnomalyThreadManager::PreProcess(std::vector logs) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("AnomalyThread::PreProcess:: Starting the pre-process..."); - size_t size, pos, i; + size_t size, i; char delimiter = ';'; std::array values{}; std::string ip, ip_dst, port, protocol; diff --git a/samples/fyara/YaraTask.cpp b/samples/fyara/YaraTask.cpp index e156879..95b9166 100644 --- a/samples/fyara/YaraTask.cpp +++ b/samples/fyara/YaraTask.cpp @@ -139,6 +139,7 @@ bool YaraTask::ParseLine(rapidjson::Value& line) { else { encoding = fields[1].GetString(); } + __attribute((fallthrough)); // No break here! case 1: if(not fields[0].IsString()){ From b3478edf7fa101fe58d22699e80f2dcd3dc5b718 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 29 Jul 2021 16:08:44 +0200 Subject: [PATCH 36/54] Fixed compilation issues on hardenedBSD Fixed issue with positional flags between test/manager and filters --- manager/Services.py | 14 ++++++++------ samples/base/Core.cpp | 4 +++- samples/base/DarwinPacket.cpp | 1 + samples/base/network/ASession.fwd.hpp | 2 +- tests/tools/filter.py | 4 +++- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manager/Services.py b/manager/Services.py index 2bed431..54f22e7 100644 --- a/manager/Services.py +++ b/manager/Services.py @@ -90,6 +90,14 @@ def _build_cmd(filt): cmd = [filt['exec_path']] + # Flags MUST be before positional arguments as the parsing on HardenedBSD is not done on all the arguments + # On BSD getopt stops at the first argument which is not in the specified flags + if filt['network']['socket_type'] == 'UDP': + cmd.append('-u') + + if filt['next_filter_network']['socket_type'] == 'UDP': + cmd.append('-v') + try: if filt['log_level'] not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", "DEVELOPER"]: logger.warning( @@ -101,12 +109,6 @@ def _build_cmd(filt): except KeyError: pass - if filt['network']['socket_type'] == 'UDP': - cmd.append('-u') - - if filt['next_filter_network']['socket_type'] == 'UDP': - cmd.append('-v') - cmd += [ filt['name'], filt['network']['address_path'], diff --git a/samples/base/Core.cpp b/samples/base/Core.cpp index 5bdf5c7..69ebc39 100644 --- a/samples/base/Core.cpp +++ b/samples/base/Core.cpp @@ -142,9 +142,11 @@ namespace darwin { // OPTIONS log.setLevel(logger::Warning); // Log level by default opt = -1; + // Flags MUST be before positional arguments as the parsing on HardenedBSD is not done on all the arguments + // On BSD getopt stops at the first argument which is not in the specified flags while((opt = getopt(ac, av, ":l:huv")) != -1) { - DARWIN_LOG_DEBUG("OPT : " + std::to_string(opt)); + DARWIN_LOG_DEBUG(std::string("OPT : ") + (char)opt); DARWIN_LOG_DEBUG("OPTIND : " + std::to_string(optind)); switch(opt) { diff --git a/samples/base/DarwinPacket.cpp b/samples/base/DarwinPacket.cpp index 58ce07e..ebc3846 100644 --- a/samples/base/DarwinPacket.cpp +++ b/samples/base/DarwinPacket.cpp @@ -1,6 +1,7 @@ #include "DarwinPacket.hpp" #include #include +#include #include #include "../../toolkit/rapidjson/writer.h" #include "../../toolkit/rapidjson/stringbuffer.h" diff --git a/samples/base/network/ASession.fwd.hpp b/samples/base/network/ASession.fwd.hpp index cbcfb17..b9a4417 100644 --- a/samples/base/network/ASession.fwd.hpp +++ b/samples/base/network/ASession.fwd.hpp @@ -1,5 +1,5 @@ #pragma once -#include +#include #include // Forward Declaration mandatory because of the circular dependency between Tasks, Sessions and Generators diff --git a/tests/tools/filter.py b/tests/tools/filter.py index 9a5f2e6..f8a086b 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -28,7 +28,9 @@ def __init__(self, path=None, config_file=None, filter_name="filter", socket_pat self.pid = pid_file if pid_file else "{}/{}.pid".format(conf.TEST_FILES_DIR, filter_name) self.cmd = [self.path, "-l", log_level, self.filter_name, self.socket, self.config, self.monitor, self.pid, output, next_filter_socket_path, str(nb_threads), str(cache_size), str(threshold)] if self.socket_type.startswith('udp'): - self.cmd += ['-u'] + # Flags MUST be before positional arguments as the parsing on HardenedBSD is not done on all the arguments + # On BSD getopt stops at the first argument which is not in the specified flags + self.cmd.insert(1, '-u') self.error_code = 99 # For valgrind testing self.pubsub = None self.prepare_log_file() From 15ef861ae87636a8b81abb13d061b910366326b8 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 30 Jul 2021 11:17:47 +0200 Subject: [PATCH 37/54] Fixed last issue with UDP NextFilterConnector --- .../base/network/UdpNextFilterConnector.cpp | 3 +-- tests/manager_socket/multi_filters_test.py | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/samples/base/network/UdpNextFilterConnector.cpp b/samples/base/network/UdpNextFilterConnector.cpp index 9a846bd..d3c937f 100644 --- a/samples/base/network/UdpNextFilterConnector.cpp +++ b/samples/base/network/UdpNextFilterConnector.cpp @@ -35,8 +35,7 @@ namespace darwin { DARWIN_LOG_DEBUG("UdpNextFilterConnector::Send : Sending packet to UDP endpoint, no retries"); - _socket.async_send_to(*packet, - _endpoint, + _socket.async_send(*packet, boost::bind(&UdpNextFilterConnector::SendCallback, this, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred, diff --git a/tests/manager_socket/multi_filters_test.py b/tests/manager_socket/multi_filters_test.py index 67f3bad..e15ac40 100644 --- a/tests/manager_socket/multi_filters_test.py +++ b/tests/manager_socket/multi_filters_test.py @@ -5,7 +5,7 @@ from tools.darwin_utils import darwin_configure, darwin_stop, darwin_start, darwin_remove_configuration from tools.filter import Filter import conf - +import psutil def run(): scenarios = [ @@ -96,10 +96,24 @@ def alerting_tests(filters_list): # socket path is given according to how it is currently generated in the manager, # this may have to change in the future - path_addr = '{}/sockets/test_1.1.sock'.format(conf.TEST_FILES_DIR) if filters_list[0] == 'unix' else '[::]:8181' + path_addr = '{}/sockets/test_1.1.sock'.format(conf.TEST_FILES_DIR) if filters_list[0] == 'unix' else '[::1]:8181' f = Filter(socket_type=filters_list[0], socket_path=path_addr) api = f.get_darwin_api() + start = time.time() + nb_test_filters=0 + # We check that all 3 filters are running, on VM HardenedBSD, it may take some time + while nb_test_filters < 3 and time.time() - start < 6: + time.sleep(1) + nb_test_filters=0 + for proc in psutil.process_iter(): + try: + # Check if process name contains the given name string. + if 'darwin_test' in proc.name().lower(): + nb_test_filters += 1 + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass + api.call("Hello", response_type="darwin") time.sleep(0.2) api.call("There", response_type="darwin") @@ -123,5 +137,6 @@ def alerting_tests(filters_list): if line1 != 2 or line2 != 2 or line3 != 2: print('test failed: ', filters_list) + print("test1 : {}, test2 : {}, test3 : {}".format(line1, line2, line3)) return False return True From 016d80fbeaddbccdb8f0b7e04eaa386b81a5f9b7 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 30 Jul 2021 14:18:16 +0200 Subject: [PATCH 38/54] Added Thread Pool Cpp dependency --- toolkit/thread-pool-cpp/LICENSE | 21 ++ toolkit/thread-pool-cpp/README.md | 32 +++ .../thread-pool-cpp/include/thread_pool.hpp | 3 + .../include/thread_pool/fixed_function.hpp | 163 ++++++++++++ .../thread_pool/mpmc_bounded_queue.hpp | 242 ++++++++++++++++++ .../include/thread_pool/thread_pool.hpp | 161 ++++++++++++ .../thread_pool/thread_pool_options.hpp | 76 ++++++ .../include/thread_pool/worker.hpp | 179 +++++++++++++ 8 files changed, 877 insertions(+) create mode 100644 toolkit/thread-pool-cpp/LICENSE create mode 100644 toolkit/thread-pool-cpp/README.md create mode 100644 toolkit/thread-pool-cpp/include/thread_pool.hpp create mode 100644 toolkit/thread-pool-cpp/include/thread_pool/fixed_function.hpp create mode 100644 toolkit/thread-pool-cpp/include/thread_pool/mpmc_bounded_queue.hpp create mode 100644 toolkit/thread-pool-cpp/include/thread_pool/thread_pool.hpp create mode 100644 toolkit/thread-pool-cpp/include/thread_pool/thread_pool_options.hpp create mode 100644 toolkit/thread-pool-cpp/include/thread_pool/worker.hpp diff --git a/toolkit/thread-pool-cpp/LICENSE b/toolkit/thread-pool-cpp/LICENSE new file mode 100644 index 0000000..e102761 --- /dev/null +++ b/toolkit/thread-pool-cpp/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Andrey Kubarkov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/toolkit/thread-pool-cpp/README.md b/toolkit/thread-pool-cpp/README.md new file mode 100644 index 0000000..afce450 --- /dev/null +++ b/toolkit/thread-pool-cpp/README.md @@ -0,0 +1,32 @@ +thread-pool-cpp +================= +[![Build Status](https://travis-ci.org/inkooboo/thread-pool-cpp.svg?branch=master)](https://travis-ci.org/inkooboo/thread-pool-cpp) +[![Codecov branch](https://img.shields.io/codecov/c/github/inkooboo/thread-pool-cpp/master.svg)](https://codecov.io/gh/inkooboo/thread-pool-cpp) +[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) + + * It is highly scalable and fast. + * It is header only. + * No external dependencies, only standard library needed. + * It implements both work-stealing and work-distribution balancing startegies. + * It implements cooperative scheduling strategy. + +Example run: +Post job to thread pool is much faster than for boost::asio based thread pool. + + Benchmark job reposting + ***thread pool cpp*** + reposted 1000001 in 61.6754 ms + reposted 1000001 in 62.0187 ms + reposted 1000001 in 62.8785 ms + reposted 1000001 in 70.2714 ms + ***asio thread pool*** + reposted 1000001 in 1381.58 ms + reposted 1000001 in 1390.35 ms + reposted 1000001 in 1391.84 ms + reposted 1000001 in 1393.19 ms + +See benchmark/benchmark.cpp for benchmark code. + +All code except [MPMCBoundedQueue](https://github.com/inkooboo/thread-pool-cpp/blob/master/include/thread_pool/mpmc_bounded_queue.hpp) +is under MIT license. + diff --git a/toolkit/thread-pool-cpp/include/thread_pool.hpp b/toolkit/thread-pool-cpp/include/thread_pool.hpp new file mode 100644 index 0000000..d8b0bc4 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include diff --git a/toolkit/thread-pool-cpp/include/thread_pool/fixed_function.hpp b/toolkit/thread-pool-cpp/include/thread_pool/fixed_function.hpp new file mode 100644 index 0000000..5237052 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/fixed_function.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include + +namespace tp +{ + +/** + * @brief The FixedFunction class implements + * functional object. + * This function is analog of 'std::function' with limited capabilities: + * - It supports only move semantics. + * - The size of functional objects is limited to storage size. + * Due to limitations above it is much faster on creation and copying than + * std::function. + */ +template +class FixedFunction; + +template +class FixedFunction +{ + + typedef R (*func_ptr_type)(ARGS...); + +public: + FixedFunction() + : m_function_ptr(nullptr), m_method_ptr(nullptr), + m_alloc_ptr(nullptr) + { + } + + /** + * @brief FixedFunction Constructor from functional object. + * @param object Functor object will be stored in the internal storage + * using move constructor. Unmovable objects are prohibited explicitly. + */ + template + FixedFunction(FUNC&& object) + : FixedFunction() + { + typedef typename std::remove_reference::type unref_type; + + static_assert(sizeof(unref_type) < STORAGE_SIZE, + "functional object doesn't fit into internal storage"); + static_assert(std::is_move_constructible::value, + "Should be of movable type"); + + m_method_ptr = []( + void* object_ptr, func_ptr_type, ARGS... args) -> R + { + return static_cast(object_ptr) + -> + operator()(args...); + }; + + m_alloc_ptr = [](void* storage_ptr, void* object_ptr) + { + if(object_ptr) + { + unref_type* x_object = static_cast(object_ptr); + new(storage_ptr) unref_type(std::move(*x_object)); + } + else + { + static_cast(storage_ptr)->~unref_type(); + } + }; + + m_alloc_ptr(&m_storage, &object); + } + + /** + * @brief FixedFunction Constructor from free function or static member. + */ + template + FixedFunction(RET (*func_ptr)(PARAMS...)) + : FixedFunction() + { + m_function_ptr = func_ptr; + m_method_ptr = [](void*, func_ptr_type f_ptr, ARGS... args) -> R + { + return static_cast(f_ptr)(args...); + }; + } + + FixedFunction(FixedFunction&& o) : FixedFunction() + { + moveFromOther(o); + } + + FixedFunction& operator=(FixedFunction&& o) + { + moveFromOther(o); + return *this; + } + + ~FixedFunction() + { + if(m_alloc_ptr) m_alloc_ptr(&m_storage, nullptr); + } + + /** + * @brief operator () Execute stored functional object. + * @throws std::runtime_error if no functional object is stored. + */ + R operator()(ARGS... args) + { + if(!m_method_ptr) throw std::runtime_error("call of empty functor"); + return m_method_ptr(&m_storage, m_function_ptr, args...); + } + +private: + FixedFunction& operator=(const FixedFunction&) = delete; + FixedFunction(const FixedFunction&) = delete; + + union + { + typename std::aligned_storage::type + m_storage; + func_ptr_type m_function_ptr; + }; + + typedef R (*method_type)( + void* object_ptr, func_ptr_type free_func_ptr, ARGS... args); + method_type m_method_ptr; + + typedef void (*alloc_type)(void* storage_ptr, void* object_ptr); + alloc_type m_alloc_ptr; + + void moveFromOther(FixedFunction& o) + { + if(this == &o) return; + + if(m_alloc_ptr) + { + m_alloc_ptr(&m_storage, nullptr); + m_alloc_ptr = nullptr; + } + else + { + m_function_ptr = nullptr; + } + + m_method_ptr = o.m_method_ptr; + o.m_method_ptr = nullptr; + + if(o.m_alloc_ptr) + { + m_alloc_ptr = o.m_alloc_ptr; + m_alloc_ptr(&m_storage, &o.m_storage); + } + else + { + m_function_ptr = o.m_function_ptr; + } + } +}; + +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/mpmc_bounded_queue.hpp b/toolkit/thread-pool-cpp/include/thread_pool/mpmc_bounded_queue.hpp new file mode 100644 index 0000000..b06ee08 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/mpmc_bounded_queue.hpp @@ -0,0 +1,242 @@ +// Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided +// that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list +// of conditions and the following disclaimer in the documentation and/or +// other materials +// provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT +// SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +// OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF +// THE POSSIBILITY OF SUCH DAMAGE. +// +// The views and conclusions contained in the software and documentation are +// those of the authors and +// should not be interpreted as representing official policies, either expressed +// or implied, of Dmitry Vyukov. + +#pragma once + +#include +#include +#include +#include + +namespace tp +{ + +/** + * @brief The MPMCBoundedQueue class implements bounded + * multi-producers/multi-consumers lock-free queue. + * Doesn't accept non-movable types as T. + * Inspired by Dmitry Vyukov's mpmc queue. + * http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue + */ +template +class MPMCBoundedQueue +{ + static_assert( + std::is_move_constructible::value, "Should be of movable type"); + +public: + /** + * @brief MPMCBoundedQueue Constructor. + * @param size Power of 2 number - queue length. + * @throws std::invalid_argument if size is bad. + */ + explicit MPMCBoundedQueue(size_t size); + + /** + * @brief Move ctor implementation. + */ + MPMCBoundedQueue(MPMCBoundedQueue&& rhs) noexcept; + + /** + * @brief Move assignment implementaion. + */ + MPMCBoundedQueue& operator=(MPMCBoundedQueue&& rhs) noexcept; + + /** + * @brief push Push data to queue. + * @param data Data to be pushed. + * @return true on success. + */ + template + bool push(U&& data); + + /** + * @brief pop Pop data from queue. + * @param data Place to store popped data. + * @return true on sucess. + */ + bool pop(T& data); + +private: + struct Cell + { + std::atomic sequence; + T data; + + Cell() = default; + + Cell(const Cell&) = delete; + Cell& operator=(const Cell&) = delete; + + Cell(Cell&& rhs) + : sequence(rhs.sequence.load()), data(std::move(rhs.data)) + { + } + + Cell& operator=(Cell&& rhs) + { + sequence = rhs.sequence.load(); + data = std::move(rhs.data); + + return *this; + } + }; + +private: + typedef char Cacheline[64]; + + Cacheline pad0; + std::vector m_buffer; + /* const */ size_t m_buffer_mask; + Cacheline pad1; + std::atomic m_enqueue_pos; + Cacheline pad2; + std::atomic m_dequeue_pos; + Cacheline pad3; +}; + + +/// Implementation + +template +inline MPMCBoundedQueue::MPMCBoundedQueue(size_t size) + : m_buffer(size), m_buffer_mask(size - 1), m_enqueue_pos(0), + m_dequeue_pos(0) +{ + bool size_is_power_of_2 = (size >= 2) && ((size & (size - 1)) == 0); + if(!size_is_power_of_2) + { + throw std::invalid_argument("buffer size should be a power of 2"); + } + + for(size_t i = 0; i < size; ++i) + { + m_buffer[i].sequence = i; + } +} + +template +inline MPMCBoundedQueue::MPMCBoundedQueue(MPMCBoundedQueue&& rhs) noexcept +{ + *this = rhs; +} + +template +inline MPMCBoundedQueue& MPMCBoundedQueue::operator=(MPMCBoundedQueue&& rhs) noexcept +{ + if (this != &rhs) + { + m_buffer = std::move(rhs.m_buffer); + m_buffer_mask = std::move(rhs.m_buffer_mask); + m_enqueue_pos = rhs.m_enqueue_pos.load(); + m_dequeue_pos = rhs.m_dequeue_pos.load(); + } + return *this; +} + +template +template +inline bool MPMCBoundedQueue::push(U&& data) +{ + Cell* cell; + size_t pos = m_enqueue_pos.load(std::memory_order_relaxed); + for(;;) + { + cell = &m_buffer[pos & m_buffer_mask]; + size_t seq = cell->sequence.load(std::memory_order_acquire); + intptr_t dif = (intptr_t)seq - (intptr_t)pos; + if(dif == 0) + { + if(m_enqueue_pos.compare_exchange_weak( + pos, pos + 1, std::memory_order_relaxed)) + { + break; + } + } + else if(dif < 0) + { + return false; + } + else + { + pos = m_enqueue_pos.load(std::memory_order_relaxed); + } + } + + cell->data = std::forward(data); + + cell->sequence.store(pos + 1, std::memory_order_release); + + return true; +} + +template +inline bool MPMCBoundedQueue::pop(T& data) +{ + Cell* cell; + size_t pos = m_dequeue_pos.load(std::memory_order_relaxed); + for(;;) + { + cell = &m_buffer[pos & m_buffer_mask]; + size_t seq = cell->sequence.load(std::memory_order_acquire); + intptr_t dif = (intptr_t)seq - (intptr_t)(pos + 1); + if(dif == 0) + { + if(m_dequeue_pos.compare_exchange_weak( + pos, pos + 1, std::memory_order_relaxed)) + { + break; + } + } + else if(dif < 0) + { + return false; + } + else + { + pos = m_dequeue_pos.load(std::memory_order_relaxed); + } + } + + data = std::move(cell->data); + + cell->sequence.store( + pos + m_buffer_mask + 1, std::memory_order_release); + + return true; +} + +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/thread_pool.hpp b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool.hpp new file mode 100644 index 0000000..4c79fdb --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool.hpp @@ -0,0 +1,161 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace tp +{ + +template class Queue> +class ThreadPoolImpl; +using ThreadPool = ThreadPoolImpl, + MPMCBoundedQueue>; + +/** + * @brief The ThreadPool class implements thread pool pattern. + * It is highly scalable and fast. + * It is header only. + * It implements both work-stealing and work-distribution balancing + * startegies. + * It implements cooperative scheduling strategy for tasks. + */ +template class Queue> +class ThreadPoolImpl { +public: + /** + * @brief ThreadPool Construct and start new thread pool. + * @param options Creation options. + */ + explicit ThreadPoolImpl( + const ThreadPoolOptions& options = ThreadPoolOptions()); + + /** + * @brief Move ctor implementation. + */ + ThreadPoolImpl(ThreadPoolImpl&& rhs) noexcept; + + /** + * @brief ~ThreadPool Stop all workers and destroy thread pool. + */ + ~ThreadPoolImpl(); + + /** + * @brief Move assignment implementaion. + */ + ThreadPoolImpl& operator=(ThreadPoolImpl&& rhs) noexcept; + + /** + * @brief post Try post job to thread pool. + * @param handler Handler to be called from thread pool worker. It has + * to be callable as 'handler()'. + * @return 'true' on success, false otherwise. + * @note All exceptions thrown by handler will be suppressed. + */ + template + bool tryPost(Handler&& handler); + + /** + * @brief post Post job to thread pool. + * @param handler Handler to be called from thread pool worker. It has + * to be callable as 'handler()'. + * @throw std::overflow_error if worker's queue is full. + * @note All exceptions thrown by handler will be suppressed. + */ + template + void post(Handler&& handler); + +private: + Worker& getWorker(); + + std::vector>> m_workers; + std::atomic m_next_worker; +}; + + +/// Implementation + +template class Queue> +inline ThreadPoolImpl::ThreadPoolImpl( + const ThreadPoolOptions& options) + : m_workers(options.threadCount()) + , m_next_worker(0) +{ + for(auto& worker_ptr : m_workers) + { + worker_ptr.reset(new Worker(options.queueSize())); + } + + for(size_t i = 0; i < m_workers.size(); ++i) + { + Worker* steal_donor = + m_workers[(i + 1) % m_workers.size()].get(); + m_workers[i]->start(i, steal_donor); + } +} + +template class Queue> +inline ThreadPoolImpl::ThreadPoolImpl(ThreadPoolImpl&& rhs) noexcept +{ + *this = rhs; +} + +template class Queue> +inline ThreadPoolImpl::~ThreadPoolImpl() +{ + for (auto& worker_ptr : m_workers) + { + worker_ptr->stop(); + } +} + +template class Queue> +inline ThreadPoolImpl& +ThreadPoolImpl::operator=(ThreadPoolImpl&& rhs) noexcept +{ + if (this != &rhs) + { + m_workers = std::move(rhs.m_workers); + m_next_worker = rhs.m_next_worker.load(); + } + return *this; +} + +template class Queue> +template +inline bool ThreadPoolImpl::tryPost(Handler&& handler) +{ + return getWorker().post(std::forward(handler)); +} + +template class Queue> +template +inline void ThreadPoolImpl::post(Handler&& handler) +{ + const auto ok = tryPost(std::forward(handler)); + if (!ok) + { + throw std::runtime_error("thread pool queue is full"); + } +} + +template class Queue> +inline Worker& ThreadPoolImpl::getWorker() +{ + auto id = Worker::getWorkerIdForCurrentThread(); + + if (id > m_workers.size()) + { + id = m_next_worker.fetch_add(1, std::memory_order_relaxed) % + m_workers.size(); + } + + return *m_workers[id]; +} +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/thread_pool_options.hpp b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool_options.hpp new file mode 100644 index 0000000..c1cde52 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/thread_pool_options.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include +#include + +namespace tp +{ + +/** + * @brief The ThreadPoolOptions class provides creation options for + * ThreadPool. + */ +class ThreadPoolOptions +{ +public: + /** + * @brief ThreadPoolOptions Construct default options for thread pool. + */ + ThreadPoolOptions(); + + /** + * @brief setThreadCount Set thread count. + * @param count Number of threads to be created. + */ + void setThreadCount(size_t count); + + /** + * @brief setQueueSize Set single worker queue size. + * @param count Maximum length of queue of single worker. + */ + void setQueueSize(size_t size); + + /** + * @brief threadCount Return thread count. + */ + size_t threadCount() const; + + /** + * @brief queueSize Return single worker queue size. + */ + size_t queueSize() const; + +private: + size_t m_thread_count; + size_t m_queue_size; +}; + +/// Implementation + +inline ThreadPoolOptions::ThreadPoolOptions() + : m_thread_count(std::max(1u, std::thread::hardware_concurrency())) + , m_queue_size(1024u) +{ +} + +inline void ThreadPoolOptions::setThreadCount(size_t count) +{ + m_thread_count = std::max(1u, count); +} + +inline void ThreadPoolOptions::setQueueSize(size_t size) +{ + m_queue_size = std::max(1u, size); +} + +inline size_t ThreadPoolOptions::threadCount() const +{ + return m_thread_count; +} + +inline size_t ThreadPoolOptions::queueSize() const +{ + return m_queue_size; +} + +} diff --git a/toolkit/thread-pool-cpp/include/thread_pool/worker.hpp b/toolkit/thread-pool-cpp/include/thread_pool/worker.hpp new file mode 100644 index 0000000..91e67a3 --- /dev/null +++ b/toolkit/thread-pool-cpp/include/thread_pool/worker.hpp @@ -0,0 +1,179 @@ +#pragma once + +#include +#include + +namespace tp +{ + +/** + * @brief The Worker class owns task queue and executing thread. + * In thread it tries to pop task from queue. If queue is empty then it tries + * to steal task from the sibling worker. If steal was unsuccessful then spins + * with one millisecond delay. + */ +template class Queue> +class Worker +{ +public: + /** + * @brief Worker Constructor. + * @param queue_size Length of undelaying task queue. + */ + explicit Worker(size_t queue_size); + + /** + * @brief Move ctor implementation. + */ + Worker(Worker&& rhs) noexcept; + + /** + * @brief Move assignment implementaion. + */ + Worker& operator=(Worker&& rhs) noexcept; + + /** + * @brief start Create the executing thread and start tasks execution. + * @param id Worker ID. + * @param steal_donor Sibling worker to steal task from it. + */ + void start(size_t id, Worker* steal_donor); + + /** + * @brief stop Stop all worker's thread and stealing activity. + * Waits until the executing thread became finished. + */ + void stop(); + + /** + * @brief post Post task to queue. + * @param handler Handler to be executed in executing thread. + * @return true on success. + */ + template + bool post(Handler&& handler); + + /** + * @brief steal Steal one task from this worker queue. + * @param task Place for stealed task to be stored. + * @return true on success. + */ + bool steal(Task& task); + + /** + * @brief getWorkerIdForCurrentThread Return worker ID associated with + * current thread if exists. + * @return Worker ID. + */ + static size_t getWorkerIdForCurrentThread(); + +private: + /** + * @brief threadFunc Executing thread function. + * @param id Worker ID to be associated with this thread. + * @param steal_donor Sibling worker to steal task from it. + */ + void threadFunc(size_t id, Worker* steal_donor); + + Queue m_queue; + std::atomic m_running_flag; + std::thread m_thread; +}; + + +/// Implementation + +namespace detail +{ + inline size_t* thread_id() + { + static thread_local size_t tss_id = -1u; + return &tss_id; + } +} + +template class Queue> +inline Worker::Worker(size_t queue_size) + : m_queue(queue_size) + , m_running_flag(true) +{ +} + +template class Queue> +inline Worker::Worker(Worker&& rhs) noexcept +{ + *this = rhs; +} + +template class Queue> +inline Worker& Worker::operator=(Worker&& rhs) noexcept +{ + if (this != &rhs) + { + m_queue = std::move(rhs.m_queue); + m_running_flag = rhs.m_running_flag.load(); + m_thread = std::move(rhs.m_thread); + } + return *this; +} + +template class Queue> +inline void Worker::stop() +{ + m_running_flag.store(false, std::memory_order_relaxed); + m_thread.join(); +} + +template class Queue> +inline void Worker::start(size_t id, Worker* steal_donor) +{ + m_thread = std::thread(&Worker::threadFunc, this, id, steal_donor); +} + +template class Queue> +inline size_t Worker::getWorkerIdForCurrentThread() +{ + return *detail::thread_id(); +} + +template class Queue> +template +inline bool Worker::post(Handler&& handler) +{ + return m_queue.push(std::forward(handler)); +} + +template class Queue> +inline bool Worker::steal(Task& task) +{ + return m_queue.pop(task); +} + +template class Queue> +inline void Worker::threadFunc(size_t id, Worker* steal_donor) +{ + *detail::thread_id() = id; + + Task handler; + + while (m_running_flag.load(std::memory_order_relaxed)) + { + if (m_queue.pop(handler) || steal_donor->steal(handler)) + { + try + { + handler(); + } + catch(...) + { + // suppress all exceptions + } + } + else + { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } +} + +} From 901e5e56045979a6b9c482a12a3e5b008f3c4f78 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 30 Aug 2021 17:48:27 +0200 Subject: [PATCH 39/54] First working version --- CMakeLists.txt | 5 + cmake/fpython.cmake | 33 ++++ samples/fpython/Generator.cpp | 302 +++++++++++++++++++++++++++++++++ samples/fpython/Generator.hpp | 93 ++++++++++ samples/fpython/PythonTask.cpp | 167 ++++++++++++++++++ samples/fpython/PythonTask.hpp | 58 +++++++ samples/fpython/example.py | 44 +++++ samples/fpython/fpython.hpp | 14 ++ 8 files changed, 716 insertions(+) create mode 100644 cmake/fpython.cmake create mode 100644 samples/fpython/Generator.cpp create mode 100644 samples/fpython/Generator.hpp create mode 100644 samples/fpython/PythonTask.cpp create mode 100644 samples/fpython/PythonTask.hpp create mode 100644 samples/fpython/example.py create mode 100644 samples/fpython/fpython.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 114bee3..d030ea2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ if (NOT DEFINED FILTER) CONTENT_INSPECTION BUFFER YARA + PYTHON ) else (NOT DEFINED FILTER) set( @@ -168,3 +169,7 @@ endif() if("VAML" IN_LIST FILTERS) include(fvaml) endif() + +if("PYTHON" IN_LIST FILTERS) +include(fpython) +endif() diff --git a/cmake/fpython.cmake b/cmake/fpython.cmake new file mode 100644 index 0000000..444cdef --- /dev/null +++ b/cmake/fpython.cmake @@ -0,0 +1,33 @@ +set(PYTHON_NAME darwin_python) + +####################### +# FILTER DEPENDENCIES # +####################### + +include_directories(SYSTEM /usr/include/python3.8) +link_directories(/usr/lib/python3.8/config-3.8-x86) + +################### +# EXECUTABLE # +################### + + +add_executable( + ${PYTHON_NAME} + ${DARWIN_SOURCES} + samples/fpython/PythonTask.cpp samples/fpython/PythonTask.hpp + samples/fpython/Generator.cpp samples/fpython/Generator.hpp +) + +target_link_libraries( + ${PYTHON_NAME} + ${DARWIN_LIBRARIES} + python3.8 + crypt + pthread + dl + util + m +) + +target_include_directories(${PYTHON_NAME} PUBLIC samples/fpython/) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp new file mode 100644 index 0000000..81c265f --- /dev/null +++ b/samples/fpython/Generator.cpp @@ -0,0 +1,302 @@ +/// \file Generator.cpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#include +#include + +#include "../../toolkit/lru_cache.hpp" +#include "base/Logger.hpp" +#include "Generator.hpp" +#include "PythonTask.hpp" +#include "AlertManager.hpp" +#include "fpython.hpp" + +Generator::Generator(): pName{nullptr}, pModule{nullptr} + { } + +bool Generator::ConfigureAlerting(const std::string& tags) { + DARWIN_LOGGER; + + DARWIN_LOG_DEBUG("Python:: ConfigureAlerting:: Configuring Alerting"); + DARWIN_ALERT_MANAGER_SET_FILTER_NAME(DARWIN_FILTER_NAME); + DARWIN_ALERT_MANAGER_SET_RULE_NAME(DARWIN_ALERT_RULE_NAME); + if (tags.empty()) { + DARWIN_LOG_DEBUG("Python:: ConfigureAlerting:: No alert tags provided in the configuration. Using default."); + DARWIN_ALERT_MANAGER_SET_TAGS(DARWIN_ALERT_TAGS); + } else { + DARWIN_ALERT_MANAGER_SET_TAGS(tags); + } + return true; +} + +bool Generator::LoadConfig(const rapidjson::Document &configuration) { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Python:: Generator:: Loading configuration..."); + + std::string python_script_path, shared_library_path; + + if (!configuration.HasMember("python_script_path")) { + DARWIN_LOG_CRITICAL("Python:: Generator:: Missing parameter: 'python_script_path'"); + return false; + } + + if (!configuration["python_script_path"].IsString()) { + DARWIN_LOG_CRITICAL("Python:: Generator:: 'python_script_path' needs to be a string"); + return false; + } + + python_script_path = configuration["python_script_path"].GetString(); + + if (!configuration.HasMember("shared_library_path")) { + DARWIN_LOG_CRITICAL("Python:: Generator:: Missing parameter: 'shared_library_path'"); + return false; + } + + if (!configuration["shared_library_path"].IsString()) { + DARWIN_LOG_CRITICAL("Python:: Generator:: 'shared_library_path' needs to be a string"); + return false; + } + + shared_library_path = configuration["shared_library_path"].GetString(); + DARWIN_LOG_DEBUG("Python:: Generator:: Loading configuration..."); + + return LoadPythonScript(python_script_path) && LoadSharedLibrary(shared_library_path) && CheckConfig(); +} + +bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { + DARWIN_LOGGER; + + if(shared_library_path.empty()){ + DARWIN_LOG_INFO("Generator::LoadSharedLibrary : No shared library to load"); + return true; + } + + void* handle = dlopen(shared_library_path.c_str(), RTLD_LAZY); + if(handle == nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadSharedLibrary : Error loading the shared library : failed to open " + shared_library_path); + return false; + } + + FunctionHolder::parse_body_t parse = (FunctionHolder::parse_body_t)dlsym(handle, "parse_body"); + + if(parse == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' function in the shared library"); + } else { + functions.parseBodyOrigin = FunctionOrigin::SHARED_LIBRARY; + functions.parseBodyFunc.so = parse; + } + + FunctionHolder::process_t preProc = (FunctionHolder::process_t)dlsym(handle, "filter_pre_process"); + if(preProc == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_pre_process' function in the shared library"); + } else { + functions.preProcessingOrigin = FunctionOrigin::SHARED_LIBRARY; + functions.preProcessingFunc.so = preProc; + } + + FunctionHolder::process_t proc = (FunctionHolder::process_t)dlsym(handle, "filter_process"); + if(proc == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_process' function in the shared library"); + } else { + functions.processingOrigin = FunctionOrigin::SHARED_LIBRARY; + functions.processingFunc.so = proc; + } + + FunctionHolder::format_t alert_format = (FunctionHolder::format_t)dlsym(handle, "alert_formating"); + if(alert_format == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' function in the shared library"); + } else { + functions.alertFormatingOrigin = FunctionOrigin::SHARED_LIBRARY; + functions.alertFormatingFunc.so = alert_format; + } + + FunctionHolder::format_t output_format = (FunctionHolder::format_t)dlsym(handle, "output_formating"); + if(parse == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' function in the shared library"); + } else { + functions.outputFormatingOrigin = FunctionOrigin::SHARED_LIBRARY; + functions.outputFormatingFunc.so = output_format; + } + + FunctionHolder::format_t resp_format = (FunctionHolder::format_t)dlsym(handle, "response_formating"); + if(resp_format == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' function in the shared library"); + } else { + functions.responseFormatingOrigin = FunctionOrigin::SHARED_LIBRARY; + functions.responseFormatingFunc.so = resp_format; + } + + return true; +} + +bool Generator::CheckConfig() { + if(functions.preProcessingOrigin == FunctionOrigin::NONE + || functions.processingOrigin == FunctionOrigin::NONE + || functions.alertFormatingOrigin == FunctionOrigin::NONE + || functions.outputFormatingOrigin == FunctionOrigin::NONE + || functions.responseFormatingOrigin == FunctionOrigin::NONE) + { + DARWIN_LOGGER; + DARWIN_LOG_CRITICAL("Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library"); + return false; + } + return true; +} + +bool Generator::LoadPythonScript(const std::string& python_script_path) { + DARWIN_LOGGER; + + if(python_script_path.empty()){ + DARWIN_LOG_INFO("Generator::LoadPythonScript : No python script to load"); + return true; + } + + std::ifstream f(python_script_path.c_str()); + if(f.bad()) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : failed to open " + python_script_path); + return false; + } + f.close(); + + std::size_t pos = python_script_path.rfind("/"); + auto folders = python_script_path.substr(0, pos); + auto filename = python_script_path.substr(pos+1, std::string::npos); + if((pos = filename.rfind(".py")) != std::string::npos) { + filename.erase(pos, std::string::npos); + } + + Py_Initialize(); + if(PyErr_Occurred() != nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post py init"); + PyErr_Print(); + } + pName = PyUnicode_DecodeFSDefault(filename.c_str()); + // PyObject* pFolders = PyUnicode_DecodeFSDefault(folders.c_str()); + // PyObject* pFilename = PyUnicode_DecodeFSDefault(filename.c_str()); + if(PyErr_Occurred() != nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post filename decode"); + PyErr_Print(); + } + if (PyRun_SimpleString("import sys") != 0) { + DARWIN_LOG_DEBUG("darwin:: pythonutils:: InitPythonProgram:: An error occurred while loading the 'sys' module"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + + return false; + } + if(PyErr_Occurred() != nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post imp sys"); + PyErr_Print(); + } + + std::string command = "sys.path.append(\"" + folders + "\")"; + if (PyRun_SimpleString(command.c_str()) != 0) { + DARWIN_LOG_DEBUG( + "darwin:: pythonutils:: InitPythonProgram:: An error occurred while appending the custom path '" + + folders + + "' to the Python path" + ); + + if(PyErr_Occurred() != nullptr) PyErr_Print(); + + return false; + } + + if(PyErr_Occurred() != nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post sys path"); + PyErr_Print(); + } + + pModule = PyImport_Import(pName); + + if(PyErr_Occurred() != nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : post import"); + + PyErr_Print(); + } + + if(pModule == nullptr) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error while attempting to load the python script module"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + + return false; + } + + functions.parseBodyFunc.py = PyObject_GetAttrString(pModule, "parse_body"); + if(functions.parseBodyFunc.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' method in the python script"); + } else if (! PyCallable_Check(functions.parseBodyFunc.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : parse_body symbol exists but is not callable"); + return false; + } else { + functions.parseBodyOrigin = FunctionOrigin::PYTHON_MODULE; + } + + functions.preProcessingFunc.py = PyObject_GetAttrString(pModule, "filter_pre_process"); + if(functions.preProcessingFunc.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_pre_process' method in the python script"); + } else if (! PyCallable_Check(functions.preProcessingFunc.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_pre_process symbol exists but is not callable"); + return false; + } else { + functions.preProcessingOrigin = FunctionOrigin::PYTHON_MODULE; + } + + functions.processingFunc.py = PyObject_GetAttrString(pModule, "filter_process"); + if(functions.processingFunc.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_process' method in the python script"); + } else if (! PyCallable_Check(functions.processingFunc.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_process symbol exists but is not callable"); + return false; + } else { + functions.processingOrigin = FunctionOrigin::PYTHON_MODULE; + } + + functions.alertFormatingFunc.py = PyObject_GetAttrString(pModule, "alert_formating"); + if(functions.alertFormatingFunc.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' method in the python script"); + } else if (! PyCallable_Check(functions.alertFormatingFunc.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : alert_formating symbol exists but is not callable"); + return false; + } else { + functions.alertFormatingOrigin = FunctionOrigin::PYTHON_MODULE; + } + + functions.outputFormatingFunc.py = PyObject_GetAttrString(pModule, "output_formating"); + if(functions.outputFormatingFunc.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' method in the python script"); + } else if (! PyCallable_Check(functions.outputFormatingFunc.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : output_formating symbol exists but is not callable"); + return false; + } else { + functions.outputFormatingOrigin = FunctionOrigin::PYTHON_MODULE; + } + + functions.responseFormatingFunc.py = PyObject_GetAttrString(pModule, "response_formating"); + if(functions.responseFormatingFunc.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' method in the python script"); + } else if (! PyCallable_Check(functions.responseFormatingFunc.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : response_formating symbol exists but is not callable"); + return false; + } else { + functions.responseFormatingOrigin = FunctionOrigin::PYTHON_MODULE; + } + + return true; +} + +darwin::session_ptr_t +Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, + darwin::Manager& manager) noexcept { + return std::static_pointer_cast( + std::make_shared(socket, manager, _cache, _cache_mutex, + functions)); +} + + +Generator::~Generator() { + Py_Finalize(); +} \ No newline at end of file diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp new file mode 100644 index 0000000..7490b5c --- /dev/null +++ b/samples/fpython/Generator.hpp @@ -0,0 +1,93 @@ +/// \file Generator.hpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#pragma once + +#include + +#include "../../toolkit/PythonUtils.hpp" +#include "Session.hpp" +#include "../../toolkit/RedisManager.hpp" +#include "../toolkit/rapidjson/document.h" +#include "AGenerator.hpp" + +template +union FunctionUnion{ + PyObject* py; + F so; + + FunctionUnion() { + this->py = nullptr; + } + + ~FunctionUnion(){} +}; + +enum FunctionOrigin { + NONE, + PYTHON_MODULE, + SHARED_LIBRARY, +}; + +struct FunctionHolder{ + FunctionHolder(): parseBodyOrigin{FunctionOrigin::NONE}, preProcessingOrigin{FunctionOrigin::NONE}, + alertFormatingOrigin{FunctionOrigin::NONE}, outputFormatingOrigin{FunctionOrigin::NONE}, + responseFormatingOrigin{FunctionOrigin::NONE} { } + ~FunctionHolder() = default; + FunctionHolder(FunctionHolder const &) = delete; + FunctionHolder(FunctionHolder &&) = delete; + FunctionHolder& operator=(FunctionHolder const &) = delete; + FunctionHolder& operator=(FunctionHolder &&) = delete; + + typedef PyObject*(*parse_body_t)(const std::string&); + typedef PyObject*(*process_t)(PyObject*); + typedef std::string(*format_t)(PyObject*); + + FunctionUnion parseBodyFunc; + FunctionOrigin parseBodyOrigin; + + FunctionUnion processingFunc; + FunctionOrigin processingOrigin; + + FunctionUnion preProcessingFunc; + FunctionOrigin preProcessingOrigin; + + FunctionUnion alertFormatingFunc; + FunctionOrigin alertFormatingOrigin; + + FunctionUnion outputFormatingFunc; + FunctionOrigin outputFormatingOrigin; + + FunctionUnion responseFormatingFunc; + FunctionOrigin responseFormatingOrigin; + +}; + +class Generator: public AGenerator { +public: + Generator(); + ~Generator(); + +public: + virtual darwin::session_ptr_t + CreateTask(boost::asio::local::stream_protocol::socket& socket, + darwin::Manager& manager) noexcept override final; + +private: + virtual bool LoadConfig(const rapidjson::Document &configuration) override final; + virtual bool ConfigureAlerting(const std::string& tags) override final; + + bool LoadPythonScript(const std::string& python_script_path); + bool LoadSharedLibrary(const std::string& shared_library_path); + bool CheckConfig(); + + PyObject* pName; + PyObject* pModule; + + FunctionHolder functions; + +}; \ No newline at end of file diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp new file mode 100644 index 0000000..39159bf --- /dev/null +++ b/samples/fpython/PythonTask.cpp @@ -0,0 +1,167 @@ +/// \file PythonExample.cpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#include + +#include "../toolkit/rapidjson/document.h" +#include "PythonTask.hpp" +#include "Logger.hpp" +#include "AlertManager.hpp" + +PythonTask::PythonTask(boost::asio::local::stream_protocol::socket& socket, + darwin::Manager& manager, + std::shared_ptr> cache, + std::mutex& cache_mutex, FunctionHolder& functions) + : Session{DARWIN_FILTER_NAME, socket, manager, cache, cache_mutex}, _functions{functions} +{ + _is_cache = _cache != nullptr; +} + +long PythonTask::GetFilterCode() noexcept{ + return DARWIN_FILTER_PYTHON_EXAMPLE; +} + +bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { + return true; +} + +void PythonTask::operator()() { + DARWIN_LOGGER; + if(PyErr_Occurred() != nullptr) PyErr_Print(); + + PyObject* py_result2 = nullptr; + if ((py_result2 = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, _result, nullptr)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + } + + if ((py_result2 = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, py_result2, nullptr)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + } + PyObject* preProc_res = nullptr, *proc_res = nullptr; + + switch(_functions.preProcessingOrigin) { + case FunctionOrigin::PYTHON_MODULE:{ + if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, _parsed_body, nullptr)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + } + break; + } + case FunctionOrigin::SHARED_LIBRARY:{ + if ((preProc_res = _functions.preProcessingFunc.so(_parsed_body)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); + } + + switch(_functions.preProcessingOrigin) { + case FunctionOrigin::PYTHON_MODULE:{ + if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, preProc_res, nullptr)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + } + break; + } + case FunctionOrigin::SHARED_LIBRARY:{ + if ((proc_res = _functions.processingFunc.so(preProc_res)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); + } + + std::string alert = GetFormated(_functions.alertFormatingFunc, _functions.alertFormatingOrigin, proc_res); + std::string output = GetFormated(_functions.outputFormatingFunc, _functions.outputFormatingOrigin, proc_res); + std::string resp = GetFormated(_functions.responseFormatingFunc, _functions.responseFormatingOrigin, proc_res); + if( ! alert.empty()) { + DARWIN_ALERT_MANAGER.Alert(alert); + } + + if( ! output.empty()) { + _raw_body = output; + } + + if( ! resp.empty()){ + _response_body = resp; + } +} + +bool PythonTask::ParseBody() { + DARWIN_LOGGER; + PyObject* raw_body = nullptr; + bool ret = false; + switch(_functions.parseBodyOrigin){ + case FunctionOrigin::NONE:{ + ret = Session::ParseBody(); + if(!ret) + return false; + std::string parsed_body = JsonStringify(_body); + if(parsed_body.empty()) + return true; + + _parsed_body = PyUnicode_DecodeFSDefault(parsed_body.c_str()); + if(PyErr_Occurred() != nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_raw_body); + PyErr_Print(); + return false; + } + return ret; + } + case FunctionOrigin::PYTHON_MODULE:{ + if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.py, raw_body, nullptr)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + return false; + } + return true; + } + case FunctionOrigin::SHARED_LIBRARY:{ + if((_parsed_body = _functions.parseBodyFunc.so(_raw_body))== nullptr){ + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); + return false; + } + return true; + } + + } + return false; +} + + +std::string PythonTask::GetFormated(FunctionUnion& func, FunctionOrigin& origin, PyObject* processedData){ + DARWIN_LOGGER; + PyObject *ret = nullptr; + char * out = nullptr; + switch(origin) { + case FunctionOrigin::PYTHON_MODULE:{ + if ((ret = PyObject_CallFunctionObjArgs(func.py, processedData, nullptr)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + return ""; + } + if((out = PyBytes_AsString(ret)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + return ""; + } + return std::string(out); + } + case FunctionOrigin::SHARED_LIBRARY: + return func.so(processedData); + default: + return ""; + } + + +} \ No newline at end of file diff --git a/samples/fpython/PythonTask.hpp b/samples/fpython/PythonTask.hpp new file mode 100644 index 0000000..45b13cf --- /dev/null +++ b/samples/fpython/PythonTask.hpp @@ -0,0 +1,58 @@ +/// \file PythonExample.hpp +/// \authors gcatto +/// \version 1.0 +/// \date 23/07/19 +/// \license GPLv3 +/// \brief Copyright (c) 2019 Advens. All rights reserved. + +#pragma once + +#include + +#include "../../toolkit/lru_cache.hpp" +#include "../../toolkit/PythonUtils.hpp" +#include "../../toolkit/xxhash.h" +#include "../../toolkit/xxhash.hpp" +#include "protocol.h" +#include "Session.hpp" +#include "Generator.hpp" + +#define DARWIN_FILTER_PYTHON_EXAMPLE 0x70797468 +#define DARWIN_FILTER_NAME "python" +#define DARWIN_ALERT_RULE_NAME "Python" +#define DARWIN_ALERT_TAGS "[]" + +class PythonTask : public darwin::Session { +public: + explicit PythonTask(boost::asio::local::stream_protocol::socket& socket, + darwin::Manager& manager, + std::shared_ptr> cache, + std::mutex& cache_mutex, FunctionHolder& functions); + + ~PythonTask() override = default; + +public: + // You need to override the functor to compile and be executed by the thread + void operator()() override; + +protected: + /// Get the result from the cache + // xxh::hash64_t GenerateHash() override; + /// Return filter code + long GetFilterCode() noexcept override; + +private: + + bool ParseLine(rapidjson::Value& line) override; + + /// Parse the body received. + bool ParseBody() override; + + std::string GetFormated(FunctionUnion& func, FunctionOrigin& origin, PyObject* processedData); + +private: + FunctionHolder& _functions; + + PyObject* _result; + PyObject* _parsed_body; +}; diff --git a/samples/fpython/example.py b/samples/fpython/example.py new file mode 100644 index 0000000..66039b0 --- /dev/null +++ b/samples/fpython/example.py @@ -0,0 +1,44 @@ +import json +from typing import List + + +class PythonFilterError(Exception): + pass + +class PythonFilterResponse: + body: str + certitudes: List[int] + alerts: List[str] + def __init__(self) -> None: + self.body = '' + self.certitudes = [] + self.alerts = [] + self.response = '' + +def parse_body(input: str) -> list: + parsed = json.loads(input) + if not isinstance(parsed, list): + raise PythonFilterError("Wrong type bro") + return parsed + +def filter_pre_process(input) -> list: + return input + +def filter_process(input: list) -> PythonFilterResponse: + resp = PythonFilterResponse() + for line in input: + print(line) + if isinstance(line, str): + resp.certitudes.append(101) + resp.alerts.append(line) + resp.response += line + return resp + +def alert_formating(input: PythonFilterResponse) -> str: + return json.dumps(input.alerts) + +def output_formating(input: PythonFilterResponse) -> str: + return json.dumps(input.response) + +def response_formating(input: PythonFilterResponse) -> str: + return json.dumps(input.response) diff --git a/samples/fpython/fpython.hpp b/samples/fpython/fpython.hpp new file mode 100644 index 0000000..c67c0e9 --- /dev/null +++ b/samples/fpython/fpython.hpp @@ -0,0 +1,14 @@ +#pragma once +#include +#include + +extern "C" { + + PyObject* parse_body(const std::string&); + PyObject* filter_pre_process(PyObject*); + PyObject* filter_process(PyObject*); + std::string alert_formating(PyObject*); + std::string output_formating(PyObject*); + std::string response_formating(PyObject*); + +} \ No newline at end of file From c7330b50988cbac848af17450f2ec8daa1a5f96e Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 30 Aug 2021 18:23:56 +0200 Subject: [PATCH 40/54] Modified function structures, added missing links --- samples/fpython/Generator.cpp | 82 +++++++++++++++++----------------- samples/fpython/Generator.hpp | 37 ++++++++------- samples/fpython/PythonTask.cpp | 54 ++++++++++------------ samples/fpython/PythonTask.hpp | 2 +- tests/filters/fpython.py | 35 +++++++++++++++ tests/filters/test.py | 2 + 6 files changed, 121 insertions(+), 91 deletions(-) create mode 100644 tests/filters/fpython.py diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 81c265f..e2c0a72 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -86,59 +86,59 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { if(parse == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' function in the shared library"); } else { - functions.parseBodyOrigin = FunctionOrigin::SHARED_LIBRARY; - functions.parseBodyFunc.so = parse; + functions.parseBodyFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.parseBodyFunc.f.so = parse; } FunctionHolder::process_t preProc = (FunctionHolder::process_t)dlsym(handle, "filter_pre_process"); if(preProc == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_pre_process' function in the shared library"); } else { - functions.preProcessingOrigin = FunctionOrigin::SHARED_LIBRARY; - functions.preProcessingFunc.so = preProc; + functions.preProcessingFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.preProcessingFunc.f.so = preProc; } FunctionHolder::process_t proc = (FunctionHolder::process_t)dlsym(handle, "filter_process"); if(proc == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_process' function in the shared library"); } else { - functions.processingOrigin = FunctionOrigin::SHARED_LIBRARY; - functions.processingFunc.so = proc; + functions.processingFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.processingFunc.f.so = proc; } FunctionHolder::format_t alert_format = (FunctionHolder::format_t)dlsym(handle, "alert_formating"); if(alert_format == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' function in the shared library"); } else { - functions.alertFormatingOrigin = FunctionOrigin::SHARED_LIBRARY; - functions.alertFormatingFunc.so = alert_format; + functions.alertFormatingFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.alertFormatingFunc.f.so = alert_format; } FunctionHolder::format_t output_format = (FunctionHolder::format_t)dlsym(handle, "output_formating"); if(parse == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' function in the shared library"); } else { - functions.outputFormatingOrigin = FunctionOrigin::SHARED_LIBRARY; - functions.outputFormatingFunc.so = output_format; + functions.outputFormatingFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.outputFormatingFunc.f.so = output_format; } FunctionHolder::format_t resp_format = (FunctionHolder::format_t)dlsym(handle, "response_formating"); if(resp_format == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' function in the shared library"); } else { - functions.responseFormatingOrigin = FunctionOrigin::SHARED_LIBRARY; - functions.responseFormatingFunc.so = resp_format; + functions.responseFormatingFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.responseFormatingFunc.f.so = resp_format; } return true; } bool Generator::CheckConfig() { - if(functions.preProcessingOrigin == FunctionOrigin::NONE - || functions.processingOrigin == FunctionOrigin::NONE - || functions.alertFormatingOrigin == FunctionOrigin::NONE - || functions.outputFormatingOrigin == FunctionOrigin::NONE - || functions.responseFormatingOrigin == FunctionOrigin::NONE) + if(functions.preProcessingFunc.loc == FunctionOrigin::NONE + || functions.processingFunc.loc == FunctionOrigin::NONE + || functions.alertFormatingFunc.loc == FunctionOrigin::NONE + || functions.outputFormatingFunc.loc == FunctionOrigin::NONE + || functions.responseFormatingFunc.loc == FunctionOrigin::NONE) { DARWIN_LOGGER; DARWIN_LOG_CRITICAL("Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library"); @@ -225,64 +225,64 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { return false; } - functions.parseBodyFunc.py = PyObject_GetAttrString(pModule, "parse_body"); - if(functions.parseBodyFunc.py == nullptr) { + functions.parseBodyFunc.f.py = PyObject_GetAttrString(pModule, "parse_body"); + if(functions.parseBodyFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' method in the python script"); - } else if (! PyCallable_Check(functions.parseBodyFunc.py)){ + } else if (! PyCallable_Check(functions.parseBodyFunc.f.py)){ DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : parse_body symbol exists but is not callable"); return false; } else { - functions.parseBodyOrigin = FunctionOrigin::PYTHON_MODULE; + functions.parseBodyFunc.loc= FunctionOrigin::PYTHON_MODULE; } - functions.preProcessingFunc.py = PyObject_GetAttrString(pModule, "filter_pre_process"); - if(functions.preProcessingFunc.py == nullptr) { + functions.preProcessingFunc.f.py = PyObject_GetAttrString(pModule, "filter_pre_process"); + if(functions.preProcessingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_pre_process' method in the python script"); - } else if (! PyCallable_Check(functions.preProcessingFunc.py)){ + } else if (! PyCallable_Check(functions.preProcessingFunc.f.py)){ DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_pre_process symbol exists but is not callable"); return false; } else { - functions.preProcessingOrigin = FunctionOrigin::PYTHON_MODULE; + functions.preProcessingFunc.loc= FunctionOrigin::PYTHON_MODULE; } - functions.processingFunc.py = PyObject_GetAttrString(pModule, "filter_process"); - if(functions.processingFunc.py == nullptr) { + functions.processingFunc.f.py = PyObject_GetAttrString(pModule, "filter_process"); + if(functions.processingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_process' method in the python script"); - } else if (! PyCallable_Check(functions.processingFunc.py)){ + } else if (! PyCallable_Check(functions.processingFunc.f.py)){ DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_process symbol exists but is not callable"); return false; } else { - functions.processingOrigin = FunctionOrigin::PYTHON_MODULE; + functions.processingFunc.loc= FunctionOrigin::PYTHON_MODULE; } - functions.alertFormatingFunc.py = PyObject_GetAttrString(pModule, "alert_formating"); - if(functions.alertFormatingFunc.py == nullptr) { + functions.alertFormatingFunc.f.py = PyObject_GetAttrString(pModule, "alert_formating"); + if(functions.alertFormatingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' method in the python script"); - } else if (! PyCallable_Check(functions.alertFormatingFunc.py)){ + } else if (! PyCallable_Check(functions.alertFormatingFunc.f.py)){ DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : alert_formating symbol exists but is not callable"); return false; } else { - functions.alertFormatingOrigin = FunctionOrigin::PYTHON_MODULE; + functions.alertFormatingFunc.loc= FunctionOrigin::PYTHON_MODULE; } - functions.outputFormatingFunc.py = PyObject_GetAttrString(pModule, "output_formating"); - if(functions.outputFormatingFunc.py == nullptr) { + functions.outputFormatingFunc.f.py = PyObject_GetAttrString(pModule, "output_formating"); + if(functions.outputFormatingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' method in the python script"); - } else if (! PyCallable_Check(functions.outputFormatingFunc.py)){ + } else if (! PyCallable_Check(functions.outputFormatingFunc.f.py)){ DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : output_formating symbol exists but is not callable"); return false; } else { - functions.outputFormatingOrigin = FunctionOrigin::PYTHON_MODULE; + functions.outputFormatingFunc.loc= FunctionOrigin::PYTHON_MODULE; } - functions.responseFormatingFunc.py = PyObject_GetAttrString(pModule, "response_formating"); - if(functions.responseFormatingFunc.py == nullptr) { + functions.responseFormatingFunc.f.py = PyObject_GetAttrString(pModule, "response_formating"); + if(functions.responseFormatingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' method in the python script"); - } else if (! PyCallable_Check(functions.responseFormatingFunc.py)){ + } else if (! PyCallable_Check(functions.responseFormatingFunc.f.py)){ DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : response_formating symbol exists but is not callable"); return false; } else { - functions.responseFormatingOrigin = FunctionOrigin::PYTHON_MODULE; + functions.responseFormatingFunc.loc= FunctionOrigin::PYTHON_MODULE; } return true; diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index 7490b5c..f791c29 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -33,11 +33,20 @@ enum FunctionOrigin { SHARED_LIBRARY, }; +template +struct FunctionPySo { + FunctionOrigin loc; + FunctionUnion f; + + FunctionPySo(): loc{FunctionOrigin::NONE} { } + ~FunctionPySo() = default; +}; + + struct FunctionHolder{ - FunctionHolder(): parseBodyOrigin{FunctionOrigin::NONE}, preProcessingOrigin{FunctionOrigin::NONE}, - alertFormatingOrigin{FunctionOrigin::NONE}, outputFormatingOrigin{FunctionOrigin::NONE}, - responseFormatingOrigin{FunctionOrigin::NONE} { } + FunctionHolder() = default; ~FunctionHolder() = default; + FunctionHolder(FunctionHolder const &) = delete; FunctionHolder(FunctionHolder &&) = delete; FunctionHolder& operator=(FunctionHolder const &) = delete; @@ -47,24 +56,14 @@ struct FunctionHolder{ typedef PyObject*(*process_t)(PyObject*); typedef std::string(*format_t)(PyObject*); - FunctionUnion parseBodyFunc; - FunctionOrigin parseBodyOrigin; + FunctionPySo parseBodyFunc; - FunctionUnion processingFunc; - FunctionOrigin processingOrigin; - - FunctionUnion preProcessingFunc; - FunctionOrigin preProcessingOrigin; - - FunctionUnion alertFormatingFunc; - FunctionOrigin alertFormatingOrigin; - - FunctionUnion outputFormatingFunc; - FunctionOrigin outputFormatingOrigin; - - FunctionUnion responseFormatingFunc; - FunctionOrigin responseFormatingOrigin; + FunctionPySo processingFunc; + FunctionPySo preProcessingFunc; + FunctionPySo alertFormatingFunc; + FunctionPySo outputFormatingFunc; + FunctionPySo responseFormatingFunc; }; class Generator: public AGenerator { diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index 39159bf..b28ae52 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -31,30 +31,18 @@ bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { void PythonTask::operator()() { DARWIN_LOGGER; - if(PyErr_Occurred() != nullptr) PyErr_Print(); - - PyObject* py_result2 = nullptr; - if ((py_result2 = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, _result, nullptr)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); - } - - if ((py_result2 = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, py_result2, nullptr)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); - } PyObject* preProc_res = nullptr, *proc_res = nullptr; - switch(_functions.preProcessingOrigin) { + switch(_functions.preProcessingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ - if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, _parsed_body, nullptr)) == nullptr) { + if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.f.py, _parsed_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); } break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((preProc_res = _functions.preProcessingFunc.so(_parsed_body)) == nullptr) { + if ((preProc_res = _functions.preProcessingFunc.f.so(_parsed_body)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); } break; @@ -63,16 +51,16 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); } - switch(_functions.preProcessingOrigin) { + switch(_functions.processingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ - if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, preProc_res, nullptr)) == nullptr) { + if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.f.py, preProc_res, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); } break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((proc_res = _functions.processingFunc.so(preProc_res)) == nullptr) { + if ((proc_res = _functions.processingFunc.f.so(preProc_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); } break; @@ -81,9 +69,9 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); } - std::string alert = GetFormated(_functions.alertFormatingFunc, _functions.alertFormatingOrigin, proc_res); - std::string output = GetFormated(_functions.outputFormatingFunc, _functions.outputFormatingOrigin, proc_res); - std::string resp = GetFormated(_functions.responseFormatingFunc, _functions.responseFormatingOrigin, proc_res); + std::string alert = GetFormated(_functions.alertFormatingFunc, proc_res); + std::string output = GetFormated(_functions.outputFormatingFunc, proc_res); + std::string resp = GetFormated(_functions.responseFormatingFunc, proc_res); if( ! alert.empty()) { DARWIN_ALERT_MANAGER.Alert(alert); } @@ -101,7 +89,7 @@ bool PythonTask::ParseBody() { DARWIN_LOGGER; PyObject* raw_body = nullptr; bool ret = false; - switch(_functions.parseBodyOrigin){ + switch(_functions.parseBodyFunc.loc){ case FunctionOrigin::NONE:{ ret = Session::ParseBody(); if(!ret) @@ -119,7 +107,13 @@ bool PythonTask::ParseBody() { return ret; } case FunctionOrigin::PYTHON_MODULE:{ - if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.py, raw_body, nullptr)) == nullptr) { + raw_body = PyUnicode_DecodeFSDefault(_raw_body.c_str()); + if((raw_body = PyUnicode_DecodeFSDefault(_raw_body.c_str())) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_raw_body); + PyErr_Print(); + return false; + } + if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.f.py, raw_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); return false; @@ -127,7 +121,7 @@ bool PythonTask::ParseBody() { return true; } case FunctionOrigin::SHARED_LIBRARY:{ - if((_parsed_body = _functions.parseBodyFunc.so(_raw_body))== nullptr){ + if((_parsed_body = _functions.parseBodyFunc.f.so(_raw_body))== nullptr){ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); return false; } @@ -139,18 +133,18 @@ bool PythonTask::ParseBody() { } -std::string PythonTask::GetFormated(FunctionUnion& func, FunctionOrigin& origin, PyObject* processedData){ +std::string PythonTask::GetFormated(FunctionPySo& func, PyObject* processedData){ DARWIN_LOGGER; PyObject *ret = nullptr; - char * out = nullptr; - switch(origin) { + const char * out = nullptr; + switch(func.loc) { case FunctionOrigin::PYTHON_MODULE:{ - if ((ret = PyObject_CallFunctionObjArgs(func.py, processedData, nullptr)) == nullptr) { + if ((ret = PyObject_CallFunctionObjArgs(func.f.py, processedData, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); return ""; } - if((out = PyBytes_AsString(ret)) == nullptr) { + if((out = PyUnicode_AS_DATA(ret)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); return ""; @@ -158,7 +152,7 @@ std::string PythonTask::GetFormated(FunctionUnion& fun return std::string(out); } case FunctionOrigin::SHARED_LIBRARY: - return func.so(processedData); + return func.f.so(processedData); default: return ""; } diff --git a/samples/fpython/PythonTask.hpp b/samples/fpython/PythonTask.hpp index 45b13cf..fa3e903 100644 --- a/samples/fpython/PythonTask.hpp +++ b/samples/fpython/PythonTask.hpp @@ -48,7 +48,7 @@ class PythonTask : public darwin::Session { /// Parse the body received. bool ParseBody() override; - std::string GetFormated(FunctionUnion& func, FunctionOrigin& origin, PyObject* processedData); + std::string GetFormated(FunctionPySo& func, PyObject* processedData); private: FunctionHolder& _functions; diff --git a/tests/filters/fpython.py b/tests/filters/fpython.py new file mode 100644 index 0000000..03b25d0 --- /dev/null +++ b/tests/filters/fpython.py @@ -0,0 +1,35 @@ +from tools.filter import Filter +from tools.output import print_result + +class PythonFilter(Filter): + def __init__(self): + super().__init__(filter_name="python") + + def configure(self, content=None): + if not content: + content = '{{\n' \ + '"python_script_path": "{python_path}",\n' \ + '"shared_library_path": "{so_path}",\n' \ + '"log_file_path": "/tmp/python_alerts.log"\n' \ + '}}'.format(python_path='/home/myadvens.lan/tcartegnie/workspace/darwin/samples/fpython/example.py', + so_path='/home/myadvens.lan/tcartegnie/workspace/pycpp/build/libfpython.so') + print(content) + super(PythonFilter, self).configure(content) + + + + +def run(): + tests = [ + test + ] + + for i in tests: + print_result("Python: " + i.__name__, i) + +def test(): + f = PythonFilter() + print(' '.join(f.cmd)) + f.configure() + f.valgrind_start() + print(f.send_single("hello")) \ No newline at end of file diff --git a/tests/filters/test.py b/tests/filters/test.py index b63432a..1485d18 100644 --- a/tests/filters/test.py +++ b/tests/filters/test.py @@ -9,6 +9,7 @@ import filters.fvast as fvast import filters.fvaml as fvaml import filters.fsession as fsession +import filters.fpython as fpython from tools.output import print_results @@ -27,6 +28,7 @@ def run(): fvast.run() fvaml.run() fsession.run() + fpython.run() print() print() From 466e3d2e2aac87982ecd883b8e517711b8d1e7c3 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 1 Sep 2021 17:08:00 +0200 Subject: [PATCH 41/54] WIP : fixed few things in the protocol python - Shared Lib --- samples/fpython/Generator.cpp | 2 +- samples/fpython/Generator.hpp | 6 ++--- samples/fpython/PythonTask.cpp | 43 ++++++++++++++++++++++------------ samples/fpython/PythonTask.hpp | 5 ++-- samples/fpython/fpython.hpp | 17 +++++++++----- 5 files changed, 46 insertions(+), 27 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index e2c0a72..f745f62 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -293,7 +293,7 @@ Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, darwin::Manager& manager) noexcept { return std::static_pointer_cast( std::make_shared(socket, manager, _cache, _cache_mutex, - functions)); + pModule, functions)); } diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index f791c29..fba98a5 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -52,9 +52,9 @@ struct FunctionHolder{ FunctionHolder& operator=(FunctionHolder const &) = delete; FunctionHolder& operator=(FunctionHolder &&) = delete; - typedef PyObject*(*parse_body_t)(const std::string&); - typedef PyObject*(*process_t)(PyObject*); - typedef std::string(*format_t)(PyObject*); + typedef PyObject*(*parse_body_t)(PyObject*, const std::string&); + typedef PyObject*(*process_t)(PyObject*, PyObject*); + typedef std::list(*format_t)(PyObject*, PyObject*); FunctionPySo parseBodyFunc; diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index b28ae52..d76be6c 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -15,8 +15,8 @@ PythonTask::PythonTask(boost::asio::local::stream_protocol::socket& socket, darwin::Manager& manager, std::shared_ptr> cache, - std::mutex& cache_mutex, FunctionHolder& functions) - : Session{DARWIN_FILTER_NAME, socket, manager, cache, cache_mutex}, _functions{functions} + std::mutex& cache_mutex, PyObject* pModule, FunctionHolder& functions) + : Session{DARWIN_FILTER_NAME, socket, manager, cache, cache_mutex}, _pModule{pModule}, _functions{functions} { _is_cache = _cache != nullptr; } @@ -42,7 +42,7 @@ void PythonTask::operator()() { break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((preProc_res = _functions.preProcessingFunc.f.so(_parsed_body)) == nullptr) { + if ((preProc_res = _functions.preProcessingFunc.f.so(_pModule, _parsed_body)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); } break; @@ -60,7 +60,7 @@ void PythonTask::operator()() { break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((proc_res = _functions.processingFunc.f.so(preProc_res)) == nullptr) { + if ((proc_res = _functions.processingFunc.f.so(_pModule, preProc_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); } break; @@ -121,7 +121,7 @@ bool PythonTask::ParseBody() { return true; } case FunctionOrigin::SHARED_LIBRARY:{ - if((_parsed_body = _functions.parseBodyFunc.f.so(_raw_body))== nullptr){ + if((_parsed_body = _functions.parseBodyFunc.f.so(_pModule, _raw_body))== nullptr){ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); return false; } @@ -133,28 +133,41 @@ bool PythonTask::ParseBody() { } -std::string PythonTask::GetFormated(FunctionPySo& func, PyObject* processedData){ +std::list PythonTask::GetFormated(FunctionPySo& func, PyObject* processedData){ DARWIN_LOGGER; - PyObject *ret = nullptr; + PyObject *pyRes = nullptr; + ssize_t pySize = 0, strSize=0; const char * out = nullptr; + std::list ret; switch(func.loc) { case FunctionOrigin::PYTHON_MODULE:{ - if ((ret = PyObject_CallFunctionObjArgs(func.f.py, processedData, nullptr)) == nullptr) { + if ((pyRes = PyObject_CallFunctionObjArgs(func.f.py, processedData, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); - return ""; + return ret; } - if((out = PyUnicode_AS_DATA(ret)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyList_Check(pyRes) == 0){ + DARWIN_LOG_ERROR("PythonTask::GetFormated : not a list"); + } + pySize = PyList_Size(pyRes); + if(PyErr_Occurred() != nullptr) { + PyErr_Print(); + return ret; + } + + for(size_t i = 0; i < pySize; i++) { + PyObject* str = PyList_GetItem(pyRes, i); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + out = PyUnicode_AsUTF8AndSize(str, &strSize); if(PyErr_Occurred() != nullptr) PyErr_Print(); - return ""; + ret.push_back(std::string(out, strSize)); } - return std::string(out); + return ret; } case FunctionOrigin::SHARED_LIBRARY: - return func.f.so(processedData); + return func.f.so(_pModule, processedData); default: - return ""; + return ret; } diff --git a/samples/fpython/PythonTask.hpp b/samples/fpython/PythonTask.hpp index fa3e903..9dfed53 100644 --- a/samples/fpython/PythonTask.hpp +++ b/samples/fpython/PythonTask.hpp @@ -27,7 +27,7 @@ class PythonTask : public darwin::Session { explicit PythonTask(boost::asio::local::stream_protocol::socket& socket, darwin::Manager& manager, std::shared_ptr> cache, - std::mutex& cache_mutex, FunctionHolder& functions); + std::mutex& cache_mutex, PyObject * pModule, FunctionHolder& functions); ~PythonTask() override = default; @@ -48,9 +48,10 @@ class PythonTask : public darwin::Session { /// Parse the body received. bool ParseBody() override; - std::string GetFormated(FunctionPySo& func, PyObject* processedData); + std::list GetFormated(FunctionPySo& func, PyObject* processedData); private: + PyObject* _pModule; FunctionHolder& _functions; PyObject* _result; diff --git a/samples/fpython/fpython.hpp b/samples/fpython/fpython.hpp index c67c0e9..c48bddc 100644 --- a/samples/fpython/fpython.hpp +++ b/samples/fpython/fpython.hpp @@ -2,13 +2,18 @@ #include #include +struct DarwinResponse{ + std::list certitudes; + std::string body; +}; + extern "C" { - PyObject* parse_body(const std::string&); - PyObject* filter_pre_process(PyObject*); - PyObject* filter_process(PyObject*); - std::string alert_formating(PyObject*); - std::string output_formating(PyObject*); - std::string response_formating(PyObject*); + PyObject* parse_body(PyObject* module, const std::string&); + PyObject* filter_pre_process(PyObject* module, PyObject*); + PyObject* filter_process(PyObject* module, PyObject*); + std::list alert_formating(PyObject* module, PyObject*input); + DarwinResponse output_formating(PyObject* module, PyObject*); + DarwinResponse response_formating(PyObject* module, PyObject*); } \ No newline at end of file From 86649fab2f880d40a4c1e34cb182b75d0a107685 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 2 Sep 2021 17:10:34 +0200 Subject: [PATCH 42/54] WIP : fixed return of operator (not completely) --- samples/fpython/Generator.cpp | 6 +-- samples/fpython/Generator.hpp | 10 ++-- samples/fpython/PythonTask.cpp | 99 +++++++++++++++++++++++++++++----- samples/fpython/PythonTask.hpp | 5 +- samples/fpython/fpython.hpp | 1 + tests/filters/fpython.py | 3 +- 6 files changed, 102 insertions(+), 22 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index f745f62..0a860dd 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -106,7 +106,7 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { functions.processingFunc.f.so = proc; } - FunctionHolder::format_t alert_format = (FunctionHolder::format_t)dlsym(handle, "alert_formating"); + FunctionHolder::alert_format_t alert_format = (FunctionHolder::alert_format_t)dlsym(handle, "alert_formating"); if(alert_format == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' function in the shared library"); } else { @@ -114,7 +114,7 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { functions.alertFormatingFunc.f.so = alert_format; } - FunctionHolder::format_t output_format = (FunctionHolder::format_t)dlsym(handle, "output_formating"); + FunctionHolder::resp_format_t output_format = (FunctionHolder::resp_format_t)dlsym(handle, "output_formating"); if(parse == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' function in the shared library"); } else { @@ -122,7 +122,7 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { functions.outputFormatingFunc.f.so = output_format; } - FunctionHolder::format_t resp_format = (FunctionHolder::format_t)dlsym(handle, "response_formating"); + FunctionHolder::resp_format_t resp_format = (FunctionHolder::resp_format_t)dlsym(handle, "response_formating"); if(resp_format == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' function in the shared library"); } else { diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index fba98a5..6236003 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -14,6 +14,7 @@ #include "../../toolkit/RedisManager.hpp" #include "../toolkit/rapidjson/document.h" #include "AGenerator.hpp" +#include "fpython.hpp" template union FunctionUnion{ @@ -54,16 +55,17 @@ struct FunctionHolder{ typedef PyObject*(*parse_body_t)(PyObject*, const std::string&); typedef PyObject*(*process_t)(PyObject*, PyObject*); - typedef std::list(*format_t)(PyObject*, PyObject*); + typedef std::list(*alert_format_t)(PyObject*, PyObject*); + typedef DarwinResponse(*resp_format_t)(PyObject*, PyObject*); FunctionPySo parseBodyFunc; FunctionPySo processingFunc; FunctionPySo preProcessingFunc; - FunctionPySo alertFormatingFunc; - FunctionPySo outputFormatingFunc; - FunctionPySo responseFormatingFunc; + FunctionPySo alertFormatingFunc; + FunctionPySo outputFormatingFunc; + FunctionPySo responseFormatingFunc; }; class Generator: public AGenerator { diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index d76be6c..40c0929 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -69,19 +69,26 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); } - std::string alert = GetFormated(_functions.alertFormatingFunc, proc_res); - std::string output = GetFormated(_functions.outputFormatingFunc, proc_res); - std::string resp = GetFormated(_functions.responseFormatingFunc, proc_res); - if( ! alert.empty()) { - DARWIN_ALERT_MANAGER.Alert(alert); + std::list alerts = GetFormatedAlerts(_functions.alertFormatingFunc, proc_res); + DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, proc_res); + DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, proc_res); + Py_DECREF(proc_res); + Py_DECREF(preProc_res); + + for(auto& alert: alerts) { + if( ! alert.empty()) { + DARWIN_ALERT_MANAGER.Alert(alert); + } } - - if( ! output.empty()) { - _raw_body = output; + for(auto cert: resp.certitudes) { + _certitudes.push_back(cert); + } + if( ! output.body.empty()) { + _raw_body = output.body; } - if( ! resp.empty()){ - _response_body = resp; + if( ! resp.body.empty()){ + _response_body = resp.body; } } @@ -133,7 +140,7 @@ bool PythonTask::ParseBody() { } -std::list PythonTask::GetFormated(FunctionPySo& func, PyObject* processedData){ +std::list PythonTask::GetFormatedAlerts(FunctionPySo& func, PyObject* processedData){ DARWIN_LOGGER; PyObject *pyRes = nullptr; ssize_t pySize = 0, strSize=0; @@ -155,7 +162,7 @@ std::list PythonTask::GetFormated(FunctionPySo PythonTask::GetFormated(FunctionPySo& func, PyObject* processedData) { + DARWIN_LOGGER; + DarwinResponse ret; + switch(func.loc) { + case FunctionOrigin::PYTHON_MODULE:{ + + PyObject * pBody = nullptr, *pCertitudes = nullptr; + ssize_t certSize = 0; + if((pBody = PyObject_GetAttrString(processedData, "response")) == nullptr){ + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + return ret; + } + + const char* cBody = nullptr; + ssize_t bodySize = 0; + + if((cBody = PyUnicode_AsUTF8AndSize(pBody, &bodySize)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + return ret; + } + + ret.body = std::string(cBody, bodySize); + Py_DECREF(pBody); + + if((pCertitudes = PyObject_GetAttrString(processedData, "certitudes")) == nullptr){ + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + if(PyErr_Occurred() != nullptr) PyErr_Print(); + return ret; + } + certSize = PyList_Size(pCertitudes); + if(PyErr_Occurred() != nullptr) { + PyErr_Print(); + return ret; + } + + for(ssize_t i = 0; i < certSize; i++) { + PyObject* cert = nullptr; + if((cert = PyList_GetItem(pCertitudes, i)) == nullptr) { + if(PyErr_Occurred() != nullptr) PyErr_Print(); + continue; + } + + if(PyLong_Check(cert) == 0) { + DARWIN_LOG_DEBUG("PythonTask:: Not a long"); + } + long lCert = PyLong_AS_LONG(cert); + if(PyErr_Occurred() != nullptr){ + DARWIN_LOG_DEBUG("PythonTask:: error getting long"); + PyErr_Print(); + continue; + } + if (lCert < 0 || lCert > UINT_MAX) { + DARWIN_LOG_DEBUG("PythonTask:: out of range certitude"); + continue; + } + ret.certitudes.push_back(static_cast(lCert)); + } + return ret; + } + case FunctionOrigin::SHARED_LIBRARY: + return func.f.so(_pModule, processedData); + default: + return ret; + } } \ No newline at end of file diff --git a/samples/fpython/PythonTask.hpp b/samples/fpython/PythonTask.hpp index 9dfed53..3736a29 100644 --- a/samples/fpython/PythonTask.hpp +++ b/samples/fpython/PythonTask.hpp @@ -16,6 +16,7 @@ #include "protocol.h" #include "Session.hpp" #include "Generator.hpp" +#include "fpython.hpp" #define DARWIN_FILTER_PYTHON_EXAMPLE 0x70797468 #define DARWIN_FILTER_NAME "python" @@ -47,8 +48,8 @@ class PythonTask : public darwin::Session { /// Parse the body received. bool ParseBody() override; - - std::list GetFormated(FunctionPySo& func, PyObject* processedData); + std::list GetFormatedAlerts(FunctionPySo& func, PyObject* processedData); + DarwinResponse GetFormatedResponse(FunctionPySo& func, PyObject* processedData); private: PyObject* _pModule; diff --git a/samples/fpython/fpython.hpp b/samples/fpython/fpython.hpp index c48bddc..00276b0 100644 --- a/samples/fpython/fpython.hpp +++ b/samples/fpython/fpython.hpp @@ -1,6 +1,7 @@ #pragma once #include #include +#include struct DarwinResponse{ std::list certitudes; diff --git a/tests/filters/fpython.py b/tests/filters/fpython.py index 03b25d0..4935324 100644 --- a/tests/filters/fpython.py +++ b/tests/filters/fpython.py @@ -32,4 +32,5 @@ def test(): print(' '.join(f.cmd)) f.configure() f.valgrind_start() - print(f.send_single("hello")) \ No newline at end of file + print('ret:' , f.send_single("hello")) + return True \ No newline at end of file From 4e1fc71beaf63ec79b8e97cda68a0b23cb19ef69 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Fri, 3 Sep 2021 10:49:34 +0200 Subject: [PATCH 43/54] Adapted filter for support_udp_tcp --- samples/fpython/Generator.cpp | 19 ++++++++++------ samples/fpython/Generator.hpp | 12 +++++----- samples/fpython/PythonTask.cpp | 41 +++++++++++++++++++++------------- samples/fpython/PythonTask.hpp | 19 +++++++++------- 4 files changed, 54 insertions(+), 37 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 0a860dd..bf18506 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -12,10 +12,12 @@ #include "base/Logger.hpp" #include "Generator.hpp" #include "PythonTask.hpp" +#include "ASession.hpp" #include "AlertManager.hpp" #include "fpython.hpp" -Generator::Generator(): pName{nullptr}, pModule{nullptr} +Generator::Generator(size_t nb_task_threads) + : AGenerator(nb_task_threads), pName{nullptr}, pModule{nullptr} { } bool Generator::ConfigureAlerting(const std::string& tags) { @@ -288,14 +290,17 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { return true; } -darwin::session_ptr_t -Generator::CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept { - return std::static_pointer_cast( - std::make_shared(socket, manager, _cache, _cache_mutex, - pModule, functions)); +std::shared_ptr +Generator::CreateTask(darwin::session_ptr_t s) noexcept { + return std::static_pointer_cast( + std::make_shared(_cache, _cache_mutex, s, s->_packet, + pModule, functions) + ); } +long Generator::GetFilterCode() const { + return DARWIN_FILTER_PYTHON; +} Generator::~Generator() { Py_Finalize(); diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index 6236003..c85b70e 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -10,7 +10,7 @@ #include #include "../../toolkit/PythonUtils.hpp" -#include "Session.hpp" +#include "ATask.hpp" #include "../../toolkit/RedisManager.hpp" #include "../toolkit/rapidjson/document.h" #include "AGenerator.hpp" @@ -70,13 +70,13 @@ struct FunctionHolder{ class Generator: public AGenerator { public: - Generator(); + Generator(size_t nb_task_threads); ~Generator(); -public: - virtual darwin::session_ptr_t - CreateTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager) noexcept override final; + virtual std::shared_ptr + CreateTask(darwin::session_ptr_t s) noexcept override final; + + virtual long GetFilterCode() const; private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index 40c0929..e43fba0 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -12,17 +12,17 @@ #include "Logger.hpp" #include "AlertManager.hpp" -PythonTask::PythonTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, PyObject* pModule, FunctionHolder& functions) - : Session{DARWIN_FILTER_NAME, socket, manager, cache, cache_mutex}, _pModule{pModule}, _functions{functions} +PythonTask::PythonTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, PyObject* pModule, FunctionHolder& functions) + : ATask{DARWIN_FILTER_NAME, cache, cache_mutex, s, packet}, _pModule{pModule}, _functions{functions} { _is_cache = _cache != nullptr; } long PythonTask::GetFilterCode() noexcept{ - return DARWIN_FILTER_PYTHON_EXAMPLE; + return DARWIN_FILTER_PYTHON; } bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { @@ -81,14 +81,20 @@ void PythonTask::operator()() { } } for(auto cert: resp.certitudes) { - _certitudes.push_back(cert); + DARWIN_LOG_DEBUG("PythonTask:: cert added"); + _packet.AddCertitude(cert); } + std::string& body = _packet.GetMutableBody(); if( ! output.body.empty()) { - _raw_body = output.body; + DARWIN_LOG_DEBUG("PythonTask:: output"); + body.clear(); + body.append(output.body); } if( ! resp.body.empty()){ - _response_body = resp.body; + DARWIN_LOG_DEBUG("PythonTask:: resp"); + body.clear(); + body.append(resp.body); } } @@ -98,37 +104,40 @@ bool PythonTask::ParseBody() { bool ret = false; switch(_functions.parseBodyFunc.loc){ case FunctionOrigin::NONE:{ - ret = Session::ParseBody(); + ret = ATask::ParseBody(); + + // TODO :WRONG! Turn to list if(!ret) return false; - std::string parsed_body = JsonStringify(_body); + std::string parsed_body;// = JsonStringify(_packet.GetBody()); if(parsed_body.empty()) return true; _parsed_body = PyUnicode_DecodeFSDefault(parsed_body.c_str()); if(PyErr_Occurred() != nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_raw_body); + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"); PyErr_Print(); return false; } return ret; } case FunctionOrigin::PYTHON_MODULE:{ - raw_body = PyUnicode_DecodeFSDefault(_raw_body.c_str()); - if((raw_body = PyUnicode_DecodeFSDefault(_raw_body.c_str())) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_raw_body); + if((raw_body = PyUnicode_DecodeFSDefault(_packet.GetBody().c_str())) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); PyErr_Print(); return false; } if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.f.py, raw_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); + Py_DECREF(raw_body); return false; } + Py_DECREF(raw_body); return true; } case FunctionOrigin::SHARED_LIBRARY:{ - if((_parsed_body = _functions.parseBodyFunc.f.so(_pModule, _raw_body))== nullptr){ + if((_parsed_body = _functions.parseBodyFunc.f.so(_pModule, _packet.GetBody()))== nullptr){ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); return false; } diff --git a/samples/fpython/PythonTask.hpp b/samples/fpython/PythonTask.hpp index 3736a29..dfa9557 100644 --- a/samples/fpython/PythonTask.hpp +++ b/samples/fpython/PythonTask.hpp @@ -13,22 +13,25 @@ #include "../../toolkit/PythonUtils.hpp" #include "../../toolkit/xxhash.h" #include "../../toolkit/xxhash.hpp" -#include "protocol.h" -#include "Session.hpp" +#include "ATask.hpp" +#include "DarwinPacket.hpp" +#include "ASession.fwd.hpp" #include "Generator.hpp" #include "fpython.hpp" -#define DARWIN_FILTER_PYTHON_EXAMPLE 0x70797468 +#define DARWIN_FILTER_PYTHON 0x70797468 #define DARWIN_FILTER_NAME "python" #define DARWIN_ALERT_RULE_NAME "Python" #define DARWIN_ALERT_TAGS "[]" -class PythonTask : public darwin::Session { +class PythonTask : public darwin::ATask { public: - explicit PythonTask(boost::asio::local::stream_protocol::socket& socket, - darwin::Manager& manager, - std::shared_ptr> cache, - std::mutex& cache_mutex, PyObject * pModule, FunctionHolder& functions); + explicit PythonTask(std::shared_ptr> cache, + std::mutex& cache_mutex, + darwin::session_ptr_t s, + darwin::DarwinPacket& packet, + PyObject * pModule, + FunctionHolder& functions); ~PythonTask() override = default; From 3064dd8f57714646fdbbf09860e64ec8ba6d8ee4 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 15 Sep 2021 10:30:24 +0200 Subject: [PATCH 44/54] WIP working threads, memory issues --- samples/fpython/Generator.cpp | 111 ++++++++++++++++++++----------- samples/fpython/Generator.hpp | 2 + samples/fpython/PythonObject.hpp | 58 ++++++++++++++++ samples/fpython/PythonTask.cpp | 65 +++++++++++++----- samples/fpython/PythonThread.hpp | 54 +++++++++++++++ tests/filters/fpython.py | 6 +- tests/tools/filter.py | 4 +- 7 files changed, 243 insertions(+), 57 deletions(-) create mode 100644 samples/fpython/PythonObject.hpp create mode 100644 samples/fpython/PythonThread.hpp diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index bf18506..859537b 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -15,6 +15,45 @@ #include "ASession.hpp" #include "AlertManager.hpp" #include "fpython.hpp" +#include "PythonObject.hpp" + +/// +/// \brief +/// +/// \param prefix_log +/// \param log_type +/// \return true if an exception was found and handled +/// \return false if no exception were risen +/// +bool PyExceptionCheckAndLog(std::string const& prefix_log, darwin::logger::log_type log_type=darwin::logger::Critical){ + if(PyErr_Occurred() == nullptr){ + return false; + } + DARWIN_LOGGER; + PyObject *errType, *err, *tb; + PyObjectOwner pErrStr, pTypeStr; + const char * pErrChars, *pTypeChars; + ssize_t errCharsLen = 0, typeCharsLen = 0; + std::string errString, typeString; + PyErr_Fetch(&errType, &err, &tb); + + if((pErrStr = PyObject_Str(err)) != nullptr){ + if((pErrChars = PyUnicode_AsUTF8AndSize(*pErrStr, &errCharsLen)) != nullptr) { + errString = std::string(pErrChars, errCharsLen); + } + } + + if((pTypeStr = PyObject_Str(errType)) != nullptr){ + if((pTypeChars = PyUnicode_AsUTF8AndSize(*pTypeStr, &typeCharsLen)) != nullptr) { + typeString = std::string(pTypeChars, typeCharsLen); + } + } + std::ostringstream oss; + oss << prefix_log; + oss << "Python error '" << typeString << "' : " << errString; + log.log(log_type, oss.str()); + return true; +} Generator::Generator(size_t nb_task_threads) : AGenerator(nb_task_threads), pName{nullptr}, pModule{nullptr} @@ -66,7 +105,14 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { shared_library_path = configuration["shared_library_path"].GetString(); DARWIN_LOG_DEBUG("Python:: Generator:: Loading configuration..."); - return LoadPythonScript(python_script_path) && LoadSharedLibrary(shared_library_path) && CheckConfig(); + bool pythonLoad = LoadPythonScript(python_script_path); + + if(Py_IsInitialized() != 0) { + // Release the GIL, This is safe to call because we just initialized the python interpreter + PyEval_SaveThread(); + } + + return pythonLoad && LoadSharedLibrary(shared_library_path) && CheckConfig(); } bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { @@ -149,6 +195,16 @@ bool Generator::CheckConfig() { return true; } +PythonThread& Generator::GetPythonThread(){ + thread_local PythonThread thread; + if(! thread.IsInit()){ + PyGILState_STATE lock = PyGILState_Ensure(); + thread.Init(PyInterpreterState_Main()); + PyGILState_Release(lock); + } + return thread; +} + bool Generator::LoadPythonScript(const std::string& python_script_path) { DARWIN_LOGGER; @@ -172,58 +228,33 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { } Py_Initialize(); - if(PyErr_Occurred() != nullptr) { - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post py init"); - PyErr_Print(); + PyEval_InitThreads(); + + if(PyExceptionCheckAndLog("Generator::LoadPythonScript : Error during python init :")) { + return false; } - pName = PyUnicode_DecodeFSDefault(filename.c_str()); - // PyObject* pFolders = PyUnicode_DecodeFSDefault(folders.c_str()); - // PyObject* pFilename = PyUnicode_DecodeFSDefault(filename.c_str()); - if(PyErr_Occurred() != nullptr) { - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post filename decode"); - PyErr_Print(); + if((pName = PyUnicode_DecodeFSDefault(filename.c_str())) == nullptr){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : error importing string " + filename + " : "); + return false; } + if (PyRun_SimpleString("import sys") != 0) { - DARWIN_LOG_DEBUG("darwin:: pythonutils:: InitPythonProgram:: An error occurred while loading the 'sys' module"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); - + DARWIN_LOG_DEBUG("Generator::LoadPythonScript : An error occurred while loading the 'sys' module"); return false; } - if(PyErr_Occurred() != nullptr) { - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post imp sys"); - PyErr_Print(); - } std::string command = "sys.path.append(\"" + folders + "\")"; if (PyRun_SimpleString(command.c_str()) != 0) { DARWIN_LOG_DEBUG( - "darwin:: pythonutils:: InitPythonProgram:: An error occurred while appending the custom path '" + + "Generator::LoadPythonScript : An error occurred while appending the custom path '" + folders + "' to the Python path" ); - - if(PyErr_Occurred() != nullptr) PyErr_Print(); - return false; } - if(PyErr_Occurred() != nullptr) { - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error post sys path"); - PyErr_Print(); - } - - pModule = PyImport_Import(pName); - - if(PyErr_Occurred() != nullptr) { - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : post import"); - - PyErr_Print(); - } - - if(pModule == nullptr) { - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error while attempting to load the python script module"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); - + if((pModule = PyImport_Import(pName)) == nullptr) { + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python script module '" + filename + "' : "); return false; } @@ -303,5 +334,7 @@ long Generator::GetFilterCode() const { } Generator::~Generator() { + PyGILState_Ensure(); Py_Finalize(); -} \ No newline at end of file + // Python environment is destroyed, we can't release the GIL +} diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index c85b70e..20e9991 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -15,6 +15,7 @@ #include "../toolkit/rapidjson/document.h" #include "AGenerator.hpp" #include "fpython.hpp" +#include "PythonThread.hpp" template union FunctionUnion{ @@ -77,6 +78,7 @@ class Generator: public AGenerator { CreateTask(darwin::session_ptr_t s) noexcept override final; virtual long GetFilterCode() const; + static PythonThread& GetPythonThread(); private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; diff --git a/samples/fpython/PythonObject.hpp b/samples/fpython/PythonObject.hpp new file mode 100644 index 0000000..ae3e068 --- /dev/null +++ b/samples/fpython/PythonObject.hpp @@ -0,0 +1,58 @@ +#pragma once +#include + +class PyObjectOwner { + public: + // no copy, move ok + PyObjectOwner(): handle{nullptr} {} + PyObjectOwner(PyObject* o): handle{o} {} + ~PyObjectOwner(){ + this->Decref(); + } + + PyObjectOwner(PyObjectOwner const&) = delete; + PyObjectOwner(PyObjectOwner&& other): handle{other.handle} { + other.handle = nullptr; + } + + PyObjectOwner& operator=(PyObjectOwner const&) = delete; + PyObjectOwner& operator=(PyObjectOwner&& other){ + this->Decref(); + this->handle = other.handle; + other.handle = nullptr; + return *this; + } + + bool operator==(std::nullptr_t){ + return handle == nullptr; + } + + bool operator!=(std::nullptr_t){ + return handle != nullptr; + } + + PyObjectOwner& operator=(PyObject* o){ + this->Decref(); + this->handle = o; + return *this; + } + + PyObject* operator* () + { + return handle; + } + + PyObject& operator-> () + { + return *handle; + } + +private: + void Decref(){ + if(handle != nullptr){ + Py_DECREF(handle); + handle = nullptr; + } + } + PyObject* handle; +}; \ No newline at end of file diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index e43fba0..e672ec0 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -29,12 +29,15 @@ bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { return true; } + + void PythonTask::operator()() { DARWIN_LOGGER; PyObject* preProc_res = nullptr, *proc_res = nullptr; switch(_functions.preProcessingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ + PythonThread::Use pylock(Generator::GetPythonThread()); if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.f.py, _parsed_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); @@ -53,6 +56,7 @@ void PythonTask::operator()() { switch(_functions.processingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ + PythonThread::Use pylock(Generator::GetPythonThread()); if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.f.py, preProc_res, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); @@ -72,8 +76,11 @@ void PythonTask::operator()() { std::list alerts = GetFormatedAlerts(_functions.alertFormatingFunc, proc_res); DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, proc_res); DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, proc_res); - Py_DECREF(proc_res); - Py_DECREF(preProc_res); + { + PythonThread::Use pylock(Generator::GetPythonThread()); + Py_DECREF(proc_res); + Py_DECREF(preProc_res); + } for(auto& alert: alerts) { if( ! alert.empty()) { @@ -102,26 +109,39 @@ bool PythonTask::ParseBody() { DARWIN_LOGGER; PyObject* raw_body = nullptr; bool ret = false; + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: helo"); + switch(_functions.parseBodyFunc.loc){ case FunctionOrigin::NONE:{ + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: None koi " "__LINE__"); ret = ATask::ParseBody(); - - // TODO :WRONG! Turn to list - if(!ret) + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: None koi2"); + if(!ret){ return false; - std::string parsed_body;// = JsonStringify(_packet.GetBody()); - if(parsed_body.empty()) - return true; - - _parsed_body = PyUnicode_DecodeFSDefault(parsed_body.c_str()); - if(PyErr_Occurred() != nullptr) { + } + PythonThread::Use pylock(Generator::GetPythonThread()); + if((_parsed_body = PyList_New(_body.Size())) == nullptr){ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"); - PyErr_Print(); + if(PyErr_Occurred() != nullptr) PyErr_Print(); return false; } + + for(ssize_t i = 0; i < _body.Size(); i++){ + PyObject *str; + if((str = PyUnicode_DecodeFSDefault(_body.GetArray()[i].GetString())) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); + PyErr_Print(); + return false; + } + PyList_SetItem(_parsed_body, i, str); + } return ret; } case FunctionOrigin::PYTHON_MODULE:{ + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koi"); + PythonThread::Use pylock(Generator::GetPythonThread()); + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koilol"); + if((raw_body = PyUnicode_DecodeFSDefault(_packet.GetBody().c_str())) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); PyErr_Print(); @@ -137,10 +157,18 @@ bool PythonTask::ParseBody() { return true; } case FunctionOrigin::SHARED_LIBRARY:{ - if((_parsed_body = _functions.parseBodyFunc.f.so(_pModule, _packet.GetBody()))== nullptr){ - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: SO koi " "__LINE__"); + try{ + if((_parsed_body = _functions.parseBodyFunc.f.so(_pModule, _packet.GetBody()))== nullptr){ + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); + return false; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: Error while calling the SO parse_body function : " + std::string(e.what())); return false; - } + } + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: SO koi 2" "__LINE__"); + return true; } @@ -157,6 +185,7 @@ std::list PythonTask::GetFormatedAlerts(FunctionPySo ret; switch(func.loc) { case FunctionOrigin::PYTHON_MODULE:{ + PythonThread::Use pylock(Generator::GetPythonThread()); if ((pyRes = PyObject_CallFunctionObjArgs(func.f.py, processedData, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); if(PyErr_Occurred() != nullptr) PyErr_Print(); @@ -194,6 +223,9 @@ DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo(lCert)); } + DARWIN_LOG_DEBUG("PythonTask::GetFormatedResponse regturn"); return ret; } case FunctionOrigin::SHARED_LIBRARY: diff --git a/samples/fpython/PythonThread.hpp b/samples/fpython/PythonThread.hpp new file mode 100644 index 0000000..96d1394 --- /dev/null +++ b/samples/fpython/PythonThread.hpp @@ -0,0 +1,54 @@ +#include +#include "Logger.hpp" +class PythonThread { +public: + PythonThread(): _init{false}, pThreadState{nullptr} {} + void Init(PyInterpreterState * interp) { + if (! _init){ + pThreadState = PyThreadState_New(interp); + _init = true; + } + } + bool IsInit(){ + return _init; + } + + PyThreadState* GetThreadState(){ + return pThreadState; + } + + void SetThreadState(PyThreadState* state){ + pThreadState = state; + } + + ~PythonThread() { + if(pThreadState != nullptr){ + PyThreadState_Clear(pThreadState); + PyThreadState_Delete(pThreadState); + } + } + + class Use { + public: + Use(PythonThread& thread): _thread{thread} { + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("Start thread"); + PyEval_RestoreThread(_thread.GetThreadState()); + } + + ~Use(){ + DARWIN_LOGGER; + DARWIN_LOG_DEBUG("End thread"); + _thread.SetThreadState(PyEval_SaveThread()); + } + + private: + PythonThread& _thread; + }; + +private: + + bool _init; + PyThreadState * pThreadState; + +}; \ No newline at end of file diff --git a/tests/filters/fpython.py b/tests/filters/fpython.py index 4935324..17a16ed 100644 --- a/tests/filters/fpython.py +++ b/tests/filters/fpython.py @@ -32,5 +32,9 @@ def test(): print(' '.join(f.cmd)) f.configure() f.valgrind_start() - print('ret:' , f.send_single("hello")) + api = f.get_darwin_api(verbose=True) + ret = api.call('hello', response_type="back") + + print(ret) + # print('ret:' , f.send_single("hello")) return True \ No newline at end of file diff --git a/tests/tools/filter.py b/tests/tools/filter.py index f8a086b..54401c4 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -123,8 +123,10 @@ def valgrind_start(self): '--errors-for-leak-kinds=definite', '-q', # Quiet mode so it doesn't pollute the output '--error-exitcode={}'.format(self.error_code), #If valgrind reports error(s) during the run, return this exit code. + '--suppressions=/home/myadvens.lan/tcartegnie/workspace/darwin/tests/valgrind-python.supp', + '--log-file=valgrind.log', ] + self.cmd - + print(' '.join(command)) self.process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sleep(3) return self.check_start() From f3e619b6140ad3c24d71f12c6cef88e6c7269bc2 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 21 Sep 2021 11:57:15 +0200 Subject: [PATCH 45/54] Fixed issues with memory usage, added config step --- samples/fpython/Generator.cpp | 130 +++++++++++++++++++++++++++---- samples/fpython/Generator.hpp | 13 +++- samples/fpython/PythonObject.hpp | 9 ++- samples/fpython/PythonTask.cpp | 61 ++++++++------- samples/fpython/PythonThread.hpp | 19 +++-- samples/fpython/example.py | 22 ++++-- 6 files changed, 197 insertions(+), 57 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 859537b..3623629 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -25,7 +25,7 @@ /// \return true if an exception was found and handled /// \return false if no exception were risen /// -bool PyExceptionCheckAndLog(std::string const& prefix_log, darwin::logger::log_type log_type=darwin::logger::Critical){ +bool Generator::PyExceptionCheckAndLog(std::string const& prefix_log, darwin::logger::log_type log_type){ if(PyErr_Occurred() == nullptr){ return false; } @@ -112,7 +112,83 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { PyEval_SaveThread(); } - return pythonLoad && LoadSharedLibrary(shared_library_path) && CheckConfig(); + return pythonLoad && LoadSharedLibrary(shared_library_path) && CheckConfig() && SendConfig(configuration); +} + +bool Generator::SendConfig(rapidjson::Document const& config) const{ + DARWIN_LOGGER; + bool pyConfigRes = false, soConfigRes = false; + switch(functions.configPyFunc.loc) { + case FunctionOrigin::NONE: + DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in python script"); + pyConfigRes = true; + break; + case FunctionOrigin::PYTHON_MODULE: + pyConfigRes = this->SendPythonConfig(config); + break; + case FunctionOrigin::SHARED_LIBRARY: + DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: Incoherent configuration"); + return false; + } + + switch(functions.configSoFunc.loc) { + case FunctionOrigin::NONE: + DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in shared object"); + soConfigRes = true; + break; + case FunctionOrigin::PYTHON_MODULE: + DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: Incoherent configuration"); + return false; + case FunctionOrigin::SHARED_LIBRARY: + soConfigRes = functions.configSoFunc.f.so(config); + break; + } + + return pyConfigRes && soConfigRes; +} + +bool Generator::SendPythonConfig(rapidjson::Document const& config) const{ + DARWIN_LOGGER; + rapidjson::StringBuffer buffer; + buffer.Clear(); + + rapidjson::Writer writer(buffer); + config.Accept(writer); + + PythonLock lock; + if(PyRun_SimpleString("import json") != 0) { + DARWIN_LOG_CRITICAL("Python:: Generator:: SendPythonConfig: import json failed"); + return false; + } + + std::string cmd = "json.loads(\"\"\""; + cmd += buffer.GetString(); + cmd += "\"\"\")"; + PyObject* globalDictionary = PyModule_GetDict(pModule); + + PyObjectOwner pConfig = PyRun_StringFlags(cmd.c_str(), Py_eval_input, globalDictionary, globalDictionary, nullptr); + if(Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: Converting config to Py Object :")){ + return false; + } + PyObjectOwner res; + if ((res = PyObject_CallFunctionObjArgs(functions.configPyFunc.f.py, *pConfig, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: "); + return false; + } + int resTruthiness = -1; + if((resTruthiness = PyObject_IsTrue(*res)) == 1) { + // 1 : true + return true; + } else if (resTruthiness == 0){ + // 0 : false + DARWIN_LOG_CRITICAL("Python:: Generator:: SendPythonConfig: filter_config returned false"); + } else { + // -1 : error while evaluating truthiness + DARWIN_LOG_CRITICAL("Python:: Generator:: SendPythonConfig: truthiness evaluation of the returned value of 'filter_config' failed"); + Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: "); + } + + return false; } bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { @@ -123,12 +199,21 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { return true; } - void* handle = dlopen(shared_library_path.c_str(), RTLD_LAZY); + void* handle = dlopen(shared_library_path.c_str(), RTLD_NOW | RTLD_GLOBAL); if(handle == nullptr) { - DARWIN_LOG_CRITICAL("Generator::LoadSharedLibrary : Error loading the shared library : failed to open " + shared_library_path); + DARWIN_LOG_CRITICAL("Generator::LoadSharedLibrary : Error loading the shared library : failed to open " + shared_library_path + " : " + std::string(dlerror())); return false; } + FunctionHolder::config_t conf = (FunctionHolder::config_t)dlsym(handle, "filter_config"); + + if(conf == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_config' function in the shared library"); + } else { + functions.configSoFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.configSoFunc.f.so = conf; + } + FunctionHolder::parse_body_t parse = (FunctionHolder::parse_body_t)dlsym(handle, "parse_body"); if(parse == nullptr) { @@ -181,7 +266,7 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { return true; } -bool Generator::CheckConfig() { +bool Generator::CheckConfig() const { if(functions.preProcessingFunc.loc == FunctionOrigin::NONE || functions.processingFunc.loc == FunctionOrigin::NONE || functions.alertFormatingFunc.loc == FunctionOrigin::NONE @@ -237,7 +322,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { PyExceptionCheckAndLog("Generator::LoadPythonScript : error importing string " + filename + " : "); return false; } - + if (PyRun_SimpleString("import sys") != 0) { DARWIN_LOG_DEBUG("Generator::LoadPythonScript : An error occurred while loading the 'sys' module"); return false; @@ -253,11 +338,30 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { return false; } + if (PyRun_SimpleString("sys.path.append('/tmp/pydebug/lib/python3.8/site-packages/')") != 0) { + DARWIN_LOG_DEBUG( + "Generator::LoadPythonScript : An error occurred while appending the custom path '" + + folders + + "' to the Python path" + ); + return false; + } + if((pModule = PyImport_Import(pName)) == nullptr) { PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python script module '" + filename + "' : "); return false; } - + + functions.configPyFunc.f.py = PyObject_GetAttrString(pModule, "filter_config"); + if(functions.configPyFunc.f.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_config' method in the python script"); + } else if (! PyCallable_Check(functions.configPyFunc.f.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_config symbol exists but is not callable"); + return false; + } else { + functions.configPyFunc.loc = FunctionOrigin::PYTHON_MODULE; + } + functions.parseBodyFunc.f.py = PyObject_GetAttrString(pModule, "parse_body"); if(functions.parseBodyFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' method in the python script"); @@ -265,7 +369,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : parse_body symbol exists but is not callable"); return false; } else { - functions.parseBodyFunc.loc= FunctionOrigin::PYTHON_MODULE; + functions.parseBodyFunc.loc = FunctionOrigin::PYTHON_MODULE; } functions.preProcessingFunc.f.py = PyObject_GetAttrString(pModule, "filter_pre_process"); @@ -275,7 +379,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_pre_process symbol exists but is not callable"); return false; } else { - functions.preProcessingFunc.loc= FunctionOrigin::PYTHON_MODULE; + functions.preProcessingFunc.loc = FunctionOrigin::PYTHON_MODULE; } functions.processingFunc.f.py = PyObject_GetAttrString(pModule, "filter_process"); @@ -285,7 +389,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_process symbol exists but is not callable"); return false; } else { - functions.processingFunc.loc= FunctionOrigin::PYTHON_MODULE; + functions.processingFunc.loc = FunctionOrigin::PYTHON_MODULE; } functions.alertFormatingFunc.f.py = PyObject_GetAttrString(pModule, "alert_formating"); @@ -295,7 +399,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : alert_formating symbol exists but is not callable"); return false; } else { - functions.alertFormatingFunc.loc= FunctionOrigin::PYTHON_MODULE; + functions.alertFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; } functions.outputFormatingFunc.f.py = PyObject_GetAttrString(pModule, "output_formating"); @@ -305,7 +409,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : output_formating symbol exists but is not callable"); return false; } else { - functions.outputFormatingFunc.loc= FunctionOrigin::PYTHON_MODULE; + functions.outputFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; } functions.responseFormatingFunc.f.py = PyObject_GetAttrString(pModule, "response_formating"); @@ -315,7 +419,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : response_formating symbol exists but is not callable"); return false; } else { - functions.responseFormatingFunc.loc= FunctionOrigin::PYTHON_MODULE; + functions.responseFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; } return true; diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index 20e9991..65b0a00 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -29,7 +29,7 @@ union FunctionUnion{ ~FunctionUnion(){} }; -enum FunctionOrigin { +enum class FunctionOrigin { NONE, PYTHON_MODULE, SHARED_LIBRARY, @@ -58,7 +58,7 @@ struct FunctionHolder{ typedef PyObject*(*process_t)(PyObject*, PyObject*); typedef std::list(*alert_format_t)(PyObject*, PyObject*); typedef DarwinResponse(*resp_format_t)(PyObject*, PyObject*); - + typedef bool(*config_t)(rapidjson::Document const &); FunctionPySo parseBodyFunc; FunctionPySo processingFunc; @@ -67,6 +67,8 @@ struct FunctionHolder{ FunctionPySo alertFormatingFunc; FunctionPySo outputFormatingFunc; FunctionPySo responseFormatingFunc; + FunctionPySo configPyFunc; + FunctionPySo configSoFunc; }; class Generator: public AGenerator { @@ -79,6 +81,7 @@ class Generator: public AGenerator { virtual long GetFilterCode() const; static PythonThread& GetPythonThread(); + static bool PyExceptionCheckAndLog(std::string const& prefix_log, darwin::logger::log_type log_type=darwin::logger::Critical); private: virtual bool LoadConfig(const rapidjson::Document &configuration) override final; @@ -86,8 +89,10 @@ class Generator: public AGenerator { bool LoadPythonScript(const std::string& python_script_path); bool LoadSharedLibrary(const std::string& shared_library_path); - bool CheckConfig(); - + bool CheckConfig() const; + bool SendConfig(rapidjson::Document const& config) const; + bool SendPythonConfig(rapidjson::Document const& config) const; + PyObject* pName; PyObject* pModule; diff --git a/samples/fpython/PythonObject.hpp b/samples/fpython/PythonObject.hpp index ae3e068..1b6c5c1 100644 --- a/samples/fpython/PythonObject.hpp +++ b/samples/fpython/PythonObject.hpp @@ -50,8 +50,13 @@ class PyObjectOwner { private: void Decref(){ if(handle != nullptr){ - Py_DECREF(handle); - handle = nullptr; + if(PyGILState_Check() == 1){ + Py_DECREF(handle); + } else { + PythonLock lock; + Py_DECREF(handle); + } + handle = nullptr; } } PyObject* handle; diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index e672ec0..25f9824 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -11,6 +11,7 @@ #include "PythonTask.hpp" #include "Logger.hpp" #include "AlertManager.hpp" +#include "PythonObject.hpp" PythonTask::PythonTask(std::shared_ptr> cache, std::mutex& cache_mutex, @@ -37,16 +38,15 @@ void PythonTask::operator()() { switch(_functions.preProcessingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ - PythonThread::Use pylock(Generator::GetPythonThread()); + PythonLock pylock; if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.f.py, _parsed_body, nullptr)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Preprocess: "); } break; } case FunctionOrigin::SHARED_LIBRARY:{ if ((preProc_res = _functions.preProcessingFunc.f.so(_pModule, _parsed_body)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO pre-process function"); } break; } @@ -56,28 +56,27 @@ void PythonTask::operator()() { switch(_functions.processingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ - PythonThread::Use pylock(Generator::GetPythonThread()); + PythonLock pylock; if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.f.py, preProc_res, nullptr)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Process: "); } break; } case FunctionOrigin::SHARED_LIBRARY:{ if ((proc_res = _functions.processingFunc.f.so(_pModule, preProc_res)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO process function"); } break; } default: - DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); } std::list alerts = GetFormatedAlerts(_functions.alertFormatingFunc, proc_res); DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, proc_res); DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, proc_res); { - PythonThread::Use pylock(Generator::GetPythonThread()); + PythonLock pylock; Py_DECREF(proc_res); Py_DECREF(preProc_res); } @@ -107,7 +106,7 @@ void PythonTask::operator()() { bool PythonTask::ParseBody() { DARWIN_LOGGER; - PyObject* raw_body = nullptr; + PyObjectOwner raw_body; bool ret = false; DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: helo"); @@ -119,10 +118,9 @@ bool PythonTask::ParseBody() { if(!ret){ return false; } - PythonThread::Use pylock(Generator::GetPythonThread()); + PythonLock pylock; if((_parsed_body = PyList_New(_body.Size())) == nullptr){ - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } @@ -130,30 +128,38 @@ bool PythonTask::ParseBody() { PyObject *str; if((str = PyUnicode_DecodeFSDefault(_body.GetArray()[i].GetString())) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); - PyErr_Print(); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } PyList_SetItem(_parsed_body, i, str); + if(Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :")){ + return false; + } } return ret; } case FunctionOrigin::PYTHON_MODULE:{ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koi"); - PythonThread::Use pylock(Generator::GetPythonThread()); + // This lock is valid because we use only ONE interpreter + // If at some point, we decide to use multiple interpreters + // (one by thread for example) + // We must switch to PythonThread utility class + PythonLock pylock; DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koilol"); - if((raw_body = PyUnicode_DecodeFSDefault(_packet.GetBody().c_str())) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); - PyErr_Print(); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } - if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.f.py, raw_body, nullptr)) == nullptr) { + std::cout << std::endl << "after unicode decode" << std::endl; + + DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koilol yess"); + + if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.f.py, *raw_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); - Py_DECREF(raw_body); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } - Py_DECREF(raw_body); + return true; } case FunctionOrigin::SHARED_LIBRARY:{ @@ -185,14 +191,13 @@ std::list PythonTask::GetFormatedAlerts(FunctionPySo ret; switch(func.loc) { case FunctionOrigin::PYTHON_MODULE:{ - PythonThread::Use pylock(Generator::GetPythonThread()); + PythonLock pylock; if ((pyRes = PyObject_CallFunctionObjArgs(func.f.py, processedData, nullptr)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the Python function"); - if(PyErr_Occurred() != nullptr) PyErr_Print(); + Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: "); return ret; } if(PyList_Check(pyRes) == 0){ - DARWIN_LOG_ERROR("PythonTask::GetFormated : not a list"); + DARWIN_LOG_ERROR("PythonTask::GetFormatedAlerts : not a list"); } pySize = PyList_Size(pyRes); if(PyErr_Occurred() != nullptr) { @@ -224,7 +229,7 @@ DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo -#include "Logger.hpp" + class PythonThread { public: PythonThread(): _init{false}, pThreadState{nullptr} {} @@ -31,14 +31,10 @@ class PythonThread { class Use { public: Use(PythonThread& thread): _thread{thread} { - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("Start thread"); PyEval_RestoreThread(_thread.GetThreadState()); } ~Use(){ - DARWIN_LOGGER; - DARWIN_LOG_DEBUG("End thread"); _thread.SetThreadState(PyEval_SaveThread()); } @@ -51,4 +47,17 @@ class PythonThread { bool _init; PyThreadState * pThreadState; +}; + +class PythonLock { +public: + PythonLock(){ + lock = PyGILState_Ensure(); + } + ~PythonLock(){ + PyGILState_Release(lock); + } +private: + PyGILState_STATE lock; + }; \ No newline at end of file diff --git a/samples/fpython/example.py b/samples/fpython/example.py index 66039b0..5db114d 100644 --- a/samples/fpython/example.py +++ b/samples/fpython/example.py @@ -1,5 +1,5 @@ import json -from typing import List +from typing import Any, List class PythonFilterError(Exception): @@ -15,6 +15,10 @@ def __init__(self) -> None: self.alerts = [] self.response = '' +def filter_config(config: Any) -> bool: + print('filter', config) + return True + def parse_body(input: str) -> list: parsed = json.loads(input) if not isinstance(parsed, list): @@ -25,20 +29,28 @@ def filter_pre_process(input) -> list: return input def filter_process(input: list) -> PythonFilterResponse: + print('py hi') + print(input) resp = PythonFilterResponse() for line in input: - print(line) + print('py line') + # print(line) if isinstance(line, str): resp.certitudes.append(101) resp.alerts.append(line) resp.response += line + + print('py ret') return resp def alert_formating(input: PythonFilterResponse) -> str: - return json.dumps(input.alerts) + print('alert') + return input.alerts def output_formating(input: PythonFilterResponse) -> str: - return json.dumps(input.response) + print('output') + return input def response_formating(input: PythonFilterResponse) -> str: - return json.dumps(input.response) + print('output') + return input From 4446f0d67580fdba744d0d3163eebc606d7f6868 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 21 Sep 2021 16:08:45 +0200 Subject: [PATCH 46/54] Fixed memory problems with python, meory usage is now stable --- samples/fpython/Generator.cpp | 28 ++++---- samples/fpython/Generator.hpp | 10 +-- samples/fpython/PythonObject.hpp | 5 +- samples/fpython/PythonTask.cpp | 112 +++++++++++++------------------ samples/fpython/PythonTask.hpp | 3 +- samples/fpython/fpython.hpp | 2 +- tests/test.py | 4 ++ 7 files changed, 75 insertions(+), 89 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 3623629..66d8161 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -18,7 +18,7 @@ #include "PythonObject.hpp" /// -/// \brief +/// \brief This expects the thread to hold the GIL /// /// \param prefix_log /// \param log_type @@ -56,7 +56,7 @@ bool Generator::PyExceptionCheckAndLog(std::string const& prefix_log, darwin::lo } Generator::Generator(size_t nb_task_threads) - : AGenerator(nb_task_threads), pName{nullptr}, pModule{nullptr} + : AGenerator(nb_task_threads) { } bool Generator::ConfigureAlerting(const std::string& tags) { @@ -115,7 +115,7 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { return pythonLoad && LoadSharedLibrary(shared_library_path) && CheckConfig() && SendConfig(configuration); } -bool Generator::SendConfig(rapidjson::Document const& config) const{ +bool Generator::SendConfig(rapidjson::Document const& config) { DARWIN_LOGGER; bool pyConfigRes = false, soConfigRes = false; switch(functions.configPyFunc.loc) { @@ -147,7 +147,7 @@ bool Generator::SendConfig(rapidjson::Document const& config) const{ return pyConfigRes && soConfigRes; } -bool Generator::SendPythonConfig(rapidjson::Document const& config) const{ +bool Generator::SendPythonConfig(rapidjson::Document const& config) { DARWIN_LOGGER; rapidjson::StringBuffer buffer; buffer.Clear(); @@ -164,7 +164,7 @@ bool Generator::SendPythonConfig(rapidjson::Document const& config) const{ std::string cmd = "json.loads(\"\"\""; cmd += buffer.GetString(); cmd += "\"\"\")"; - PyObject* globalDictionary = PyModule_GetDict(pModule); + PyObject* globalDictionary = PyModule_GetDict(*pModule); PyObjectOwner pConfig = PyRun_StringFlags(cmd.c_str(), Py_eval_input, globalDictionary, globalDictionary, nullptr); if(Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: Converting config to Py Object :")){ @@ -347,12 +347,12 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { return false; } - if((pModule = PyImport_Import(pName)) == nullptr) { + if((pModule = PyImport_Import(*pName)) == nullptr) { PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python script module '" + filename + "' : "); return false; } - functions.configPyFunc.f.py = PyObject_GetAttrString(pModule, "filter_config"); + functions.configPyFunc.f.py = PyObject_GetAttrString(*pModule, "filter_config"); if(functions.configPyFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_config' method in the python script"); } else if (! PyCallable_Check(functions.configPyFunc.f.py)){ @@ -362,7 +362,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { functions.configPyFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.parseBodyFunc.f.py = PyObject_GetAttrString(pModule, "parse_body"); + functions.parseBodyFunc.f.py = PyObject_GetAttrString(*pModule, "parse_body"); if(functions.parseBodyFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' method in the python script"); } else if (! PyCallable_Check(functions.parseBodyFunc.f.py)){ @@ -372,7 +372,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { functions.parseBodyFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.preProcessingFunc.f.py = PyObject_GetAttrString(pModule, "filter_pre_process"); + functions.preProcessingFunc.f.py = PyObject_GetAttrString(*pModule, "filter_pre_process"); if(functions.preProcessingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_pre_process' method in the python script"); } else if (! PyCallable_Check(functions.preProcessingFunc.f.py)){ @@ -382,7 +382,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { functions.preProcessingFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.processingFunc.f.py = PyObject_GetAttrString(pModule, "filter_process"); + functions.processingFunc.f.py = PyObject_GetAttrString(*pModule, "filter_process"); if(functions.processingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_process' method in the python script"); } else if (! PyCallable_Check(functions.processingFunc.f.py)){ @@ -392,7 +392,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { functions.processingFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.alertFormatingFunc.f.py = PyObject_GetAttrString(pModule, "alert_formating"); + functions.alertFormatingFunc.f.py = PyObject_GetAttrString(*pModule, "alert_formating"); if(functions.alertFormatingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' method in the python script"); } else if (! PyCallable_Check(functions.alertFormatingFunc.f.py)){ @@ -402,7 +402,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { functions.alertFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.outputFormatingFunc.f.py = PyObject_GetAttrString(pModule, "output_formating"); + functions.outputFormatingFunc.f.py = PyObject_GetAttrString(*pModule, "output_formating"); if(functions.outputFormatingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' method in the python script"); } else if (! PyCallable_Check(functions.outputFormatingFunc.f.py)){ @@ -412,7 +412,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { functions.outputFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.responseFormatingFunc.f.py = PyObject_GetAttrString(pModule, "response_formating"); + functions.responseFormatingFunc.f.py = PyObject_GetAttrString(*pModule, "response_formating"); if(functions.responseFormatingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' method in the python script"); } else if (! PyCallable_Check(functions.responseFormatingFunc.f.py)){ @@ -429,7 +429,7 @@ std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) noexcept { return std::static_pointer_cast( std::make_shared(_cache, _cache_mutex, s, s->_packet, - pModule, functions) + *pModule, functions) ); } diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index 65b0a00..c5f9185 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -9,6 +9,7 @@ #include +#include "Logger.hpp" #include "../../toolkit/PythonUtils.hpp" #include "ATask.hpp" #include "../../toolkit/RedisManager.hpp" @@ -16,6 +17,7 @@ #include "AGenerator.hpp" #include "fpython.hpp" #include "PythonThread.hpp" +#include "PythonObject.hpp" template union FunctionUnion{ @@ -90,11 +92,11 @@ class Generator: public AGenerator { bool LoadPythonScript(const std::string& python_script_path); bool LoadSharedLibrary(const std::string& shared_library_path); bool CheckConfig() const; - bool SendConfig(rapidjson::Document const& config) const; - bool SendPythonConfig(rapidjson::Document const& config) const; + bool SendConfig(rapidjson::Document const& config); + bool SendPythonConfig(rapidjson::Document const& config); - PyObject* pName; - PyObject* pModule; + PyObjectOwner pName; + PyObjectOwner pModule; FunctionHolder functions; diff --git a/samples/fpython/PythonObject.hpp b/samples/fpython/PythonObject.hpp index 1b6c5c1..941078a 100644 --- a/samples/fpython/PythonObject.hpp +++ b/samples/fpython/PythonObject.hpp @@ -47,9 +47,8 @@ class PyObjectOwner { return *handle; } -private: void Decref(){ - if(handle != nullptr){ + if(handle != nullptr && Py_IsInitialized() != 0){ if(PyGILState_Check() == 1){ Py_DECREF(handle); } else { @@ -59,5 +58,7 @@ class PyObjectOwner { handle = nullptr; } } + +private: PyObject* handle; }; \ No newline at end of file diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index 25f9824..d40d794 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -30,22 +30,20 @@ bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { return true; } - - void PythonTask::operator()() { DARWIN_LOGGER; - PyObject* preProc_res = nullptr, *proc_res = nullptr; + PyObjectOwner preProc_res, proc_res; switch(_functions.preProcessingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ PythonLock pylock; - if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.f.py, _parsed_body, nullptr)) == nullptr) { + if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.f.py, *_parsed_body, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Preprocess: "); } break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((preProc_res = _functions.preProcessingFunc.f.so(_pModule, _parsed_body)) == nullptr) { + if ((preProc_res = _functions.preProcessingFunc.f.so(_pModule, *_parsed_body)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO pre-process function"); } break; @@ -57,13 +55,13 @@ void PythonTask::operator()() { switch(_functions.processingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ PythonLock pylock; - if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.f.py, preProc_res, nullptr)) == nullptr) { + if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.f.py, *preProc_res, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Process: "); } break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((proc_res = _functions.processingFunc.f.so(_pModule, preProc_res)) == nullptr) { + if ((proc_res = _functions.processingFunc.f.so(_pModule, *preProc_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO process function"); } break; @@ -72,14 +70,9 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); } - std::list alerts = GetFormatedAlerts(_functions.alertFormatingFunc, proc_res); - DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, proc_res); - DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, proc_res); - { - PythonLock pylock; - Py_DECREF(proc_res); - Py_DECREF(preProc_res); - } + std::list alerts = GetFormatedAlerts(_functions.alertFormatingFunc, *proc_res); + DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, *proc_res); + DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, *proc_res); for(auto& alert: alerts) { if( ! alert.empty()) { @@ -87,18 +80,15 @@ void PythonTask::operator()() { } } for(auto cert: resp.certitudes) { - DARWIN_LOG_DEBUG("PythonTask:: cert added"); _packet.AddCertitude(cert); } std::string& body = _packet.GetMutableBody(); if( ! output.body.empty()) { - DARWIN_LOG_DEBUG("PythonTask:: output"); body.clear(); body.append(output.body); } if( ! resp.body.empty()){ - DARWIN_LOG_DEBUG("PythonTask:: resp"); body.clear(); body.append(resp.body); } @@ -108,13 +98,9 @@ bool PythonTask::ParseBody() { DARWIN_LOGGER; PyObjectOwner raw_body; bool ret = false; - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: helo"); - switch(_functions.parseBodyFunc.loc){ case FunctionOrigin::NONE:{ - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: None koi " "__LINE__"); ret = ATask::ParseBody(); - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: None koi2"); if(!ret){ return false; } @@ -125,45 +111,39 @@ bool PythonTask::ParseBody() { } for(ssize_t i = 0; i < _body.Size(); i++){ - PyObject *str; + PyObject *str; // This reference is "stolen" by PyList_SetItem, we don't have to DECREF it if((str = PyUnicode_DecodeFSDefault(_body.GetArray()[i].GetString())) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } - PyList_SetItem(_parsed_body, i, str); - if(Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :")){ + if(PyList_SetItem(*_parsed_body, i, str) != 0){ + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } } return ret; } case FunctionOrigin::PYTHON_MODULE:{ - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koi"); // This lock is valid because we use only ONE interpreter // If at some point, we decide to use multiple interpreters // (one by thread for example) // We must switch to PythonThread utility class PythonLock pylock; - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koilol"); if((raw_body = PyUnicode_DecodeFSDefault(_packet.GetBody().c_str())) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } - std::cout << std::endl << "after unicode decode" << std::endl; - - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: koilol yess"); if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.f.py, *raw_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; } - + return true; } case FunctionOrigin::SHARED_LIBRARY:{ - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: SO koi " "__LINE__"); try{ if((_parsed_body = _functions.parseBodyFunc.f.so(_pModule, _packet.GetBody()))== nullptr){ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); @@ -173,7 +153,6 @@ bool PythonTask::ParseBody() { DARWIN_LOG_ERROR("PythonTask:: ParseBody:: Error while calling the SO parse_body function : " + std::string(e.what())); return false; } - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: SO koi 2" "__LINE__"); return true; } @@ -185,31 +164,36 @@ bool PythonTask::ParseBody() { std::list PythonTask::GetFormatedAlerts(FunctionPySo& func, PyObject* processedData){ DARWIN_LOGGER; - PyObject *pyRes = nullptr; ssize_t pySize = 0, strSize=0; const char * out = nullptr; std::list ret; switch(func.loc) { case FunctionOrigin::PYTHON_MODULE:{ PythonLock pylock; + PyObjectOwner pyRes; if ((pyRes = PyObject_CallFunctionObjArgs(func.f.py, processedData, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: "); return ret; } - if(PyList_Check(pyRes) == 0){ - DARWIN_LOG_ERROR("PythonTask::GetFormatedAlerts : not a list"); + if(PyList_Check(*pyRes) == 0){ + DARWIN_LOG_ERROR("PythonTask::GetFormatedAlerts : 'format_alert' did not return a list"); + return ret; } - pySize = PyList_Size(pyRes); - if(PyErr_Occurred() != nullptr) { - PyErr_Print(); + pySize = PyList_Size(*pyRes); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: ")) { return ret; } for(ssize_t i = 0; i < pySize; i++) { - PyObject* str = PyList_GetItem(pyRes, i); - if(PyErr_Occurred() != nullptr) PyErr_Print(); + // str is a borrowed reference, we don't have to DECREF it + PyObject* str = PyList_GetItem(*pyRes, i); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: ")){ + continue; + } out = PyUnicode_AsUTF8AndSize(str, &strSize); - if(PyErr_Occurred() != nullptr) PyErr_Print(); + if(Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: ")){ + continue; + } ret.push_back(std::string(out, strSize)); } return ret; @@ -228,64 +212,60 @@ DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo UINT_MAX) { - DARWIN_LOG_DEBUG("PythonTask:: out of range certitude"); + DARWIN_LOG_DEBUG("PythonTask:: GetFormatedResponse:: out of range certitude"); continue; } ret.certitudes.push_back(static_cast(lCert)); } - DARWIN_LOG_DEBUG("PythonTask::GetFormatedResponse regturn"); return ret; } case FunctionOrigin::SHARED_LIBRARY: diff --git a/samples/fpython/PythonTask.hpp b/samples/fpython/PythonTask.hpp index dfa9557..a985381 100644 --- a/samples/fpython/PythonTask.hpp +++ b/samples/fpython/PythonTask.hpp @@ -58,6 +58,5 @@ class PythonTask : public darwin::ATask { PyObject* _pModule; FunctionHolder& _functions; - PyObject* _result; - PyObject* _parsed_body; + PyObjectOwner _parsed_body; }; diff --git a/samples/fpython/fpython.hpp b/samples/fpython/fpython.hpp index 00276b0..3ca9090 100644 --- a/samples/fpython/fpython.hpp +++ b/samples/fpython/fpython.hpp @@ -9,7 +9,7 @@ struct DarwinResponse{ }; extern "C" { - + bool filter_config(rapidjson::Document const&); PyObject* parse_body(PyObject* module, const std::string&); PyObject* filter_pre_process(PyObject* module, PyObject*); PyObject* filter_process(PyObject* module, PyObject*); diff --git a/tests/test.py b/tests/test.py index b6e984f..6f7ecd0 100644 --- a/tests/test.py +++ b/tests/test.py @@ -16,6 +16,7 @@ socket_arg_group.add_argument("--unix", action="store_true") parser.add_argument("ip_address", nargs='?', help="(For tcp and udp only) The IP address formatted like this: ipv4: 'x.x.x.x:port', ipv6: '[x:x:x:x:x:x:x]:port'") +parser.add_argument("--valgrind", action='store_true', help="Run valgrind on the filters") args = parser.parse_args() if args.tcp: @@ -28,6 +29,9 @@ if args.ip_address != None: conf.DEFAULT_ADDRESS = args.ip_address +if args.valgrind: + conf.VALGRIND_MEMCHECK = True + if __name__ == "__main__": logging.basicConfig(filename="test_error.log", filemode='w', level=logging.ERROR) From 71c133df8b43e57b445c0c114cc436300c990d5e Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 6 Oct 2021 09:53:48 +0200 Subject: [PATCH 47/54] Added configuration and contextualization phases --- samples/fpython/Generator.cpp | 56 +++++++++++++--- samples/fpython/Generator.hpp | 72 ++++++++++++++++++-- samples/fpython/PythonObject.hpp | 6 ++ samples/fpython/PythonTask.cpp | 109 ++++++++++++++++++++++++++----- samples/fpython/PythonTask.hpp | 2 +- samples/fpython/example.py | 63 ++++++++++-------- samples/fpython/fpython.hpp | 34 ++++++++-- 7 files changed, 277 insertions(+), 65 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 66d8161..9d9008d 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -140,7 +140,12 @@ bool Generator::SendConfig(rapidjson::Document const& config) { DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: Incoherent configuration"); return false; case FunctionOrigin::SHARED_LIBRARY: - soConfigRes = functions.configSoFunc.f.so(config); + try{ + soConfigRes = functions.configSoFunc.f.so(config); + } catch(std::exception const& e){ + DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: error while sending config :" + std::string(e.what())); + soConfigRes = false; + } break; } @@ -239,6 +244,22 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { functions.processingFunc.f.so = proc; } + FunctionHolder::process_t context = (FunctionHolder::process_t)dlsym(handle, "filter_contextualize"); + if(context == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_contextualize' function in the shared library"); + } else { + functions.processingFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.processingFunc.f.so = context; + } + + FunctionHolder::process_t alert_cont = (FunctionHolder::process_t)dlsym(handle, "alert_contextualize"); + if(alert_cont == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_contextualize' function in the shared library"); + } else { + functions.processingFunc.loc = FunctionOrigin::SHARED_LIBRARY; + functions.processingFunc.f.so = alert_cont; + } + FunctionHolder::alert_format_t alert_format = (FunctionHolder::alert_format_t)dlsym(handle, "alert_formating"); if(alert_format == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' function in the shared library"); @@ -269,6 +290,8 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { bool Generator::CheckConfig() const { if(functions.preProcessingFunc.loc == FunctionOrigin::NONE || functions.processingFunc.loc == FunctionOrigin::NONE + || functions.contextualizeFunc.loc == FunctionOrigin::NONE + || functions.alertContextualizeFunc.loc == FunctionOrigin::NONE || functions.alertFormatingFunc.loc == FunctionOrigin::NONE || functions.outputFormatingFunc.loc == FunctionOrigin::NONE || functions.responseFormatingFunc.loc == FunctionOrigin::NONE) @@ -318,6 +341,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { if(PyExceptionCheckAndLog("Generator::LoadPythonScript : Error during python init :")) { return false; } + PyObjectOwner pName; if((pName = PyUnicode_DecodeFSDefault(filename.c_str())) == nullptr){ PyExceptionCheckAndLog("Generator::LoadPythonScript : error importing string " + filename + " : "); return false; @@ -338,19 +362,11 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { return false; } - if (PyRun_SimpleString("sys.path.append('/tmp/pydebug/lib/python3.8/site-packages/')") != 0) { - DARWIN_LOG_DEBUG( - "Generator::LoadPythonScript : An error occurred while appending the custom path '" + - folders + - "' to the Python path" - ); - return false; - } - if((pModule = PyImport_Import(*pName)) == nullptr) { PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python script module '" + filename + "' : "); return false; } + pName.Decref(); functions.configPyFunc.f.py = PyObject_GetAttrString(*pModule, "filter_config"); if(functions.configPyFunc.f.py == nullptr) { @@ -392,6 +408,26 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { functions.processingFunc.loc = FunctionOrigin::PYTHON_MODULE; } + functions.contextualizeFunc.f.py = PyObject_GetAttrString(*pModule, "filter_contextualize"); + if(functions.contextualizeFunc.f.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_contextualize' method in the python script"); + } else if (! PyCallable_Check(functions.contextualizeFunc.f.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_contextualize symbol exists but is not callable"); + return false; + } else { + functions.contextualizeFunc.loc = FunctionOrigin::PYTHON_MODULE; + } + + functions.alertContextualizeFunc.f.py = PyObject_GetAttrString(*pModule, "alert_contextualize"); + if(functions.alertContextualizeFunc.f.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_contextualize' method in the python script"); + } else if (! PyCallable_Check(functions.alertContextualizeFunc.f.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : alert_contextualize symbol exists but is not callable"); + return false; + } else { + functions.alertContextualizeFunc.loc = FunctionOrigin::PYTHON_MODULE; + } + functions.alertFormatingFunc.f.py = PyObject_GetAttrString(*pModule, "alert_formating"); if(functions.alertFormatingFunc.f.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' method in the python script"); diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index c5f9185..9f241a9 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -19,6 +19,11 @@ #include "PythonThread.hpp" #include "PythonObject.hpp" +/// +/// \brief Unions representing either a function pointer from a shared object or a python function reference +/// +/// \tparam F the prototype of the function (if it comes from a shared object) +/// template union FunctionUnion{ PyObject* py; @@ -37,6 +42,11 @@ enum class FunctionOrigin { SHARED_LIBRARY, }; +/// +/// \brief tagged union containing a FunctionUnion and its tag FunctionOrigin +/// +/// \tparam F prototype of the function if loaded from a shared object +/// template struct FunctionPySo { FunctionOrigin loc; @@ -46,7 +56,11 @@ struct FunctionPySo { ~FunctionPySo() = default; }; - +/// +/// \brief Struct holding the function pointers for the python filter +/// It should be unique and passed by reference +/// +/// struct FunctionHolder{ FunctionHolder() = default; ~FunctionHolder() = default; @@ -56,21 +70,28 @@ struct FunctionHolder{ FunctionHolder& operator=(FunctionHolder const &) = delete; FunctionHolder& operator=(FunctionHolder &&) = delete; + typedef bool(*config_t)(rapidjson::Document const &); typedef PyObject*(*parse_body_t)(PyObject*, const std::string&); typedef PyObject*(*process_t)(PyObject*, PyObject*); - typedef std::list(*alert_format_t)(PyObject*, PyObject*); + typedef std::vector(*alert_format_t)(PyObject*, PyObject*); typedef DarwinResponse(*resp_format_t)(PyObject*, PyObject*); - typedef bool(*config_t)(rapidjson::Document const &); + + // There are 2 config functions as we accept to pass informations to both the python module and the shared library + // Note that the shared library has a direct access to the python module + FunctionPySo configPyFunc; + FunctionPySo configSoFunc; + FunctionPySo parseBodyFunc; FunctionPySo processingFunc; FunctionPySo preProcessingFunc; + FunctionPySo contextualizeFunc; + FunctionPySo alertContextualizeFunc; + FunctionPySo alertFormatingFunc; FunctionPySo outputFormatingFunc; FunctionPySo responseFormatingFunc; - FunctionPySo configPyFunc; - FunctionPySo configSoFunc; }; class Generator: public AGenerator { @@ -89,13 +110,52 @@ class Generator: public AGenerator { virtual bool LoadConfig(const rapidjson::Document &configuration) override final; virtual bool ConfigureAlerting(const std::string& tags) override final; + /// + /// \brief loads the python script which path is given and attempts to load all defined functions + /// If the path is not empty, a python interpreter will be initialized (by a call to Py_Initiliaze()) + /// + /// \param python_script_path + /// \return true if the script was correctly loaded or if the path is empty + /// \return false if the path can't be read, if loading the module triggered an error + /// or if a symbol with the name of a defined function is found but not callable + /// bool LoadPythonScript(const std::string& python_script_path); + + /// + /// \brief loads the shared library which path is given and attempts to load all defined functions + /// + /// \param shared_library_path + /// \return true if the SO was correctly loaded or if the path is empty + /// \return false if the shared library can't be read or if loading the SO triggered an error + /// bool LoadSharedLibrary(const std::string& shared_library_path); + + /// + /// \brief Verifies that the configuration if valid + /// filter_preprocess, filter_process, alert_formating, + /// response_formating and output_formating must be set + /// parse_body and filter_config are optional + /// + /// \return true the configuration is valid + /// bool CheckConfig() const; + + /// + /// \brief Dispatches the configuration to the Shared object (if any) and the python module (if any) + /// + /// \param config parsed configuration + /// \return true the module and/or the SO sent back true + /// bool SendConfig(rapidjson::Document const& config); + + /// + /// \brief Sends the configuration to the Python module + /// + /// \param config + /// \return false an error occured whil calling the python function or it sent back false + /// bool SendPythonConfig(rapidjson::Document const& config); - PyObjectOwner pName; PyObjectOwner pModule; FunctionHolder functions; diff --git a/samples/fpython/PythonObject.hpp b/samples/fpython/PythonObject.hpp index 941078a..283df2c 100644 --- a/samples/fpython/PythonObject.hpp +++ b/samples/fpython/PythonObject.hpp @@ -1,6 +1,12 @@ #pragma once #include +/// +/// \brief Helper class that takes the ownership of a PyObject pointer +/// when detroyed, if it still holds a reference, it will attempt to DECREF it to the interpreter +/// Calls to destructor or Move operator or Decref will query the GIL if it does not have it. +/// These methods are always safe to call +/// class PyObjectOwner { public: // no copy, move ok diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index d40d794..8749cfa 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -32,19 +32,26 @@ bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { void PythonTask::operator()() { DARWIN_LOGGER; - PyObjectOwner preProc_res, proc_res; + PyObjectOwner preProc_res, proc_res, context_res, alert_context_res; switch(_functions.preProcessingFunc.loc) { case FunctionOrigin::PYTHON_MODULE:{ PythonLock pylock; if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.f.py, *_parsed_body, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Preprocess: "); + return; } break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((preProc_res = _functions.preProcessingFunc.f.so(_pModule, *_parsed_body)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO pre-process function"); + try { + if ((preProc_res = _functions.preProcessingFunc.f.so(_pModule, *_parsed_body)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_pre_process function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO filter_pre_process function : " + std::string(e.what())); + return; } break; } @@ -57,12 +64,44 @@ void PythonTask::operator()() { PythonLock pylock; if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.f.py, *preProc_res, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Process: "); + return; + } + break; + } + case FunctionOrigin::SHARED_LIBRARY:{ + try{ + if ((proc_res = _functions.processingFunc.f.so(_pModule, *preProc_res)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_process function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO filter_process function : " + std::string(e.what())); + return; + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); + } + + switch(_functions.contextualizeFunc.loc) { + case FunctionOrigin::PYTHON_MODULE:{ + PythonLock pylock; + if ((context_res = PyObject_CallFunctionObjArgs(_functions.contextualizeFunc.f.py, *proc_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Contextualize: "); + return; } break; } case FunctionOrigin::SHARED_LIBRARY:{ - if ((proc_res = _functions.processingFunc.f.so(_pModule, *preProc_res)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO process function"); + try{ + if ((context_res = _functions.contextualizeFunc.f.so(_pModule, *proc_res)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_contextualize function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO filter_contextualize function : " + std::string(e.what())); + return; } break; } @@ -70,7 +109,33 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); } - std::list alerts = GetFormatedAlerts(_functions.alertFormatingFunc, *proc_res); + switch(_functions.alertContextualizeFunc.loc) { + case FunctionOrigin::PYTHON_MODULE:{ + PythonLock pylock; + if ((alert_context_res = PyObject_CallFunctionObjArgs(_functions.alertContextualizeFunc.f.py, *context_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Alert Contextualize: "); + return; + } + break; + } + case FunctionOrigin::SHARED_LIBRARY:{ + try{ + if ((alert_context_res = _functions.alertContextualizeFunc.f.so(_pModule, *context_res)) == nullptr) { + DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO alert_contextualize function"); + return; + } + } catch(std::exception const& e){ + DARWIN_LOG_ERROR("PythonTask:: Error while calling the SO alert_contextualize function : " + std::string(e.what())); + return; + } + break; + } + default: + DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); + } + + + std::vector alerts = GetFormatedAlerts(_functions.alertFormatingFunc, *alert_context_res); DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, *proc_res); DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, *proc_res); @@ -162,11 +227,11 @@ bool PythonTask::ParseBody() { } -std::list PythonTask::GetFormatedAlerts(FunctionPySo& func, PyObject* processedData){ +std::vector PythonTask::GetFormatedAlerts(FunctionPySo& func, PyObject* processedData){ DARWIN_LOGGER; ssize_t pySize = 0, strSize=0; const char * out = nullptr; - std::list ret; + std::vector ret; switch(func.loc) { case FunctionOrigin::PYTHON_MODULE:{ PythonLock pylock; @@ -198,8 +263,15 @@ std::list PythonTask::GetFormatedAlerts(FunctionPySo GetFormatedAlerts(FunctionPySo& func, PyObject* processedData); + std::vector GetFormatedAlerts(FunctionPySo& func, PyObject* processedData); DarwinResponse GetFormatedResponse(FunctionPySo& func, PyObject* processedData); private: diff --git a/samples/fpython/example.py b/samples/fpython/example.py index 5db114d..cf1c23d 100644 --- a/samples/fpython/example.py +++ b/samples/fpython/example.py @@ -1,56 +1,65 @@ import json -from typing import Any, List - +from typing import List, Union +from dataclasses import dataclass class PythonFilterError(Exception): pass +@dataclass class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ body: str certitudes: List[int] alerts: List[str] - def __init__(self) -> None: - self.body = '' - self.certitudes = [] - self.alerts = [] - self.response = '' -def filter_config(config: Any) -> bool: +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +def filter_config(config: dict) -> bool: print('filter', config) return True -def parse_body(input: str) -> list: - parsed = json.loads(input) +def parse_body(body: str) -> Union[list, CustomData]: + parsed = json.loads(body) if not isinstance(parsed, list): - raise PythonFilterError("Wrong type bro") + raise PythonFilterError("Parse Body: Wrong type") return parsed -def filter_pre_process(input) -> list: - return input +def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + return parsed_data -def filter_process(input: list) -> PythonFilterResponse: - print('py hi') - print(input) - resp = PythonFilterResponse() - for line in input: - print('py line') - # print(line) +def filter_process(pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: if isinstance(line, str): resp.certitudes.append(101) resp.alerts.append(line) - resp.response += line + resp.body += line print('py ret') return resp -def alert_formating(input: PythonFilterResponse) -> str: +def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + return processed_data + +def alert_contextualization(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + return contextualized_data + +def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: print('alert') - return input.alerts + return contextualized_data.alerts -def output_formating(input: PythonFilterResponse) -> str: +def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: print('output') - return input + return contextualized_data -def response_formating(input: PythonFilterResponse) -> str: +def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: print('output') - return input + return contextualized_data diff --git a/samples/fpython/fpython.hpp b/samples/fpython/fpython.hpp index 3ca9090..1c9b209 100644 --- a/samples/fpython/fpython.hpp +++ b/samples/fpython/fpython.hpp @@ -1,20 +1,42 @@ #pragma once #include -#include -#include +#include "../../toolkit/rapidjson/document.h" +#ifdef __cplusplus +#include +#include struct DarwinResponse{ - std::list certitudes; + std::vector certitudes; std::string body; }; +#endif +struct DarwinResponseC{ + unsigned int *certitudes; + size_t certitudes_size; + char* body; + size_t body_size; +}; extern "C" { + // C++ linkage +#ifdef __cplusplus bool filter_config(rapidjson::Document const&); PyObject* parse_body(PyObject* module, const std::string&); - PyObject* filter_pre_process(PyObject* module, PyObject*); - PyObject* filter_process(PyObject* module, PyObject*); - std::list alert_formating(PyObject* module, PyObject*input); + + std::vector alert_formating(PyObject* module, PyObject*input); DarwinResponse output_formating(PyObject* module, PyObject*); DarwinResponse response_formating(PyObject* module, PyObject*); +#endif + // C compatible linkage + PyObject* filter_pre_process(PyObject* module, PyObject*); + PyObject* filter_process(PyObject* module, PyObject*); + PyObject* filter_contextualize(PyObject* module, PyObject*); + PyObject* alert_contextualize(PyObject* module, PyObject*); + + bool filter_config_c(const char*, const size_t); + PyObject* parse_body_c(PyObject* module, const char*, const size_t); + void alert_formating_c(PyObject* module, PyObject* input, char ** alerts, size_t nb_alerts, size_t* alerts_sizes); + DarwinResponseC* output_formating_c(PyObject* module, PyObject*); + DarwinResponseC* response_formating_c(PyObject* module, PyObject*); } \ No newline at end of file From ad2d7fd6180eedf3a47a0ea1a8d4c309bfb91b1a Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Wed, 6 Oct 2021 11:29:41 +0200 Subject: [PATCH 48/54] Refactored FunctionPySo and functions loading methods --- samples/fpython/Generator.cpp | 220 ++++++++++----------------------- samples/fpython/Generator.hpp | 51 ++++---- samples/fpython/PythonTask.cpp | 70 +++++------ 3 files changed, 129 insertions(+), 212 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 9d9008d..2ca7e4e 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -118,30 +118,30 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { bool Generator::SendConfig(rapidjson::Document const& config) { DARWIN_LOGGER; bool pyConfigRes = false, soConfigRes = false; - switch(functions.configPyFunc.loc) { - case FunctionOrigin::NONE: + switch(functions.configPyFunc.origin) { + case FunctionOrigin::none: DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in python script"); pyConfigRes = true; break; - case FunctionOrigin::PYTHON_MODULE: + case FunctionOrigin::python_module: pyConfigRes = this->SendPythonConfig(config); break; - case FunctionOrigin::SHARED_LIBRARY: + case FunctionOrigin::shared_library: DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: Incoherent configuration"); return false; } - switch(functions.configSoFunc.loc) { - case FunctionOrigin::NONE: + switch(functions.configSoFunc.origin) { + case FunctionOrigin::none: DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in shared object"); soConfigRes = true; break; - case FunctionOrigin::PYTHON_MODULE: + case FunctionOrigin::python_module: DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: Incoherent configuration"); return false; - case FunctionOrigin::SHARED_LIBRARY: + case FunctionOrigin::shared_library: try{ - soConfigRes = functions.configSoFunc.f.so(config); + soConfigRes = functions.configSoFunc.so(config); } catch(std::exception const& e){ DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: error while sending config :" + std::string(e.what())); soConfigRes = false; @@ -176,7 +176,7 @@ bool Generator::SendPythonConfig(rapidjson::Document const& config) { return false; } PyObjectOwner res; - if ((res = PyObject_CallFunctionObjArgs(functions.configPyFunc.f.py, *pConfig, nullptr)) == nullptr) { + if ((res = PyObject_CallFunctionObjArgs(functions.configPyFunc.py, *pConfig, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: "); return false; } @@ -210,91 +210,38 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { return false; } - FunctionHolder::config_t conf = (FunctionHolder::config_t)dlsym(handle, "filter_config"); + LoadFunctionFromSO(handle, functions.configSoFunc, "filter_config"); + LoadFunctionFromSO(handle, functions.parseBodyFunc, "parse_body"); + LoadFunctionFromSO(handle, functions.preProcessingFunc, "filter_pre_process"); + LoadFunctionFromSO(handle, functions.processingFunc, "filter_process"); + LoadFunctionFromSO(handle, functions.contextualizeFunc, "filter_contextualize"); + LoadFunctionFromSO(handle, functions.alertContextualizeFunc, "alert_contextualize"); + LoadFunctionFromSO(handle, functions.alertFormatingFunc, "alert_formating"); + LoadFunctionFromSO(handle, functions.outputFormatingFunc, "output_formating"); + LoadFunctionFromSO(handle, functions.responseFormatingFunc, "response_formating"); - if(conf == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_config' function in the shared library"); - } else { - functions.configSoFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.configSoFunc.f.so = conf; - } - - FunctionHolder::parse_body_t parse = (FunctionHolder::parse_body_t)dlsym(handle, "parse_body"); - - if(parse == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' function in the shared library"); - } else { - functions.parseBodyFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.parseBodyFunc.f.so = parse; - } - - FunctionHolder::process_t preProc = (FunctionHolder::process_t)dlsym(handle, "filter_pre_process"); - if(preProc == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_pre_process' function in the shared library"); - } else { - functions.preProcessingFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.preProcessingFunc.f.so = preProc; - } - - FunctionHolder::process_t proc = (FunctionHolder::process_t)dlsym(handle, "filter_process"); - if(proc == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_process' function in the shared library"); - } else { - functions.processingFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.processingFunc.f.so = proc; - } - - FunctionHolder::process_t context = (FunctionHolder::process_t)dlsym(handle, "filter_contextualize"); - if(context == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_contextualize' function in the shared library"); - } else { - functions.processingFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.processingFunc.f.so = context; - } - - FunctionHolder::process_t alert_cont = (FunctionHolder::process_t)dlsym(handle, "alert_contextualize"); - if(alert_cont == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_contextualize' function in the shared library"); - } else { - functions.processingFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.processingFunc.f.so = alert_cont; - } - - FunctionHolder::alert_format_t alert_format = (FunctionHolder::alert_format_t)dlsym(handle, "alert_formating"); - if(alert_format == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' function in the shared library"); - } else { - functions.alertFormatingFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.alertFormatingFunc.f.so = alert_format; - } - - FunctionHolder::resp_format_t output_format = (FunctionHolder::resp_format_t)dlsym(handle, "output_formating"); - if(parse == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' function in the shared library"); - } else { - functions.outputFormatingFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.outputFormatingFunc.f.so = output_format; - } - - FunctionHolder::resp_format_t resp_format = (FunctionHolder::resp_format_t)dlsym(handle, "response_formating"); - if(resp_format == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' function in the shared library"); + return true; +} +template +inline void Generator::LoadFunctionFromSO(void * lib_handle, FunctionPySo &function_holder, const std::string& function_name) { + DARWIN_LOGGER; + F func = (F)dlsym(lib_handle, function_name.c_str()); + if(func == nullptr){ + DARWIN_LOG_INFO("Generator::LoadPythonScript : No '" + function_name + "' function in the shared library"); } else { - functions.responseFormatingFunc.loc = FunctionOrigin::SHARED_LIBRARY; - functions.responseFormatingFunc.f.so = resp_format; + function_holder.origin = FunctionOrigin::shared_library; + function_holder.so = func; } - - return true; } bool Generator::CheckConfig() const { - if(functions.preProcessingFunc.loc == FunctionOrigin::NONE - || functions.processingFunc.loc == FunctionOrigin::NONE - || functions.contextualizeFunc.loc == FunctionOrigin::NONE - || functions.alertContextualizeFunc.loc == FunctionOrigin::NONE - || functions.alertFormatingFunc.loc == FunctionOrigin::NONE - || functions.outputFormatingFunc.loc == FunctionOrigin::NONE - || functions.responseFormatingFunc.loc == FunctionOrigin::NONE) + if(functions.preProcessingFunc.origin == FunctionOrigin::none + || functions.processingFunc.origin == FunctionOrigin::none + || functions.contextualizeFunc.origin == FunctionOrigin::none + || functions.alertContextualizeFunc.origin == FunctionOrigin::none + || functions.alertFormatingFunc.origin == FunctionOrigin::none + || functions.outputFormatingFunc.origin == FunctionOrigin::none + || functions.responseFormatingFunc.origin == FunctionOrigin::none) { DARWIN_LOGGER; DARWIN_LOG_CRITICAL("Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library"); @@ -368,96 +315,65 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { } pName.Decref(); - functions.configPyFunc.f.py = PyObject_GetAttrString(*pModule, "filter_config"); - if(functions.configPyFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_config' method in the python script"); - } else if (! PyCallable_Check(functions.configPyFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_config symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.configPyFunc, "filter_config")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_config' : "); return false; - } else { - functions.configPyFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.parseBodyFunc.f.py = PyObject_GetAttrString(*pModule, "parse_body"); - if(functions.parseBodyFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'parse_body' method in the python script"); - } else if (! PyCallable_Check(functions.parseBodyFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : parse_body symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.parseBodyFunc, "parse_body")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'parse_body' : "); return false; - } else { - functions.parseBodyFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.preProcessingFunc.f.py = PyObject_GetAttrString(*pModule, "filter_pre_process"); - if(functions.preProcessingFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_pre_process' method in the python script"); - } else if (! PyCallable_Check(functions.preProcessingFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_pre_process symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.preProcessingFunc, "filter_pre_process")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_pre_process' : "); + return false; + } + if( ! LoadFunctionFromPython(*pModule, functions.processingFunc, "filter_process")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_process' : "); return false; - } else { - functions.preProcessingFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.processingFunc.f.py = PyObject_GetAttrString(*pModule, "filter_process"); - if(functions.processingFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_process' method in the python script"); - } else if (! PyCallable_Check(functions.processingFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_process symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.contextualizeFunc, "filter_contextualize")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_contextualize' : "); return false; - } else { - functions.processingFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.contextualizeFunc.f.py = PyObject_GetAttrString(*pModule, "filter_contextualize"); - if(functions.contextualizeFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'filter_contextualize' method in the python script"); - } else if (! PyCallable_Check(functions.contextualizeFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : filter_contextualize symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.alertContextualizeFunc, "alert_contextualize")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_contextualize' : "); return false; - } else { - functions.contextualizeFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.alertContextualizeFunc.f.py = PyObject_GetAttrString(*pModule, "alert_contextualize"); - if(functions.alertContextualizeFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_contextualize' method in the python script"); - } else if (! PyCallable_Check(functions.alertContextualizeFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : alert_contextualize symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.alertFormatingFunc, "alert_formating")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_formating' : "); return false; - } else { - functions.alertContextualizeFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.alertFormatingFunc.f.py = PyObject_GetAttrString(*pModule, "alert_formating"); - if(functions.alertFormatingFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'alert_formating' method in the python script"); - } else if (! PyCallable_Check(functions.alertFormatingFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : alert_formating symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.outputFormatingFunc, "output_formating")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'output_formating' : "); return false; - } else { - functions.alertFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; } - functions.outputFormatingFunc.f.py = PyObject_GetAttrString(*pModule, "output_formating"); - if(functions.outputFormatingFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'output_formating' method in the python script"); - } else if (! PyCallable_Check(functions.outputFormatingFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : output_formating symbol exists but is not callable"); + if( ! LoadFunctionFromPython(*pModule, functions.responseFormatingFunc, "response_formating")){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'response_formating' : "); return false; - } else { - functions.outputFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; } + + return true; +} - functions.responseFormatingFunc.f.py = PyObject_GetAttrString(*pModule, "response_formating"); - if(functions.responseFormatingFunc.f.py == nullptr) { - DARWIN_LOG_INFO("Generator::LoadPythonScript : No 'response_formating' method in the python script"); - } else if (! PyCallable_Check(functions.responseFormatingFunc.f.py)){ - DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : response_formating symbol exists but is not callable"); +template +inline bool Generator::LoadFunctionFromPython(PyObject* pModule, FunctionPySo& function_holder, const std::string& function_name){ + DARWIN_LOGGER; + function_holder.py = PyObject_GetAttrString(pModule, function_name.c_str()); + if(function_holder.py == nullptr) { + DARWIN_LOG_INFO("Generator::LoadPythonScript : No '" + function_name + "' method in the python script"); + } else if (! PyCallable_Check(function_holder.py)){ + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : '" + function_name + "' symbol exists but is not callable"); return false; } else { - functions.responseFormatingFunc.loc = FunctionOrigin::PYTHON_MODULE; + function_holder.origin = FunctionOrigin::python_module; } - return true; } diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index 9f241a9..a35296b 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -20,40 +20,36 @@ #include "PythonObject.hpp" /// -/// \brief Unions representing either a function pointer from a shared object or a python function reference -/// -/// \tparam F the prototype of the function (if it comes from a shared object) +/// \brief Tag for the FunctionPySo struct +/// It is declared outside the struct to stay unaffected by the template grammar /// -template -union FunctionUnion{ - PyObject* py; - F so; - - FunctionUnion() { - this->py = nullptr; - } - - ~FunctionUnion(){} -}; - enum class FunctionOrigin { - NONE, - PYTHON_MODULE, - SHARED_LIBRARY, + none, + python_module, + shared_library, }; /// -/// \brief tagged union containing a FunctionUnion and its tag FunctionOrigin +/// \brief tagged union containing either a function pointer from a shared object or a python function reference +/// along with a tag FunctionOrigin /// /// \tparam F prototype of the function if loaded from a shared object /// template struct FunctionPySo { - FunctionOrigin loc; - FunctionUnion f; - - FunctionPySo(): loc{FunctionOrigin::NONE} { } - ~FunctionPySo() = default; +public: + FunctionOrigin origin; + union { + PyObject* py; + F so; + }; + FunctionPySo(): origin{FunctionOrigin::none}, py {nullptr} { } + ~FunctionPySo() { + if(origin == FunctionOrigin::python_module){ + //We use PyObjectOwner to decrement the reference counter of the function + PyObjectOwner _{py}; + } + }; }; /// @@ -156,8 +152,13 @@ class Generator: public AGenerator { /// bool SendPythonConfig(rapidjson::Document const& config); - PyObjectOwner pModule; + template + inline void LoadFunctionFromSO(void* lib_handle, FunctionPySo& function_holder, const std::string& function_name); + template + inline bool LoadFunctionFromPython(PyObject* pModule, FunctionPySo& function_holder, const std::string& function_name); + + PyObjectOwner pModule; FunctionHolder functions; }; \ No newline at end of file diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index 8749cfa..8adcb66 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -34,18 +34,18 @@ void PythonTask::operator()() { DARWIN_LOGGER; PyObjectOwner preProc_res, proc_res, context_res, alert_context_res; - switch(_functions.preProcessingFunc.loc) { - case FunctionOrigin::PYTHON_MODULE:{ + switch(_functions.preProcessingFunc.origin) { + case FunctionOrigin::python_module:{ PythonLock pylock; - if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.f.py, *_parsed_body, nullptr)) == nullptr) { + if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, *_parsed_body, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Preprocess: "); return; } break; } - case FunctionOrigin::SHARED_LIBRARY:{ + case FunctionOrigin::shared_library:{ try { - if ((preProc_res = _functions.preProcessingFunc.f.so(_pModule, *_parsed_body)) == nullptr) { + if ((preProc_res = _functions.preProcessingFunc.so(_pModule, *_parsed_body)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_pre_process function"); return; } @@ -59,18 +59,18 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for preProcessing Origin"); } - switch(_functions.processingFunc.loc) { - case FunctionOrigin::PYTHON_MODULE:{ + switch(_functions.processingFunc.origin) { + case FunctionOrigin::python_module:{ PythonLock pylock; - if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.f.py, *preProc_res, nullptr)) == nullptr) { + if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, *preProc_res, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Process: "); return; } break; } - case FunctionOrigin::SHARED_LIBRARY:{ + case FunctionOrigin::shared_library:{ try{ - if ((proc_res = _functions.processingFunc.f.so(_pModule, *preProc_res)) == nullptr) { + if ((proc_res = _functions.processingFunc.so(_pModule, *preProc_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_process function"); return; } @@ -84,18 +84,18 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); } - switch(_functions.contextualizeFunc.loc) { - case FunctionOrigin::PYTHON_MODULE:{ + switch(_functions.contextualizeFunc.origin) { + case FunctionOrigin::python_module:{ PythonLock pylock; - if ((context_res = PyObject_CallFunctionObjArgs(_functions.contextualizeFunc.f.py, *proc_res, nullptr)) == nullptr) { + if ((context_res = PyObject_CallFunctionObjArgs(_functions.contextualizeFunc.py, *proc_res, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Contextualize: "); return; } break; } - case FunctionOrigin::SHARED_LIBRARY:{ + case FunctionOrigin::shared_library:{ try{ - if ((context_res = _functions.contextualizeFunc.f.so(_pModule, *proc_res)) == nullptr) { + if ((context_res = _functions.contextualizeFunc.so(_pModule, *proc_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_contextualize function"); return; } @@ -109,18 +109,18 @@ void PythonTask::operator()() { DARWIN_LOG_CRITICAL("PythonTask:: Corrupted state for processing Origin"); } - switch(_functions.alertContextualizeFunc.loc) { - case FunctionOrigin::PYTHON_MODULE:{ + switch(_functions.alertContextualizeFunc.origin) { + case FunctionOrigin::python_module:{ PythonLock pylock; - if ((alert_context_res = PyObject_CallFunctionObjArgs(_functions.alertContextualizeFunc.f.py, *context_res, nullptr)) == nullptr) { + if ((alert_context_res = PyObject_CallFunctionObjArgs(_functions.alertContextualizeFunc.py, *context_res, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Alert Contextualize: "); return; } break; } - case FunctionOrigin::SHARED_LIBRARY:{ + case FunctionOrigin::shared_library:{ try{ - if ((alert_context_res = _functions.alertContextualizeFunc.f.so(_pModule, *context_res)) == nullptr) { + if ((alert_context_res = _functions.alertContextualizeFunc.so(_pModule, *context_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO alert_contextualize function"); return; } @@ -163,8 +163,8 @@ bool PythonTask::ParseBody() { DARWIN_LOGGER; PyObjectOwner raw_body; bool ret = false; - switch(_functions.parseBodyFunc.loc){ - case FunctionOrigin::NONE:{ + switch(_functions.parseBodyFunc.origin){ + case FunctionOrigin::none:{ ret = ATask::ParseBody(); if(!ret){ return false; @@ -189,7 +189,7 @@ bool PythonTask::ParseBody() { } return ret; } - case FunctionOrigin::PYTHON_MODULE:{ + case FunctionOrigin::python_module:{ // This lock is valid because we use only ONE interpreter // If at some point, we decide to use multiple interpreters // (one by thread for example) @@ -200,7 +200,7 @@ bool PythonTask::ParseBody() { return false; } - if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.f.py, *raw_body, nullptr)) == nullptr) { + if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.py, *raw_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); return false; @@ -208,9 +208,9 @@ bool PythonTask::ParseBody() { return true; } - case FunctionOrigin::SHARED_LIBRARY:{ + case FunctionOrigin::shared_library:{ try{ - if((_parsed_body = _functions.parseBodyFunc.f.so(_pModule, _packet.GetBody()))== nullptr){ + if((_parsed_body = _functions.parseBodyFunc.so(_pModule, _packet.GetBody()))== nullptr){ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); return false; } @@ -232,11 +232,11 @@ std::vector PythonTask::GetFormatedAlerts(FunctionPySo ret; - switch(func.loc) { - case FunctionOrigin::PYTHON_MODULE:{ + switch(func.origin) { + case FunctionOrigin::python_module:{ PythonLock pylock; PyObjectOwner pyRes; - if ((pyRes = PyObject_CallFunctionObjArgs(func.f.py, processedData, nullptr)) == nullptr) { + if ((pyRes = PyObject_CallFunctionObjArgs(func.py, processedData, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("PythonTask:: GetFormatedAlerts:: "); return ret; } @@ -263,9 +263,9 @@ std::vector PythonTask::GetFormatedAlerts(FunctionPySo PythonTask::GetFormatedAlerts(FunctionPySo& func, PyObject* processedData) { DARWIN_LOGGER; DarwinResponse ret; - switch(func.loc) { - case FunctionOrigin::PYTHON_MODULE:{ + switch(func.origin) { + case FunctionOrigin::python_module:{ PythonLock pylock; PyObjectOwner pBody, pCertitudes; @@ -340,9 +340,9 @@ DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo Date: Fri, 8 Oct 2021 18:37:51 +0200 Subject: [PATCH 49/54] Added possibility to log through darwin --- samples/fpython/Generator.cpp | 61 ++++++++++++++++++++++++++++++++++ samples/fpython/PythonTask.cpp | 27 ++++++++------- samples/fpython/example.py | 3 +- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 2ca7e4e..71ebf73 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -55,6 +55,38 @@ bool Generator::PyExceptionCheckAndLog(std::string const& prefix_log, darwin::lo return true; } +static PyObject* darwin_log(PyObject *self __attribute__((__unused__)), PyObject* args) { + DARWIN_LOGGER; + int tag = 0; + const char* msg = nullptr; + ssize_t msgLen = 0; + if(PyArg_ParseTuple(args, "is#", &tag, &msg, &msgLen) != 1){ + Generator::PyExceptionCheckAndLog("darwin_log: Error parsing arguments :"); + Py_RETURN_NONE; + } + + log.log(static_cast(tag), std::string(msg, msgLen)); + + Py_RETURN_NONE; +} + +static PyMethodDef logMethod[] = { + {"log", darwin_log, METH_VARARGS, "sends log to darwin"}, + {nullptr, nullptr, 0, nullptr} /* Sentinel */ +}; + +static PyModuleDef darwin_log_mod = { + PyModuleDef_HEAD_INIT, + "darwin_logger", + "Python interface for the fdarwin C++ logger", + -1, + logMethod, + nullptr, + nullptr, + nullptr, + nullptr +}; + Generator::Generator(size_t nb_task_threads) : AGenerator(nb_task_threads) { } @@ -293,6 +325,26 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { PyExceptionCheckAndLog("Generator::LoadPythonScript : error importing string " + filename + " : "); return false; } + + if(PyImport_AddModule("darlog") == nullptr){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to import the logger module : "); + return false; + } + + PyObjectOwner logmod; + if((logmod = PyModule_Create(&darwin_log_mod))== nullptr){ + PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the logger module : "); + return false; + } + + PyObject* sys_modules = PyImport_GetModuleDict(); + if (PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while calling PyImport_GetModuleDict(): ")){ + return false; + } + PyDict_SetItemString(sys_modules, "darwin_logger", *logmod); + if (PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while calling PyDict_SetItemString(): ")){ + return false; + } if (PyRun_SimpleString("import sys") != 0) { DARWIN_LOG_DEBUG("Generator::LoadPythonScript : An error occurred while loading the 'sys' module"); @@ -314,6 +366,15 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { return false; } pName.Decref(); + // DARWIN_LOG_DEBUG("yes"); + // DARWIN_LOG_DEBUG("after log"); + // int a = 0; + // if((a=PyModule_AddFunctions(*pModule, logMethod)) != 0) { + // DARWIN_LOG_DEBUG("error func" + std::to_string(a)); + // PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while adding 'darwin_log' function to the module : "); + // return false; + // } + // DARWIN_LOG_DEBUG("after addfunc"); if( ! LoadFunctionFromPython(*pModule, functions.configPyFunc, "filter_config")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_config' : "); diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index 8adcb66..100f821 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -136,8 +136,8 @@ void PythonTask::operator()() { std::vector alerts = GetFormatedAlerts(_functions.alertFormatingFunc, *alert_context_res); - DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, *proc_res); - DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, *proc_res); + // DarwinResponse output = GetFormatedResponse(_functions.outputFormatingFunc, *context_res); + DarwinResponse resp = GetFormatedResponse(_functions.responseFormatingFunc, *context_res); for(auto& alert: alerts) { if( ! alert.empty()) { @@ -147,15 +147,14 @@ void PythonTask::operator()() { for(auto cert: resp.certitudes) { _packet.AddCertitude(cert); } - std::string& body = _packet.GetMutableBody(); - if( ! output.body.empty()) { - body.clear(); - body.append(output.body); - } + // if( ! output.body.empty()) { + // _response_body.clear(); + // _response_body.append(output.body); + // } if( ! resp.body.empty()){ - body.clear(); - body.append(resp.body); + _response_body.clear(); + _response_body.append(resp.body); } } @@ -286,9 +285,13 @@ DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo bool: - print('filter', config) + if 'dummy' not in config: + return False return True def parse_body(body: str) -> Union[list, CustomData]: From 8a6df55ee60f17ce6870887672c1c7f1dc980652 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 7 Feb 2022 11:28:04 +0100 Subject: [PATCH 50/54] Fix Compile issues : added cmake findPython module example.py is made a better example to test the filter Added logger to example added support for venv updated cmake min version --- CMakeLists.txt | 2 +- cmake/fpython.cmake | 15 ++-- samples/fpython/Generator.cpp | 41 ++++++--- samples/fpython/Generator.hpp | 8 +- samples/fpython/algorithm_module.py | 123 ++++++++++++++++++++++++++ samples/fpython/example.py | 89 ++++++++++++++++--- tests/filters/fpython.py | 128 ++++++++++++++++++++++++++-- 7 files changed, 364 insertions(+), 42 deletions(-) create mode 100644 samples/fpython/algorithm_module.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 93d3176..12d2983 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.7.2) +cmake_minimum_required(VERSION 3.12) project(darwin) diff --git a/cmake/fpython.cmake b/cmake/fpython.cmake index 444cdef..773794e 100644 --- a/cmake/fpython.cmake +++ b/cmake/fpython.cmake @@ -4,8 +4,10 @@ set(PYTHON_NAME darwin_python) # FILTER DEPENDENCIES # ####################### -include_directories(SYSTEM /usr/include/python3.8) -link_directories(/usr/lib/python3.8/config-3.8-x86) +find_package(Python3 REQUIRED COMPONENTS Development) + +include_directories(SYSTEM ${Python3_INCLUDE_DIRS}) +link_directories(${Python3_LIBRARY_DIRS}) ################### # EXECUTABLE # @@ -22,12 +24,9 @@ add_executable( target_link_libraries( ${PYTHON_NAME} ${DARWIN_LIBRARIES} - python3.8 - crypt - pthread - dl - util - m + ${Python3_LIBRARIES} + # DL libs are used for loading shared objects + ${CMAKE_DL_LIBS} ) target_include_directories(${PYTHON_NAME} PUBLIC samples/fpython/) diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index 71ebf73..d9a6a6e 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -6,6 +6,7 @@ /// \brief Copyright (c) 2019 Advens. All rights reserved. #include +#include #include #include "../../toolkit/lru_cache.hpp" @@ -110,7 +111,7 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { DARWIN_LOGGER; DARWIN_LOG_DEBUG("Python:: Generator:: Loading configuration..."); - std::string python_script_path, shared_library_path; + std::string python_script_path, shared_library_path, python_venv_folder; if (!configuration.HasMember("python_script_path")) { DARWIN_LOG_CRITICAL("Python:: Generator:: Missing parameter: 'python_script_path'"); @@ -135,9 +136,19 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { } shared_library_path = configuration["shared_library_path"].GetString(); + + if(configuration.HasMember("python_venv_folder")){ + if (!configuration["python_venv_folder"].IsString()) { + DARWIN_LOG_CRITICAL("Python:: Generator:: 'python_venv_folder' needs to be a string"); + return false; + } else { + python_venv_folder = configuration["python_venv_folder"].GetString(); + } + } + DARWIN_LOG_DEBUG("Python:: Generator:: Loading configuration..."); - bool pythonLoad = LoadPythonScript(python_script_path); + bool pythonLoad = LoadPythonScript(python_script_path, python_venv_folder); if(Py_IsInitialized() != 0) { // Release the GIL, This is safe to call because we just initialized the python interpreter @@ -292,7 +303,7 @@ PythonThread& Generator::GetPythonThread(){ return thread; } -bool Generator::LoadPythonScript(const std::string& python_script_path) { +bool Generator::LoadPythonScript(const std::string& python_script_path, const std::string& python_venv_folder) { DARWIN_LOGGER; if(python_script_path.empty()){ @@ -361,20 +372,26 @@ bool Generator::LoadPythonScript(const std::string& python_script_path) { return false; } + if(!python_venv_folder.empty()){ + std::filesystem::path venv_path(python_venv_folder); + venv_path /= std::filesystem::path("lib") / (std::string("python") + PYTHON_VERSION) / "site-packages"; + command = "sys.path.insert(0, \"" + venv_path.string() + "\")"; + // PyRun_SimpleString("sys.path.insert(0, \"/home/myadvens.lan/tcartegnie/workspace/darwin/.venv/lib/python3.8/site-packages\")"); + if (PyRun_SimpleString(command.c_str()) != 0) { + DARWIN_LOG_DEBUG( + "Generator::LoadPythonScript : An error occurred while inserting the custom venv '" + + venv_path.string() + + "' to the Python path" + ); + return false; + } + } + if((pModule = PyImport_Import(*pName)) == nullptr) { PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python script module '" + filename + "' : "); return false; } pName.Decref(); - // DARWIN_LOG_DEBUG("yes"); - // DARWIN_LOG_DEBUG("after log"); - // int a = 0; - // if((a=PyModule_AddFunctions(*pModule, logMethod)) != 0) { - // DARWIN_LOG_DEBUG("error func" + std::to_string(a)); - // PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while adding 'darwin_log' function to the module : "); - // return false; - // } - // DARWIN_LOG_DEBUG("after addfunc"); if( ! LoadFunctionFromPython(*pModule, functions.configPyFunc, "filter_config")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_config' : "); diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index a35296b..1353bdb 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -19,6 +19,11 @@ #include "PythonThread.hpp" #include "PythonObject.hpp" +// Preprocessor hack to extract the major and minor version of python +#define PY_VER_STR_(maj, min) #maj "." #min +#define PY_VER_STR(maj, min) PY_VER_STR_(maj, min) +#define PYTHON_VERSION PY_VER_STR(PY_MAJOR_VERSION,PY_MINOR_VERSION) + /// /// \brief Tag for the FunctionPySo struct /// It is declared outside the struct to stay unaffected by the template grammar @@ -111,11 +116,12 @@ class Generator: public AGenerator { /// If the path is not empty, a python interpreter will be initialized (by a call to Py_Initiliaze()) /// /// \param python_script_path + /// \param python_venv_folder /// \return true if the script was correctly loaded or if the path is empty /// \return false if the path can't be read, if loading the module triggered an error /// or if a symbol with the name of a defined function is found but not callable /// - bool LoadPythonScript(const std::string& python_script_path); + bool LoadPythonScript(const std::string& python_script_path, const std::string& python_venv_folder); /// /// \brief loads the shared library which path is given and attempts to load all defined functions diff --git a/samples/fpython/algorithm_module.py b/samples/fpython/algorithm_module.py new file mode 100644 index 0000000..48f5ada --- /dev/null +++ b/samples/fpython/algorithm_module.py @@ -0,0 +1,123 @@ +from typing import List, Union +from dataclasses import dataclass +from enum import IntEnum +import darwin_logger + + + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + """ + Function to call to push logs to Darwin + the module 'darwin_logger' is automatically loaded by Darwin + """ + darwin_logger.log(int(level), msg) + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +##################### +## ## +## API ## +## ## +##################### + +def filter_config(config: dict) -> bool: + """ + Function called with the filter configuration + This function is called only once when starting the filter + """ + pass + +def parse_body(body: str) -> Union[list, CustomData]: + """ + Function called when receiving a new packet + This implementation is optional, the default implementation parses a list of objects + """ + pass + +def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + """ + Function called after parse_body with the output of parse_body + """ + pass + +def filter_process(pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + """ + Function called after filter_pre_process with the output of filter_pre_process + This output will be forwarded to alert_formating, output_formating and response_formating + """ + pass + +def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + """ + Function called after filter_process with the output of filter_process + It should usually call query_context to gather information about the processed_date in order to contextualize it + The output of this function can extend PythonFilterResponse or may be any structure that has the + attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' + The output of the functions is forwarded to alert_contextualize, output_formating and response_formating + """ + pass + +def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + """ + Function called between filter_contextualize and alert_formating + It should be used to contextualize the alerts that should be risen by darwin + The output of this function can extend PythonFilterResponse or may be any structure that has the + attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' + """ + pass + + +def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: + """ + Function called after filter_process, each 'str' in the output will raise an alert + """ + pass + +# WIP This function is not wired, use only response_formating for now +def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + """ + Function called after filter_process, the output will be sent to the next filter as the body + with the certitudes returned by filter_process + """ + pass + +def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + """ + Function called after filter_process, the output will be sent to the sender as the body with + the certitudes returned by filter_process + """ + pass + + +# Context API + +def write_context(key, value): + """Writes an information about the current task performed""" + pass + +def query_context(key): + """Query information about the current task performed""" + pass \ No newline at end of file diff --git a/samples/fpython/example.py b/samples/fpython/example.py index 193c8b0..eb2f20a 100644 --- a/samples/fpython/example.py +++ b/samples/fpython/example.py @@ -1,7 +1,26 @@ import json +import os +import datetime +from enum import IntEnum from typing import List, Union from dataclasses import dataclass +import darwin_logger + + + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + class PythonFilterError(Exception): pass @@ -22,45 +41,93 @@ class CustomData: """ pass +threshold = 3 + def filter_config(config: dict) -> bool: + import numpy + print(numpy) + darwin_log(DarwinLogLevel.Critical, 'version:' + numpy.__version__) + global threshold if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') return False + if 'threshold' in config: + threshold = int(config['threshold']) return True def parse_body(body: str) -> Union[list, CustomData]: parsed = json.loads(body) if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') raise PythonFilterError("Parse Body: Wrong type") return parsed def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() return parsed_data def filter_process(pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: resp = PythonFilterResponse('', [], []) for line in pre_processed_data: if isinstance(line, str): - resp.certitudes.append(101) - resp.alerts.append(line) - resp.body += line + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) - print('py ret') return resp def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body return processed_data -def alert_contextualization(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: +def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) return contextualized_data def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: - print('alert') - return contextualized_data.alerts + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts -def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - print('output') - return contextualized_data def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - print('output') return contextualized_data + + +# WIP unused for now +def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/fpython.py b/tests/filters/fpython.py index 17a16ed..a8df464 100644 --- a/tests/filters/fpython.py +++ b/tests/filters/fpython.py @@ -1,7 +1,13 @@ from tools.filter import Filter from tools.output import print_result +import pandas as pd +import json +import datetime +import logging class PythonFilter(Filter): + alert_file='/tmp/python_alerts.log' + threshold=3 def __init__(self): super().__init__(filter_name="python") @@ -10,9 +16,15 @@ def configure(self, content=None): content = '{{\n' \ '"python_script_path": "{python_path}",\n' \ '"shared_library_path": "{so_path}",\n' \ - '"log_file_path": "/tmp/python_alerts.log"\n' \ - '}}'.format(python_path='/home/myadvens.lan/tcartegnie/workspace/darwin/samples/fpython/example.py', - so_path='/home/myadvens.lan/tcartegnie/workspace/pycpp/build/libfpython.so') + '"log_file_path": "{alert_file}",\n' \ + '"python_venv_folder": "/home/myadvens.lan/tcartegnie/workspace/darwin/.venv/",' \ + '"dummy":"",\n' \ + '"threshold":{threshold}\n' \ + '}}'.format(python_path='samples/fpython/example.py', + alert_file=self.alert_file, + threshold=self.threshold, + so_path='') + # so_path='/home/myadvens.lan/tcartegnie/workspace/pycpp/build/libfpython.so') print(content) super(PythonFilter, self).configure(content) @@ -21,20 +33,118 @@ def configure(self, content=None): def run(): tests = [ - test + full_python_functional_test, + # test_ueba ] for i in tests: print_result("Python: " + i.__name__, i) -def test(): +# Functional test with exemple.py, it should go through all python steps +def full_python_functional_test(): f = PythonFilter() print(' '.join(f.cmd)) + f.threshold=5 f.configure() f.valgrind_start() - api = f.get_darwin_api(verbose=True) - ret = api.call('hello', response_type="back") + api = f.get_darwin_api(verbose=False) + is_test_ok =True + for i in range(100): + line = 'hello ! ' + str(i) + ret = api.call(line , response_type="back") + if ret != len(line): + logging.error('line {} returned a bad certitude : {} instead of {}'.format(str(i), ret, len(line))) + is_test_ok = False - print(ret) - # print('ret:' , f.send_single("hello")) + line = 'alert:hello wow ! ' + str(i%10) + ret = api.call(line , response_type="back") + if ret != len(line): + logging.error('line {} returned a bad certitude : {} instead of {}'.format(str(i), ret, len(line))) + is_test_ok = False + # This next line should be too long, hence it should return a 101 certitude + ret = api.call('hello ! '*20 , response_type="back") + if ret != 101: + logging.error('line {} returned a bad certitude : {} instead of {}'.format(str(i), ret, 101)) + is_test_ok = False + if not is_test_ok: + return False + + lines = [] + for i in range(100): + lines.append('bulk hello ! ' + str(i)) + lines.append('alert:bulk hello wow ! ' + str(i%10)) + lines.append('hello ! '*20) + + # empty the log file + with open(f.alert_file, 'w') as _: + pass + + ret=api.bulk_call(lines,response_type='back') + predicted_alerts={} + for i, l in enumerate(lines): + if l.startswith('alert:'): + tmp = l.replace('alert:', '') + if tmp in predicted_alerts: + predicted_alerts[tmp] += 1 + else: + predicted_alerts[tmp] = 1 + + with open(f.alert_file, 'r') as file: + alerts_risen = file.readlines() + + is_test_ok = True + for alert in alerts_risen: + try: + json_alert = json.loads(alert) + except json.JSONDecodeError as e: + is_test_ok=False + logging.error('Malformed alert : should be a valid json', exc_info=e) + continue + if "date" not in json_alert or "alert" not in json_alert: + is_test_ok=False + logging.error('Malformed alert : "date" or "alert" not present ' + alert) + continue + if json_alert['alert'] not in predicted_alerts: + is_test_ok=False + logging.error('Malformed alert : alert should not have been triggered ' + alert) + continue + if predicted_alerts[json_alert['alert']] < f.threshold: + is_test_ok=False + logging.error('Malformed alert : alert should not have been triggered (below threshold) ' + alert) + continue + + api.close() + f.valgrind_stop() + + if not is_test_ok: + return False + return True + +def test_ueba(): + f = PythonFilter() + f.configure('{{\n' \ + '"python_script_path": "{python_path}",\n' \ + '"shared_library_path": "{so_path}",\n' \ + '"log_file_path": "/tmp/python_alerts.log",\n' \ + '"model_path": "/tmp/aggro.pkl",\n' \ + '"data_path": "/tmp/anomalies.csv"\n' + '}}'.format(python_path='/home/myadvens.lan/tcartegnie/workspace/ueba_poc-feature-ueba_bootstrap/aggro_ueba.py', + # so_path='/home/myadvens.lan/tcartegnie/workspace/ueba_aggro_rs/target/debug/libueba_aggro_rs.so')) + so_path='')) + + f.valgrind_start() + # sleep(60) + api = f.get_darwin_api(verbose=False) + + df_raw = pd.read_csv('/home/myadvens.lan/tcartegnie/workspace/ueba_poc-feature-ueba_bootstrap/samples/win_ad_hashed.csv') + + for day in pd.date_range(datetime.date.fromisoformat(df_raw.date.min()), datetime.date.fromisoformat(df_raw.date.max())): + batch = df_raw[df_raw.date == str(day.strftime('%Y-%m-%d'))].copy() + payload = [json.loads(batch.to_json())] + + start = datetime.datetime.now() + ret = api.bulk_call(payload, response_type="back") + end = datetime.datetime.now() + + print(day, ret, end-start, len(batch)) return True \ No newline at end of file From 5291f7be855057b17734cb6568ddfc07f576c058 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 7 Feb 2022 11:57:04 +0100 Subject: [PATCH 51/54] Moved example.py --- {samples/fpython => tests/filters/data}/example.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {samples/fpython => tests/filters/data}/example.py (100%) diff --git a/samples/fpython/example.py b/tests/filters/data/example.py similarity index 100% rename from samples/fpython/example.py rename to tests/filters/data/example.py From fd769202946ff575c394fab25c5c394e275e21f7 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Mon, 7 Feb 2022 18:08:55 +0100 Subject: [PATCH 52/54] Added tests and corrections --- samples/fpython/Generator.cpp | 13 +- tests/filters/data/bad_example.py | 121 ++++++++++++ tests/filters/data/bad_example_bad_import.py | 139 ++++++++++++++ tests/filters/data/example.py | 21 ++- tests/filters/fpython.py | 183 +++++++++++++++---- 5 files changed, 427 insertions(+), 50 deletions(-) create mode 100644 tests/filters/data/bad_example.py create mode 100644 tests/filters/data/bad_example_bad_import.py diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index d9a6a6e..ff3b9c7 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -312,7 +312,7 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st } std::ifstream f(python_script_path.c_str()); - if(f.bad()) { + if(! f.is_open()) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error loading the python script : failed to open " + python_script_path); return false; } @@ -358,13 +358,13 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st } if (PyRun_SimpleString("import sys") != 0) { - DARWIN_LOG_DEBUG("Generator::LoadPythonScript : An error occurred while loading the 'sys' module"); + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : An error occurred while loading the 'sys' module"); return false; } std::string command = "sys.path.append(\"" + folders + "\")"; if (PyRun_SimpleString(command.c_str()) != 0) { - DARWIN_LOG_DEBUG( + DARWIN_LOG_CRITICAL( "Generator::LoadPythonScript : An error occurred while appending the custom path '" + folders + "' to the Python path" @@ -375,10 +375,15 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st if(!python_venv_folder.empty()){ std::filesystem::path venv_path(python_venv_folder); venv_path /= std::filesystem::path("lib") / (std::string("python") + PYTHON_VERSION) / "site-packages"; + std::error_code _ec; + if(!std::filesystem::exists(venv_path, _ec)) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Virtual Env does not exist : " + venv_path.string()); + return false; + } command = "sys.path.insert(0, \"" + venv_path.string() + "\")"; // PyRun_SimpleString("sys.path.insert(0, \"/home/myadvens.lan/tcartegnie/workspace/darwin/.venv/lib/python3.8/site-packages\")"); if (PyRun_SimpleString(command.c_str()) != 0) { - DARWIN_LOG_DEBUG( + DARWIN_LOG_CRITICAL( "Generator::LoadPythonScript : An error occurred while inserting the custom venv '" + venv_path.string() + "' to the Python path" diff --git a/tests/filters/data/bad_example.py b/tests/filters/data/bad_example.py new file mode 100644 index 0000000..c0d320b --- /dev/null +++ b/tests/filters/data/bad_example.py @@ -0,0 +1,121 @@ +import json +import os +import datetime +from enum import IntEnum +from typing import List, Union +from dataclasses import dataclass + +# DOES NOT WORK , Used in tests to check if missing methods are detected by the filter + +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level, msg): + print(level, msg) + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + +class PythonFilterError(Exception): + pass + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +threshold = 3 + +def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + return True + +def parse_body(body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + +def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + +# MISSING METHOD filter_process + +def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + +def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + +def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + +def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +# WIP unused for now +def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/data/bad_example_bad_import.py b/tests/filters/data/bad_example_bad_import.py new file mode 100644 index 0000000..e8d2bc0 --- /dev/null +++ b/tests/filters/data/bad_example_bad_import.py @@ -0,0 +1,139 @@ +import json +import os +import datetime +from enum import IntEnum +from typing import List, Union +from dataclasses import dataclass + +# DOES NOT WORK , Used in tests to check if bad imports are detected by the filter +import trucmuche + + +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level, msg): + print(level, msg) + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + +class PythonFilterError(Exception): + pass + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +threshold = 3 + +def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + return True + +def parse_body(body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + +def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + +def filter_process(pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: + if isinstance(line, str): + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) + + return resp + +def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + +def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + +def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + +def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +# WIP unused for now +def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/data/example.py b/tests/filters/data/example.py index eb2f20a..fc28420 100644 --- a/tests/filters/data/example.py +++ b/tests/filters/data/example.py @@ -5,9 +5,14 @@ from typing import List, Union from dataclasses import dataclass -import darwin_logger - - +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level, msg): + print(level, msg) class DarwinLogLevel(IntEnum): Debug=0 @@ -44,15 +49,19 @@ class CustomData: threshold = 3 def filter_config(config: dict) -> bool: - import numpy - print(numpy) - darwin_log(DarwinLogLevel.Critical, 'version:' + numpy.__version__) global threshold if 'dummy' not in config: darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') return False if 'threshold' in config: threshold = int(config['threshold']) + if config.get('fail_conf', '') == 'yes': + raise Exception('FAILED PYTHON SCRIPT CONFIGURATION') + venv = config.get('python_venv_folder', '') + if len(venv) != 0: + import setuptools + if venv in str(setuptools): + darwin_log(DarwinLogLevel.Debug, 'WE ARE IN VIRTUAL ENV') return True def parse_body(body: str) -> Union[list, CustomData]: diff --git a/tests/filters/fpython.py b/tests/filters/fpython.py index a8df464..359d948 100644 --- a/tests/filters/fpython.py +++ b/tests/filters/fpython.py @@ -1,9 +1,9 @@ +import logging +import json +import tempfile +import venv from tools.filter import Filter from tools.output import print_result -import pandas as pd -import json -import datetime -import logging class PythonFilter(Filter): alert_file='/tmp/python_alerts.log' @@ -11,21 +11,22 @@ class PythonFilter(Filter): def __init__(self): super().__init__(filter_name="python") - def configure(self, content=None): + def configure(self, content=None, venv='', other_conf=''): if not content: content = '{{\n' \ '"python_script_path": "{python_path}",\n' \ '"shared_library_path": "{so_path}",\n' \ '"log_file_path": "{alert_file}",\n' \ - '"python_venv_folder": "/home/myadvens.lan/tcartegnie/workspace/darwin/.venv/",' \ + '"python_venv_folder": "{venv}",\n' \ '"dummy":"",\n' \ + '{other_conf}' \ '"threshold":{threshold}\n' \ - '}}'.format(python_path='samples/fpython/example.py', + '}}'.format(python_path='tests/filters/data/example.py', alert_file=self.alert_file, threshold=self.threshold, + venv=venv, + other_conf=other_conf, so_path='') - # so_path='/home/myadvens.lan/tcartegnie/workspace/pycpp/build/libfpython.so') - print(content) super(PythonFilter, self).configure(content) @@ -33,17 +34,148 @@ def configure(self, content=None): def run(): tests = [ + test_bad_config_no_script, + test_bad_config_no_shared_obj, + test_bad_config_nothing, + test_bad_config_no_venv, + test_wrong_config_missing_method, + test_wrong_python_missing_requirement, + test_wrong_python_exception_during_conf, + test_wrong_python_exception_during_steps, + test_venv, full_python_functional_test, - # test_ueba ] for i in tests: print_result("Python: " + i.__name__, i) +def test_bad_config_no_script(): + f = PythonFilter() + + f.configure('{"python_script_path":"/tmp/no_script_there.bad_ext", "shared_library_path":""}') + if f.valgrind_start(): + logging.error('test_bad_config_no_script: Filter should not start') + return False + if not f.check_line_in_filter_log('Generator::LoadPythonScript : Error loading the python script : failed to open'): + logging.error('test_bad_config_no_script: Filter should have failed with a log') + return False + return True + +def test_bad_config_no_shared_obj(): + f = PythonFilter() + + f.configure('{"shared_library_path":"/tmp/no_script_there.bad_ext", "python_script_path":""}') + if f.valgrind_start(): + logging.error('test_bad_config_no_shared_obj: Filter should not start') + return False + if not f.check_line_in_filter_log('Generator::LoadSharedLibrary : Error loading the shared library : failed to open'): + logging.error('test_bad_config_no_shared_obj: Filter should have failed with a log') + return False + return True + +def test_bad_config_nothing(): + f = PythonFilter() + + f.configure('{"shared_library_path":"", "python_script_path":""}') + if f.valgrind_start(): + logging.error('test_bad_config_nothing: Filter should not start') + return False + if not f.check_line_in_filter_log('Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library'): + logging.error('test_bad_config_nothing: Filter should have failed with a log') + return False + return True + +def test_bad_config_no_venv(): + f = PythonFilter() + + f.configure(venv='/tmp/VenvPythonFilterDoesNotExist') + if f.valgrind_start(): + logging.error('test_bad_config_no_venv: Filter should not start') + return False + + if not f.check_line_in_filter_log('Generator::LoadPythonScript : Virtual Env does not exist : '): + logging.error('test_bad_config_no_venv: Filter should have failed with a log') + return False + return True + +def test_wrong_config_missing_method(): + f = PythonFilter() + + f.configure('{"python_script_path":"tests/filters/data/bad_example.py", "shared_library_path":""}') + if f.valgrind_start(): + logging.error('test_wrong_config_missing_method: Filter should not start') + return False + + if not f.check_line_in_filter_log('Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library'): + logging.error('test_wrong_config_missing_method: Filter should have failed with a log') + return False + return True + +def test_wrong_python_missing_requirement(): + f = PythonFilter() + + f.configure('{"python_script_path":"tests/filters/data/bad_example_bad_import.py", "shared_library_path":""}') + if f.valgrind_start(): + logging.error('test_wrong_python_missing_requirement: Filter should not start') + return False + + if not f.check_line_in_filter_log('Generator::'): + logging.error('test_wrong_python_missing_requirement: Filter should have failed with a log') + return False + return True + +def test_wrong_python_exception_during_conf(): + f = PythonFilter() + + f.configure(other_conf='"fail_conf":"yes",\n') + if f.valgrind_start(): + logging.error('test_wrong_python_exception_during_conf: Filter should not start') + return False + + if not f.check_line_in_filter_log('FAILED PYTHON SCRIPT CONFIGURATION'): + logging.error('test_wrong_python_exception_during_conf: Filter should have failed with a log') + return False + return True + +def test_wrong_python_exception_during_steps(): + f = PythonFilter() + + f.configure() + if not f.valgrind_start(): + logging.error('test_wrong_python_exception_during_steps: Filter should start') + return False + # Error: line too long + res = f.send_single("fail"*25) + if res != 101: + logging.error('test_wrong_python_exception_during_steps: Filter should fail with an error cerittude (101), but we received: ' + str(res)) + return False + + if not f.check_run(): + logging.error('test_wrong_python_exception_during_steps: Filter should still be running') + return False + + return True + +def test_venv(): + with tempfile.TemporaryDirectory() as tmpdir: + e = venv.EnvBuilder(with_pip=True) + e.create(tmpdir) + + f = PythonFilter() + f.configure(venv=tmpdir) + if not f.valgrind_start(): + logging.error('test_venv: Filter should start') + return False + + if not f.check_line_in_filter_log('WE ARE IN VIRTUAL ENV'): + logging.error('test_venv: Filter should have logged') + return False + + return True + # Functional test with exemple.py, it should go through all python steps def full_python_functional_test(): f = PythonFilter() - print(' '.join(f.cmd)) f.threshold=5 f.configure() f.valgrind_start() @@ -119,32 +251,3 @@ def full_python_functional_test(): if not is_test_ok: return False return True - -def test_ueba(): - f = PythonFilter() - f.configure('{{\n' \ - '"python_script_path": "{python_path}",\n' \ - '"shared_library_path": "{so_path}",\n' \ - '"log_file_path": "/tmp/python_alerts.log",\n' \ - '"model_path": "/tmp/aggro.pkl",\n' \ - '"data_path": "/tmp/anomalies.csv"\n' - '}}'.format(python_path='/home/myadvens.lan/tcartegnie/workspace/ueba_poc-feature-ueba_bootstrap/aggro_ueba.py', - # so_path='/home/myadvens.lan/tcartegnie/workspace/ueba_aggro_rs/target/debug/libueba_aggro_rs.so')) - so_path='')) - - f.valgrind_start() - # sleep(60) - api = f.get_darwin_api(verbose=False) - - df_raw = pd.read_csv('/home/myadvens.lan/tcartegnie/workspace/ueba_poc-feature-ueba_bootstrap/samples/win_ad_hashed.csv') - - for day in pd.date_range(datetime.date.fromisoformat(df_raw.date.min()), datetime.date.fromisoformat(df_raw.date.max())): - batch = df_raw[df_raw.date == str(day.strftime('%Y-%m-%d'))].copy() - payload = [json.loads(batch.to_json())] - - start = datetime.datetime.now() - ret = api.bulk_call(payload, response_type="back") - end = datetime.datetime.now() - - print(day, ret, end-start, len(batch)) - return True \ No newline at end of file From 6d30340a85f8588d79265a6003a344614e46f6db Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Tue, 15 Mar 2022 16:50:36 +0100 Subject: [PATCH 53/54] Changed structure of expected python script - now expects a class (see algorithm_module.py) - added tests for class specific issues --- samples/fpython/Generator.cpp | 37 +++-- samples/fpython/Generator.hpp | 11 +- samples/fpython/PythonTask.cpp | 98 ++++++----- samples/fpython/PythonTask.hpp | 7 +- samples/fpython/algorithm_module.py | 158 ++++++++++-------- tests/filters/data/bad_example.py | 107 ++++++------ tests/filters/data/bad_example_bad_import.py | 139 ++++++++-------- tests/filters/data/bad_example_fail_init.py | 157 ++++++++++++++++++ tests/filters/data/example.py | 161 ++++++++++--------- tests/filters/fpython.py | 45 +++++- 10 files changed, 588 insertions(+), 332 deletions(-) create mode 100644 tests/filters/data/bad_example_fail_init.py diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index ff3b9c7..b051904 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -307,8 +307,8 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st DARWIN_LOGGER; if(python_script_path.empty()){ - DARWIN_LOG_INFO("Generator::LoadPythonScript : No python script to load"); - return true; + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : No python script to load"); + return false; } std::ifstream f(python_script_path.c_str()); @@ -398,46 +398,55 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st } pName.Decref(); - if( ! LoadFunctionFromPython(*pModule, functions.configPyFunc, "filter_config")){ + pClass = PyObject_GetAttrString(*pModule, "Execution"); + if (PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python class 'Execution' : ")){ + return false; + } + if(*pClass == nullptr || ! PyType_Check(*pClass)) { + DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error while attempting to load the python class 'Execution' : Not a Class"); + return false; + } + + if( ! LoadFunctionFromPython(*pClass, functions.configPyFunc, "filter_config")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_config' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.parseBodyFunc, "parse_body")){ + if( ! LoadFunctionFromPython(*pClass, functions.parseBodyFunc, "parse_body")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'parse_body' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.preProcessingFunc, "filter_pre_process")){ + if( ! LoadFunctionFromPython(*pClass, functions.preProcessingFunc, "filter_pre_process")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_pre_process' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.processingFunc, "filter_process")){ + if( ! LoadFunctionFromPython(*pClass, functions.processingFunc, "filter_process")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_process' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.contextualizeFunc, "filter_contextualize")){ + if( ! LoadFunctionFromPython(*pClass, functions.contextualizeFunc, "filter_contextualize")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_contextualize' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.alertContextualizeFunc, "alert_contextualize")){ + if( ! LoadFunctionFromPython(*pClass, functions.alertContextualizeFunc, "alert_contextualize")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_contextualize' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.alertFormatingFunc, "alert_formating")){ + if( ! LoadFunctionFromPython(*pClass, functions.alertFormatingFunc, "alert_formating")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_formating' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.outputFormatingFunc, "output_formating")){ + if( ! LoadFunctionFromPython(*pClass, functions.outputFormatingFunc, "output_formating")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'output_formating' : "); return false; } - if( ! LoadFunctionFromPython(*pModule, functions.responseFormatingFunc, "response_formating")){ + if( ! LoadFunctionFromPython(*pClass, functions.responseFormatingFunc, "response_formating")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'response_formating' : "); return false; } @@ -446,9 +455,9 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st } template -inline bool Generator::LoadFunctionFromPython(PyObject* pModule, FunctionPySo& function_holder, const std::string& function_name){ +inline bool Generator::LoadFunctionFromPython(PyObject* pClass, FunctionPySo& function_holder, const std::string& function_name){ DARWIN_LOGGER; - function_holder.py = PyObject_GetAttrString(pModule, function_name.c_str()); + function_holder.py = PyObject_GetAttrString(pClass, function_name.c_str()); if(function_holder.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No '" + function_name + "' method in the python script"); } else if (! PyCallable_Check(function_holder.py)){ @@ -464,7 +473,7 @@ std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) noexcept { return std::static_pointer_cast( std::make_shared(_cache, _cache_mutex, s, s->_packet, - *pModule, functions) + *pClass, functions) ); } diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index 1353bdb..416779c 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -72,10 +72,10 @@ struct FunctionHolder{ FunctionHolder& operator=(FunctionHolder &&) = delete; typedef bool(*config_t)(rapidjson::Document const &); - typedef PyObject*(*parse_body_t)(PyObject*, const std::string&); - typedef PyObject*(*process_t)(PyObject*, PyObject*); - typedef std::vector(*alert_format_t)(PyObject*, PyObject*); - typedef DarwinResponse(*resp_format_t)(PyObject*, PyObject*); + typedef PyObject*(*parse_body_t)(PyObject*, PyObject*, const std::string&); + typedef PyObject*(*process_t)(PyObject*, PyObject*, PyObject*); + typedef std::vector(*alert_format_t)(PyObject*, PyObject*, PyObject*); + typedef DarwinResponse(*resp_format_t)(PyObject*, PyObject*, PyObject*); // There are 2 config functions as we accept to pass informations to both the python module and the shared library // Note that the shared library has a direct access to the python module @@ -162,9 +162,10 @@ class Generator: public AGenerator { inline void LoadFunctionFromSO(void* lib_handle, FunctionPySo& function_holder, const std::string& function_name); template - inline bool LoadFunctionFromPython(PyObject* pModule, FunctionPySo& function_holder, const std::string& function_name); + inline bool LoadFunctionFromPython(PyObject* pClass, FunctionPySo& function_holder, const std::string& function_name); PyObjectOwner pModule; + PyObjectOwner pClass; FunctionHolder functions; }; \ No newline at end of file diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index 100f821..897d063 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -16,10 +16,13 @@ PythonTask::PythonTask(std::shared_ptr> cache, std::mutex& cache_mutex, darwin::session_ptr_t s, - darwin::DarwinPacket& packet, PyObject* pModule, FunctionHolder& functions) - : ATask{DARWIN_FILTER_NAME, cache, cache_mutex, s, packet}, _pModule{pModule}, _functions{functions} + darwin::DarwinPacket& packet, PyObject* pClass, FunctionHolder& functions) + : ATask{DARWIN_FILTER_NAME, cache, cache_mutex, s, packet}, _pClass{pClass}, _functions{functions} { _is_cache = _cache != nullptr; + PythonLock lock; + _pSelf = PyObject_CallObject(_pClass, nullptr); + Generator::PyExceptionCheckAndLog("PythonTask:: PythonTask:: Error creating Execution object: ", darwin::logger::Error); } long PythonTask::GetFilterCode() noexcept{ @@ -33,19 +36,22 @@ bool PythonTask::ParseLine(rapidjson::Value& line __attribute((unused))) { void PythonTask::operator()() { DARWIN_LOGGER; PyObjectOwner preProc_res, proc_res, context_res, alert_context_res; - + if (*_pSelf == nullptr) { + DARWIN_LOG_ERROR("No Execution object was created, Lines skipped."); + return; + } switch(_functions.preProcessingFunc.origin) { case FunctionOrigin::python_module:{ PythonLock pylock; - if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, *_parsed_body, nullptr)) == nullptr) { - Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Preprocess: "); + if ((preProc_res = PyObject_CallFunctionObjArgs(_functions.preProcessingFunc.py, *_pSelf, *_parsed_body, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Preprocess: ", darwin::logger::Error); return; } break; } case FunctionOrigin::shared_library:{ try { - if ((preProc_res = _functions.preProcessingFunc.so(_pModule, *_parsed_body)) == nullptr) { + if ((preProc_res = _functions.preProcessingFunc.so(_pClass, *_pSelf, *_parsed_body)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_pre_process function"); return; } @@ -62,15 +68,15 @@ void PythonTask::operator()() { switch(_functions.processingFunc.origin) { case FunctionOrigin::python_module:{ PythonLock pylock; - if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, *preProc_res, nullptr)) == nullptr) { - Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Process: "); + if ((proc_res = PyObject_CallFunctionObjArgs(_functions.processingFunc.py, *_pSelf, *preProc_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Process: ", darwin::logger::Error); return; } break; } case FunctionOrigin::shared_library:{ try{ - if ((proc_res = _functions.processingFunc.so(_pModule, *preProc_res)) == nullptr) { + if ((proc_res = _functions.processingFunc.so(_pClass, *_pSelf, *preProc_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_process function"); return; } @@ -87,15 +93,15 @@ void PythonTask::operator()() { switch(_functions.contextualizeFunc.origin) { case FunctionOrigin::python_module:{ PythonLock pylock; - if ((context_res = PyObject_CallFunctionObjArgs(_functions.contextualizeFunc.py, *proc_res, nullptr)) == nullptr) { - Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Contextualize: "); + if ((context_res = PyObject_CallFunctionObjArgs(_functions.contextualizeFunc.py, *_pSelf, *proc_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Contextualize: ", darwin::logger::Error); return; } break; } case FunctionOrigin::shared_library:{ try{ - if ((context_res = _functions.contextualizeFunc.so(_pModule, *proc_res)) == nullptr) { + if ((context_res = _functions.contextualizeFunc.so(_pClass, *_pSelf, *proc_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO filter_contextualize function"); return; } @@ -112,15 +118,15 @@ void PythonTask::operator()() { switch(_functions.alertContextualizeFunc.origin) { case FunctionOrigin::python_module:{ PythonLock pylock; - if ((alert_context_res = PyObject_CallFunctionObjArgs(_functions.alertContextualizeFunc.py, *context_res, nullptr)) == nullptr) { - Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Alert Contextualize: "); + if ((alert_context_res = PyObject_CallFunctionObjArgs(_functions.alertContextualizeFunc.py, *_pSelf, *context_res, nullptr)) == nullptr) { + Generator::PyExceptionCheckAndLog("PythonTask:: Operator:: Alert Contextualize: ", darwin::logger::Error); return; } break; } case FunctionOrigin::shared_library:{ try{ - if ((alert_context_res = _functions.alertContextualizeFunc.so(_pModule, *context_res)) == nullptr) { + if ((alert_context_res = _functions.alertContextualizeFunc.so(_pClass, *_pSelf, *context_res)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: An error occurred while calling the SO alert_contextualize function"); return; } @@ -161,6 +167,12 @@ void PythonTask::operator()() { bool PythonTask::ParseBody() { DARWIN_LOGGER; PyObjectOwner raw_body; + + if (*_pSelf == nullptr) { + DARWIN_LOG_ERROR("No Execution object was created, Lines skipped."); + return false; + } + bool ret = false; switch(_functions.parseBodyFunc.origin){ case FunctionOrigin::none:{ @@ -170,7 +182,7 @@ bool PythonTask::ParseBody() { } PythonLock pylock; if((_parsed_body = PyList_New(_body.Size())) == nullptr){ - Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); return false; } @@ -178,11 +190,11 @@ bool PythonTask::ParseBody() { PyObject *str; // This reference is "stolen" by PyList_SetItem, we don't have to DECREF it if((str = PyUnicode_DecodeFSDefault(_body.GetArray()[i].GetString())) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); - Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); return false; } if(PyList_SetItem(*_parsed_body, i, str) != 0){ - Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); return false; } } @@ -195,13 +207,13 @@ bool PythonTask::ParseBody() { // We must switch to PythonThread utility class PythonLock pylock; if((raw_body = PyUnicode_DecodeFSDefault(_packet.GetBody().c_str())) == nullptr) { - Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); return false; } - if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.py, *raw_body, nullptr)) == nullptr) { + if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.py, *_pSelf, *raw_body, nullptr)) == nullptr) { DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); - Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :"); + Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); return false; } @@ -209,7 +221,7 @@ bool PythonTask::ParseBody() { } case FunctionOrigin::shared_library:{ try{ - if((_parsed_body = _functions.parseBodyFunc.so(_pModule, _packet.GetBody()))== nullptr){ + if((_parsed_body = _functions.parseBodyFunc.so(_pClass, *_pSelf, _packet.GetBody()))== nullptr){ DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); return false; } @@ -235,8 +247,8 @@ std::vector PythonTask::GetFormatedAlerts(FunctionPySo PythonTask::GetFormatedAlerts(FunctionPySo PythonTask::GetFormatedAlerts(FunctionPySo UINT_MAX) { @@ -345,7 +357,7 @@ DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo& func, PyObject* processedData); private: - PyObject* _pModule; + PyObject* _pClass; FunctionHolder& _functions; + PyObjectOwner _pSelf; PyObjectOwner _parsed_body; }; diff --git a/samples/fpython/algorithm_module.py b/samples/fpython/algorithm_module.py index 48f5ada..5bf7d43 100644 --- a/samples/fpython/algorithm_module.py +++ b/samples/fpython/algorithm_module.py @@ -1,8 +1,17 @@ +from abc import abstractmethod from typing import List, Union from dataclasses import dataclass from enum import IntEnum -import darwin_logger +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level: int, msg: str): + import sys + outpipe = sys.stderr if level >= 4 else sys.stdout + print(DarwinLogLevel(level).name + ' : ' + msg,file=outpipe) class DarwinLogLevel(IntEnum): @@ -42,74 +51,85 @@ class CustomData: ## API ## ## ## ##################### - -def filter_config(config: dict) -> bool: - """ - Function called with the filter configuration - This function is called only once when starting the filter - """ - pass - -def parse_body(body: str) -> Union[list, CustomData]: - """ - Function called when receiving a new packet - This implementation is optional, the default implementation parses a list of objects - """ - pass - -def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: - """ - Function called after parse_body with the output of parse_body - """ - pass - -def filter_process(pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: - """ - Function called after filter_pre_process with the output of filter_pre_process - This output will be forwarded to alert_formating, output_formating and response_formating - """ - pass - -def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - """ - Function called after filter_process with the output of filter_process - It should usually call query_context to gather information about the processed_date in order to contextualize it - The output of this function can extend PythonFilterResponse or may be any structure that has the - attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' - The output of the functions is forwarded to alert_contextualize, output_formating and response_formating - """ - pass - -def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - """ - Function called between filter_contextualize and alert_formating - It should be used to contextualize the alerts that should be risen by darwin - The output of this function can extend PythonFilterResponse or may be any structure that has the - attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' - """ - pass - - -def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: - """ - Function called after filter_process, each 'str' in the output will raise an alert - """ - pass - -# WIP This function is not wired, use only response_formating for now -def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - """ - Function called after filter_process, the output will be sent to the next filter as the body - with the certitudes returned by filter_process - """ - pass - -def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - """ - Function called after filter_process, the output will be sent to the sender as the body with - the certitudes returned by filter_process - """ - pass +class Execution: + + @staticmethod + @abstractmethod + def filter_config(config: dict) -> bool: + """ + Function called with the filter configuration + This function is called only once when starting the filter + """ + pass + + @abstractmethod + def parse_body(self, body: str) -> Union[list, CustomData]: + """ + Function called when receiving a new packet + This implementation is optional, the default implementation parses a list of objects + """ + pass + + @abstractmethod + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + """ + Function called after parse_body with the output of parse_body + """ + pass + + @abstractmethod + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + """ + Function called after filter_pre_process with the output of filter_pre_process + This output will be forwarded to alert_formating, output_formating and response_formating + """ + pass + + @abstractmethod + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + """ + Function called after filter_process with the output of filter_process + It should usually call query_context to gather information about the processed_date in order to contextualize it + The output of this function can extend PythonFilterResponse or may be any structure that has the + attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' + The output of the functions is forwarded to alert_contextualize, output_formating and response_formating + """ + pass + + @abstractmethod + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + """ + Function called between filter_contextualize and alert_formating + It should be used to contextualize the alerts that should be risen by darwin + The output of this function can extend PythonFilterResponse or may be any structure that has the + attributes 'body: str', 'certitudes: List[int]' and 'alerts: List[str]' + """ + pass + + + @abstractmethod + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + """ + Function called after filter_process, each 'str' in the output will raise an alert + """ + pass + + @abstractmethod + # WIP This function is not wired, use only response_formating for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + """ + Function called after filter_process, the output will be sent to the next filter as the body + with the certitudes returned by filter_process + """ + pass + + @abstractmethod + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + """ + Function called after filter_process, the output will be sent to the sender as the body with + the certitudes returned by filter_process + """ + pass # Context API diff --git a/tests/filters/data/bad_example.py b/tests/filters/data/bad_example.py index c0d320b..8cc7369 100644 --- a/tests/filters/data/bad_example.py +++ b/tests/filters/data/bad_example.py @@ -49,59 +49,60 @@ class CustomData: pass threshold = 3 - -def filter_config(config: dict) -> bool: - global threshold - if 'dummy' not in config: - darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') - return False - if 'threshold' in config: - threshold = int(config['threshold']) - return True - -def parse_body(body: str) -> Union[list, CustomData]: - parsed = json.loads(body) - if not isinstance(parsed, list): - darwin_log(DarwinLogLevel.Error, 'input body is not a list') - raise PythonFilterError("Parse Body: Wrong type") - return parsed - -def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: - for d in parsed_data: - d = d.lower() - return parsed_data - -# MISSING METHOD filter_process - -def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - new_body = '' - for line in processed_data.body.splitlines(): - new_body += str(query_context(line)) + ':' + line + os.linesep - processed_data.body = new_body - return processed_data - -def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - for alert in contextualized_data.alerts: - if query_context('alert:' + alert) < threshold: - darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') - contextualized_data.alerts.remove(alert) - return contextualized_data - -def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: - formated_alerts = [] - for alert in contextualized_data.alerts: - date = datetime.datetime.now() - formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) - return formated_alerts - - -def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - return contextualized_data - - -# WIP unused for now -def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - return contextualized_data +class Execution: + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + # MISSING METHOD filter_process + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data context = {} diff --git a/tests/filters/data/bad_example_bad_import.py b/tests/filters/data/bad_example_bad_import.py index e8d2bc0..05f969a 100644 --- a/tests/filters/data/bad_example_bad_import.py +++ b/tests/filters/data/bad_example_bad_import.py @@ -51,75 +51,76 @@ class CustomData: pass threshold = 3 - -def filter_config(config: dict) -> bool: - global threshold - if 'dummy' not in config: - darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') - return False - if 'threshold' in config: - threshold = int(config['threshold']) - return True - -def parse_body(body: str) -> Union[list, CustomData]: - parsed = json.loads(body) - if not isinstance(parsed, list): - darwin_log(DarwinLogLevel.Error, 'input body is not a list') - raise PythonFilterError("Parse Body: Wrong type") - return parsed - -def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: - for d in parsed_data: - d = d.lower() - return parsed_data - -def filter_process(pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: - resp = PythonFilterResponse('', [], []) - for line in pre_processed_data: - if isinstance(line, str): - s = len(line) - if s < 80: - resp.certitudes.append(s) - resp.body += line + os.linesep - write_context(line) - else: - darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') - resp.certitudes.append(101) - if 'alert:' in line: - alert = line.replace('alert:', '') - resp.alerts.append(alert) - - return resp - -def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - new_body = '' - for line in processed_data.body.splitlines(): - new_body += str(query_context(line)) + ':' + line + os.linesep - processed_data.body = new_body - return processed_data - -def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - for alert in contextualized_data.alerts: - if query_context('alert:' + alert) < threshold: - darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') - contextualized_data.alerts.remove(alert) - return contextualized_data - -def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: - formated_alerts = [] - for alert in contextualized_data.alerts: - date = datetime.datetime.now() - formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) - return formated_alerts - - -def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - return contextualized_data - - -# WIP unused for now -def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - return contextualized_data +class Execution: + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: + if isinstance(line, str): + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) + + return resp + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data context = {} diff --git a/tests/filters/data/bad_example_fail_init.py b/tests/filters/data/bad_example_fail_init.py new file mode 100644 index 0000000..42b3a30 --- /dev/null +++ b/tests/filters/data/bad_example_fail_init.py @@ -0,0 +1,157 @@ +import json +import os +import datetime +from enum import IntEnum +from random import randint +from typing import List, Union +from dataclasses import dataclass + +# This import will be resolved in the darwin python runtime +try: + import darwin_logger +except ImportError: + class darwin_logger: + @staticmethod + def log(level: int, msg: str): + import sys + outpipe = sys.stderr if level >= 4 else sys.stdout + print(DarwinLogLevel(level).name, msg,file=outpipe) + + +class DarwinLogLevel(IntEnum): + Debug=0 + Info=1 + Notice=2 + Warning=3 + Error=4 + Critical=5 + +def darwin_log(level: DarwinLogLevel, msg:str): + darwin_logger.log(int(level), msg) + + +class PythonFilterError(Exception): + pass + +@dataclass +class PythonFilterResponse: + """ + Users may use this class or extend it or even use an entirely different class that has the same properties + """ + body: str + certitudes: List[int] + alerts: List[str] + +@dataclass +class CustomData: + """ + Placeholder class, the output of parse_body is passed to filter_pre_process and the + output of filter_pre_process is passed to filter_process + """ + pass + +threshold = 3 +nb = 0 # Fail at 3 +class Execution: + + def __init__(self) -> None: + global nb + if nb == 3: + nb += 1 + raise Exception("Fail at init") + nb += 1 + + + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + if config.get('fail_conf', '') == 'yes': + raise Exception('FAILED PYTHON SCRIPT CONFIGURATION') + venv = config.get('python_venv_folder', '') + if len(venv) != 0: + import setuptools + if venv in str(setuptools): + darwin_log(DarwinLogLevel.Debug, 'WE ARE IN VIRTUAL ENV') + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: + if isinstance(line, str): + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) + + return resp + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + +context = {} + +def write_context(key:str, value=None): + """Writes an information about the current task performed""" + if key in context: + context[key] += 1 + else: + context[key] = 1 + +def query_context(key: str) -> int: + """Query information about the current task performed""" + if key in context: + return context[key] + else: + return 0 diff --git a/tests/filters/data/example.py b/tests/filters/data/example.py index fc28420..ec29937 100644 --- a/tests/filters/data/example.py +++ b/tests/filters/data/example.py @@ -2,6 +2,7 @@ import os import datetime from enum import IntEnum +from random import randint from typing import List, Union from dataclasses import dataclass @@ -11,8 +12,11 @@ except ImportError: class darwin_logger: @staticmethod - def log(level, msg): - print(level, msg) + def log(level: int, msg: str): + import sys + outpipe = sys.stderr if level >= 4 else sys.stdout + print(DarwinLogLevel(level).name, msg,file=outpipe) + class DarwinLogLevel(IntEnum): Debug=0 @@ -48,81 +52,84 @@ class CustomData: threshold = 3 -def filter_config(config: dict) -> bool: - global threshold - if 'dummy' not in config: - darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') - return False - if 'threshold' in config: - threshold = int(config['threshold']) - if config.get('fail_conf', '') == 'yes': - raise Exception('FAILED PYTHON SCRIPT CONFIGURATION') - venv = config.get('python_venv_folder', '') - if len(venv) != 0: - import setuptools - if venv in str(setuptools): - darwin_log(DarwinLogLevel.Debug, 'WE ARE IN VIRTUAL ENV') - return True - -def parse_body(body: str) -> Union[list, CustomData]: - parsed = json.loads(body) - if not isinstance(parsed, list): - darwin_log(DarwinLogLevel.Error, 'input body is not a list') - raise PythonFilterError("Parse Body: Wrong type") - return parsed - -def filter_pre_process(parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: - for d in parsed_data: - d = d.lower() - return parsed_data - -def filter_process(pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: - resp = PythonFilterResponse('', [], []) - for line in pre_processed_data: - if isinstance(line, str): - s = len(line) - if s < 80: - resp.certitudes.append(s) - resp.body += line + os.linesep - write_context(line) - else: - darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') - resp.certitudes.append(101) - if 'alert:' in line: - alert = line.replace('alert:', '') - resp.alerts.append(alert) - - return resp - -def filter_contextualize(processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - new_body = '' - for line in processed_data.body.splitlines(): - new_body += str(query_context(line)) + ':' + line + os.linesep - processed_data.body = new_body - return processed_data - -def alert_contextualize(contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: - for alert in contextualized_data.alerts: - if query_context('alert:' + alert) < threshold: - darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') - contextualized_data.alerts.remove(alert) - return contextualized_data - -def alert_formating(contextualized_data: PythonFilterResponse) -> List[str]: - formated_alerts = [] - for alert in contextualized_data.alerts: - date = datetime.datetime.now() - formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) - return formated_alerts - - -def response_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - return contextualized_data - - -# WIP unused for now -def output_formating(contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: - return contextualized_data +class Execution: + + @staticmethod + def filter_config(config: dict) -> bool: + global threshold + if 'dummy' not in config: + darwin_log(DarwinLogLevel.Critical, 'The field \\"dummy\\" is not in the config') + return False + if 'threshold' in config: + threshold = int(config['threshold']) + if config.get('fail_conf', '') == 'yes': + raise Exception('FAILED PYTHON SCRIPT CONFIGURATION') + venv = config.get('python_venv_folder', '') + if len(venv) != 0: + import setuptools + if venv in str(setuptools): + darwin_log(DarwinLogLevel.Debug, 'WE ARE IN VIRTUAL ENV') + return True + + def parse_body(self, body: str) -> Union[list, CustomData]: + parsed = json.loads(body) + if not isinstance(parsed, list): + darwin_log(DarwinLogLevel.Error, 'input body is not a list') + raise PythonFilterError("Parse Body: Wrong type") + return parsed + + def filter_pre_process(self, parsed_data: Union[list, CustomData]) -> Union[list, CustomData]: + for d in parsed_data: + d = d.lower() + return parsed_data + + def filter_process(self, pre_processed_data: Union[list, CustomData]) -> Union[list, CustomData, PythonFilterResponse]: + resp = PythonFilterResponse('', [], []) + for line in pre_processed_data: + if isinstance(line, str): + s = len(line) + if s < 80: + resp.certitudes.append(s) + resp.body += line + os.linesep + write_context(line) + else: + darwin_log(DarwinLogLevel.Warning, 'Line too long, skipping it, returning 101') + resp.certitudes.append(101) + if 'alert:' in line: + alert = line.replace('alert:', '') + resp.alerts.append(alert) + + return resp + + def filter_contextualize(self, processed_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + new_body = '' + for line in processed_data.body.splitlines(): + new_body += str(query_context(line)) + ':' + line + os.linesep + processed_data.body = new_body + return processed_data + + def alert_contextualize(self, contextualized_data: Union[list, CustomData, PythonFilterResponse]) -> Union[CustomData, PythonFilterResponse]: + for alert in contextualized_data.alerts: + if query_context('alert:' + alert) < threshold: + darwin_log(DarwinLogLevel.Info, 'alert below threshold, skipping it') + contextualized_data.alerts.remove(alert) + return contextualized_data + + def alert_formating(self, contextualized_data: PythonFilterResponse) -> List[str]: + formated_alerts = [] + for alert in contextualized_data.alerts: + date = datetime.datetime.now() + formated_alerts.append('{{"date":"{date}","alert":"{alert}"}}'.format(date=str(date), alert=alert)) + return formated_alerts + + + def response_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data + + + # WIP unused for now + def output_formating(self, contextualized_data: Union[CustomData, PythonFilterResponse]) -> PythonFilterResponse: + return contextualized_data context = {} diff --git a/tests/filters/fpython.py b/tests/filters/fpython.py index 359d948..52d65ec 100644 --- a/tests/filters/fpython.py +++ b/tests/filters/fpython.py @@ -35,13 +35,15 @@ def configure(self, content=None, venv='', other_conf=''): def run(): tests = [ test_bad_config_no_script, - test_bad_config_no_shared_obj, + test_bad_config_empty_python_script_path, test_bad_config_nothing, test_bad_config_no_venv, test_wrong_config_missing_method, test_wrong_python_missing_requirement, test_wrong_python_exception_during_conf, + test_wrong_python_exception_during_init, test_wrong_python_exception_during_steps, + test_wrong_python_exception_during_delete, test_venv, full_python_functional_test, ] @@ -61,14 +63,14 @@ def test_bad_config_no_script(): return False return True -def test_bad_config_no_shared_obj(): +def test_bad_config_empty_python_script_path(): f = PythonFilter() f.configure('{"shared_library_path":"/tmp/no_script_there.bad_ext", "python_script_path":""}') if f.valgrind_start(): logging.error('test_bad_config_no_shared_obj: Filter should not start') return False - if not f.check_line_in_filter_log('Generator::LoadSharedLibrary : Error loading the shared library : failed to open'): + if not f.check_line_in_filter_log('Generator::LoadPythonScript : No python script to load'): logging.error('test_bad_config_no_shared_obj: Filter should have failed with a log') return False return True @@ -80,7 +82,7 @@ def test_bad_config_nothing(): if f.valgrind_start(): logging.error('test_bad_config_nothing: Filter should not start') return False - if not f.check_line_in_filter_log('Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library'): + if not f.check_line_in_filter_log('Generator::LoadPythonScript : No python script to load'): logging.error('test_bad_config_nothing: Filter should have failed with a log') return False return True @@ -156,6 +158,41 @@ def test_wrong_python_exception_during_steps(): return True +def test_wrong_python_exception_during_init(): + f = PythonFilter() + + f.configure('{"python_script_path":"tests/filters/data/bad_example_fail_init.py", "shared_library_path":"", "dummy":""}') + if not f.valgrind_start(): + logging.error('test_wrong_python_exception_during_init: Filter should start') + return False + # Error: exception raised on third init + is_test_ok = True + for i in range(5): + line = 'hello' + res = f.send_single(line) + if i == 3 and res != None: + is_test_ok = False + logging.error('test_wrong_python_exception_during_init: Filter should fail with an error cerittude (101), but we received: ' + str(res)) + if i == 3 and not f.check_line_in_filter_log("PythonTask:: PythonTask:: Error creating Execution object: Python error '' : Fail at init"): + is_test_ok = False + logging.error('test_wrong_python_exception_during_init: Filter should fail with a specific log, but was not found') + if i != 3 and res != len(line): + is_test_ok = False + logging.error(f'test_wrong_python_exception_during_init: Filter should succced with a certitude of ({str(len(line))}), but we received: {str(res)}') + + if not f.check_run(): + logging.error('test_wrong_python_exception_during_init: Filter should still be running') + return False + + return is_test_ok + +def test_wrong_python_exception_during_delete(): + """ + Cannot be tested as exceptions during __del__ are ignored by the python interpreter + """ + return True + + def test_venv(): with tempfile.TemporaryDirectory() as tmpdir: e = venv.EnvBuilder(with_pip=True) From ff920e90effea620f9021db14c72562192f0b0b8 Mon Sep 17 00:00:00 2001 From: Thibaud Cartegnie Date: Thu, 17 Mar 2022 09:16:59 +0100 Subject: [PATCH 54/54] Resolve comments --- samples/base/Logger.cpp | 4 ++ samples/base/Logger.hpp | 7 +++ samples/fpython/Generator.cpp | 77 +++++++++++++++++--------------- samples/fpython/Generator.hpp | 10 ++--- samples/fpython/PythonTask.cpp | 20 +++++---- samples/fpython/PythonThread.hpp | 22 ++++----- samples/fpython/fpython.hpp | 24 +++++----- tests/tools/filter.py | 3 -- 8 files changed, 92 insertions(+), 75 deletions(-) diff --git a/samples/base/Logger.cpp b/samples/base/Logger.cpp index 7a28497..ee66e2a 100644 --- a/samples/base/Logger.cpp +++ b/samples/base/Logger.cpp @@ -84,6 +84,10 @@ namespace darwin { return false; } + log_type Logger::getLevel() const { + return this->_logLevel; + } + void Logger::setName(std::string const& name) { _name = name; } diff --git a/samples/base/Logger.hpp b/samples/base/Logger.hpp index 352a741..41c23b0 100644 --- a/samples/base/Logger.hpp +++ b/samples/base/Logger.hpp @@ -95,6 +95,13 @@ namespace darwin { /// \return true if success, false otherwise. bool setLevel(std::string level); + /// + /// \brief Get the Log Level + /// + /// \return log_type + /// + log_type getLevel() const; + /// Set the name of the module in the logger. /// /// \param name The name to set as the module name. diff --git a/samples/fpython/Generator.cpp b/samples/fpython/Generator.cpp index b051904..ddd654b 100644 --- a/samples/fpython/Generator.cpp +++ b/samples/fpython/Generator.cpp @@ -38,6 +38,11 @@ bool Generator::PyExceptionCheckAndLog(std::string const& prefix_log, darwin::lo std::string errString, typeString; PyErr_Fetch(&errType, &err, &tb); + // Early return if we don't have to log + if (log_type < log.getLevel()) { + return true; + } + if((pErrStr = PyObject_Str(err)) != nullptr){ if((pErrChars = PyUnicode_AsUTF8AndSize(*pErrStr, &errCharsLen)) != nullptr) { errString = std::string(pErrChars, errCharsLen); @@ -161,7 +166,7 @@ bool Generator::LoadConfig(const rapidjson::Document &configuration) { bool Generator::SendConfig(rapidjson::Document const& config) { DARWIN_LOGGER; bool pyConfigRes = false, soConfigRes = false; - switch(functions.configPyFunc.origin) { + switch(_functions.configPyFunc.origin) { case FunctionOrigin::none: DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in python script"); pyConfigRes = true; @@ -174,7 +179,7 @@ bool Generator::SendConfig(rapidjson::Document const& config) { return false; } - switch(functions.configSoFunc.origin) { + switch(_functions.configSoFunc.origin) { case FunctionOrigin::none: DARWIN_LOG_INFO("Python:: Generator:: SendConfig: No configuration function found in shared object"); soConfigRes = true; @@ -184,7 +189,7 @@ bool Generator::SendConfig(rapidjson::Document const& config) { return false; case FunctionOrigin::shared_library: try{ - soConfigRes = functions.configSoFunc.so(config); + soConfigRes = _functions.configSoFunc.so(config); } catch(std::exception const& e){ DARWIN_LOG_CRITICAL("Python:: Generator:: SendConfig: error while sending config :" + std::string(e.what())); soConfigRes = false; @@ -212,14 +217,14 @@ bool Generator::SendPythonConfig(rapidjson::Document const& config) { std::string cmd = "json.loads(\"\"\""; cmd += buffer.GetString(); cmd += "\"\"\")"; - PyObject* globalDictionary = PyModule_GetDict(*pModule); + PyObject* globalDictionary = PyModule_GetDict(*_pModule); PyObjectOwner pConfig = PyRun_StringFlags(cmd.c_str(), Py_eval_input, globalDictionary, globalDictionary, nullptr); if(Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: Converting config to Py Object :")){ return false; } PyObjectOwner res; - if ((res = PyObject_CallFunctionObjArgs(functions.configPyFunc.py, *pConfig, nullptr)) == nullptr) { + if ((res = PyObject_CallFunctionObjArgs(_functions.configPyFunc.py, *pConfig, nullptr)) == nullptr) { Generator::PyExceptionCheckAndLog("Python:: Generator:: SendPythonConfig: "); return false; } @@ -253,15 +258,15 @@ bool Generator::LoadSharedLibrary(const std::string& shared_library_path) { return false; } - LoadFunctionFromSO(handle, functions.configSoFunc, "filter_config"); - LoadFunctionFromSO(handle, functions.parseBodyFunc, "parse_body"); - LoadFunctionFromSO(handle, functions.preProcessingFunc, "filter_pre_process"); - LoadFunctionFromSO(handle, functions.processingFunc, "filter_process"); - LoadFunctionFromSO(handle, functions.contextualizeFunc, "filter_contextualize"); - LoadFunctionFromSO(handle, functions.alertContextualizeFunc, "alert_contextualize"); - LoadFunctionFromSO(handle, functions.alertFormatingFunc, "alert_formating"); - LoadFunctionFromSO(handle, functions.outputFormatingFunc, "output_formating"); - LoadFunctionFromSO(handle, functions.responseFormatingFunc, "response_formating"); + LoadFunctionFromSO(handle, _functions.configSoFunc, "filter_config"); + LoadFunctionFromSO(handle, _functions.parseBodyFunc, "parse_body"); + LoadFunctionFromSO(handle, _functions.preProcessingFunc, "filter_pre_process"); + LoadFunctionFromSO(handle, _functions.processingFunc, "filter_process"); + LoadFunctionFromSO(handle, _functions.contextualizeFunc, "filter_contextualize"); + LoadFunctionFromSO(handle, _functions.alertContextualizeFunc, "alert_contextualize"); + LoadFunctionFromSO(handle, _functions.alertFormatingFunc, "alert_formating"); + LoadFunctionFromSO(handle, _functions.outputFormatingFunc, "output_formating"); + LoadFunctionFromSO(handle, _functions.responseFormatingFunc, "response_formating"); return true; } @@ -278,13 +283,13 @@ inline void Generator::LoadFunctionFromSO(void * lib_handle, FunctionPySo &fu } bool Generator::CheckConfig() const { - if(functions.preProcessingFunc.origin == FunctionOrigin::none - || functions.processingFunc.origin == FunctionOrigin::none - || functions.contextualizeFunc.origin == FunctionOrigin::none - || functions.alertContextualizeFunc.origin == FunctionOrigin::none - || functions.alertFormatingFunc.origin == FunctionOrigin::none - || functions.outputFormatingFunc.origin == FunctionOrigin::none - || functions.responseFormatingFunc.origin == FunctionOrigin::none) + if(_functions.preProcessingFunc.origin == FunctionOrigin::none + || _functions.processingFunc.origin == FunctionOrigin::none + || _functions.contextualizeFunc.origin == FunctionOrigin::none + || _functions.alertContextualizeFunc.origin == FunctionOrigin::none + || _functions.alertFormatingFunc.origin == FunctionOrigin::none + || _functions.outputFormatingFunc.origin == FunctionOrigin::none + || _functions.responseFormatingFunc.origin == FunctionOrigin::none) { DARWIN_LOGGER; DARWIN_LOG_CRITICAL("Generator::CheckConfig : Mandatory methods were not found in the python script or the shared library"); @@ -392,61 +397,61 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st } } - if((pModule = PyImport_Import(*pName)) == nullptr) { + if((_pModule = PyImport_Import(*pName)) == nullptr) { PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python script module '" + filename + "' : "); return false; } pName.Decref(); - pClass = PyObject_GetAttrString(*pModule, "Execution"); + _pClass = PyObject_GetAttrString(*_pModule, "Execution"); if (PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python class 'Execution' : ")){ return false; } - if(*pClass == nullptr || ! PyType_Check(*pClass)) { + if(*_pClass == nullptr || ! PyType_Check(*_pClass)) { DARWIN_LOG_CRITICAL("Generator::LoadPythonScript : Error while attempting to load the python class 'Execution' : Not a Class"); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.configPyFunc, "filter_config")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.configPyFunc, "filter_config")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_config' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.parseBodyFunc, "parse_body")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.parseBodyFunc, "parse_body")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'parse_body' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.preProcessingFunc, "filter_pre_process")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.preProcessingFunc, "filter_pre_process")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_pre_process' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.processingFunc, "filter_process")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.processingFunc, "filter_process")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_process' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.contextualizeFunc, "filter_contextualize")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.contextualizeFunc, "filter_contextualize")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'filter_contextualize' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.alertContextualizeFunc, "alert_contextualize")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.alertContextualizeFunc, "alert_contextualize")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_contextualize' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.alertFormatingFunc, "alert_formating")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.alertFormatingFunc, "alert_formating")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'alert_formating' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.outputFormatingFunc, "output_formating")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.outputFormatingFunc, "output_formating")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'output_formating' : "); return false; } - if( ! LoadFunctionFromPython(*pClass, functions.responseFormatingFunc, "response_formating")){ + if( ! LoadFunctionFromPython(*_pClass, _functions.responseFormatingFunc, "response_formating")){ PyExceptionCheckAndLog("Generator::LoadPythonScript : Error while attempting to load the python function 'response_formating' : "); return false; } @@ -455,9 +460,9 @@ bool Generator::LoadPythonScript(const std::string& python_script_path, const st } template -inline bool Generator::LoadFunctionFromPython(PyObject* pClass, FunctionPySo& function_holder, const std::string& function_name){ +inline bool Generator::LoadFunctionFromPython(PyObject* _pClass, FunctionPySo& function_holder, const std::string& function_name){ DARWIN_LOGGER; - function_holder.py = PyObject_GetAttrString(pClass, function_name.c_str()); + function_holder.py = PyObject_GetAttrString(_pClass, function_name.c_str()); if(function_holder.py == nullptr) { DARWIN_LOG_INFO("Generator::LoadPythonScript : No '" + function_name + "' method in the python script"); } else if (! PyCallable_Check(function_holder.py)){ @@ -473,7 +478,7 @@ std::shared_ptr Generator::CreateTask(darwin::session_ptr_t s) noexcept { return std::static_pointer_cast( std::make_shared(_cache, _cache_mutex, s, s->_packet, - *pClass, functions) + *_pClass, _functions) ); } diff --git a/samples/fpython/Generator.hpp b/samples/fpython/Generator.hpp index 416779c..89c73c6 100644 --- a/samples/fpython/Generator.hpp +++ b/samples/fpython/Generator.hpp @@ -113,7 +113,7 @@ class Generator: public AGenerator { /// /// \brief loads the python script which path is given and attempts to load all defined functions - /// If the path is not empty, a python interpreter will be initialized (by a call to Py_Initiliaze()) + /// If the path is not empty, a python interpreter will be initialized (by a call to Py_Initialize()) /// /// \param python_script_path /// \param python_venv_folder @@ -154,7 +154,7 @@ class Generator: public AGenerator { /// \brief Sends the configuration to the Python module /// /// \param config - /// \return false an error occured whil calling the python function or it sent back false + /// \return false an error occurred while calling the python function or it sent back false /// bool SendPythonConfig(rapidjson::Document const& config); @@ -164,8 +164,8 @@ class Generator: public AGenerator { template inline bool LoadFunctionFromPython(PyObject* pClass, FunctionPySo& function_holder, const std::string& function_name); - PyObjectOwner pModule; - PyObjectOwner pClass; - FunctionHolder functions; + PyObjectOwner _pModule; + PyObjectOwner _pClass; + FunctionHolder _functions; }; \ No newline at end of file diff --git a/samples/fpython/PythonTask.cpp b/samples/fpython/PythonTask.cpp index 897d063..b655c09 100644 --- a/samples/fpython/PythonTask.cpp +++ b/samples/fpython/PythonTask.cpp @@ -12,6 +12,7 @@ #include "Logger.hpp" #include "AlertManager.hpp" #include "PythonObject.hpp" +#include "Stats.hpp" PythonTask::PythonTask(std::shared_ptr> cache, std::mutex& cache_mutex, @@ -151,12 +152,11 @@ void PythonTask::operator()() { } } for(auto cert: resp.certitudes) { + if(cert > _threshold) { + STAT_MATCH_INC; + } _packet.AddCertitude(cert); } - // if( ! output.body.empty()) { - // _response_body.clear(); - // _response_body.append(output.body); - // } if( ! resp.body.empty()){ _response_body.clear(); @@ -188,8 +188,12 @@ bool PythonTask::ParseBody() { for(ssize_t i = 0; i < _body.Size(); i++){ PyObject *str; // This reference is "stolen" by PyList_SetItem, we don't have to DECREF it + if(! _body.GetArray()[i].IsString()){ + DARWIN_LOG_ERROR("PythonTask::ParseBody: Item on the list is not a string : " + std::to_string(i)); + return false; + } if((str = PyUnicode_DecodeFSDefault(_body.GetArray()[i].GetString())) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: An error occurred while raw body:"+_packet.GetBody()); Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); return false; } @@ -212,7 +216,7 @@ bool PythonTask::ParseBody() { } if ((_parsed_body = PyObject_CallFunctionObjArgs(_functions.parseBodyFunc.py, *_pSelf, *raw_body, nullptr)) == nullptr) { - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the Python function"); + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: An error occurred while calling the Python function"); Generator::PyExceptionCheckAndLog("PythonTask:: ParseBody:: An error occurred while parsing body :", darwin::logger::Error); return false; } @@ -222,7 +226,7 @@ bool PythonTask::ParseBody() { case FunctionOrigin::shared_library:{ try{ if((_parsed_body = _functions.parseBodyFunc.so(_pClass, *_pSelf, _packet.GetBody()))== nullptr){ - DARWIN_LOG_DEBUG("PythonTask:: ParseBody:: An error occurred while calling the SO function"); + DARWIN_LOG_ERROR("PythonTask:: ParseBody:: An error occurred while calling the SO function"); return false; } } catch(std::exception const& e){ @@ -300,7 +304,7 @@ DarwinResponse PythonTask::GetFormatedResponse(FunctionPySo alert_formating(PyObject* module, PyObject*input); - DarwinResponse output_formating(PyObject* module, PyObject*); - DarwinResponse response_formating(PyObject* module, PyObject*); + std::vector alert_formating(PyObject* pClass, PyObject* pSelf, PyObject*); + DarwinResponse output_formating(PyObject* pClass, PyObject* pSelf, PyObject*); + DarwinResponse response_formating(PyObject* pClass, PyObject* pSelf, PyObject*); #endif // C compatible linkage - PyObject* filter_pre_process(PyObject* module, PyObject*); - PyObject* filter_process(PyObject* module, PyObject*); - PyObject* filter_contextualize(PyObject* module, PyObject*); - PyObject* alert_contextualize(PyObject* module, PyObject*); + PyObject* filter_pre_process(PyObject* pClass, PyObject* pSelf, PyObject*); + PyObject* filter_process(PyObject* pClass, PyObject* pSelf, PyObject*); + PyObject* filter_contextualize(PyObject* pClass, PyObject* pSelf, PyObject*); + PyObject* alert_contextualize(PyObject* pClass, PyObject* pSelf, PyObject*); bool filter_config_c(const char*, const size_t); - PyObject* parse_body_c(PyObject* module, const char*, const size_t); - void alert_formating_c(PyObject* module, PyObject* input, char ** alerts, size_t nb_alerts, size_t* alerts_sizes); - DarwinResponseC* output_formating_c(PyObject* module, PyObject*); - DarwinResponseC* response_formating_c(PyObject* module, PyObject*); + PyObject* parse_body_c(PyObject* pClass, PyObject* pSelf, const char*, const size_t); + void alert_formating_c(PyObject* pClass, PyObject* pSelf, PyObject* input, char ** alerts, size_t nb_alerts, size_t* alerts_sizes); + DarwinResponseC* output_formating_c(PyObject* pClass, PyObject* pSelf, PyObject*); + DarwinResponseC* response_formating_c(PyObject* pClass, PyObject* pSelf, PyObject*); } \ No newline at end of file diff --git a/tests/tools/filter.py b/tests/tools/filter.py index 54401c4..d680da5 100644 --- a/tests/tools/filter.py +++ b/tests/tools/filter.py @@ -123,10 +123,7 @@ def valgrind_start(self): '--errors-for-leak-kinds=definite', '-q', # Quiet mode so it doesn't pollute the output '--error-exitcode={}'.format(self.error_code), #If valgrind reports error(s) during the run, return this exit code. - '--suppressions=/home/myadvens.lan/tcartegnie/workspace/darwin/tests/valgrind-python.supp', - '--log-file=valgrind.log', ] + self.cmd - print(' '.join(command)) self.process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sleep(3) return self.check_start()