diff --git a/code/__DEFINES/elastic.dm b/code/__DEFINES/elastic.dm index ca254d84ca9..b6ca62e210a 100644 --- a/code/__DEFINES/elastic.dm +++ b/code/__DEFINES/elastic.dm @@ -15,6 +15,8 @@ #define ELASCAT_BALANCE "balance" #define ELASCAT_MEDICAL "medical" #define ELASCAT_ENCHANTING "enchanting" + #define ELASCAT_HEARTBEAT "heartbeat" + #define ELASCAT_ROUND "round" /* Abstract Data */ /* Combat */ @@ -22,14 +24,19 @@ #define ELASDATA_DECAPITATIONS "decapitations" /// An animal mob has eaten a corpse. #define ELASDATA_EATEN_BODIES "eaten_bodies" - /* Underworld */ - /// An underworld spirit has been revived with a Toll. - #define ELASDATA_COIN_REVIVES "coin_revive" - /// An underworld spirit has won a pit fight. - #define ELASDATA_FIGHT_REVIVES "fight_revives" /* Economy */ #define ELASDATA_MAMMONS_GAINED "mammons_gained" #define ELASDATA_MAMMONS_SPENT "mammons_spent" + #define ELASDATA_TAXES_EVADED "taxes_evaded" + #define ELASDATA_TAXES_COLLECTED "taxes_collected" + #define ELASDATA_WAGES_PAID "wages_paid" + #define ELASDATA_FINE_INCOME "fine_income" + #define ELASDATA_IMPORT_VALUE "import_value" + #define ELASDATA_EXPORT_VALUE "export_value" + #define ELASDATA_GOLDFACE_SPENT "goldface_spent" + #define ELASDATA_NOBLE_INCOME "noble_income" + #define ELASDATA_TRIUMPH_AWARDED "triumph_awarded" + #define ELASDATA_TRIUMPH_SPENT "triumph_spent" /* Medical */ #define ELASDATA_ANASTASIS_REVIVE "anastasis" @@ -39,3 +46,8 @@ #define ELASDATA_LUX_REVIVE "lux_revive" #define ELASDATA_LUX_EXTRACT "lux_extract_npc" #define ELASDATA_LUX_EXTRACT_PLAYER "lux_extract_player" + /// An underworld spirit has been revived with a Toll. + #define ELASDATA_COIN_REVIVES "coin_revive" + /// An underworld spirit has won a pit fight. + #define ELASDATA_FIGHT_REVIVES "fight_revives" + #define ELASDATA_DEATH "deaths" diff --git a/code/controllers/subsystem/death_arena.dm b/code/controllers/subsystem/death_arena.dm index 1a7c00c4201..6d8a83dbfda 100644 --- a/code/controllers/subsystem/death_arena.dm +++ b/code/controllers/subsystem/death_arena.dm @@ -199,7 +199,7 @@ SUBSYSTEM_DEF(death_arena) /obj/structure/table/wood/fine/altar/after_added_effects(obj/item/item, mob/user) if(!istype(item, /obj/item/bodypart/head)) return - add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_FIGHT_REVIVES, 1) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_FIGHT_REVIVES, 1) record_round_statistic(STATS_UNDERWORLD_DUELS) SSdeath_arena.process_fight_end(item, user) diff --git a/code/controllers/subsystem/elastic.dm b/code/controllers/subsystem/elastic.dm deleted file mode 100644 index 7611e864533..00000000000 --- a/code/controllers/subsystem/elastic.dm +++ /dev/null @@ -1,141 +0,0 @@ -/datum/config_entry/flag/elastic_middleware_enabled - -/datum/config_entry/string/elastic_endpoint - protection = CONFIG_ENTRY_HIDDEN - -/datum/config_entry/string/metrics_api_token - protection = CONFIG_ENTRY_HIDDEN - -SUBSYSTEM_DEF(elastic) - name = "Elastic Middleware" - wait = 30 SECONDS - runlevels = RUNLEVEL_LOBBY | RUNLEVEL_SETUP | RUNLEVEL_GAME | RUNLEVEL_POSTGAME - flags = SS_KEEP_TIMING // This needs to ingest every 30 IRL seconds, not ingame seconds. - /// The TIMEOFDAY when /world was created. Set in Genesis. - var/world_init_time = 0 - /// Compiled round list data. Interfaced with [proc/add_list_data] - var/list/assoc_list_data = list() //! ### This NEEDS NEEDS NEEDS NEEDS NEEDS to be an assoclist. When 516 is in this will be an alist - ///abstract information - basically want to keep track of spell casts over the round? do it like this - var/list/abstract_information = list() - /// list of /datum/http_request that we check to ensure they don't leak memory - var/list/active_requests = list() - var/shutting_down = FALSE - -/datum/controller/subsystem/elastic/Initialize(start_timeofday) - if(!CONFIG_GET(flag/elastic_middleware_enabled)) - flags |= SS_NO_FIRE // Disable firing to save CPU - set_abstract_data_zeros() - return ..() - -/datum/controller/subsystem/elastic/Shutdown() - shutting_down = TRUE - var/end_time = REALTIMEOFDAY + (5 SECONDS) - while((length(active_requests) > 0) && (REALTIMEOFDAY < end_time)) - for(var/datum/http_request/request as anything in active_requests) - if(request.is_complete()) - active_requests -= request - qdel(request) - active_requests.Cut() - -/datum/controller/subsystem/elastic/fire(resumed) - send_data() - for(var/datum/http_request/request as anything in active_requests) - if(request.is_complete()) // rust-g will clear the job once it's complete - active_requests -= request - qdel(request) - -/datum/controller/subsystem/elastic/proc/send_data() - if(shutting_down) - return - var/datum/http_request/request = new() - request.prepare(RUSTG_HTTP_METHOD_POST, CONFIG_GET(string/elastic_endpoint), get_compiled_data(), list( - "Authorization" = "ApiKey [CONFIG_GET(string/metrics_api_token)]", - "Content-Type" = "application/json" - )) - request.begin_async() - active_requests += request - -/datum/controller/subsystem/elastic/proc/get_compiled_data() - var/list/compiled = list() - //DON'T CHANGE ANY OF THIS BLOCK EVER OR THIS WILL ALL BREAK - compiled["@timestamp"] = time_stamp_metric() - compiled["cpu"] = world.cpu - compiled["elapsed_process_time"] = world.time - compiled["elapsed_real_time"] = (REALTIMEOFDAY - world_init_time) - compiled["client_count"] = length(GLOB.clients) - compiled["round_id"] = GLOB.rogue_round_id // if you are on literally any other server change this to a text2num(GLOB.round_id) - compiled |= assoc_list_data // you see why this needs to be an assoc list now? - - // down here is specific to vanderlin so if you are porting this you can take this out - compiled["round_data"] = get_round_data() - - assoc_list_data = list() - return json_encode(compiled) - -/datum/controller/subsystem/elastic/proc/get_round_data() - var/list/round_data = list() - - for(var/patron_name in GLOB.patron_follower_counts) - round_data["[patron_name]_followers"] = GLOB.patron_follower_counts[patron_name] - - for(var/stat in GLOB.vanderlin_round_stats) - round_data[stat] = GLOB.vanderlin_round_stats[stat] - - return round_data - -/datum/controller/subsystem/elastic/proc/add_list_data(main_cat = ELASCAT_GENERIC, list/assoc_data) - if(!main_cat || !length(assoc_data)) - return - - assoc_list_data |= main_cat - if(!length(assoc_list_data[main_cat])) - assoc_list_data[main_cat] = list() - assoc_list_data[main_cat] |= assoc_data - -/// Inserts `(|=)` a datapoint into an elasticsearch category's data packet. -/proc/add_elastic_data(main_cat, list/assoc_data) - if(!main_cat || !length(assoc_data)) - return - SSelastic.add_list_data(main_cat, assoc_data) - return TRUE - -/// Inserts `(|=)` and immediately sends the provided data packet to elasticsearch. -/// This should be used for logging purposes, such as runtimes or wanting to track "player x did y". -/proc/add_elastic_data_immediate(main_cat, list/assoc_data) - if(add_elastic_data(main_cat, assoc_data)) - SSelastic.send_data() - return TRUE - -/// Adds `(+=)` a numerical value to an elasticsearch data point. -/// Think "x event ran 12 times this packet" since you're updating the number with the total ran anyway. -/proc/add_abstract_elastic_data(main_cat, abstract_name, abstract_value, maximum) - if(!main_cat || !isnum(abstract_value)) - return - - SSelastic.abstract_information |= abstract_name - SSelastic.abstract_information[abstract_name] += abstract_value - if(maximum) - SSelastic.abstract_information[abstract_name] = min(maximum, SSelastic.abstract_information[abstract_name]) - - var/list/data = list("[abstract_name]" = SSelastic.abstract_information[abstract_name]) - SSelastic.add_list_data(main_cat, data) - return TRUE - -/// Zeroes out some abstract data values. -/// This really exists if you want data to start at 0, useful for timeseries data without round filtering. -/proc/set_abstract_data_zeros() - add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_FIGHT_REVIVES, 0) - add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_COIN_REVIVES, 0) - add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_EATEN_BODIES, 0) - add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_DECAPITATIONS, 0) - - add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_MAMMONS_GAINED, 0) - add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_MAMMONS_SPENT, 0) - - add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_ANASTASIS_REVIVE, 0) - add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_CPR_REVIVE, 0) - add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_ABSOLVE_REVIVE, 0) - add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_ULTIMATE_REVIVE, 0) - add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_LUX_REVIVE, 0) - add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_LUX_EXTRACT, 0) - add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_LUX_EXTRACT_PLAYER, 0) diff --git a/code/controllers/subsystem/elastic/_helpers.dm b/code/controllers/subsystem/elastic/_helpers.dm new file mode 100644 index 00000000000..ed85a90bb5c --- /dev/null +++ b/code/controllers/subsystem/elastic/_helpers.dm @@ -0,0 +1,47 @@ +/proc/add_elastic_data(main_cat, list/assoc_data) + if(!main_cat || !length(assoc_data)) + return + var/datum/elastic_shard/S = SSelastic.get_shard(main_cat) + if(!S) + return + S.add_list_data(assoc_data) + return TRUE + +/proc/add_elastic_data_immediate(main_cat, list/assoc_data) + if(!main_cat || !length(assoc_data)) + return + var/datum/elastic_shard/S = SSelastic.get_shard(main_cat) + if(!S) + return + S.add_list_data(assoc_data) + var/compiled = S.get_compiled_data(SSelastic) + if(compiled) + SSelastic.dispatch_request(compiled) + S.reset() + return TRUE + +/proc/add_abstract_elastic_data(main_cat, abstract_name, abstract_value, maximum) + if(!main_cat || !isnum(abstract_value)) + return + var/datum/elastic_shard/S = SSelastic.get_shard(main_cat) + if(!S) + return + S.add_abstract_data(abstract_name, abstract_value, maximum) + return TRUE + +/proc/init_abstract_zeros() + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_FIGHT_REVIVES, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_COIN_REVIVES, 0) + add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_EATEN_BODIES, 0) + add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_DECAPITATIONS, 0) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_MAMMONS_GAINED, 0) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_MAMMONS_SPENT, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_ANASTASIS_REVIVE, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_CPR_REVIVE, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_ABSOLVE_REVIVE, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_ULTIMATE_REVIVE, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_LUX_REVIVE, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_LUX_EXTRACT, 0) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_LUX_EXTRACT_PLAYER, 0) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TRIUMPH_SPENT, 0) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TRIUMPH_AWARDED, 0) diff --git a/code/controllers/subsystem/elastic/config.dm b/code/controllers/subsystem/elastic/config.dm new file mode 100644 index 00000000000..df0f54372c0 --- /dev/null +++ b/code/controllers/subsystem/elastic/config.dm @@ -0,0 +1,31 @@ +/datum/config_entry/flag/elastic_middleware_enabled + +/datum/config_entry/string/elastic_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/combat_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/economy_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/medical_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/crafting_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/enchanting_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/round_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/storyteller_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/heartbeat_endpoint + protection = CONFIG_ENTRY_HIDDEN + +/datum/config_entry/string/metrics_api_token + protection = CONFIG_ENTRY_HIDDEN diff --git a/code/controllers/subsystem/elastic/elastic.dm b/code/controllers/subsystem/elastic/elastic.dm new file mode 100644 index 00000000000..43619479a68 --- /dev/null +++ b/code/controllers/subsystem/elastic/elastic.dm @@ -0,0 +1,50 @@ +SUBSYSTEM_DEF(elastic) + name = "Elastic Middleware" + wait = 5 SECONDS + runlevels = RUNLEVEL_LOBBY | RUNLEVEL_SETUP | RUNLEVEL_GAME | RUNLEVEL_POSTGAME + flags = SS_KEEP_TIMING + var/world_init_time = 0 + var/list/active_requests = list() + var/list/shards = list() + var/shutting_down = FALSE + +/datum/controller/subsystem/elastic/Initialize(start_timeofday) + if(!CONFIG_GET(flag/elastic_middleware_enabled)) + flags |= SS_NO_FIRE + return ..() + register_shards() + init_abstract_zeros() + return ..() + +/datum/controller/subsystem/elastic/proc/register_shards() + for(var/datum/elastic_shard/shard as anything in subtypesof(/datum/elastic_shard)) + shards += new shard() + +/datum/controller/subsystem/elastic/proc/get_shard(shard_category) + for(var/datum/elastic_shard/S as anything in shards) + if(S.shard_category == shard_category) + return S + +/datum/controller/subsystem/elastic/fire(resumed) + for(var/datum/elastic_shard/S as anything in shards) + if(S.should_fire()) + S.fire(src) + // Clean up completed requests + for(var/datum/http_request/request as anything in active_requests) + if(request.is_complete()) + active_requests -= request + qdel(request) + +/datum/controller/subsystem/elastic/proc/dispatch_request(datum/elastic_shard/shard, json_body) + if(shutting_down) + return + var/endpoint = shard.get_endpoint() + if(!endpoint) + return + var/datum/http_request/request = new() + request.prepare(RUSTG_HTTP_METHOD_POST, endpoint, json_body, list( + "Authorization" = "ApiKey [CONFIG_GET(string/metrics_api_token)]", + "Content-Type" = "application/json" + )) + request.begin_async() + active_requests += request diff --git a/code/controllers/subsystem/elastic/shards/_base_shard.dm b/code/controllers/subsystem/elastic/shards/_base_shard.dm new file mode 100644 index 00000000000..552403b62cf --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/_base_shard.dm @@ -0,0 +1,57 @@ +/datum/elastic_shard + /// Display name for debugging + var/name = "Generic Shard" + /// How often this shard fires, in real seconds + var/upload_frequency = 30 SECONDS + /// Elasticsearch index/category this shard posts to (can be a different endpoint suffix) + var/shard_category = ELASCAT_GENERIC + /// Tracks the last REALTIMEOFDAY this shard fired + var/last_fired = 0 + /// Accumulated assoc data for this shard + var/list/assoc_list_data = list() + /// Abstract (cumulative numeric) data, keyed by ELASDATA_ defines + var/list/abstract_information = list() + ///should we keep things dated? (send on loop if its being triggered once) + var/should_keep_dated = FALSE + +/datum/elastic_shard/proc/should_fire() + return (REALTIMEOFDAY - last_fired) >= upload_frequency + +/datum/elastic_shard/proc/get_endpoint() + return CONFIG_GET(string/elastic_endpoint) + +/datum/elastic_shard/proc/fire(datum/controller/subsystem/elastic/SS) + last_fired = REALTIMEOFDAY + var/compiled = get_compiled_data(SS) + if(!compiled) + return + SS.dispatch_request(src, compiled) + reset() + +/datum/elastic_shard/proc/get_compiled_data(datum/controller/subsystem/elastic/SS) + if(!length(assoc_list_data) && (should_keep_dated && !length(abstract_information))) + return null + var/list/compiled = list() + compiled["@timestamp"] = time_stamp_metric() + compiled["shard"] = shard_category + compiled["round_id"] = GLOB.rogue_round_id + compiled["elapsed_real_time"] = (REALTIMEOFDAY - SS.world_init_time) + compiled |= assoc_list_data + return json_encode(compiled) + +/datum/elastic_shard/proc/reset() + assoc_list_data = list() + // Note: abstract_information is NOT reset - it's cumulative + +/datum/elastic_shard/proc/add_list_data(list/assoc_data) + assoc_list_data |= assoc_data + +/datum/elastic_shard/proc/add_abstract_data(abstract_name, abstract_value, maximum) + if(!isnum(abstract_value)) + return + abstract_information |= abstract_name + abstract_information[abstract_name] += abstract_value + if(maximum) + abstract_information[abstract_name] = min(maximum, abstract_information[abstract_name]) + // Mirror into assoc so it gets compiled + assoc_list_data["[abstract_name]"] = abstract_information[abstract_name] diff --git a/code/controllers/subsystem/elastic/shards/combat.dm b/code/controllers/subsystem/elastic/shards/combat.dm new file mode 100644 index 00000000000..ad313ffdc38 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/combat.dm @@ -0,0 +1,7 @@ +/datum/elastic_shard/combat + name = "Combat" + upload_frequency = 2 MINUTES + shard_category = ELASCAT_COMBAT + +/datum/elastic_shard/combat/get_endpoint() + return CONFIG_GET(string/combat_endpoint) diff --git a/code/controllers/subsystem/elastic/shards/crafting.dm b/code/controllers/subsystem/elastic/shards/crafting.dm new file mode 100644 index 00000000000..2d2ec827f20 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/crafting.dm @@ -0,0 +1,7 @@ +/datum/elastic_shard/crafting + name = "Crafting" + upload_frequency = 5 MINUTES + shard_category = ELASCAT_CRAFTING + +/datum/elastic_shard/crafting/get_endpoint() + return CONFIG_GET(string/crafting_endpoint) diff --git a/code/controllers/subsystem/elastic/shards/economy.dm b/code/controllers/subsystem/elastic/shards/economy.dm new file mode 100644 index 00000000000..2967a1b11a5 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/economy.dm @@ -0,0 +1,7 @@ +/datum/elastic_shard/economy + name = "Economy" + upload_frequency = 4 MINUTES + shard_category = ELASCAT_ECONOMY + +/datum/elastic_shard/economy/get_endpoint() + return CONFIG_GET(string/economy_endpoint) diff --git a/code/controllers/subsystem/elastic/shards/enchanting.dm b/code/controllers/subsystem/elastic/shards/enchanting.dm new file mode 100644 index 00000000000..24a4a8d5ea1 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/enchanting.dm @@ -0,0 +1,7 @@ +/datum/elastic_shard/enchanting + name = "Enchantments" + upload_frequency = 5 MINUTES + shard_category = ELASCAT_ENCHANTING + +/datum/elastic_shard/enchanting/get_endpoint() + return CONFIG_GET(string/enchanting_endpoint) diff --git a/code/controllers/subsystem/elastic/shards/heartbeat.dm b/code/controllers/subsystem/elastic/shards/heartbeat.dm new file mode 100644 index 00000000000..38820364c16 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/heartbeat.dm @@ -0,0 +1,19 @@ +/datum/elastic_shard/heartbeat + name = "Heartbeat" + upload_frequency = 30 SECONDS + shard_category = ELASCAT_HEARTBEAT + +/datum/elastic_shard/heartbeat/get_compiled_data(datum/controller/subsystem/elastic/SS) + var/list/compiled = list() + compiled["@timestamp"] = time_stamp_metric() + compiled["shard"] = shard_category + compiled["round_id"] = GLOB.rogue_round_id + compiled["cpu"] = world.cpu + compiled["elapsed_process_time"] = world.time + compiled["elapsed_real_time"] = (REALTIMEOFDAY - SS.world_init_time) + compiled["client_count"] = length(GLOB.clients) + compiled |= assoc_list_data + return json_encode(compiled) + +/datum/elastic_shard/heartbeat/get_endpoint() + return CONFIG_GET(string/heartbeat_endpoint) diff --git a/code/controllers/subsystem/elastic/shards/medical.dm b/code/controllers/subsystem/elastic/shards/medical.dm new file mode 100644 index 00000000000..ad977c97204 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/medical.dm @@ -0,0 +1,7 @@ +/datum/elastic_shard/medical + name = "Medical" + upload_frequency = 60 SECONDS + shard_category = ELASCAT_MEDICAL + +/datum/elastic_shard/medical/get_endpoint() + return CONFIG_GET(string/medical_endpoint) diff --git a/code/controllers/subsystem/elastic/shards/round_data.dm b/code/controllers/subsystem/elastic/shards/round_data.dm new file mode 100644 index 00000000000..b5172ba26c0 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/round_data.dm @@ -0,0 +1,19 @@ +/datum/elastic_shard/round_data + name = "Round Data" + upload_frequency = 5 MINUTES + shard_category = ELASCAT_ROUND + +/datum/elastic_shard/round_data/get_compiled_data(datum/controller/subsystem/elastic/SS) + var/list/compiled = list() + compiled["@timestamp"] = time_stamp_metric() + compiled["shard"] = shard_category + compiled["round_id"] = GLOB.rogue_round_id + for(var/patron_name in GLOB.patron_follower_counts) + compiled["[patron_name]_followers"] = GLOB.patron_follower_counts[patron_name] + for(var/stat in GLOB.vanderlin_round_stats) + compiled[stat] = GLOB.vanderlin_round_stats[stat] + compiled |= assoc_list_data + return json_encode(compiled) + +/datum/elastic_shard/round_data/get_endpoint() + return CONFIG_GET(string/round_endpoint) diff --git a/code/controllers/subsystem/elastic/shards/storytellers.dm b/code/controllers/subsystem/elastic/shards/storytellers.dm new file mode 100644 index 00000000000..fa17e33d756 --- /dev/null +++ b/code/controllers/subsystem/elastic/shards/storytellers.dm @@ -0,0 +1,7 @@ +/datum/elastic_shard/storytellers + name = "Storytellers" + upload_frequency = 5 MINUTES + shard_category = ELASCAT_STORYTELLER + +/datum/elastic_shard/storytellers/get_endpoint() + return CONFIG_GET(string/storyteller_endpoint) diff --git a/code/controllers/subsystem/elastic/so_you_wanna_port_this.md b/code/controllers/subsystem/elastic/so_you_wanna_port_this.md new file mode 100644 index 00000000000..d5703827a7e --- /dev/null +++ b/code/controllers/subsystem/elastic/so_you_wanna_port_this.md @@ -0,0 +1,42 @@ +# What is this? +This is a subsystem dedicated to sending data to [elasticsearch](https://www.elastic.co/elasticsearch), more specifically sending data to multiple indices inside your elastic cluster. +## Why? +Because if you want to collect lots of data with lots of fields its better to split things into multiple indices, it also allows you to setup different times for sending data see heartbeat vs round_data. +## How? +This is entirely self contained except the category defines as I assume you want to do those yourself, so just drop this folder into your codebase. After that you will want to create an API key see the curl request below (my homies hate the kibana dashboard) + +1 thing to note is rouge_round_id will almost certainly need to be replaced with round_id in round_id compiled data sections. +``` +curl -X POST "https://{YOUR_BACKEND}:9200/_security/api_key" \ + -H "Content-Type: application/json" \ + -u "elastic:your-password" \ + -d '{ + "name": "{INDICE_NAME}-metrics", + "role_descriptors": { + "{INDICE_NAME}_writer": { + "cluster": ["monitor"], + "index": [ + { + "names": ["{INDICE_NAME}*"], + "privileges": ["create_index", "index", "auto_configure"] + } + ] + } + } + }' +``` +this will create you an api key thats wildcarded to your indice + anything which is ideal, each cat can be stuff like: CODEBASE_heartbeat + +You will use the encoded output from the above command as your metric key. +For the Endpoint urls that depends on your structure but it will always be: +https://{YOUR_BACKEND}:9200/{INDICE}/_doc +assuming you have default ports setup. + +While you can have multiple shards pointing to 1 endpoint I highly recommend against as the entire reason you are doing shards is to contain the data to that shard. + +## That's cool but how do I use this data? +The easiest way is to use something like [grafana](https://grafana.com/), you can add each of the endpoints as a source and build dashboards from it, quite useful and alot faster then blackbox was. + +# Things to Note +- By default all the shards are set to not report data if no new data is added, this means you will want to bridge nulls on your dashboards or have it send regardless (I recommend just bridging in most cases) +- Crafting and Enchanting shards probably shouldn't be replicated by you since this creates alot of fields but genuinely lazy and like this. diff --git a/code/controllers/subsystem/treasury.dm b/code/controllers/subsystem/treasury.dm index 34558b2fa26..66aea63b447 100644 --- a/code/controllers/subsystem/treasury.dm +++ b/code/controllers/subsystem/treasury.dm @@ -298,5 +298,6 @@ SUBSYSTEM_DEF(treasury) for(var/mob/living/welfare_dependant in noble_incomes) var/how_much = noble_incomes[welfare_dependant] record_round_statistic(STATS_NOBLE_INCOME_TOTAL, how_much) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_NOBLE_INCOME, how_much) give_money_treasury(how_much, silent = TRUE) give_money_account(how_much, welfare_dependant, "Vanderlin Noble Estate") diff --git a/code/controllers/subsystem/triumph/triumph_adjust_procs.dm b/code/controllers/subsystem/triumph/triumph_adjust_procs.dm index 3c3865c3c0f..c3e9095608b 100644 --- a/code/controllers/subsystem/triumph/triumph_adjust_procs.dm +++ b/code/controllers/subsystem/triumph/triumph_adjust_procs.dm @@ -41,10 +41,12 @@ adjustment_verb = "awarded" if(counted) record_round_statistic(STATS_TRIUMPHS_AWARDED, amount) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TRIUMPH_AWARDED, amount) else adjustment_verb = "lost" if(counted) record_round_statistic(STATS_TRIUMPHS_STOLEN, amount) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TRIUMPH_SPENT, amount) var/final_text = "[abs(amount)] TRIUMPH\s [adjustment_verb]." if(reason) diff --git a/code/datums/components/crafting/crafting.dm b/code/datums/components/crafting/crafting.dm index 43305d03982..60b2f7aa69a 100644 --- a/code/datums/components/crafting/crafting.dm +++ b/code/datums/components/crafting/crafting.dm @@ -16,7 +16,7 @@ SEND_SIGNAL(user, COMSIG_ITEM_CRAFTED, user, type) record_featured_stat(FEATURED_STATS_CRAFTERS, user) record_featured_object_stat(FEATURED_STATS_CRAFTED_ITEMS, name) - add_abstract_elastic_data(ELASCAT_CRAFTING, "[name]", 1) + add_abstract_elastic_data(ELASCAT_CRAFTING, "[initial(name)]", 1) return /obj/OnCrafted(dirin, mob/user) diff --git a/code/game/machinery/trams_and_elevators/lift_master.dm b/code/game/machinery/trams_and_elevators/lift_master.dm index b9d0a7bd743..32843cf9638 100644 --- a/code/game/machinery/trams_and_elevators/lift_master.dm +++ b/code/game/machinery/trams_and_elevators/lift_master.dm @@ -797,6 +797,7 @@ GLOBAL_LIST_EMPTY(active_lifts_by_type) if(spent_amount) record_round_statistic(STATS_TRADE_VALUE_IMPORTED, spent_amount) add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_MAMMONS_SPENT, spent_amount, 1) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_IMPORT_VALUE, total_coin_value) /datum/lift_master/tram/proc/get_valid_turfs(obj/structure/industrial_lift/tram/platform) var/list/valid_turfs = list() @@ -951,6 +952,7 @@ GLOBAL_LIST_EMPTY(active_lifts_by_type) var/atom/location = spawn_coins(total_coin_value, platform) // try_process_order will eat these coins, so don't spawn a chest record_round_statistic(STATS_TRADE_VALUE_EXPORTED, total_coin_value) add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_MAMMONS_GAINED, total_coin_value) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_EXPORT_VALUE, total_coin_value) if(length(sold_items) && !fence) var/scrolls_to_spawn = CEILING(length(sold_items) / 6, 1) for(var/i = 1 to scrolls_to_spawn) diff --git a/code/game/objects/structures/fake_machines/ATM.dm b/code/game/objects/structures/fake_machines/ATM.dm index e22426b5fa7..7ff98233711 100644 --- a/code/game/objects/structures/fake_machines/ATM.dm +++ b/code/game/objects/structures/fake_machines/ATM.dm @@ -97,6 +97,7 @@ say("Your deposit was taxed [deposit_results[2]] mammon.") record_featured_stat(FEATURED_STATS_TAX_PAYERS, H, deposit_results[2]) record_round_statistic(STATS_TAXES_COLLECTED, deposit_results[2]) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TAXES_COLLECTED, deposit_results[2]) qdel(P) playsound(src, 'sound/misc/coininsert.ogg', 100, FALSE, -1) return diff --git a/code/game/objects/structures/fake_machines/drugmachine.dm b/code/game/objects/structures/fake_machines/drugmachine.dm index c34201421e0..1d8ddb9e5cd 100644 --- a/code/game/objects/structures/fake_machines/drugmachine.dm +++ b/code/game/objects/structures/fake_machines/drugmachine.dm @@ -168,8 +168,10 @@ SStreasury.give_money_treasury(tax_amt, "goldface import tax") record_featured_stat(FEATURED_STATS_TAX_PAYERS, human_mob, tax_amt) record_round_statistic(STATS_TAXES_COLLECTED, tax_amt) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TAXES_COLLECTED, tax_amt) else record_round_statistic(STATS_TAXES_EVADED, tax_amt) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TAXES_EVADED, tax_amt) else say("Not enough!") return diff --git a/code/game/objects/structures/fake_machines/merchant.dm b/code/game/objects/structures/fake_machines/merchant.dm index cea970dd439..8a42ecc720f 100644 --- a/code/game/objects/structures/fake_machines/merchant.dm +++ b/code/game/objects/structures/fake_machines/merchant.dm @@ -167,12 +167,15 @@ if(budget >= final_price) budget -= final_price record_round_statistic(STATS_GOLDFACE_VALUE_SPENT, final_price) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_GOLDFACE_SPENT, final_price) if(!(upgrade_flags & UPGRADE_NOTAX)) SStreasury.give_money_treasury(taxes, "goldface import tax") record_featured_stat(FEATURED_STATS_TAX_PAYERS, human_mob, taxes) record_round_statistic(STATS_TAXES_COLLECTED, taxes) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TAXES_COLLECTED, taxes) else record_round_statistic(STATS_TAXES_EVADED, taxes) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_TAXES_EVADED, taxes) else say("Not enough!") return diff --git a/code/game/objects/structures/fake_machines/steward.dm b/code/game/objects/structures/fake_machines/steward.dm index f5065983dd9..40b537fc3cb 100644 --- a/code/game/objects/structures/fake_machines/steward.dm +++ b/code/game/objects/structures/fake_machines/steward.dm @@ -276,6 +276,7 @@ if(newtax < 1) return record_round_statistic(STATS_FINES_INCOME, newtax) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_FINE_INCOME, newtax) SStreasury.give_money_account(-newtax, A) break if(href_list["payroll"]) @@ -303,6 +304,7 @@ var/datum/job/job_pay = SSjob.GetJob(job_to_pay) if(job_check && job_check.type == job_pay.type) record_round_statistic(STATS_WAGES_PAID, amount_to_pay) + add_abstract_elastic_data(ELASCAT_ECONOMY, ELASDATA_WAGES_PAID, amount_to_pay) SStreasury.give_money_account(amount_to_pay, H) if(href_list["compact"]) compact = !compact diff --git a/code/game/objects/structures/stairs.dm b/code/game/objects/structures/stairs.dm index 405c3a40e06..cbf746157ce 100644 --- a/code/game/objects/structures/stairs.dm +++ b/code/game/objects/structures/stairs.dm @@ -257,7 +257,7 @@ SEND_SIGNAL(user, COMSIG_ITEM_CRAFTED, user, type) record_featured_stat(FEATURED_STATS_CRAFTERS, user) record_featured_object_stat(FEATURED_STATS_CRAFTED_ITEMS, name) - add_abstract_elastic_data(ELASCAT_CRAFTING, "[name]", 1) + add_abstract_elastic_data(ELASCAT_CRAFTING, "[initial(name)]", 1) dir = dirin var/turf/partner = get_step(src, turn(dir, 180)) @@ -276,7 +276,7 @@ SEND_SIGNAL(user, COMSIG_ITEM_CRAFTED, user, type) record_featured_stat(FEATURED_STATS_CRAFTERS, user) record_featured_object_stat(FEATURED_STATS_CRAFTED_ITEMS, name) - add_abstract_elastic_data(ELASCAT_CRAFTING, "[name]", 1) + add_abstract_elastic_data(ELASCAT_CRAFTING, "[initial(name)]", 1) dir = dirin var/turf/partner = get_step(src, turn(dir, 180)) diff --git a/code/game/turfs/closed/wall/walls.dm b/code/game/turfs/closed/wall/walls.dm index 726d59d4c85..4fcd87899e6 100644 --- a/code/game/turfs/closed/wall/walls.dm +++ b/code/game/turfs/closed/wall/walls.dm @@ -262,7 +262,7 @@ SEND_SIGNAL(user, COMSIG_ITEM_CRAFTED, user, type) record_featured_stat(FEATURED_STATS_CRAFTERS, user) record_featured_object_stat(FEATURED_STATS_CRAFTED_ITEMS, name) - add_abstract_elastic_data(ELASCAT_CRAFTING, "[name]", 1) + add_abstract_elastic_data(ELASCAT_CRAFTING, "[initial(name)]", 1) return /turf/closed/wall/mineral/roofwall diff --git a/code/modules/mob/living/carbon/human/death.dm b/code/modules/mob/living/carbon/human/death.dm index ac493a73710..a4628a0900c 100644 --- a/code/modules/mob/living/carbon/human/death.dm +++ b/code/modules/mob/living/carbon/human/death.dm @@ -42,6 +42,7 @@ if(client || mind) record_round_statistic(STATS_DEATHS) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_DEATH, 1) var/area_of_death = lowertext(get_area_name(src)) if(area_of_death == "wilderness") record_round_statistic(STATS_FOREST_DEATHS) diff --git a/code/modules/underworld/underworld.dm b/code/modules/underworld/underworld.dm index 23136d8a0f2..c6abcee1691 100644 --- a/code/modules/underworld/underworld.dm +++ b/code/modules/underworld/underworld.dm @@ -119,7 +119,7 @@ switch(tgui_alert(user, "Are you ready to be judged?","Ready", list("Yes","No"))) if("Yes") playsound(user, 'sound/misc/deadbell.ogg', 50, TRUE, -2, ignore_walls = TRUE) - add_abstract_elastic_data(ELASCAT_COMBAT, ELASDATA_COIN_REVIVES, 1) + add_abstract_elastic_data(ELASCAT_MEDICAL, ELASDATA_COIN_REVIVES, 1) record_round_statistic(STATS_SOULS_REINCARNATED) user.returntolobby() if("No") diff --git a/vanderlin.dme b/vanderlin.dme index 0f104123918..baeafa51729 100644 --- a/vanderlin.dme +++ b/vanderlin.dme @@ -454,7 +454,6 @@ #include "code\controllers\subsystem\discord.dm" #include "code\controllers\subsystem\dungeon_generator.dm" #include "code\controllers\subsystem\economy.dm" -#include "code\controllers\subsystem\elastic.dm" #include "code\controllers\subsystem\events.dm" #include "code\controllers\subsystem\familytree.dm" #include "code\controllers\subsystem\fire_burning.dm" @@ -524,6 +523,18 @@ #include "code\controllers\subsystem\assets\asset_loading.dm" #include "code\controllers\subsystem\assets\assets.dm" #include "code\controllers\subsystem\assets\early_assets.dm" +#include "code\controllers\subsystem\elastic\_helpers.dm" +#include "code\controllers\subsystem\elastic\config.dm" +#include "code\controllers\subsystem\elastic\elastic.dm" +#include "code\controllers\subsystem\elastic\shards\_base_shard.dm" +#include "code\controllers\subsystem\elastic\shards\combat.dm" +#include "code\controllers\subsystem\elastic\shards\crafting.dm" +#include "code\controllers\subsystem\elastic\shards\economy.dm" +#include "code\controllers\subsystem\elastic\shards\enchanting.dm" +#include "code\controllers\subsystem\elastic\shards\heartbeat.dm" +#include "code\controllers\subsystem\elastic\shards\medical.dm" +#include "code\controllers\subsystem\elastic\shards\round_data.dm" +#include "code\controllers\subsystem\elastic\shards\storytellers.dm" #include "code\controllers\subsystem\mobs\islander.dm" #include "code\controllers\subsystem\mobs\matthios.dm" #include "code\controllers\subsystem\mobs\mobs.dm"