From 840ad43453f328e266fb2402ac5384aa2e4ad4cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Lidwin?= Date: Sat, 1 Apr 2023 19:00:49 +0200 Subject: [PATCH] improv: refactor structure, add files generator --- Cargo.lock | 117 ++++++++++++++++ Cargo.toml | 5 +- README.md | 20 ++- src/bin/generator.rs | 213 ++++++++++++++++++++++++++++++ src/{main.rs => bin/gog_proxy.rs} | 9 +- src/lib.rs | 2 + src/structs.rs | 12 +- src/utils.rs | 22 +-- 8 files changed, 369 insertions(+), 31 deletions(-) create mode 100644 src/bin/generator.rs rename src/{main.rs => bin/gog_proxy.rs} (85%) create mode 100644 src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6bce769..aa6b51c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "0.7.20" @@ -107,6 +113,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "console" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.42.0", +] + [[package]] name = "cookie" version = "0.17.0" @@ -134,6 +153,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "devise" version = "0.4.1" @@ -173,6 +201,12 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -226,6 +260,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -355,11 +399,14 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" name = "gog_proxy" version = "0.1.0" dependencies = [ + "flate2", + "indicatif", "regex", "reqwest", "rocket", "serde", "serde_json", + "sqlite", "tokio", ] @@ -495,6 +542,19 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "tokio", + "unicode-width", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -627,6 +687,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.6" @@ -697,6 +766,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.17.1" @@ -824,6 +899,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "portable-atomic" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1291,6 +1372,36 @@ version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0959fd6f767df20b231736396e4f602171e00d95205676286e79d4a4eb67bef" +[[package]] +name = "sqlite" +version = "0.30.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1908664131c21a38e5b531344d52a196ec338af5bf44f7fa2c83d539e9561d" +dependencies = [ + "libc", + "sqlite3-sys", +] + +[[package]] +name = "sqlite3-src" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1815a7a02c996eb8e5c64f61fcb6fd9b12e593ce265c512c5853b2513635691" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "sqlite3-sys" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d47c99824fc55360ba00caf28de0b8a0458369b832e016a64c13af0ad9fbb9ee" +dependencies = [ + "libc", + "sqlite3-src", +] + [[package]] name = "stable-pattern" version = "0.1.0" @@ -1595,6 +1706,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "unicode-xid" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index 7aa5b6a..40bea03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,7 @@ reqwest = { version = "0.11", features = ["json"] } tokio = { version = "1", features = ["full"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -regex = "1" \ No newline at end of file +regex = "1" +sqlite = "0.30.4" +flate2 = "1.0" +indicatif = { version = "0.17.3", features = ["tokio"] } diff --git a/README.md b/README.md index 9dce39e..2365a67 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ GET /api/config/client_id ## Contributing -The `games_data/` directory contains `ClientId.json` files for each game. Contributions to that directory are welcome. All json files are formatted using Prettier. +The `games_data/` directory contains automatically generated `ClientId.json` files for each game. That are disabled by default. Contributions to that directory are welcome. All json files are formatted using Prettier. ### Obtaining the relevant game data @@ -30,7 +30,7 @@ Let's say, you want to add a new linux native game to suport cloud saves. 4. Copy client id 5. Make a [request to the API](#api) with it 6. Find corresponding paths on Linux -7. Write config file +7. Write/Update config file ### Sample config @@ -43,7 +43,7 @@ Let's say, you want to add a new linux native game to suport cloud saves. { "name": "saves", "location": "/unity3d/Team Cherry/Hollow Knight", - "wildcard": "*.dat" // This can be null + "wildcard": "*.dat" } ] } @@ -58,7 +58,7 @@ Let's say, you want to add a new linux native game to suport cloud saves. - location - place where files are stored - this can use env vars like `$HOME`, values that have fallbacks (like `$XDG_CONFIG_HOME` falls back to `$HOME/.config`) should be closed in `` -- wildcard - value can be set to `null`. Due to nature of some games on Linux that's required to not push unrelated junk to the cloud (that's usually a case for Unity games) +- wildcard - value can be set to `null`. Due to nature of some games on Linux that's required to not push unrelated junk to the cloud (that's usually a case for Unity games on Linux) **IMPORTANT NOTE** Some games can have enabled `cloudStorage` but empty locations array. This is caused by the fact that such games use `__default` location which is only available on Windows and Mac, since it's handled by Galaxy SDK (which doesn't exist on Linux btw). [SDK Documentation related to this](https://docs.gog.com/sdk-storage/#cloud-saves) @@ -78,3 +78,15 @@ APPLICATION_DATA_ROAMING APPLICATION_SUPPORT DOCUMENTS ``` + +## Running the server + +``` +cargo run --bin gog_proxy +``` + +## Generate json files + +``` +cargo run --bin generator +``` diff --git a/src/bin/generator.rs b/src/bin/generator.rs new file mode 100644 index 0000000..8383853 --- /dev/null +++ b/src/bin/generator.rs @@ -0,0 +1,213 @@ +use flate2::bufread::ZlibDecoder; +use gog_proxy::{structs::PlatformConfig, utils}; +use indicatif::{ProgressBar, ProgressStyle}; +use serde::Deserialize; +use sqlite::State; +use std::io::prelude::*; +use std::{cmp::min, env}; +use tokio::{fs, io::AsyncWriteExt}; + +const GET_LINUX_NATIVE_GAMES_QUERY: &str = + "SELECT product_id FROM products WHERE comp_systems LIKE '%l%' AND product_type='game'"; + +#[derive(Deserialize, Debug)] +struct GalaxyMeta { + #[serde(rename = "clientId")] + pub client_id: Option, +} + +#[derive(Deserialize, Debug)] +struct GalaxyBuild { + pub link: String, + pub generation: u8, +} + +#[derive(Deserialize, Debug)] +struct GalaxyBuilds { + pub items: Vec, +} + +async fn get_existing_clients() -> Vec { + let mut clients: Vec = Vec::new(); + if let Ok(mut dir) = fs::read_dir("./games_data").await { + if let Ok(Some(file)) = dir.next_entry().await { + let fname = file.file_name(); + let fname = fname.to_str().unwrap(); + + if let Some(id) = fname.split('.').next() { + clients.push(String::from(id)); + } + } + } + + clients +} + +async fn obtain_client_id(client: reqwest::Client, product: String) -> Option { + let prepared_request = client + .get(format!( + "https://content-system.gog.com/products/{product}/os/windows/builds?generation=2" + )) + .build() + .expect("Failed to prepare request to builds"); + + let response = client + .execute(prepared_request) + .await + .expect("Failed to get builds"); + let data: GalaxyBuilds = response.json().await.unwrap(); + + let url = if let Some(data) = data.items.get(0) { + data.link.clone() + } else { + String::new() + }; + + if url.is_empty() { + return None; + } + + // Get meta + + let prepared_request = client + .get(url) + .build() + .expect("Failed to prepare request to meta"); + + let response = client + .execute(prepared_request) + .await + .expect("Failed to get meta "); + + let json_data: GalaxyMeta = match data.items.get(0).unwrap().generation { + 1 => response.json().await.unwrap(), + + 2 => { + let data = response.bytes().await.expect("Failed to download meta"); + let mut decoder = ZlibDecoder::new(&data[..]); + let mut s = String::new(); + decoder + .read_to_string(&mut s) + .expect("Failed to decompress meta"); + + serde_json::from_str(s.as_str()).unwrap() + } + + _ => { + panic!("Unsupported version"); + } + }; + + json_data.client_id +} + +#[tokio::main] +async fn main() { + let client = reqwest::Client::new(); + let defined_clients = get_existing_clients().await; + let mut native_products: Vec = Vec::new(); + + let tmp_dir = env::temp_dir(); + let sqlite_path = tmp_dir.join("gogdb.sqlite"); + + if fs::metadata(&sqlite_path).await.is_err() { + // File doesn't exist, downlad it + + println!("Downloading sqlite index"); + let mut response = client + .execute( + client + .get("https://www.gogdb.org/data/index.sqlite3") + .build() + .unwrap(), + ) + .await + .expect("Failed to obtain sqlite index"); + let mut file = fs::File::create(&sqlite_path) + .await + .expect("Failed to create file sqlite file handle"); + + let total_size: u64 = response + .headers() + .get("Content-Length") + .unwrap() + .to_str() + .unwrap() + .parse() + .unwrap(); + let mut downloaded: u64 = 0; + + let pb = ProgressBar::new(total_size); + pb.set_style( + ProgressStyle::with_template( + "{msg} [{elapsed_precise}] {wide_bar} {bytes}/{total_bytes}", + ) + .unwrap(), + ); + + while let Ok(Some(mut chunk)) = response.chunk().await { + let new_downloaded = downloaded + chunk.len() as u64; + let new = min(new_downloaded, total_size); + downloaded = new; + pb.set_message("Downloading"); + pb.set_position(new); + file.write_buf(&mut chunk) + .await + .expect("Failed to write chunk"); + } + + file.flush().await.unwrap(); + file.shutdown().await.unwrap(); + pb.finish_with_message("Downloaded"); + } + + let connection = sqlite::open(&sqlite_path).expect("Failed to open connection to sqlite"); + + let mut query = connection + .prepare(GET_LINUX_NATIVE_GAMES_QUERY) + .expect("Failed to prepare query"); + + while let Ok(State::Row) = query.next() { + let id = query.read::("product_id").unwrap(); + native_products.push(id); + } + + let pb = ProgressBar::new(native_products.len() as u64); + + let futures = native_products + .iter() + .map(|product| tokio::spawn(obtain_client_id(client.clone(), product.to_string()))); + + let default_platform = PlatformConfig::default(); + + for future in futures { + let joined = future.await.expect("Failed to join future"); + pb.inc(1); + if let Some(id) = joined { + if !defined_clients.contains(&id) { + let remote_config = utils::get_gog_remote_config(&id).await; + let mut f_handle = fs::File::create(format!("./games_data/{id}.json")) + .await + .expect("Failed to create a file"); + + let mut platform_cfg = default_platform.clone(); + + if let Some(windows_config) = remote_config { + platform_cfg.cloud_storage.locations = + windows_config.content.windows.cloud_storage.locations; + } + + f_handle + .write_all( + serde_json::to_string_pretty(&platform_cfg) + .unwrap() + .as_bytes(), + ) + .await + .expect("Failed to write default config"); + f_handle.shutdown().await.unwrap(); + } + } + } + pb.finish(); +} diff --git a/src/main.rs b/src/bin/gog_proxy.rs similarity index 85% rename from src/main.rs rename to src/bin/gog_proxy.rs index 73eca65..beeb264 100644 --- a/src/main.rs +++ b/src/bin/gog_proxy.rs @@ -1,10 +1,9 @@ #[macro_use] extern crate rocket; -use rocket::{http::Status, serde::json}; -use structs::{GOGConfig, PlatformConfig}; -mod structs; -mod utils; +use gog_proxy::structs::{GOGConfig, PlatformConfig}; +use gog_proxy::utils; +use rocket::{http::Status, serde::json}; #[get("/")] fn index() -> &'static str { @@ -13,7 +12,7 @@ fn index() -> &'static str { #[get("/config/")] async fn get_config(id: String) -> (Status, Option>) { - let gog_config = utils::get_gog_remote_config(id.clone()).await; + let gog_config = utils::get_gog_remote_config(&id).await; if gog_config.is_none() { return (Status::NotFound, None); diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..4f0446a --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,2 @@ +pub mod structs; +pub mod utils; diff --git a/src/structs.rs b/src/structs.rs index 7ad4fbd..8405efe 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -1,18 +1,18 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct SaveLocation { pub name: String, pub location: String, pub wildcard: Option, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct Overlay { pub supported: bool, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct CloudStorage { pub enabled: bool, pub locations: Vec, @@ -23,15 +23,15 @@ pub struct QuotaConfig { pub quota: u32, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct PlatformConfig { pub overlay: Overlay, #[serde(rename = "cloudStorage")] pub cloud_storage: CloudStorage, } -impl PlatformConfig { - pub fn default() -> PlatformConfig { +impl Default for PlatformConfig { + fn default() -> PlatformConfig { PlatformConfig { overlay: Overlay { supported: false }, cloud_storage: CloudStorage { diff --git a/src/utils.rs b/src/utils.rs index afb22c1..098e23f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,7 +1,8 @@ -use crate::structs::{GOGConfig, PlatformConfig}; +use super::structs::{GOGConfig, PlatformConfig}; +use rocket::async_test; use tokio::fs; -pub async fn get_gog_remote_config(id: String) -> Option { +pub async fn get_gog_remote_config(id: &String) -> Option { let response = reqwest::get(format!("https://remote-config.gog.com/components/galaxy_client/clients/{id}?component_version=2.0.50")).await; match response { @@ -34,18 +35,9 @@ async fn check_configs() { .await .expect("Failed to read games data config"); - loop { - let file = read_dir.next_entry().await; - if file.is_err() { - break; - } - - if let Some(file) = file.unwrap() { - let path = file.path(); - let raw_data = fs::read_to_string(&path).await.unwrap(); - serde_json::from_str::(&raw_data).expect("Failed to parse json file"); - } else { - break; - } + while let Ok(Some(file)) = read_dir.next_entry().await { + let path = file.path(); + let raw_data = fs::read_to_string(&path).await.unwrap(); + serde_json::from_str::(&raw_data).expect("Failed to parse json file"); } }