Devel (#237)

BlinkyStitt · web-flow · commit 94c802d7a343 · 2023-11-22T00:46:46.000-04:00
* add internal_bearer_token to config

* cargo upgrade &amp;&amp; cargo update

* 1.43.91

* explicit eth_blockNumber

* simpler Debug impl

* 1.43.92

* save the db less often

* 1.43.93

* skip biased semaphore

* 1.43.94
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/entities/Cargo.toml b/entities/Cargo.toml
@@ -10,7 +10,7 @@ path = "src/mod.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ethers = { version = "2.0.10", default-features = false }
+ethers = { version = "2.0.11", default-features = false }
 sea-orm = "0.12.6"
-serde = "1.0.192"
+serde = "1.0.193"
 ulid = "1.1.0"
diff --git a/latency/Cargo.toml b/latency/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 portable-atomic = { version = "1.5.1", features = ["float"] }
-serde = { version = "1.0.192", features = [] }
+serde = { version = "1.0.193", features = [] }
 tokio = { version = "1.34.0", features = ["full"] }
 tracing = "0.1.40"
 watermill = "0.1.1"
diff --git a/payment-contracts/Cargo.toml b/payment-contracts/Cargo.toml
@@ -6,8 +6,8 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [build-dependencies]
-ethers = { version = "2.0.10", default-features = false }
+ethers = { version = "2.0.11", default-features = false }
 glob = "0.3.1"
 
 [dependencies]
-ethers = { version = "2.0.10", default-features = false }
+ethers = { version = "2.0.11", default-features = false }
diff --git a/web3_proxy/Cargo.toml b/web3_proxy/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "web3_proxy"
-version = "1.43.90"
+version = "1.43.94"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -49,15 +49,15 @@ bytes = "1.5.0"
 chrono = { version = "0.4.31" }
 derivative = "2.2.0"
 derive_more = { version = "0.99.17", features = ["nightly"] }
-ethers = { version = "2.0.10", default-features = false, features = ["rustls", "ws"] }
+ethers = { version = "2.0.11", default-features = false, features = ["rustls", "ws"] }
 fdlimit = "0.3.0"
 fstrings = "0.2"
 futures = { version = "0.3.29" }
 futures-util = "0.3.29"
 glob = "0.3.1"
 handlebars = "4.5.0"
 hashbrown = { version = "0.14.2", features = ["serde", "nightly"] }
-hdrhistogram = "7.5.3"
+hdrhistogram = "7.5.4"
 hostname = "0.3.1"
 http = "0.2.11"
 hyper = { version = "0.14.27", features = ["full", "nightly"] }
@@ -75,10 +75,10 @@ pagerduty-rs = { version = "0.1.6", default-features = false, features = ["async
 parking_lot = { version = "0.12.1", features = ["arc_lock", "nightly"] }
 rdkafka = { version = "0.36.0", default-features = false, features = ["tokio", "tracing"], optional = true }
 reqwest = { version = "0.11.22", default-features = false, features = ["json", "rustls"] }
-rust_decimal = { version = "1.33.0" }
+rust_decimal = { version = "1.33.1" }
 sentry = { version = "0.31.8", default-features = false, features = ["anyhow", "backtrace", "contexts", "panic", "reqwest", "rustls", "serde_json", "tracing"] }
 sentry-tracing = "0.31.8"
-serde = { version = "1.0.192" }
+serde = { version = "1.0.193" }
 serde-inline-default = "0.1.1"
 serde_json = { version = "1.0.108", default-features = false, features = ["raw_value"] }
 serde_prometheus = "0.2.4"
@@ -93,7 +93,7 @@ tower-service = "0.3.2"
 tracing = "0.1"
 ulid = { version = "1.1.0", features = ["rand", "uuid", "serde"] }
 url = { version = "2.4.1" }
-uuid = { version = "1.5.0", default-features = false }
+uuid = { version = "1.6.1", default-features = false }
 
 # TODO: why doesn't this work in dev-dependencies. i think because of how we split web3_proxy and web3_proxy_cli. im not sure that is even helping anymore
 test-log = { version = "0.2.13", default-features = false, features = ["trace"] }
diff --git a/web3_proxy/src/app/mod.rs b/web3_proxy/src/app/mod.rs
@@ -307,7 +307,7 @@ impl App {
         let stat_sender = if let Some(spawned_stat_buffer) = StatBuffer::try_spawn(
             BILLING_PERIOD_SECONDS,
             top_config.app.chain_id,
-            30,
+            120,
             top_config.app.influxdb_bucket.clone(),
             influxdb_client.clone(),
             rpc_secret_key_cache.clone(),
diff --git a/web3_proxy/src/block_number.rs b/web3_proxy/src/block_number.rs
@@ -355,6 +355,11 @@ impl CacheMode {
                 // TODO: make sure re-orgs work properly!
                 Ok(Self::SuccessForever)
             }
+            "eth_blockNumber" => Ok(Self::Standard {
+                block_needed: head_block.into(),
+                cache_block: head_block.into(),
+                cache_errors: true,
+            }),
             "eth_gasPrice" => Ok(Self::Never),
             "eth_getBlockByHash" => {
                 // TODO: double check that any node can serve this
diff --git a/web3_proxy/src/config.rs b/web3_proxy/src/config.rs
@@ -150,6 +150,9 @@ pub struct AppConfig {
     /// percentage to increase eth_estimateGas results. 100 == 100%
     pub gas_increase_percent: Option<U256>,
 
+    /// bearer token for internal requests. keep this secret
+    pub internal_bearer_token: Option<String>,
+
     /// Restrict user registration.
     /// None = no code needed
     pub invite_code: Option<String>,
diff --git a/web3_proxy/src/frontend/admin.rs b/web3_proxy/src/frontend/admin.rs
@@ -56,7 +56,10 @@ pub async fn admin_increase_balance(
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
     Json(payload): Json<AdminIncreaseBalancePost>,
 ) -> Web3ProxyResponse {
-    let caller = app.bearer_is_authorized(bearer).await?;
+    let caller = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     // Establish connections
     let db_conn = global_db_conn()?;
diff --git a/web3_proxy/src/frontend/authorization.rs b/web3_proxy/src/frontend/authorization.rs
@@ -551,16 +551,18 @@ impl App {
                 })
                 .await;
 
-            let semaphore_permit = tokio::select! {
-                biased;
+            let semaphore_permit = semaphore.acquire_owned().await?;
 
-                p = semaphore.acquire_owned() => {
-                    p
-                }
-                p = self.bonus_ip_concurrency.clone().acquire_owned() => {
-                    p
-                }
-            }?;
+            // let semaphore_permit = tokio::select! {
+            //     biased;
+
+            //     p = semaphore.acquire_owned() => {
+            //         p
+            //     }
+            //     p = self.bonus_ip_concurrency.clone().acquire_owned() => {
+            //         p
+            //     }
+            // }?;
 
             Ok(Some(semaphore_permit))
         } else {
@@ -590,19 +592,21 @@ impl App {
                 })
                 .await;
 
-            let semaphore_permit = tokio::select! {
-                biased;
+            let semaphore_permit = semaphore.acquire_owned().await?;
 
-                p = semaphore.acquire_owned() => {
-                    p
-                }
-                p = self.bonus_user_concurrency.clone().acquire_owned() => {
-                    p
-                }
-                p = self.bonus_ip_concurrency.clone().acquire_owned() => {
-                    p
-                }
-            }?;
+            // let semaphore_permit = tokio::select! {
+            //     biased;
+
+            //     p = semaphore.acquire_owned() => {
+            //         p
+            //     }
+            //     p = self.bonus_user_concurrency.clone().acquire_owned() => {
+            //         p
+            //     }
+            //     p = self.bonus_ip_concurrency.clone().acquire_owned() => {
+            //         p
+            //     }
+            // }?;
 
             Ok(Some(semaphore_permit))
         } else {
@@ -614,7 +618,16 @@ impl App {
     /// Verify that the given bearer token and address are allowed to take the specified action.
     /// This includes concurrent request limiting.
     /// keep the semaphore alive until the user's request is entirely complete
-    pub async fn bearer_is_authorized(&self, bearer: Bearer) -> Web3ProxyResult<user::Model> {
+    pub async fn bearer_is_authorized(
+        &self,
+        bearer: Bearer,
+    ) -> Web3ProxyResult<Option<user::Model>> {
+        if let Some(internal_token) = &self.config.internal_bearer_token {
+            if internal_token == bearer.token() {
+                return Ok(None);
+            }
+        }
+
         // get the user id for this bearer token
         let user_bearer_token = UserBearerToken::try_from(bearer)?;
 
@@ -631,7 +644,7 @@ impl App {
             .web3_context("fetching user from db by bearer token")?
             .web3_context("unknown bearer token")?;
 
-        Ok(user)
+        Ok(Some(user))
     }
 
     pub async fn rate_limit_login(
diff --git a/web3_proxy/src/frontend/users/mod.rs b/web3_proxy/src/frontend/users/mod.rs
@@ -56,7 +56,10 @@ pub async fn user_post(
     TypedHeader(Authorization(bearer_token)): TypedHeader<Authorization<Bearer>>,
     Json(payload): Json<UserPost>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer_token).await?;
+    let user = app
+        .bearer_is_authorized(bearer_token)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let user_id = user.id;
 
diff --git a/web3_proxy/src/frontend/users/payment.rs b/web3_proxy/src/frontend/users/payment.rs
@@ -44,7 +44,10 @@ pub async fn user_balance_get(
     State(app): State<Arc<App>>,
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
@@ -64,7 +67,10 @@ pub async fn user_chain_deposits_get(
     State(app): State<Arc<App>>,
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
@@ -103,7 +109,10 @@ pub async fn user_stripe_deposits_get(
     State(app): State<Arc<App>>,
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
@@ -146,7 +155,10 @@ pub async fn user_admin_deposits_get(
     State(app): State<Arc<App>>,
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
diff --git a/web3_proxy/src/frontend/users/referral.rs b/web3_proxy/src/frontend/users/referral.rs
@@ -1,5 +1,5 @@
 //! Handle registration, logins, and managing account data.
-use crate::errors::Web3ProxyResponse;
+use crate::errors::{Web3ProxyError, Web3ProxyResponse};
 use crate::globals::global_db_conn;
 use crate::referral_code::ReferralCode;
 use crate::{app::App, globals::global_db_replica_conn};
@@ -34,7 +34,10 @@ pub async fn user_referral_link_get(
     Query(_params): Query<HashMap<String, String>>,
 ) -> Web3ProxyResponse {
     // First get the bearer token and check if the user is logged in
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
@@ -79,7 +82,10 @@ pub async fn user_used_referral_stats(
     Query(_params): Query<HashMap<String, String>>,
 ) -> Web3ProxyResponse {
     // First get the bearer token and check if the user is logged in
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
@@ -137,7 +143,10 @@ pub async fn user_shared_referral_stats(
     Query(_params): Query<HashMap<String, String>>,
 ) -> Web3ProxyResponse {
     // First get the bearer token and check if the user is logged in
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
diff --git a/web3_proxy/src/frontend/users/rpc_keys.rs b/web3_proxy/src/frontend/users/rpc_keys.rs
@@ -31,7 +31,10 @@ pub async fn rpc_keys_get(
     State(app): State<Arc<App>>,
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
@@ -160,7 +163,10 @@ pub async fn rpc_keys_management(
 ) -> Web3ProxyResponse {
     // TODO: is there a way we can know if this is a PUT or POST? right now we can modify or create keys with either. though that probably doesn't matter
 
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let db_replica = global_db_replica_conn()?;
 
diff --git a/web3_proxy/src/frontend/users/stats.rs b/web3_proxy/src/frontend/users/stats.rs
@@ -1,6 +1,6 @@
 //! Handle registration, logins, and managing account data.
 use crate::app::App;
-use crate::errors::{Web3ProxyErrorContext, Web3ProxyResponse};
+use crate::errors::{Web3ProxyError, Web3ProxyErrorContext, Web3ProxyResponse};
 use crate::globals::global_db_replica_conn;
 use crate::http_params::{
     get_chain_id_from_params, get_page_from_params, get_query_start_from_params,
@@ -32,7 +32,10 @@ pub async fn user_revert_logs_get(
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
     Query(params): Query<HashMap<String, String>>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
 
     let chain_id = get_chain_id_from_params(app.as_ref(), &params)?;
     let query_start = get_query_start_from_params(&params)?;
@@ -140,7 +143,10 @@ pub async fn user_mysql_stats_get(
     State(app): State<Arc<App>>,
     TypedHeader(Authorization(bearer)): TypedHeader<Authorization<Bearer>>,
 ) -> Web3ProxyResponse {
-    let user = app.bearer_is_authorized(bearer).await?;
+    let user = app
+        .bearer_is_authorized(bearer)
+        .await?
+        .ok_or(Web3ProxyError::InvalidUserKey)?;
     let db_replica = global_db_replica_conn()?;
 
     // Fetch everything from mysql, joined
diff --git a/web3_proxy/src/frontend/users/subuser.rs b/web3_proxy/src/frontend/users/subuser.rs
diff --git a/web3_proxy/src/rpcs/blockchain.rs b/web3_proxy/src/rpcs/blockchain.rs
diff --git a/web3_proxy/src/stats/influxdb_queries.rs b/web3_proxy/src/stats/influxdb_queries.rs
diff --git a/web3_proxy_cli/Cargo.toml b/web3_proxy_cli/Cargo.toml
