From 437734ab6b28858dd7c794325add04e3db8075fa Mon Sep 17 00:00:00 2001 From: pranavdaa Date: Wed, 15 May 2024 02:53:10 +0530 Subject: [PATCH 1/3] Added rate limit to cf-worker example --- examples/nodejs-cf-worker/.gitignore | 1 - examples/nodejs-cf-worker/src/index.ts | 12 ++++++++++++ examples/nodejs-cf-worker/wrangler.toml | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/examples/nodejs-cf-worker/.gitignore b/examples/nodejs-cf-worker/.gitignore index 3b0fe33..6bffef3 100644 --- a/examples/nodejs-cf-worker/.gitignore +++ b/examples/nodejs-cf-worker/.gitignore @@ -7,7 +7,6 @@ yarn-debug.log* yarn-error.log* lerna-debug.log* .pnpm-debug.log* - # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json diff --git a/examples/nodejs-cf-worker/src/index.ts b/examples/nodejs-cf-worker/src/index.ts index fe14b90..b4e2c57 100644 --- a/examples/nodejs-cf-worker/src/index.ts +++ b/examples/nodejs-cf-worker/src/index.ts @@ -26,6 +26,11 @@ import { z } from 'zod'; +export interface Env { + MY_RATE_LIMITER: any; + SUBGRAPH_ENDPOINT: any; +} + const GraphqlReqSchema = z.object({ query: z.string().min(1), operationName: z.string().optional().nullable(), @@ -46,6 +51,13 @@ export default { return new Response(response.statusText, { status: response.status }); } const body = await response.json(); + + const { pathname } = new URL(env.SUBGRAPH_ENDPOINT) + + const { success } = await env.MY_RATE_LIMITER.limit({ key: pathname }) // key can be any string of your choosing + if (!success) { + return new Response(`429 Failure – rate limit exceeded for ${pathname}`, { status: 429 }) + } return new Response(JSON.stringify(body), { headers: { 'Content-Type': 'application/json' }, status: 200 }); }, diff --git a/examples/nodejs-cf-worker/wrangler.toml b/examples/nodejs-cf-worker/wrangler.toml index 330621d..bfef9bd 100644 --- a/examples/nodejs-cf-worker/wrangler.toml +++ b/examples/nodejs-cf-worker/wrangler.toml @@ -17,3 +17,15 @@ mode = "smart" # Replace this with your subgraph endpoint [vars] SUBGRAPH_ENDPOINT = "https://gateway-arbitrum.network.thegraph.com/api/subgraphs/id/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp" + +[[unsafe.bindings]] +name = "MY_RATE_LIMITER" +type = "ratelimit" +# An identifier you define, that is unique to your Cloudflare account. +# Must be an integer. +namespace_id = "1001" + +# Limit: the number of tokens allowed within a given period in a single +# Cloudflare location +# Period: the duration of the period, in seconds. Must be either 10 or 60 +simple = { limit = 100, period = 60 } \ No newline at end of file From 6b7f9929142af677d6402768f53027fba93cb9ad Mon Sep 17 00:00:00 2001 From: Simon Emanuel Schmid Date: Fri, 17 May 2024 19:00:16 +0200 Subject: [PATCH 2/3] Working example but seems buggy on Cloudflare --- examples/nodejs-cf-worker/src/index.ts | 39 ++++++++++++------- .../worker-configuration.d.ts | 5 ++- examples/nodejs-cf-worker/wrangler.toml | 8 ++-- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/examples/nodejs-cf-worker/src/index.ts b/examples/nodejs-cf-worker/src/index.ts index b4e2c57..12dd9db 100644 --- a/examples/nodejs-cf-worker/src/index.ts +++ b/examples/nodejs-cf-worker/src/index.ts @@ -26,11 +26,6 @@ import { z } from 'zod'; -export interface Env { - MY_RATE_LIMITER: any; - SUBGRAPH_ENDPOINT: any; -} - const GraphqlReqSchema = z.object({ query: z.string().min(1), operationName: z.string().optional().nullable(), @@ -46,18 +41,36 @@ export default { return new Response('Unsupported', { status: 400 }); } + console.log('Hello world'); + + // const { pathname } = new URL(env.SUBGRAPH_ENDPOINT) + + // const { success } = await env.MY_RATE_LIMITER.limit({ key: pathname }) // key can be any string of your choosing + + // if (!success) { + // return new Response(`429 Failure - rate limit exceeded for ${pathname}`, { status: 429 }) + // } + + const { success } = await env.MY_RATE_LIMITER.limit({ key: 'limiter' }); // key can be any string of your choosing + if (!success) { + return new Response( + JSON.stringify({ + errors: [ + { + message: + 'Rate limit on ENS communtiy key exceeded. Try again later or got to https://thegraph.com/studio to create your own API key. Find the ENS subgraph here: https://thegraph.com/explorer/subgraphs/5XqPmWe6gjyrJtFn9cLy237i4cWw2j9HcUJEXsP5qGtH?v=1&view=Overview&chain=arbitrum-one', + }, + ], + }), + { status: 429 } + ); + } + const response = await querySubgraph(req, env); if (response.status !== 200) { return new Response(response.statusText, { status: response.status }); } const body = await response.json(); - - const { pathname } = new URL(env.SUBGRAPH_ENDPOINT) - - const { success } = await env.MY_RATE_LIMITER.limit({ key: pathname }) // key can be any string of your choosing - if (!success) { - return new Response(`429 Failure – rate limit exceeded for ${pathname}`, { status: 429 }) - } return new Response(JSON.stringify(body), { headers: { 'Content-Type': 'application/json' }, status: 200 }); }, @@ -78,7 +91,7 @@ async function querySubgraph(req: Request, env: Env) { method: 'POST', body: JSON.stringify(gqlRequest), headers: { - authorization: `Bearer ${env.API_KEY}`, + // authorization: `Bearer ${env.API_KEY}`, 'Content-Type': 'application/json', }, }); diff --git a/examples/nodejs-cf-worker/worker-configuration.d.ts b/examples/nodejs-cf-worker/worker-configuration.d.ts index 1a4360a..c5ea1f3 100644 --- a/examples/nodejs-cf-worker/worker-configuration.d.ts +++ b/examples/nodejs-cf-worker/worker-configuration.d.ts @@ -1,7 +1,8 @@ -// Generated by Wrangler on Mon May 13 2024 08:41:07 GMT-1000 (Hawaii-Aleutian Standard Time) +// Generated by Wrangler on Fri May 17 2024 18:36:19 GMT+0200 (Central European Summer Time) // by running `wrangler types` interface Env { - SUBGRAPH_ENDPOINT: "https://gateway-arbitrum.network.thegraph.com/api/subgraphs/id/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp"; + SUBGRAPH_ENDPOINT: "https://gateway-arbitrum.network.thegraph.com/api/25eb15e49c08702d60d90be66dd9a9a3/subgraphs/id/74z4nnW6cr62ox3fLFaD3muieJzMXJimi6EZsG52yChM"; API_KEY: string; + MY_RATE_LIMITER: any; } diff --git a/examples/nodejs-cf-worker/wrangler.toml b/examples/nodejs-cf-worker/wrangler.toml index bfef9bd..6c479da 100644 --- a/examples/nodejs-cf-worker/wrangler.toml +++ b/examples/nodejs-cf-worker/wrangler.toml @@ -1,5 +1,5 @@ #:schema node_modules/wrangler/config-schema.json -name = "nodejs-cf-worker" +name = "redirect-rate-limit-network-test" main = "src/index.ts" compatibility_date = "2024-05-12" @@ -16,16 +16,16 @@ mode = "smart" # Replace this with your subgraph endpoint [vars] -SUBGRAPH_ENDPOINT = "https://gateway-arbitrum.network.thegraph.com/api/subgraphs/id/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp" +SUBGRAPH_ENDPOINT = "https://gateway-arbitrum.network.thegraph.com/api/25eb15e49c08702d60d90be66dd9a9a3/subgraphs/id/74z4nnW6cr62ox3fLFaD3muieJzMXJimi6EZsG52yChM" [[unsafe.bindings]] name = "MY_RATE_LIMITER" type = "ratelimit" # An identifier you define, that is unique to your Cloudflare account. # Must be an integer. -namespace_id = "1001" +namespace_id = "1234123429324" # Limit: the number of tokens allowed within a given period in a single # Cloudflare location # Period: the duration of the period, in seconds. Must be either 10 or 60 -simple = { limit = 100, period = 60 } \ No newline at end of file +simple = { limit = 1, period = 10 } \ No newline at end of file From 9afe01f6c69e1acd87d4564de0b51fe1e2032005 Mon Sep 17 00:00:00 2001 From: Simon Emanuel Schmid Date: Fri, 17 May 2024 19:39:23 +0200 Subject: [PATCH 3/3] First steps with DurableObjects --- examples/nodejs-cf-worker/src/index.ts | 84 ++++++++++++++----- .../worker-configuration.d.ts | 4 +- examples/nodejs-cf-worker/wrangler.toml | 16 ++-- 3 files changed, 69 insertions(+), 35 deletions(-) diff --git a/examples/nodejs-cf-worker/src/index.ts b/examples/nodejs-cf-worker/src/index.ts index 12dd9db..a5bd8f6 100644 --- a/examples/nodejs-cf-worker/src/index.ts +++ b/examples/nodejs-cf-worker/src/index.ts @@ -24,8 +24,17 @@ * @see https://developers.cloudflare.com/workers/configuration/secrets/ */ +import { DurableObject } from 'cloudflare:workers'; import { z } from 'zod'; +// const WHITEGLOVE_MAP = { +// "api.thegraph.com/subgraphs/name/schmidsi/anudit-lens": { +// "" +// "rateLimit": 100, +// "rateLimitWindow": 6 +// } +// } + const GraphqlReqSchema = z.object({ query: z.string().min(1), operationName: z.string().optional().nullable(), @@ -41,29 +50,34 @@ export default { return new Response('Unsupported', { status: 400 }); } - console.log('Hello world'); - - // const { pathname } = new URL(env.SUBGRAPH_ENDPOINT) - - // const { success } = await env.MY_RATE_LIMITER.limit({ key: pathname }) // key can be any string of your choosing - - // if (!success) { - // return new Response(`429 Failure - rate limit exceeded for ${pathname}`, { status: 429 }) - // } - - const { success } = await env.MY_RATE_LIMITER.limit({ key: 'limiter' }); // key can be any string of your choosing - if (!success) { - return new Response( - JSON.stringify({ - errors: [ - { - message: - 'Rate limit on ENS communtiy key exceeded. Try again later or got to https://thegraph.com/studio to create your own API key. Find the ENS subgraph here: https://thegraph.com/explorer/subgraphs/5XqPmWe6gjyrJtFn9cLy237i4cWw2j9HcUJEXsP5qGtH?v=1&view=Overview&chain=arbitrum-one', - }, - ], - }), - { status: 429 } - ); + // TODO: per subgraph rate limiting + const id = env.RATE_LIMITER.idFromName('global'); + + console.log('Hello Pranav'); + + try { + const stub = env.RATE_LIMITER.get(id); + // const milliseconds_to_next_request = await ; + + // console.log(milliseconds_to_next_request, 'milliseconds_to_next_request') + + if (await stub.rateLimit()) { + // Alternatively one could sleep for the necessary length of time + return new Response( + JSON.stringify({ + errors: [ + { + message: + 'Rate limit on ENS communtiy key exceeded. Try again later or got to https://thegraph.com/studio to create your own API key. Find the ENS subgraph here: https://thegraph.com/explorer/subgraphs/5XqPmWe6gjyrJtFn9cLy237i4cWw2j9HcUJEXsP5qGtH?v=1&view=Overview&chain=arbitrum-one', + }, + ], + }), + { status: 429 } + ); + } + } catch (error) { + console.log(error, 'error') + return new Response('Could not connect to rate limiter', { status: 502 }); } const response = await querySubgraph(req, env); @@ -96,3 +110,27 @@ async function querySubgraph(req: Request, env: Env) { }, }); } + +export class RateLimiter extends DurableObject { + static milliseconds_per_request = 10000; + + lastRequest: number; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.lastRequest = 0; + } + + async rateLimit(): Promise { + const now = Date.now(); + + console.log(now, this.lastRequest, RateLimiter.milliseconds_per_request); + + if (now - this.lastRequest > RateLimiter.milliseconds_per_request) { + this.lastRequest = now; + return false; + } else { + return true + } + } + } \ No newline at end of file diff --git a/examples/nodejs-cf-worker/worker-configuration.d.ts b/examples/nodejs-cf-worker/worker-configuration.d.ts index c5ea1f3..06854bc 100644 --- a/examples/nodejs-cf-worker/worker-configuration.d.ts +++ b/examples/nodejs-cf-worker/worker-configuration.d.ts @@ -1,8 +1,8 @@ -// Generated by Wrangler on Fri May 17 2024 18:36:19 GMT+0200 (Central European Summer Time) +// Generated by Wrangler on Fri May 17 2024 19:36:49 GMT+0200 (Central European Summer Time) // by running `wrangler types` interface Env { SUBGRAPH_ENDPOINT: "https://gateway-arbitrum.network.thegraph.com/api/25eb15e49c08702d60d90be66dd9a9a3/subgraphs/id/74z4nnW6cr62ox3fLFaD3muieJzMXJimi6EZsG52yChM"; API_KEY: string; - MY_RATE_LIMITER: any; + RATE_LIMITER: DurableObjectNamespace; } diff --git a/examples/nodejs-cf-worker/wrangler.toml b/examples/nodejs-cf-worker/wrangler.toml index 6c479da..bac35e4 100644 --- a/examples/nodejs-cf-worker/wrangler.toml +++ b/examples/nodejs-cf-worker/wrangler.toml @@ -18,14 +18,10 @@ mode = "smart" [vars] SUBGRAPH_ENDPOINT = "https://gateway-arbitrum.network.thegraph.com/api/25eb15e49c08702d60d90be66dd9a9a3/subgraphs/id/74z4nnW6cr62ox3fLFaD3muieJzMXJimi6EZsG52yChM" -[[unsafe.bindings]] -name = "MY_RATE_LIMITER" -type = "ratelimit" -# An identifier you define, that is unique to your Cloudflare account. -# Must be an integer. -namespace_id = "1234123429324" +[[durable_objects.bindings]] +name = "RATE_LIMITER" +class_name = "RateLimiter" -# Limit: the number of tokens allowed within a given period in a single -# Cloudflare location -# Period: the duration of the period, in seconds. Must be either 10 or 60 -simple = { limit = 1, period = 10 } \ No newline at end of file +[[migrations]] +tag = "v1" +new_classes = ["RateLimiter"] \ No newline at end of file