diff --git a/build.gradle.kts b/build.gradle.kts index bf6f37e..61ae7b8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -32,6 +32,10 @@ dependencies { implementation("org.mybatis.spring.boot:mybatis-spring-boot-starter:3.0.3") implementation("org.postgresql:postgresql:42.7.3") + // Caching + implementation("org.springframework.boot:spring-boot-starter-cache") + implementation("com.github.ben-manes.caffeine:caffeine") + runtimeOnly("org.postgresql:postgresql") // Development tools diff --git a/src/main/java/io/paradaux/api/ParadauxApiApplication.java b/src/main/java/io/paradaux/api/ParadauxApiApplication.java index 0e0ceb6..ca3e724 100644 --- a/src/main/java/io/paradaux/api/ParadauxApiApplication.java +++ b/src/main/java/io/paradaux/api/ParadauxApiApplication.java @@ -2,8 +2,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableScheduling +@EnableCaching public class ParadauxApiApplication { public static void main(String[] args) { diff --git a/src/main/java/io/paradaux/api/controllers/IfumController.java b/src/main/java/io/paradaux/api/controllers/IfumController.java deleted file mode 100644 index d4655c5..0000000 --- a/src/main/java/io/paradaux/api/controllers/IfumController.java +++ /dev/null @@ -1,28 +0,0 @@ -package io.paradaux.api.controllers; - -import io.paradaux.api.models.IfumVisit; -import io.paradaux.api.models.annotations.ProtectedRoute; -import io.paradaux.api.services.IfumService; -import org.springframework.web.bind.annotation.*; - -@RestController -@RequestMapping("/api/ifum") -public class IfumController { - - private final IfumService ifumService; - - public IfumController(IfumService ifumService) { - this.ifumService = ifumService; - } - - @ProtectedRoute - @PostMapping("/visit") - public void insertVisit(@RequestBody IfumVisit visit) { - ifumService.insertVisit(visit); - } - - @GetMapping("/visits") - public Integer getVisitCount() { - return ifumService.getVisitCount(); - } -} diff --git a/src/main/java/io/paradaux/api/controllers/VisitsController.java b/src/main/java/io/paradaux/api/controllers/VisitsController.java new file mode 100644 index 0000000..6584d1b --- /dev/null +++ b/src/main/java/io/paradaux/api/controllers/VisitsController.java @@ -0,0 +1,28 @@ +package io.paradaux.api.controllers; + +import io.paradaux.api.models.VisitEntry; +import io.paradaux.api.models.annotations.ProtectedRoute; +import io.paradaux.api.services.VisitsService; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping("/api/analytics/visits") +public class VisitsController { + + private final VisitsService visitsService; + + public VisitsController(VisitsService visitsService) { + this.visitsService = visitsService; + } + + @ProtectedRoute + @PostMapping("/visit") + public void insertVisit(@RequestBody VisitEntry visit) { + visitsService.insertVisit(visit); + } + + @GetMapping("/project/{project}") + public Integer getVisitCountByProject(@PathVariable String project) { + return visitsService.getVisitCount(project); + } +} diff --git a/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java b/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java new file mode 100644 index 0000000..a92c0ee --- /dev/null +++ b/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java @@ -0,0 +1,23 @@ +package io.paradaux.api.jobs; + +import io.paradaux.api.mappers.VisitsMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class VisitCacheRefreshJob { + + private final VisitsMapper visitsMapper; + + public VisitCacheRefreshJob(VisitsMapper visitsMapper) { + this.visitsMapper = visitsMapper; + } + + @Scheduled(fixedRate = 5 * 60 * 1000) // 5 mins + public void refreshMaterializedView() { + visitsMapper.refreshVisitCache(); + log.info("Visit cache refreshed successfully."); + } +} diff --git a/src/main/java/io/paradaux/api/mappers/IfumMapper.java b/src/main/java/io/paradaux/api/mappers/IfumMapper.java deleted file mode 100644 index 9e4b31f..0000000 --- a/src/main/java/io/paradaux/api/mappers/IfumMapper.java +++ /dev/null @@ -1,10 +0,0 @@ -package io.paradaux.api.mappers; - -import io.paradaux.api.models.IfumVisit; -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface IfumMapper { - void insertVisit(IfumVisit visit); - Integer getVisitCount(); -} diff --git a/src/main/java/io/paradaux/api/mappers/VisitsMapper.java b/src/main/java/io/paradaux/api/mappers/VisitsMapper.java new file mode 100644 index 0000000..cba676d --- /dev/null +++ b/src/main/java/io/paradaux/api/mappers/VisitsMapper.java @@ -0,0 +1,11 @@ +package io.paradaux.api.mappers; + +import io.paradaux.api.models.VisitEntry; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface VisitsMapper { + void insertVisit(VisitEntry visit); + Integer getVisitCount(String project); + void refreshVisitCache(); +} diff --git a/src/main/java/io/paradaux/api/models/IfumVisit.java b/src/main/java/io/paradaux/api/models/VisitEntry.java similarity index 78% rename from src/main/java/io/paradaux/api/models/IfumVisit.java rename to src/main/java/io/paradaux/api/models/VisitEntry.java index ef4b1f0..6422ba9 100644 --- a/src/main/java/io/paradaux/api/models/IfumVisit.java +++ b/src/main/java/io/paradaux/api/models/VisitEntry.java @@ -5,9 +5,10 @@ import java.sql.Timestamp; @Data -public class IfumVisit { +public class VisitEntry { private String ipAddress; private String userAgent; private int id; private Timestamp createdAt; + private String project; } diff --git a/src/main/java/io/paradaux/api/services/IfumService.java b/src/main/java/io/paradaux/api/services/IfumService.java deleted file mode 100644 index bfa293c..0000000 --- a/src/main/java/io/paradaux/api/services/IfumService.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.paradaux.api.services; - -import io.paradaux.api.models.IfumVisit; - -public interface IfumService { - /** - * Inserts a new IfumVisit record into the database. - */ - void insertVisit(IfumVisit visit); - - /** - * Retrieves the total count of IfumVisit records. - * - * @return the count of IfumVisit records - */ - Integer getVisitCount(); -} diff --git a/src/main/java/io/paradaux/api/services/VisitsService.java b/src/main/java/io/paradaux/api/services/VisitsService.java new file mode 100644 index 0000000..5247e91 --- /dev/null +++ b/src/main/java/io/paradaux/api/services/VisitsService.java @@ -0,0 +1,22 @@ +package io.paradaux.api.services; + +import io.paradaux.api.models.VisitEntry; + +public interface VisitsService { + /** + * Inserts a new IfumVisit record into the database. + */ + void insertVisit(VisitEntry visit); + + /** + * Retrieves the total count of IfumVisit records. + * + * @return the count of IfumVisit records + */ + Integer getVisitCount(String project); + + /** + * Refreshes the visit view which groups visit counts by project. + */ + void refreshVisitCache(); +} diff --git a/src/main/java/io/paradaux/api/services/impl/IfumServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/IfumServiceImpl.java deleted file mode 100644 index d0117cc..0000000 --- a/src/main/java/io/paradaux/api/services/impl/IfumServiceImpl.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.paradaux.api.services.impl; - -import io.paradaux.api.mappers.IfumMapper; -import io.paradaux.api.models.IfumVisit; -import io.paradaux.api.services.IfumService; -import org.springframework.stereotype.Service; - -@Service -public class IfumServiceImpl implements IfumService { - - private final IfumMapper ifumMapper; - - public IfumServiceImpl(IfumMapper ifumMapper) { - this.ifumMapper = ifumMapper; - } - - @Override - public void insertVisit(IfumVisit visit) { - ifumMapper.insertVisit(visit); - } - - @Override - public Integer getVisitCount() { - return ifumMapper.getVisitCount(); - } -} diff --git a/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java new file mode 100644 index 0000000..df2c057 --- /dev/null +++ b/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java @@ -0,0 +1,60 @@ +package io.paradaux.api.services.impl; + +import io.paradaux.api.mappers.VisitsMapper; +import io.paradaux.api.models.VisitEntry; +import io.paradaux.api.services.VisitsService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Service; + +import java.util.Arrays; +import java.util.Optional; +import java.util.TreeSet; + +@Slf4j +@Service +public class VisitsServiceImpl implements VisitsService { + + private static final TreeSet IGNORED_USER_AGENTS = new TreeSet<>(Arrays.asList( + "bot", "crawler", "spider", "scanner", "curl", "wget", + "python", "java", "php", "unknown", "discord", "(compatible)", + "zgrab", "scrapy", "censys", "okhttp", "axios", "go-http-client", + "google", "bing", "yahoo" + )); + + private final VisitsMapper visitsMapper; + + public VisitsServiceImpl(VisitsMapper visitsMapper) { + this.visitsMapper = visitsMapper; + } + + @CacheEvict(value = "visitCounts", key = "#visit.project") + @Override + public void insertVisit(VisitEntry visit) { + String userAgent = Optional.ofNullable(visit.getUserAgent()) + .map(String::toLowerCase) + .orElse("unknown"); + + // If the user agent matches any of the ignored partial agents, skip the insert + if (IGNORED_USER_AGENTS.stream().anyMatch(userAgent::contains)) { + log.warn("Blocked bot visit for project: {} with user agent: {}", visit.getProject(), userAgent); + return; + } + + log.info("Attempting to inserting seemingly valid visit for: {}", visit.getProject()); + visitsMapper.insertVisit(visit); + } + + @Cacheable(value = "visitCounts", key = "#project") + @Override + public Integer getVisitCount(String project) { + return visitsMapper.getVisitCount(project); + } + + @CacheEvict(value = "visitCounts", allEntries = true) + @Override + public void refreshVisitCache() { + visitsMapper.refreshVisitCache(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 9c00a21..fffb3a9 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,10 @@ spring: application: name: paradaux-api + cache: + type: caffeine + caffeine: + spec: maximumSize=1000,expireAfterWrite=10m cloudflare: turnstile: @@ -11,4 +15,4 @@ discord: mybatis: type-aliases-package: io.paradaux.api.models - mapper-locations: classpath:mappers/*.xml \ No newline at end of file + mapper-locations: classpath:mappers/*.xml diff --git a/src/main/resources/db/2.sql b/src/main/resources/db/2.sql new file mode 100644 index 0000000..3d24e27 --- /dev/null +++ b/src/main/resources/db/2.sql @@ -0,0 +1,16 @@ +CREATE SCHEMA IF NOT EXISTS analytics; +ALTER TABLE ifum.visits SET SCHEMA analytics; +ALTER TABLE analytics.visits ADD COLUMN project TEXT NOT NULL DEFAULT 'ifuckedur.mom'; +ALTER TABLE analytics.visits ALTER COLUMN project DROP DEFAULT; +ALTER TABLE analytics.visits DROP CONSTRAINT IF EXISTS visits_ip_address_key; +ALTER TABLE analytics.visits ADD CONSTRAINT visits_ip_project_unique UNIQUE (ip_address, project); +CREATE MATERIALIZED VIEW analytics.visit_count_view AS +SELECT + project, + COUNT(*) AS total_count +FROM + analytics.visits +GROUP BY + project; +CREATE UNIQUE INDEX visit_count_view_project_idx ON analytics.visit_count_view(project); +DROP SCHEMA IF EXISTS ifum CASCADE; \ No newline at end of file diff --git a/src/main/resources/mappers/IfumMapper.xml b/src/main/resources/mappers/IfumMapper.xml deleted file mode 100644 index fe539a5..0000000 --- a/src/main/resources/mappers/IfumMapper.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - INSERT INTO ifum.visits (ip_address, user_agent) - SELECT #{ipAddress}, #{userAgent} - WHERE NOT EXISTS ( - SELECT 1 FROM ifum.visits WHERE ip_address = #{ipAddress} - ) - - - - \ No newline at end of file diff --git a/src/main/resources/mappers/VisitsMapper.xml b/src/main/resources/mappers/VisitsMapper.xml new file mode 100644 index 0000000..4c1c599 --- /dev/null +++ b/src/main/resources/mappers/VisitsMapper.xml @@ -0,0 +1,24 @@ + + + + + + + + INSERT INTO analytics.visits (ip_address, user_agent, project) + SELECT #{ipAddress}, #{userAgent}, #{project} + WHERE NOT EXISTS ( + SELECT 1 FROM analytics.visits WHERE ip_address = #{ipAddress} AND project = #{project} + ) + + + + + + REFRESH MATERIALIZED VIEW CONCURRENTLY analytics.visit_count_view; + + \ No newline at end of file diff --git a/test-api.py b/test-api.py new file mode 100644 index 0000000..f03a6d8 --- /dev/null +++ b/test-api.py @@ -0,0 +1,74 @@ +import asyncio +import http.client +import time +from urllib.parse import urlparse +from statistics import mean, median + +# Config +# URL = "https://api.paradaux.io/api/ifum/visits" +URL = "http://127.0.0.1:8080/api/visits/project/ifuckedur.mom" +TOTAL_REQUESTS = 1500 +MAX_CONCURRENCY = 50 + +response_times = [] +failures = 0 + +semaphore = asyncio.Semaphore(MAX_CONCURRENCY) + +def sync_http_get(url): + parsed = urlparse(url) + conn_class = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection + conn = conn_class(parsed.netloc, timeout=10) + full_path = parsed.path or "/" + if parsed.query: + full_path += "?" + parsed.query + headers = { + "Host": parsed.hostname, + "User-Agent": "BenchmarkClient/1.0" + } + try: + start = time.perf_counter() + conn.request("GET", full_path, headers=headers) + response = conn.getresponse() + response.read() + duration = time.perf_counter() - start + conn.close() + return response.status, duration + except Exception as e: + conn.close() + print(f"[ERROR] Exception during request: {e}") + return None, None + + +async def fetch(i): + global failures + async with semaphore: + loop = asyncio.get_running_loop() + print(f"Starting request {i}") + status, duration = await loop.run_in_executor(None, sync_http_get, URL) + if status == 200: + print(f"Request {i} succeeded in {duration:.4f}s") + response_times.append(duration) + else: + print(f"Request {i} failed (status={status})") + failures += 1 + +async def main(): + print("Starting benchmark...") + tasks = [fetch(i) for i in range(TOTAL_REQUESTS)] + await asyncio.gather(*tasks) + + if response_times: + print(f"\n--- Benchmark Report for {URL} ---") + print(f"Total Requests: {TOTAL_REQUESTS}") + print(f"Successful: {len(response_times)}") + print(f"Failed: {failures}") + print(f"Average Response Time: {mean(response_times):.4f} seconds") + print(f"Median Response Time: {median(response_times):.4f} seconds") + print(f"Min Response Time: {min(response_times):.4f} seconds") + print(f"Max Response Time: {max(response_times):.4f} seconds") + else: + print("All requests failed.") + +if __name__ == "__main__": + asyncio.run(main())