From a6ca7c9612f363ef948e1d33bbcee1b191444c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Mon, 14 Jul 2025 14:47:33 +0100 Subject: [PATCH 01/17] Add some useful prod queries --- src/main/resources/db/runbook_queries.sql | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/main/resources/db/runbook_queries.sql diff --git a/src/main/resources/db/runbook_queries.sql b/src/main/resources/db/runbook_queries.sql new file mode 100644 index 0000000..7bb14ac --- /dev/null +++ b/src/main/resources/db/runbook_queries.sql @@ -0,0 +1,31 @@ +SELECT + os, + COUNT(*) AS count, + ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (), 2) AS percentage_share +FROM ( + SELECT + CASE + + WHEN user_agent ~* 'Windows NT 10.0' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.3' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.2' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.1' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.0' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 5.1' THEN 'Windows' + WHEN user_agent ~* 'iPhone|iPad' THEN 'iOS' + WHEN user_agent ~* 'Macintosh|Mac OS X' THEN 'macOS' + WHEN user_agent ~* 'Android' THEN 'Android' + WHEN user_agent ~* 'Linux' THEN 'Linux' + ELSE 'Other/Unknown' + END AS os + FROM analytics.visits + ) AS derived +GROUP BY os +ORDER BY count DESC; + +REFRESH MATERIALIZED VIEW CONCURRENTLY analytics.visit_count_view; + + +SELECT ROW_NUMBER() OVER (ORDER BY id) AS row_number, ip_address, user_agent +FROM analytics.visits +WHERE project = 'ifuckedur.mom'; \ No newline at end of file From a37fe27304a5614e32746e0b331f96acc15714bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 18:49:01 +0100 Subject: [PATCH 02/17] Rate limiting + rate limit bypass for authenticated requests --- build.gradle.kts | 14 ++++- .../io/paradaux/api/config/RedisConfig.java | 16 ++++++ .../io/paradaux/api/config/WebConfig.java | 10 ++-- .../ProtectedRouteInterceptor.java | 52 +++++++++++++------ .../interceptors/RateLimitingInterceptor.java | 38 ++++++++++++++ .../mappers/typehandlers/UUIDTypeHandler.java | 31 +++++++++++ .../api/services/RateLimitingService.java | 5 ++ .../impl/RateLimitingServiceImpl.java | 33 ++++++++++++ .../api/services/impl/VisitsServiceImpl.java | 2 +- .../java/io/paradaux/api/utils/IPUtils.java | 14 +++++ src/main/resources/application.yml | 14 +++++ src/main/resources/db/runbook_queries.sql | 30 ++++++++++- test-api.py | 2 +- 13 files changed, 236 insertions(+), 25 deletions(-) create mode 100644 src/main/java/io/paradaux/api/config/RedisConfig.java create mode 100644 src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java create mode 100644 src/main/java/io/paradaux/api/mappers/typehandlers/UUIDTypeHandler.java create mode 100644 src/main/java/io/paradaux/api/services/RateLimitingService.java create mode 100644 src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java create mode 100644 src/main/java/io/paradaux/api/utils/IPUtils.java diff --git a/build.gradle.kts b/build.gradle.kts index 61ae7b8..dcf3f79 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,8 +30,16 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-jdbc") implementation("org.mybatis.spring.boot:mybatis-spring-boot-starter:3.0.3") + implementation("org.springframework.boot:spring-boot-starter-data-redis") implementation("org.postgresql:postgresql:42.7.3") + implementation("com.maxmind.geoip2:geoip2:4.2.0") + implementation("com.opencsv:opencsv:5.8") + + // Utilities + implementation("org.apache.commons:commons-compress:1.24.0") + implementation("org.apache.commons:commons-lang3:3.13.0") + // Caching implementation("org.springframework.boot:spring-boot-starter-cache") implementation("com.github.ben-manes.caffeine:caffeine") @@ -43,14 +51,16 @@ dependencies { compileOnly("org.projectlombok:lombok") annotationProcessor("org.projectlombok:lombok") - - // Test testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } +configurations.all { + exclude(group = "commons-logging", module = "commons-logging") +} + tasks.withType { useJUnitPlatform() } diff --git a/src/main/java/io/paradaux/api/config/RedisConfig.java b/src/main/java/io/paradaux/api/config/RedisConfig.java new file mode 100644 index 0000000..67da12b --- /dev/null +++ b/src/main/java/io/paradaux/api/config/RedisConfig.java @@ -0,0 +1,16 @@ +package io.paradaux.api.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; + +@Configuration +public class RedisConfig { + @Bean + public RedisTemplate redisTemplate(RedisConnectionFactory factory) { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(factory); + return template; + } +} diff --git a/src/main/java/io/paradaux/api/config/WebConfig.java b/src/main/java/io/paradaux/api/config/WebConfig.java index 55ac869..02ea962 100644 --- a/src/main/java/io/paradaux/api/config/WebConfig.java +++ b/src/main/java/io/paradaux/api/config/WebConfig.java @@ -1,19 +1,23 @@ package io.paradaux.api.config; import io.paradaux.api.interceptors.ProtectedRouteInterceptor; -import org.springframework.beans.factory.annotation.Autowired; +import io.paradaux.api.interceptors.RateLimitingInterceptor; +import lombok.RequiredArgsConstructor; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration +@RequiredArgsConstructor public class WebConfig implements WebMvcConfigurer { - @Autowired - private ProtectedRouteInterceptor protectedRouteInterceptor; + private final ProtectedRouteInterceptor protectedRouteInterceptor; + private final RateLimitingInterceptor rateLimitingInterceptor; @Override public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(protectedRouteInterceptor); + registry.addInterceptor(rateLimitingInterceptor) + .addPathPatterns("/**"); } } diff --git a/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java b/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java index 02bd441..fbe177b 100644 --- a/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java +++ b/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java @@ -8,31 +8,51 @@ import org.springframework.web.method.HandlerMethod; import org.springframework.web.servlet.HandlerInterceptor; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + @Component public class ProtectedRouteInterceptor implements HandlerInterceptor { + private final ConcurrentMap protectedRouteCache = new ConcurrentHashMap<>(); + @Value("${api.secret}") private String secret; + /** + * Intercepts requests to check if they are protected routes. + * If the route is protected, it checks for a valid secret token in the request header "X-SECRET". + * If the secret is invalid or missing, it responds with HTTP 401 Unauthorized. + */ @Override - public boolean preHandle(HttpServletRequest request, - HttpServletResponse response, - Object handler) throws Exception { - if (handler instanceof HandlerMethod) { - HandlerMethod method = (HandlerMethod) handler; - - boolean methodProtected = method.hasMethodAnnotation(ProtectedRoute.class); - boolean classProtected = method.getBeanType().isAnnotationPresent(ProtectedRoute.class); - - if (methodProtected || classProtected) { - String secretHeader = request.getHeader("X-SECRET"); - if (secretHeader == null || !secretHeader.equals(secret)) { - response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); - response.getWriter().write("Unauthorized: invalid or missing secret"); - return false; - } + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + if (handler instanceof HandlerMethod method) { + if (!isProtectedRoute(method)) { + return true; + } + if (!hasValidSecret(request)) { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.getWriter().write("Unauthorized: invalid or missing secret"); + return false; } } return true; } + + /** + * A route is protected if the method or its class is annotated with @ProtectedRoute. + * This requires a secret token in the request header "X-SECRET". + * */ + private boolean isProtectedRoute(HandlerMethod method) { + return protectedRouteCache.computeIfAbsent(method, hm -> hm.hasMethodAnnotation(ProtectedRoute.class) + || hm.getBeanType().isAnnotationPresent(ProtectedRoute.class)); + } + + /** + * Checks if the request has a valid secret token. + */ + public boolean hasValidSecret(HttpServletRequest request) { + String secretHeader = request.getHeader("X-SECRET"); + return secretHeader != null && secretHeader.equals(secret); + } } diff --git a/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java b/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java new file mode 100644 index 0000000..a7f4310 --- /dev/null +++ b/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java @@ -0,0 +1,38 @@ +package io.paradaux.api.interceptors; + +import io.paradaux.api.services.RateLimitingService; +import io.paradaux.api.utils.IPUtils; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.HandlerInterceptor; + +import java.io.IOException; + +@Component +@RequiredArgsConstructor +public class RateLimitingInterceptor implements HandlerInterceptor { + + private final RateLimitingService rateLimiterService; + private final ProtectedRouteInterceptor protectedRouteInterceptor; + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws IOException { + String ip = IPUtils.getClientIp(request); + + // Allow requests that contain a valid secret token + if (protectedRouteInterceptor.hasValidSecret(request)) { + return true; + } + + if (!rateLimiterService.isAllowed(ip)) { + response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + response.getWriter().write("Rate limit exceeded"); + return false; + } + + return true; + } +} diff --git a/src/main/java/io/paradaux/api/mappers/typehandlers/UUIDTypeHandler.java b/src/main/java/io/paradaux/api/mappers/typehandlers/UUIDTypeHandler.java new file mode 100644 index 0000000..758bc41 --- /dev/null +++ b/src/main/java/io/paradaux/api/mappers/typehandlers/UUIDTypeHandler.java @@ -0,0 +1,31 @@ +package io.paradaux.api.mappers.typehandlers; + +import org.apache.ibatis.type.BaseTypeHandler; +import org.apache.ibatis.type.JdbcType; + +import java.sql.*; +import java.util.UUID; + +public class UUIDTypeHandler extends BaseTypeHandler { + + @Override + public void setNonNullParameter(PreparedStatement ps, int i, UUID parameter, JdbcType jdbcType) + throws SQLException { + ps.setObject(i, parameter, Types.OTHER); + } + + @Override + public UUID getNullableResult(ResultSet rs, String columnName) throws SQLException { + return (UUID) rs.getObject(columnName); + } + + @Override + public UUID getNullableResult(ResultSet rs, int columnIndex) throws SQLException { + return (UUID) rs.getObject(columnIndex); + } + + @Override + public UUID getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { + return (UUID) cs.getObject(columnIndex); + } +} diff --git a/src/main/java/io/paradaux/api/services/RateLimitingService.java b/src/main/java/io/paradaux/api/services/RateLimitingService.java new file mode 100644 index 0000000..a56518d --- /dev/null +++ b/src/main/java/io/paradaux/api/services/RateLimitingService.java @@ -0,0 +1,5 @@ +package io.paradaux.api.services; + +public interface RateLimitingService { + boolean isAllowed(String key); +} diff --git a/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java new file mode 100644 index 0000000..1ebdb2f --- /dev/null +++ b/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java @@ -0,0 +1,33 @@ +package io.paradaux.api.services.impl; + +import io.paradaux.api.services.RateLimitingService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +@Slf4j +public class RateLimitingServiceImpl implements RateLimitingService { + + private final RedisTemplate redisTemplate; + + private static final long TIME_WINDOW = 60_000; // 1 min + private static final int MAX_REQUESTS = 10; + + @Override + public boolean isAllowed(String key) { + long now = System.currentTimeMillis(); + String redisKey = "rate:" + key; + redisTemplate.opsForZSet().removeRangeByScore(redisKey, 0, now - TIME_WINDOW); + redisTemplate.opsForZSet().add(redisKey, UUID.randomUUID().toString(), now); + redisTemplate.expire(redisKey, Duration.ofMinutes(1)); + Long count = redisTemplate.opsForZSet().zCard(redisKey); + log.info("Rate limiter check for key: {}, count: {}", key, count); + return count != null && count <= MAX_REQUESTS; + } +} diff --git a/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java index 8755335..c226699 100644 --- a/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java @@ -22,7 +22,7 @@ public class VisitsServiceImpl implements VisitsService { "bot", "crawler", "spider", "scanner", "curl", "wget", "python", "java", "php", "unknown", "discord", "(compatible;", "zgrab", "scrapy", "censys", "okhttp", "axios", "go-http-client", - "google", "bing", "yahoo", "WhatCMS" + "google", "bing", "yahoo", "WhatCMS", "Palo Alto" )); private final VisitsMapper visitsMapper; diff --git a/src/main/java/io/paradaux/api/utils/IPUtils.java b/src/main/java/io/paradaux/api/utils/IPUtils.java new file mode 100644 index 0000000..ff80cc6 --- /dev/null +++ b/src/main/java/io/paradaux/api/utils/IPUtils.java @@ -0,0 +1,14 @@ +package io.paradaux.api.utils; + +import jakarta.servlet.http.HttpServletRequest; + +public class IPUtils { + + public static String getClientIp(HttpServletRequest request) { + String xfHeader = request.getHeader("X-Forwarded-For"); + if (xfHeader != null && !xfHeader.isEmpty()) { + return xfHeader.split(",")[0].trim(); // real client IP + } + return request.getRemoteAddr(); // fallback + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index fffb3a9..c2df1b8 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -6,6 +6,16 @@ spring: caffeine: spec: maximumSize=1000,expireAfterWrite=10m + data: + redis: + timeout: 2000ms + lettuce: + pool: + max-active: 8 + max-wait: -1ms + max-idle: 8 + min-idle: 0 + cloudflare: turnstile: secret: ${CLOUDFLARE_TURNSTILE_SECRET} @@ -16,3 +26,7 @@ discord: mybatis: type-aliases-package: io.paradaux.api.models mapper-locations: classpath:mappers/*.xml + type-handlers-package: io.paradaux.api.mappers.typehandlers + +maxmind: + base-url: https://download.maxmind.com/app/geoip_download \ No newline at end of file diff --git a/src/main/resources/db/runbook_queries.sql b/src/main/resources/db/runbook_queries.sql index 7bb14ac..cbdf547 100644 --- a/src/main/resources/db/runbook_queries.sql +++ b/src/main/resources/db/runbook_queries.sql @@ -25,7 +25,33 @@ ORDER BY count DESC; REFRESH MATERIALIZED VIEW CONCURRENTLY analytics.visit_count_view; - +-- Get the IP/User-Agent pairs for the project 'ifuckedur.mom' with their viewer number SELECT ROW_NUMBER() OVER (ORDER BY id) AS row_number, ip_address, user_agent FROM analytics.visits -WHERE project = 'ifuckedur.mom'; \ No newline at end of file +WHERE project = 'ifuckedur.mom'; + +-- Get visitor counts for each project in the last 24 hours +SELECT + project, + COUNT(*) AS total_count +FROM + analytics.visits +WHERE created_at > CURRENT_TIMESTAMP - INTERVAL '1 day' +GROUP BY + project; + +-- Get visitor counts for each project in the last 7 days +SELECT + TO_CHAR(created_at, 'FMDay') AS day_of_week, + COUNT(*) FILTER (WHERE project = 'cans.ie') AS cans_ie, + COUNT(*) FILTER (WHERE project = 'ifuckedur.mom') AS ifuckedur_mom, + COUNT(*) FILTER (WHERE project = 'isbetterthandubl.in') AS isbetterthandublin +FROM + analytics.visits +WHERE + created_at >= CURRENT_DATE - INTERVAL '6 days' +GROUP BY + day_of_week, + EXTRACT(DOW FROM created_at) +ORDER BY + EXTRACT(DOW FROM created_at); \ No newline at end of file diff --git a/test-api.py b/test-api.py index f03a6d8..b6f596b 100644 --- a/test-api.py +++ b/test-api.py @@ -6,7 +6,7 @@ # Config # URL = "https://api.paradaux.io/api/ifum/visits" -URL = "http://127.0.0.1:8080/api/visits/project/ifuckedur.mom" +URL = "http://127.0.0.1:8080/api/analytics/visits/project/ifuckedur.mom" TOTAL_REQUESTS = 1500 MAX_CONCURRENCY = 50 From 4a0d922882e1a085d050b0680087aaa7690eacc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 19:51:03 +0100 Subject: [PATCH 03/17] first pass at geoip --- .../api/controllers/GeoIPController.java | 29 ++++ .../io/paradaux/api/jobs/MaxMindSyncJob.java | 11 ++ .../io/paradaux/api/mappers/GeoIPMapper.java | 14 ++ .../io/paradaux/api/models/geoip/ASN.java | 9 + .../io/paradaux/api/models/geoip/IPBlock.java | 19 +++ .../paradaux/api/models/geoip/IPLocation.java | 21 +++ .../api/services/GeoIPInformationService.java | 4 + .../impl/GeoIPInformationServiceImpl.java | 157 ++++++++++++++++++ src/main/resources/application-dev.yml | 6 +- src/main/resources/application-local.yml | 12 +- src/main/resources/db/3.sql | 44 +++++ src/main/resources/mappers/GeoIPMapper.xml | 52 ++++++ 12 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 src/main/java/io/paradaux/api/controllers/GeoIPController.java create mode 100644 src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java create mode 100644 src/main/java/io/paradaux/api/mappers/GeoIPMapper.java create mode 100644 src/main/java/io/paradaux/api/models/geoip/ASN.java create mode 100644 src/main/java/io/paradaux/api/models/geoip/IPBlock.java create mode 100644 src/main/java/io/paradaux/api/models/geoip/IPLocation.java create mode 100644 src/main/java/io/paradaux/api/services/GeoIPInformationService.java create mode 100644 src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java create mode 100644 src/main/resources/db/3.sql create mode 100644 src/main/resources/mappers/GeoIPMapper.xml diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java new file mode 100644 index 0000000..8d82040 --- /dev/null +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -0,0 +1,29 @@ +package io.paradaux.api.controllers; + +import com.opencsv.exceptions.CsvValidationException; +import io.paradaux.api.models.annotations.ProtectedRoute; +import io.paradaux.api.services.impl.GeoIPInformationServiceImpl; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; + +@RestController +@RequestMapping("/api/geoip") +@Validated +@Slf4j +@RequiredArgsConstructor +public class GeoIPController { + + private final GeoIPInformationServiceImpl geoIPInformationService; + + @PostMapping("/sync") + @ProtectedRoute + public void syncGeoIPData() { + geoIPInformationService.importAllData(); + } +} diff --git a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java new file mode 100644 index 0000000..d0f95c5 --- /dev/null +++ b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java @@ -0,0 +1,11 @@ +package io.paradaux.api.jobs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class MaxMindSyncJob { + + +} diff --git a/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java new file mode 100644 index 0000000..78a2e9c --- /dev/null +++ b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java @@ -0,0 +1,14 @@ +package io.paradaux.api.mappers; + +import io.paradaux.api.models.geoip.ASN; +import io.paradaux.api.models.geoip.IPBlock; +import io.paradaux.api.models.geoip.IPLocation; +import org.apache.ibatis.annotations.Mapper; +import java.util.List; + +@Mapper +public interface GeoIPMapper { + void insertLocations(List locations); + void insertASNs(List asns); + void insertIPBlocks(List blocks); +} diff --git a/src/main/java/io/paradaux/api/models/geoip/ASN.java b/src/main/java/io/paradaux/api/models/geoip/ASN.java new file mode 100644 index 0000000..830e61e --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/ASN.java @@ -0,0 +1,9 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class ASN { + private Integer autonomousSystemNumber; + private String autonomousSystemOrganization; +} diff --git a/src/main/java/io/paradaux/api/models/geoip/IPBlock.java b/src/main/java/io/paradaux/api/models/geoip/IPBlock.java new file mode 100644 index 0000000..a90401d --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/IPBlock.java @@ -0,0 +1,19 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class IPBlock { + private String network; + private Integer geonameId; + private Integer registeredCountryGeonameId; + private Integer representedCountryGeonameId; + private Boolean isAnonymousProxy; + private Boolean isSatelliteProvider; + private String postalCode; + private Double latitude; + private Double longitude; + private Integer accuracyRadius; + private Boolean isAnycast; + private Integer autonomousSystemNumber; +} diff --git a/src/main/java/io/paradaux/api/models/geoip/IPLocation.java b/src/main/java/io/paradaux/api/models/geoip/IPLocation.java new file mode 100644 index 0000000..8ecd8e0 --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/IPLocation.java @@ -0,0 +1,21 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class IPLocation { + private Integer geonameId; + private String localeCode; + private String continentCode; + private String continentName; + private String countryIsoCode; + private String countryName; + private String subdivision1IsoCode; + private String subdivision1Name; + private String subdivision2IsoCode; + private String subdivision2Name; + private String cityName; + private String metroCode; + private String timeZone; + private Boolean isInEuropeanUnion; +} diff --git a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java new file mode 100644 index 0000000..72a3559 --- /dev/null +++ b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java @@ -0,0 +1,4 @@ +package io.paradaux.api.services; + +public interface GeoIPInformationService { +} diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java new file mode 100644 index 0000000..01b7691 --- /dev/null +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -0,0 +1,157 @@ +package io.paradaux.api.services.impl; + +import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvValidationException; +import io.paradaux.api.mappers.GeoIPMapper; +import io.paradaux.api.models.geoip.ASN; +import io.paradaux.api.models.geoip.IPBlock; +import io.paradaux.api.models.geoip.IPLocation; +import io.paradaux.api.services.GeoIPInformationService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +@Slf4j +@Service +@RequiredArgsConstructor +public class GeoIPInformationServiceImpl implements GeoIPInformationService { + + private static final int BATCH_SIZE = 1000; + + private final GeoIPMapper geoIPMapper; + + String cityLocationsFile = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-City-CSV_20250718\\GeoLite2-City-Locations-en.csv"; + String asnBlocksIPv4File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-ASN-CSV_20250720\\GeoLite2-ASN-Blocks-IPv4.csv"; + String asnBlocksIPv6File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-ASN-CSV_20250720\\GeoLite2-ASN-Blocks-IPv6.csv"; + String cityBlocksIPv4File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-City-CSV_20250718\\GeoLite2-City-Blocks-IPv4.csv"; + String cityBlocksIPv6File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-City-CSV_20250718\\GeoLite2-City-Blocks-IPv6.csv"; + + public void importAllData() { + try { + importLocations(cityLocationsFile); + } catch (Exception e) { + log.error("Failed to import locations", e); + return; // or handle fail-fast + } + + CompletableFuture asnsFuture = CompletableFuture.runAsync(() -> { + try { + importASNs(asnBlocksIPv4File, asnBlocksIPv6File); + } catch (Exception e) { + log.error("Failed to import ASN", e); + } + }); + + try { + importIPBlocks(cityBlocksIPv4File, cityBlocksIPv6File); + } catch (Exception e) { + log.error("Failed to import IP blocks", e); + } + + asnsFuture.join(); + log.info("All GeoIP data imported"); + } + + private void importLocations(String csvPath) throws IOException, CsvValidationException { + List locations = new ArrayList<>(); + try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { + reader.readNext(); // skip header + String[] line; + while ((line = reader.readNext()) != null) { + IPLocation loc = new IPLocation(); + loc.setGeonameId(parseIntOrNull(line[0])); + loc.setLocaleCode(line[1]); + loc.setContinentCode(line[2]); + loc.setContinentName(line[3]); + loc.setCountryIsoCode(line[4]); + loc.setCountryName(line[5]); + loc.setSubdivision1IsoCode(line[6]); + loc.setSubdivision1Name(line[7]); + loc.setSubdivision2IsoCode(line[8]); + loc.setSubdivision2Name(line[9]); + loc.setCityName(line[10]); + loc.setMetroCode(line[11]); + loc.setTimeZone(line[12]); + loc.setIsInEuropeanUnion("1".equals(line[13])); + locations.add(loc); + } + } + batchInsert(locations, geoIPMapper::insertLocations); + } + + private void importASNs(String... csvPaths) throws IOException, CsvValidationException { + Map uniqueAsns = new HashMap<>(); + + for (String csvPath : csvPaths) { + try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { + reader.readNext(); // skip header + String[] line; + while ((line = reader.readNext()) != null) { + Integer asnNumber = parseIntOrNull(line[1]); + if (asnNumber == null || uniqueAsns.containsKey(asnNumber)) continue; + + ASN asn = new ASN(); + asn.setAutonomousSystemNumber(asnNumber); + asn.setAutonomousSystemOrganization(line[2]); + uniqueAsns.put(asnNumber, asn); + } + } + } + + batchInsert(new ArrayList<>(uniqueAsns.values()), geoIPMapper::insertASNs); + } + + private void importIPBlocks(String... csvPaths) throws IOException, CsvValidationException { + List blocks = new ArrayList<>(); + + for (String csvPath : csvPaths) { + try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { + reader.readNext(); // skip header + String[] line; + while ((line = reader.readNext()) != null) { + IPBlock block = new IPBlock(); + block.setNetwork(line[0]); + block.setGeonameId(parseIntOrNull(line[1])); + block.setRegisteredCountryGeonameId(parseIntOrNull(line[2])); + block.setRepresentedCountryGeonameId(parseIntOrNull(line[3])); + block.setIsAnonymousProxy("1".equals(line[4])); + block.setIsSatelliteProvider("1".equals(line[5])); + block.setPostalCode(line[6]); + block.setLatitude(parseDoubleOrNull(line[7])); + block.setLongitude(parseDoubleOrNull(line[8])); + block.setAccuracyRadius(parseIntOrNull(line[9])); + block.setIsAnycast("1".equals(line[10])); + blocks.add(block); + } + } + } + + batchInsert(blocks, geoIPMapper::insertIPBlocks); + } + + private void batchInsert(List list, Consumer> insertFunction) { + for (int i = 0; i < list.size(); i += BATCH_SIZE) { + int end = Math.min(i + BATCH_SIZE, list.size()); + insertFunction.accept(list.subList(i, end)); + } + } + + private Integer parseIntOrNull(String s) { + try { return s == null || s.isBlank() ? null : Integer.parseInt(s); } + catch (NumberFormatException e) { return null; } + } + + private Double parseDoubleOrNull(String s) { + try { return s == null || s.isBlank() ? null : Double.parseDouble(s); } + catch (NumberFormatException e) { return null; } + } +} \ No newline at end of file diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 7e1e77b..73f690a 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -5,4 +5,8 @@ spring: password: ${SPRING_DATASOURCE_PASSWORD} api: - secret: ${API_SECRET} \ No newline at end of file + secret: ${API_SECRET} + +maxmind: + license-key: ${MAXMIND_LICENSE_KEY} + user-id: ${MAXMIND_USER_ID} \ No newline at end of file diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index a6e8645..d78eaf7 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -5,5 +5,15 @@ spring: password: example driver-class-name: org.postgresql.Driver + data: + redis: + host: ${REDIS_HOST:localhost} + port: ${REDIS_PORT:6379} + password: ${REDIS_PASSWORD:} + api: - secret: test \ No newline at end of file + secret: test + +maxmind: + license-key: ${MAXMIND_LICENSE_KEY} + user-id: ${MAXMIND_USER_ID} \ No newline at end of file diff --git a/src/main/resources/db/3.sql b/src/main/resources/db/3.sql new file mode 100644 index 0000000..08f38e3 --- /dev/null +++ b/src/main/resources/db/3.sql @@ -0,0 +1,44 @@ +CREATE SCHEMA IF NOT EXISTS geoip; + +CREATE TABLE geoip.location +( + geoname_id INTEGER PRIMARY KEY, + locale_code TEXT, + continent_code TEXT, + continent_name TEXT, + country_iso_code TEXT, + country_name TEXT, + subdivision_1_iso_code TEXT, + subdivision_1_name TEXT, + subdivision_2_iso_code TEXT, + subdivision_2_name TEXT, + city_name TEXT, + metro_code TEXT, + time_zone TEXT, + is_in_european_union BOOLEAN +); + +CREATE TABLE geoip.autonomous_system +( + autonomous_system_number INTEGER PRIMARY KEY, + autonomous_system_organization TEXT +); + +CREATE TABLE geoip.ip_block +( + network CIDR PRIMARY KEY, + geoname_id INTEGER REFERENCES geoip.location (geoname_id), + registered_country_geoname_id INTEGER, + represented_country_geoname_id INTEGER, + is_anonymous_proxy BOOLEAN, + is_satellite_provider BOOLEAN, + postal_code TEXT, + latitude DOUBLE PRECISION, + longitude DOUBLE PRECISION, + accuracy_radius INTEGER, + is_anycast BOOLEAN, + autonomous_system_number INTEGER REFERENCES geoip.autonomous_system (autonomous_system_number) +); + +-- Optional index for faster geo lookups +CREATE INDEX idx_ip_block_network ON geoip.ip_block USING GIST (network inet_ops); \ No newline at end of file diff --git a/src/main/resources/mappers/GeoIPMapper.xml b/src/main/resources/mappers/GeoIPMapper.xml new file mode 100644 index 0000000..454d6db --- /dev/null +++ b/src/main/resources/mappers/GeoIPMapper.xml @@ -0,0 +1,52 @@ + + + + + + + INSERT INTO geoip.location ( + geoname_id, locale_code, continent_code, continent_name, + country_iso_code, country_name, subdivision_1_iso_code, + subdivision_1_name, subdivision_2_iso_code, subdivision_2_name, + city_name, metro_code, time_zone, is_in_european_union + ) VALUES + + ( + #{loc.geonameId}, #{loc.localeCode}, #{loc.continentCode}, #{loc.continentName}, + #{loc.countryIsoCode}, #{loc.countryName}, #{loc.subdivision1IsoCode}, + #{loc.subdivision1Name}, #{loc.subdivision2IsoCode}, #{loc.subdivision2Name}, + #{loc.cityName}, #{loc.metroCode}, #{loc.timeZone}, #{loc.isInEuropeanUnion} + ) + + ON CONFLICT (geoname_id) DO NOTHING + + + + INSERT INTO geoip.autonomous_system ( + autonomous_system_number, autonomous_system_organization + ) VALUES + + ( + #{asn.autonomousSystemNumber}, #{asn.autonomousSystemOrganization} + ) + + ON CONFLICT DO NOTHING + + + + INSERT INTO geoip.ip_block ( + network, geoname_id, registered_country_geoname_id, represented_country_geoname_id, + is_anonymous_proxy, is_satellite_provider, postal_code, + latitude, longitude, accuracy_radius, is_anycast, autonomous_system_number + ) VALUES + + ( + CAST(#{block.network} AS cidr), #{block.geonameId}, #{block.registeredCountryGeonameId}, #{block.representedCountryGeonameId}, + #{block.isAnonymousProxy}, #{block.isSatelliteProvider}, #{block.postalCode}, + #{block.latitude}, #{block.longitude}, #{block.accuracyRadius}, #{block.isAnycast}, #{block.autonomousSystemNumber} + ) + + ON CONFLICT DO NOTHING + + \ No newline at end of file From 874125878d3d473e18e52a9cc64801902292548b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 20:05:02 +0100 Subject: [PATCH 04/17] second pass at IP info --- .../io/paradaux/api/mappers/GeoIPMapper.java | 14 +- .../paradaux/api/models/geoip/ASNBlock.java | 9 + .../geoip/{IPBlock.java => CityBlock.java} | 3 +- .../impl/GeoIPInformationServiceImpl.java | 164 ++++++++++++------ src/main/resources/db/3.sql | 66 ++++++- src/main/resources/mappers/GeoIPMapper.xml | 56 +++++- 6 files changed, 245 insertions(+), 67 deletions(-) create mode 100644 src/main/java/io/paradaux/api/models/geoip/ASNBlock.java rename src/main/java/io/paradaux/api/models/geoip/{IPBlock.java => CityBlock.java} (87%) diff --git a/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java index 78a2e9c..4e84850 100644 --- a/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java +++ b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java @@ -1,14 +1,22 @@ package io.paradaux.api.mappers; import io.paradaux.api.models.geoip.ASN; -import io.paradaux.api.models.geoip.IPBlock; +import io.paradaux.api.models.geoip.ASNBlock; +import io.paradaux.api.models.geoip.CityBlock; import io.paradaux.api.models.geoip.IPLocation; import org.apache.ibatis.annotations.Mapper; import java.util.List; +import java.util.Map; @Mapper public interface GeoIPMapper { void insertLocations(List locations); void insertASNs(List asns); - void insertIPBlocks(List blocks); -} + void insertCityBlocks(List cityBlocks); + void insertASNBlocks(List asnBlocks); + + Map getIPInfo(String ipAddress); + CityBlock getCityBlockByIP(String ipAddress); + Map getASNByIP(String ipAddress); + IPLocation getLocationById(Integer geonameId); +} \ No newline at end of file diff --git a/src/main/java/io/paradaux/api/models/geoip/ASNBlock.java b/src/main/java/io/paradaux/api/models/geoip/ASNBlock.java new file mode 100644 index 0000000..3654898 --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/ASNBlock.java @@ -0,0 +1,9 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class ASNBlock { + private String network; + private Integer autonomousSystemNumber; +} diff --git a/src/main/java/io/paradaux/api/models/geoip/IPBlock.java b/src/main/java/io/paradaux/api/models/geoip/CityBlock.java similarity index 87% rename from src/main/java/io/paradaux/api/models/geoip/IPBlock.java rename to src/main/java/io/paradaux/api/models/geoip/CityBlock.java index a90401d..15e39ab 100644 --- a/src/main/java/io/paradaux/api/models/geoip/IPBlock.java +++ b/src/main/java/io/paradaux/api/models/geoip/CityBlock.java @@ -3,7 +3,7 @@ import lombok.Data; @Data -public class IPBlock { +public class CityBlock { private String network; private Integer geonameId; private Integer registeredCountryGeonameId; @@ -15,5 +15,4 @@ public class IPBlock { private Double longitude; private Integer accuracyRadius; private Boolean isAnycast; - private Integer autonomousSystemNumber; } diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java index 01b7691..5acf540 100644 --- a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -4,7 +4,8 @@ import com.opencsv.exceptions.CsvValidationException; import io.paradaux.api.mappers.GeoIPMapper; import io.paradaux.api.models.geoip.ASN; -import io.paradaux.api.models.geoip.IPBlock; +import io.paradaux.api.models.geoip.ASNBlock; +import io.paradaux.api.models.geoip.CityBlock; import io.paradaux.api.models.geoip.IPLocation; import io.paradaux.api.services.GeoIPInformationService; import lombok.RequiredArgsConstructor; @@ -37,28 +38,33 @@ public class GeoIPInformationServiceImpl implements GeoIPInformationService { public void importAllData() { try { + // Import locations first importLocations(cityLocationsFile); - } catch (Exception e) { - log.error("Failed to import locations", e); - return; // or handle fail-fast - } - CompletableFuture asnsFuture = CompletableFuture.runAsync(() -> { - try { - importASNs(asnBlocksIPv4File, asnBlocksIPv6File); - } catch (Exception e) { - log.error("Failed to import ASN", e); - } - }); + // Import ASNs and ASN blocks in parallel with city blocks + CompletableFuture asnFuture = CompletableFuture.runAsync(() -> { + try { + importASNsAndBlocks(asnBlocksIPv4File, asnBlocksIPv6File); + } catch (Exception e) { + log.error("Failed to import ASN data", e); + } + }); - try { - importIPBlocks(cityBlocksIPv4File, cityBlocksIPv6File); + CompletableFuture cityFuture = CompletableFuture.runAsync(() -> { + try { + importCityBlocks(cityBlocksIPv4File, cityBlocksIPv6File); + } catch (Exception e) { + log.error("Failed to import city blocks", e); + } + }); + + // Wait for both to complete + CompletableFuture.allOf(asnFuture, cityFuture).join(); + + log.info("All GeoIP data imported successfully"); } catch (Exception e) { - log.error("Failed to import IP blocks", e); + log.error("Failed to import locations", e); } - - asnsFuture.join(); - log.info("All GeoIP data imported"); } private void importLocations(String csvPath) throws IOException, CsvValidationException { @@ -67,91 +73,151 @@ private void importLocations(String csvPath) throws IOException, CsvValidationEx reader.readNext(); // skip header String[] line; while ((line = reader.readNext()) != null) { + if (line.length < 14) { + log.warn("Skipping location row with insufficient columns: {}", line.length); + continue; + } + IPLocation loc = new IPLocation(); loc.setGeonameId(parseIntOrNull(line[0])); - loc.setLocaleCode(line[1]); - loc.setContinentCode(line[2]); - loc.setContinentName(line[3]); - loc.setCountryIsoCode(line[4]); - loc.setCountryName(line[5]); - loc.setSubdivision1IsoCode(line[6]); - loc.setSubdivision1Name(line[7]); - loc.setSubdivision2IsoCode(line[8]); - loc.setSubdivision2Name(line[9]); - loc.setCityName(line[10]); - loc.setMetroCode(line[11]); - loc.setTimeZone(line[12]); + loc.setLocaleCode(parseStringOrNull(line[1])); + loc.setContinentCode(parseStringOrNull(line[2])); + loc.setContinentName(parseStringOrNull(line[3])); + loc.setCountryIsoCode(parseStringOrNull(line[4])); + loc.setCountryName(parseStringOrNull(line[5])); + loc.setSubdivision1IsoCode(parseStringOrNull(line[6])); + loc.setSubdivision1Name(parseStringOrNull(line[7])); + loc.setSubdivision2IsoCode(parseStringOrNull(line[8])); + loc.setSubdivision2Name(parseStringOrNull(line[9])); + loc.setCityName(parseStringOrNull(line[10])); + loc.setMetroCode(parseStringOrNull(line[11])); + loc.setTimeZone(parseStringOrNull(line[12])); loc.setIsInEuropeanUnion("1".equals(line[13])); locations.add(loc); } } + log.info("Imported {} locations", locations.size()); batchInsert(locations, geoIPMapper::insertLocations); } - private void importASNs(String... csvPaths) throws IOException, CsvValidationException { + private void importASNsAndBlocks(String... csvPaths) throws IOException, CsvValidationException { Map uniqueAsns = new HashMap<>(); + List asnBlocks = new ArrayList<>(); for (String csvPath : csvPaths) { try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { reader.readNext(); // skip header String[] line; while ((line = reader.readNext()) != null) { - Integer asnNumber = parseIntOrNull(line[1]); - if (asnNumber == null || uniqueAsns.containsKey(asnNumber)) continue; + if (line.length < 3) { + log.warn("Skipping ASN row with insufficient columns: {}", line.length); + continue; + } - ASN asn = new ASN(); - asn.setAutonomousSystemNumber(asnNumber); - asn.setAutonomousSystemOrganization(line[2]); - uniqueAsns.put(asnNumber, asn); + String network = parseStringOrNull(line[0]); + Integer asnNumber = parseIntOrNull(line[1]); + String asnOrg = parseStringOrNull(line[2]); + + if (network == null || asnNumber == null) { + continue; + } + + // Store unique ASN + if (!uniqueAsns.containsKey(asnNumber)) { + ASN asn = new ASN(); + asn.setAutonomousSystemNumber(asnNumber); + asn.setAutonomousSystemOrganization(asnOrg); + uniqueAsns.put(asnNumber, asn); + } + + // Store ASN block + ASNBlock asnBlock = new ASNBlock(); + asnBlock.setNetwork(network); + asnBlock.setAutonomousSystemNumber(asnNumber); + asnBlocks.add(asnBlock); } } } + log.info("Imported {} unique ASNs and {} ASN blocks", uniqueAsns.size(), asnBlocks.size()); + + // Insert ASNs first (due to foreign key constraint) batchInsert(new ArrayList<>(uniqueAsns.values()), geoIPMapper::insertASNs); + + // Then insert ASN blocks + batchInsert(asnBlocks, geoIPMapper::insertASNBlocks); } - private void importIPBlocks(String... csvPaths) throws IOException, CsvValidationException { - List blocks = new ArrayList<>(); + private void importCityBlocks(String... csvPaths) throws IOException, CsvValidationException { + List cityBlocks = new ArrayList<>(); for (String csvPath : csvPaths) { try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { reader.readNext(); // skip header String[] line; while ((line = reader.readNext()) != null) { - IPBlock block = new IPBlock(); - block.setNetwork(line[0]); + if (line.length < 11) { + log.warn("Skipping city block row with insufficient columns: {}", line.length); + continue; + } + + CityBlock block = new CityBlock(); + block.setNetwork(parseStringOrNull(line[0])); block.setGeonameId(parseIntOrNull(line[1])); block.setRegisteredCountryGeonameId(parseIntOrNull(line[2])); block.setRepresentedCountryGeonameId(parseIntOrNull(line[3])); block.setIsAnonymousProxy("1".equals(line[4])); block.setIsSatelliteProvider("1".equals(line[5])); - block.setPostalCode(line[6]); + block.setPostalCode(parseStringOrNull(line[6])); block.setLatitude(parseDoubleOrNull(line[7])); block.setLongitude(parseDoubleOrNull(line[8])); block.setAccuracyRadius(parseIntOrNull(line[9])); block.setIsAnycast("1".equals(line[10])); - blocks.add(block); + + cityBlocks.add(block); } } } - batchInsert(blocks, geoIPMapper::insertIPBlocks); + log.info("Imported {} city blocks", cityBlocks.size()); + batchInsert(cityBlocks, geoIPMapper::insertCityBlocks); } private void batchInsert(List list, Consumer> insertFunction) { for (int i = 0; i < list.size(); i += BATCH_SIZE) { int end = Math.min(i + BATCH_SIZE, list.size()); - insertFunction.accept(list.subList(i, end)); + List batch = list.subList(i, end); + try { + insertFunction.accept(batch); + if (i % (BATCH_SIZE * 10) == 0) { + log.info("Processed {} records", i + batch.size()); + } + } catch (Exception e) { + log.error("Failed to insert batch starting at index {}", i, e); + } } } private Integer parseIntOrNull(String s) { - try { return s == null || s.isBlank() ? null : Integer.parseInt(s); } - catch (NumberFormatException e) { return null; } + if (s == null || s.trim().isEmpty()) return null; + try { + return Integer.parseInt(s.trim()); + } catch (NumberFormatException e) { + return null; + } } private Double parseDoubleOrNull(String s) { - try { return s == null || s.isBlank() ? null : Double.parseDouble(s); } - catch (NumberFormatException e) { return null; } + if (s == null || s.trim().isEmpty()) return null; + try { + return Double.parseDouble(s.trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private String parseStringOrNull(String s) { + if (s == null || s.trim().isEmpty()) return null; + return s.trim(); } } \ No newline at end of file diff --git a/src/main/resources/db/3.sql b/src/main/resources/db/3.sql index 08f38e3..19b2296 100644 --- a/src/main/resources/db/3.sql +++ b/src/main/resources/db/3.sql @@ -24,7 +24,8 @@ CREATE TABLE geoip.autonomous_system autonomous_system_organization TEXT ); -CREATE TABLE geoip.ip_block +-- Separate tables for different data sources since they have different network ranges +CREATE TABLE geoip.city_block ( network CIDR PRIMARY KEY, geoname_id INTEGER REFERENCES geoip.location (geoname_id), @@ -36,9 +37,64 @@ CREATE TABLE geoip.ip_block latitude DOUBLE PRECISION, longitude DOUBLE PRECISION, accuracy_radius INTEGER, - is_anycast BOOLEAN, - autonomous_system_number INTEGER REFERENCES geoip.autonomous_system (autonomous_system_number) + is_anycast BOOLEAN ); --- Optional index for faster geo lookups -CREATE INDEX idx_ip_block_network ON geoip.ip_block USING GIST (network inet_ops); \ No newline at end of file +CREATE TABLE geoip.asn_block +( + network CIDR PRIMARY KEY, + autonomous_system_number INTEGER REFERENCES geoip.autonomous_system (autonomous_system_number) +); + +-- Indexes for performance +CREATE INDEX idx_city_block_network ON geoip.city_block USING GIST (network inet_ops); +CREATE INDEX idx_asn_block_network ON geoip.asn_block USING GIST (network inet_ops); +CREATE INDEX idx_city_block_geoname ON geoip.city_block (geoname_id); + +-- Function to get complete IP information +CREATE OR REPLACE FUNCTION geoip.get_ip_info(ip_address INET) + RETURNS TABLE + ( + network CIDR, + geoname_id INTEGER, + city_name TEXT, + country_name TEXT, + latitude DOUBLE PRECISION, + longitude DOUBLE PRECISION, + asn_number INTEGER, + asn_organization TEXT + ) +AS +$$ +BEGIN + RETURN QUERY + WITH city_data AS (SELECT cb.network, + cb.geoname_id, + cb.latitude, + cb.longitude, + l.city_name, + l.country_name + FROM geoip.city_block cb + LEFT JOIN geoip.location l ON cb.geoname_id = l.geoname_id + WHERE cb.network >>= ip_address + ORDER BY masklen(cb.network) DESC + LIMIT 1), + asn_data AS (SELECT ab.autonomous_system_number, aut.autonomous_system_organization + FROM geoip.asn_block ab + LEFT JOIN geoip.autonomous_system aut + ON ab.autonomous_system_number = aut.autonomous_system_number + WHERE ab.network >>= ip_address + ORDER BY masklen(ab.network) DESC + LIMIT 1) + SELECT city_data.network, + city_data.geoname_id, + city_data.city_name, + city_data.country_name, + city_data.latitude, + city_data.longitude, + asn_data.autonomous_system_number, + asn_data.autonomous_system_organization + FROM city_data + FULL OUTER JOIN asn_data ON true; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/src/main/resources/mappers/GeoIPMapper.xml b/src/main/resources/mappers/GeoIPMapper.xml index 454d6db..f8623e6 100644 --- a/src/main/resources/mappers/GeoIPMapper.xml +++ b/src/main/resources/mappers/GeoIPMapper.xml @@ -31,22 +31,62 @@ #{asn.autonomousSystemNumber}, #{asn.autonomousSystemOrganization} ) - ON CONFLICT DO NOTHING + ON CONFLICT (autonomous_system_number) DO NOTHING - - INSERT INTO geoip.ip_block ( + + INSERT INTO geoip.city_block ( network, geoname_id, registered_country_geoname_id, represented_country_geoname_id, is_anonymous_proxy, is_satellite_provider, postal_code, - latitude, longitude, accuracy_radius, is_anycast, autonomous_system_number + latitude, longitude, accuracy_radius, is_anycast ) VALUES ( - CAST(#{block.network} AS cidr), #{block.geonameId}, #{block.registeredCountryGeonameId}, #{block.representedCountryGeonameId}, - #{block.isAnonymousProxy}, #{block.isSatelliteProvider}, #{block.postalCode}, - #{block.latitude}, #{block.longitude}, #{block.accuracyRadius}, #{block.isAnycast}, #{block.autonomousSystemNumber} + CAST(#{block.network} AS cidr), #{block.geonameId}, #{block.registeredCountryGeonameId}, + #{block.representedCountryGeonameId}, #{block.isAnonymousProxy}, #{block.isSatelliteProvider}, + #{block.postalCode}, #{block.latitude}, #{block.longitude}, #{block.accuracyRadius}, #{block.isAnycast} ) - ON CONFLICT DO NOTHING + ON CONFLICT (network) DO NOTHING + + + INSERT INTO geoip.asn_block ( + network, autonomous_system_number + ) VALUES + + ( + CAST(#{asnBlock.network} AS cidr), #{asnBlock.autonomousSystemNumber} + ) + + ON CONFLICT (network) DO NOTHING + + + + + + + + + + + \ No newline at end of file From dbbf7076fedcfe7d0fa1e7d76403030570c9e5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 21:29:46 +0100 Subject: [PATCH 05/17] Third pass at IP Info, now has a method to import directly from web --- build.gradle.kts | 1 + .../api/controllers/GeoIPController.java | 7 +- .../io/paradaux/api/mappers/GeoIPMapper.java | 2 + .../api/services/GeoIPInformationService.java | 4 ++ .../impl/GeoIPInformationServiceImpl.java | 70 +++++++++++++------ src/main/resources/mappers/GeoIPMapper.xml | 3 + 6 files changed, 66 insertions(+), 21 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index dcf3f79..fc29cd4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -39,6 +39,7 @@ dependencies { // Utilities implementation("org.apache.commons:commons-compress:1.24.0") implementation("org.apache.commons:commons-lang3:3.13.0") + implementation("com.squareup.okhttp3:okhttp:4.12.0") // Caching implementation("org.springframework.boot:spring-boot-starter-cache") diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java index 8d82040..b71ff3f 100644 --- a/src/main/java/io/paradaux/api/controllers/GeoIPController.java +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -24,6 +24,11 @@ public class GeoIPController { @PostMapping("/sync") @ProtectedRoute public void syncGeoIPData() { - geoIPInformationService.importAllData(); + try { + log.info("Starting GeoIP data synchronization..."); + geoIPInformationService.importAllData(); + } catch (IOException e) { + log.error("Failed to import GeoIP data", e); + } } } diff --git a/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java index 4e84850..6eda650 100644 --- a/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java +++ b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java @@ -19,4 +19,6 @@ public interface GeoIPMapper { CityBlock getCityBlockByIP(String ipAddress); Map getASNByIP(String ipAddress); IPLocation getLocationById(Integer geonameId); + + void truncateAll(); } \ No newline at end of file diff --git a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java index 72a3559..ddf5367 100644 --- a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java +++ b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java @@ -1,4 +1,8 @@ package io.paradaux.api.services; +import java.io.IOException; + public interface GeoIPInformationService { + + void importAllData() throws IOException; } diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java index 5acf540..fce1dd4 100644 --- a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -8,12 +8,16 @@ import io.paradaux.api.models.geoip.CityBlock; import io.paradaux.api.models.geoip.IPLocation; import io.paradaux.api.services.GeoIPInformationService; +import io.paradaux.api.utils.FileUtils; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.FileReader; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -30,21 +34,43 @@ public class GeoIPInformationServiceImpl implements GeoIPInformationService { private final GeoIPMapper geoIPMapper; - String cityLocationsFile = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-City-CSV_20250718\\GeoLite2-City-Locations-en.csv"; - String asnBlocksIPv4File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-ASN-CSV_20250720\\GeoLite2-ASN-Blocks-IPv4.csv"; - String asnBlocksIPv6File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-ASN-CSV_20250720\\GeoLite2-ASN-Blocks-IPv6.csv"; - String cityBlocksIPv4File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-City-CSV_20250718\\GeoLite2-City-Blocks-IPv4.csv"; - String cityBlocksIPv6File = "C:\\Workspace\\data-wrangling\\geoip\\GeoLite2-City-CSV_20250718\\GeoLite2-City-Blocks-IPv6.csv"; + @Value("${maxmind.license-key}") + private String maxMindLicenseKey; - public void importAllData() { + private static final String MAXMIND_DOWNLOAD_URL = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-%s-CSV&license_key=%s&suffix=zip"; + + + public void importAllData() throws IOException { + // Clear existing data + geoIPMapper.truncateAll(); + + // Download zips from MaxMind + Path dataDir = Files.createTempDirectory("geoip-"); + Path cityDir = dataDir.resolve("city"); + Path asnDir = dataDir.resolve("asn"); + + String url = String.format(MAXMIND_DOWNLOAD_URL, "City", maxMindLicenseKey); + FileUtils.downloadAndExtractZip(url, cityDir); + url = String.format(MAXMIND_DOWNLOAD_URL, "ASN", maxMindLicenseKey); + FileUtils.downloadAndExtractZip(url, asnDir); + + // Process and import data from MaxMind + importAllData(dataDir); + } + + + public void importAllData(Path dataDir) { try { - // Import locations first - importLocations(cityLocationsFile); + // Import locations first - use Path instead of String + importLocations(dataDir.resolve("city/GeoLite2-City-Locations-en.csv")); // Import ASNs and ASN blocks in parallel with city blocks CompletableFuture asnFuture = CompletableFuture.runAsync(() -> { try { - importASNsAndBlocks(asnBlocksIPv4File, asnBlocksIPv6File); + importASNsAndBlocks( + dataDir.resolve("asn/GeoLite2-ASN-Blocks-IPv4.csv"), + dataDir.resolve("asn/GeoLite2-ASN-Blocks-IPv6.csv") + ); } catch (Exception e) { log.error("Failed to import ASN data", e); } @@ -52,7 +78,10 @@ public void importAllData() { CompletableFuture cityFuture = CompletableFuture.runAsync(() -> { try { - importCityBlocks(cityBlocksIPv4File, cityBlocksIPv6File); + importCityBlocks( + dataDir.resolve("city/GeoLite2-City-Blocks-IPv4.csv"), + dataDir.resolve("city/GeoLite2-City-Blocks-IPv6.csv") + ); } catch (Exception e) { log.error("Failed to import city blocks", e); } @@ -60,16 +89,17 @@ public void importAllData() { // Wait for both to complete CompletableFuture.allOf(asnFuture, cityFuture).join(); + log.info("All GeoIP data imported successfully from: " + dataDir); - log.info("All GeoIP data imported successfully"); } catch (Exception e) { - log.error("Failed to import locations", e); + log.error("Failed to import GeoIP data from: " + dataDir, e); } } - private void importLocations(String csvPath) throws IOException, CsvValidationException { + // Update method signatures to accept Path instead of String + private void importLocations(Path csvPath) throws IOException, CsvValidationException { List locations = new ArrayList<>(); - try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { + try (CSVReader reader = new CSVReader(new FileReader(csvPath.toFile()))) { reader.readNext(); // skip header String[] line; while ((line = reader.readNext()) != null) { @@ -100,12 +130,12 @@ private void importLocations(String csvPath) throws IOException, CsvValidationEx batchInsert(locations, geoIPMapper::insertLocations); } - private void importASNsAndBlocks(String... csvPaths) throws IOException, CsvValidationException { + private void importASNsAndBlocks(Path... csvPaths) throws IOException, CsvValidationException { Map uniqueAsns = new HashMap<>(); List asnBlocks = new ArrayList<>(); - for (String csvPath : csvPaths) { - try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { + for (Path csvPath : csvPaths) { + try (CSVReader reader = new CSVReader(new FileReader(csvPath.toFile()))) { reader.readNext(); // skip header String[] line; while ((line = reader.readNext()) != null) { @@ -148,11 +178,11 @@ private void importASNsAndBlocks(String... csvPaths) throws IOException, CsvVali batchInsert(asnBlocks, geoIPMapper::insertASNBlocks); } - private void importCityBlocks(String... csvPaths) throws IOException, CsvValidationException { + private void importCityBlocks(Path... csvPaths) throws IOException, CsvValidationException { List cityBlocks = new ArrayList<>(); - for (String csvPath : csvPaths) { - try (CSVReader reader = new CSVReader(new FileReader(csvPath))) { + for (Path csvPath : csvPaths) { + try (CSVReader reader = new CSVReader(new FileReader(csvPath.toFile()))) { reader.readNext(); // skip header String[] line; while ((line = reader.readNext()) != null) { diff --git a/src/main/resources/mappers/GeoIPMapper.xml b/src/main/resources/mappers/GeoIPMapper.xml index f8623e6..a75ddb9 100644 --- a/src/main/resources/mappers/GeoIPMapper.xml +++ b/src/main/resources/mappers/GeoIPMapper.xml @@ -89,4 +89,7 @@ SELECT * FROM geoip.location WHERE geoname_id = #{geonameId} + + TRUNCATE TABLE geoip.location, geoip.autonomous_system, geoip.city_block, geoip.asn_block CASCADE + \ No newline at end of file From f67a5974437b1ac244b5cd0128c77192ea4139f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 21:29:57 +0100 Subject: [PATCH 06/17] Download and flatten logic --- .../java/io/paradaux/api/utils/FileUtils.java | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 src/main/java/io/paradaux/api/utils/FileUtils.java diff --git a/src/main/java/io/paradaux/api/utils/FileUtils.java b/src/main/java/io/paradaux/api/utils/FileUtils.java new file mode 100644 index 0000000..1ee1e41 --- /dev/null +++ b/src/main/java/io/paradaux/api/utils/FileUtils.java @@ -0,0 +1,121 @@ +package io.paradaux.api.utils; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class FileUtils { + private static final int BUFFER_SIZE = 8192; // 8KB buffer + private static final OkHttpClient client = new OkHttpClient.Builder() + .followRedirects(true) + .followSslRedirects(true) + .build(); + + /** + * Downloads a ZIP file from the given URL and extracts it to the specified directory. + * If the ZIP contains a single root folder, its contents are flattened to the extract directory. + * + * @param url The URL to download from + * @param extractDir The directory path where files should be extracted + * @throws IOException if download or extraction fails + */ + public static void downloadAndExtractZip(String url, Path extractDir) throws IOException { + Request request = new Request.Builder() + .url(url) + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Download failed: " + response.code() + " " + response.message()); + } + + ResponseBody body = response.body(); + if (body == null) { + throw new IOException("Response body is null"); + } + + // First pass: collect all entries to determine structure + List entries = new ArrayList<>(); + String commonRoot = null; + + try (InputStream inputStream = body.byteStream(); + BufferedInputStream bufferedInput = new BufferedInputStream(inputStream, BUFFER_SIZE); + ZipInputStream zipStream = new ZipInputStream(bufferedInput)) { + + ZipEntry entry; + while ((entry = zipStream.getNextEntry()) != null) { + entries.add(new ZipEntry(entry)); + zipStream.closeEntry(); + } + } + + // Determine if there's a single root folder + if (!entries.isEmpty()) { + String firstEntryName = entries.get(0).getName(); + int firstSlash = firstEntryName.indexOf('/'); + + if (firstSlash > 0) { + String potentialRoot = firstEntryName.substring(0, firstSlash + 1); + boolean allEntriesShareRoot = entries.stream() + .allMatch(e -> e.getName().startsWith(potentialRoot)); + + if (allEntriesShareRoot) { + commonRoot = potentialRoot; + } + } + } + + // Second pass: extract files + try (InputStream inputStream2 = client.newCall(request).execute().body().byteStream(); + BufferedInputStream bufferedInput2 = new BufferedInputStream(inputStream2, BUFFER_SIZE); + ZipInputStream zipStream2 = new ZipInputStream(bufferedInput2)) { + + ZipEntry entry; + while ((entry = zipStream2.getNextEntry()) != null) { + String entryName = entry.getName(); + + // Remove common root if present + if (commonRoot != null && entryName.startsWith(commonRoot)) { + entryName = entryName.substring(commonRoot.length()); + } + + // Skip empty paths after root removal + if (entryName.isEmpty()) { + zipStream2.closeEntry(); + continue; + } + + if (entry.isDirectory()) { + // Create directory + Path dirPath = extractDir.resolve(entryName); + Files.createDirectories(dirPath); + } else { + // Extract file + Path filePath = extractDir.resolve(entryName); + Files.createDirectories(filePath.getParent()); + + try (FileOutputStream fos = new FileOutputStream(filePath.toFile()); + BufferedOutputStream bos = new BufferedOutputStream(fos, BUFFER_SIZE)) { + + byte[] buffer = new byte[BUFFER_SIZE]; + int bytesRead; + while ((bytesRead = zipStream2.read(buffer)) != -1) { + bos.write(buffer, 0, bytesRead); + } + } + } + zipStream2.closeEntry(); + } + } + } + } +} From 29266364c67e8e9785e1aa1ce07582c3bbad1c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 21:35:03 +0100 Subject: [PATCH 07/17] Reformat --- src/main/java/io/paradaux/api/config/WebConfig.java | 3 +-- .../io/paradaux/api/controllers/ContactController.java | 6 +++--- .../java/io/paradaux/api/controllers/GeoIPController.java | 3 ++- .../api/interceptors/ProtectedRouteInterceptor.java | 8 ++++---- .../api/interceptors/RateLimitingInterceptor.java | 3 ++- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/paradaux/api/config/WebConfig.java b/src/main/java/io/paradaux/api/config/WebConfig.java index 02ea962..881fcd2 100644 --- a/src/main/java/io/paradaux/api/config/WebConfig.java +++ b/src/main/java/io/paradaux/api/config/WebConfig.java @@ -17,7 +17,6 @@ public class WebConfig implements WebMvcConfigurer { @Override public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(protectedRouteInterceptor); - registry.addInterceptor(rateLimitingInterceptor) - .addPathPatterns("/**"); + registry.addInterceptor(rateLimitingInterceptor).addPathPatterns("/**"); } } diff --git a/src/main/java/io/paradaux/api/controllers/ContactController.java b/src/main/java/io/paradaux/api/controllers/ContactController.java index e41145e..a0d05ef 100644 --- a/src/main/java/io/paradaux/api/controllers/ContactController.java +++ b/src/main/java/io/paradaux/api/controllers/ContactController.java @@ -2,7 +2,7 @@ import io.paradaux.api.models.ContactFormRequest; import io.paradaux.api.services.ContactService; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -12,10 +12,10 @@ @RestController @RequestMapping("/api/contact") +@RequiredArgsConstructor public class ContactController { - @Autowired - private ContactService contactService; + private final ContactService contactService; @PostMapping public Mono> handleContactForm(@RequestBody ContactFormRequest request) { diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java index b71ff3f..e7cb805 100644 --- a/src/main/java/io/paradaux/api/controllers/GeoIPController.java +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -2,6 +2,7 @@ import com.opencsv.exceptions.CsvValidationException; import io.paradaux.api.models.annotations.ProtectedRoute; +import io.paradaux.api.services.GeoIPInformationService; import io.paradaux.api.services.impl.GeoIPInformationServiceImpl; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -19,7 +20,7 @@ @RequiredArgsConstructor public class GeoIPController { - private final GeoIPInformationServiceImpl geoIPInformationService; + private final GeoIPInformationService geoIPInformationService; @PostMapping("/sync") @ProtectedRoute diff --git a/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java b/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java index fbe177b..4296457 100644 --- a/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java +++ b/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java @@ -3,6 +3,7 @@ import io.paradaux.api.models.annotations.ProtectedRoute; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.method.HandlerMethod; @@ -25,7 +26,7 @@ public class ProtectedRouteInterceptor implements HandlerInterceptor { * If the secret is invalid or missing, it responds with HTTP 401 Unauthorized. */ @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) throws Exception { if (handler instanceof HandlerMethod method) { if (!isProtectedRoute(method)) { return true; @@ -42,10 +43,9 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons /** * A route is protected if the method or its class is annotated with @ProtectedRoute. * This requires a secret token in the request header "X-SECRET". - * */ + */ private boolean isProtectedRoute(HandlerMethod method) { - return protectedRouteCache.computeIfAbsent(method, hm -> hm.hasMethodAnnotation(ProtectedRoute.class) - || hm.getBeanType().isAnnotationPresent(ProtectedRoute.class)); + return protectedRouteCache.computeIfAbsent(method, hm -> hm.hasMethodAnnotation(ProtectedRoute.class) || hm.getBeanType().isAnnotationPresent(ProtectedRoute.class)); } /** diff --git a/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java b/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java index a7f4310..395e8c9 100644 --- a/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java +++ b/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java @@ -5,6 +5,7 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.springframework.web.servlet.HandlerInterceptor; @@ -19,7 +20,7 @@ public class RateLimitingInterceptor implements HandlerInterceptor { private final ProtectedRouteInterceptor protectedRouteInterceptor; @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws IOException { + public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) throws IOException { String ip = IPUtils.getClientIp(request); // Allow requests that contain a valid secret token From a438b2aca31ed8c873ac50c7e5639ddf68e76525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 21:35:24 +0100 Subject: [PATCH 08/17] Optimize imports --- src/main/java/io/paradaux/api/controllers/GeoIPController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java index e7cb805..4727ba2 100644 --- a/src/main/java/io/paradaux/api/controllers/GeoIPController.java +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -1,9 +1,7 @@ package io.paradaux.api.controllers; -import com.opencsv.exceptions.CsvValidationException; import io.paradaux.api.models.annotations.ProtectedRoute; import io.paradaux.api.services.GeoIPInformationService; -import io.paradaux.api.services.impl.GeoIPInformationServiceImpl; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.validation.annotation.Validated; From 1f3b8fe479a902611713f60bddf5e626135cfcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 21:51:42 +0100 Subject: [PATCH 09/17] MaxMind Sync Job setup --- .../io/paradaux/api/jobs/MaxMindSyncJob.java | 27 +++++++++++++ .../api/jobs/VisitCacheRefreshJob.java | 8 ++-- .../paradaux/api/services/DiscordService.java | 3 ++ .../api/services/impl/DiscordServiceImpl.java | 38 +++++++++++++++++++ .../impl/GeoIPInformationServiceImpl.java | 34 +++++++++-------- 5 files changed, 91 insertions(+), 19 deletions(-) diff --git a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java index d0f95c5..41b2037 100644 --- a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java +++ b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java @@ -1,11 +1,38 @@ package io.paradaux.api.jobs; +import io.paradaux.api.mappers.GeoIPMapper; +import io.paradaux.api.services.DiscordService; +import io.paradaux.api.services.GeoIPInformationService; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import java.io.IOException; +import java.util.HashMap; + @Service @Slf4j +@RequiredArgsConstructor public class MaxMindSyncJob { + private final GeoIPInformationService geoIPInformationService; + private final GeoIPMapper geoIPMapper; + private final DiscordService discordService; + + @Scheduled(cron = "0 0 3 * * WED") + public void refreshMaxMindDb() { + // Download all GeoIP data from MaxMind + log.info("Starting MaxMind GeoIP data synchronization..."); + discordService.sendMessage("Starting MaxMind GeoIP data synchronization...", "", new HashMap<>()); + + try { + geoIPInformationService.importAllData(); + } catch (IOException e) { + log.error("Failed to import MaxMind GeoIP data", e); + } + discordService.sendMessage("MaxMind GeoIP data synchronization completed successfully", "", new HashMap<>()); + log.info("MaxMind GeoIP data synchronization completed successfully."); + } } diff --git a/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java b/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java index a92c0ee..9910c2a 100644 --- a/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java +++ b/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java @@ -1,23 +1,23 @@ package io.paradaux.api.jobs; import io.paradaux.api.mappers.VisitsMapper; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @Service @Slf4j +@RequiredArgsConstructor public class VisitCacheRefreshJob { private final VisitsMapper visitsMapper; - public VisitCacheRefreshJob(VisitsMapper visitsMapper) { - this.visitsMapper = visitsMapper; - } - @Scheduled(fixedRate = 5 * 60 * 1000) // 5 mins public void refreshMaterializedView() { + log.info("Starting visit cache refresh..."); visitsMapper.refreshVisitCache(); log.info("Visit cache refreshed successfully."); + } } diff --git a/src/main/java/io/paradaux/api/services/DiscordService.java b/src/main/java/io/paradaux/api/services/DiscordService.java index 4de476d..a17c797 100644 --- a/src/main/java/io/paradaux/api/services/DiscordService.java +++ b/src/main/java/io/paradaux/api/services/DiscordService.java @@ -3,6 +3,9 @@ import io.paradaux.api.models.ContactFormRequest; import reactor.core.publisher.Mono; +import java.util.Map; + public interface DiscordService { Mono sendContactForm(ContactFormRequest request); + void sendMessage(String title, String description, Map fields); } diff --git a/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java index f37489d..0422009 100644 --- a/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java @@ -2,12 +2,15 @@ import io.paradaux.api.models.ContactFormRequest; import io.paradaux.api.services.DiscordService; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; @Service @@ -48,4 +51,39 @@ public Mono sendContactForm(ContactFormRequest request) { .retrieve() .bodyToMono(Void.class); } + + @Override + public void sendMessage(String title, String description, Map fields) { + Map embed = getEmbed(title, description, fields); + Map body = new HashMap<>(); + body.put("embeds", List.of(embed)); + + String webhookPath = webhookUrl.replace("https://discord.com/api/webhooks", ""); + + webClient.post() + .uri(webhookPath) + .bodyValue(body) + .retrieve() + .toBodilessEntity() + .block(); // blocking call + } + + @NotNull + private static Map getEmbed(String title, String description, Map fields) { + List> embedFields = new ArrayList<>(); + for (Map.Entry entry : fields.entrySet()) { + Map field = new HashMap<>(); + field.put("name", entry.getKey()); + field.put("value", entry.getValue()); + field.put("inline", false); + embedFields.add(field); + } + + Map embed = new HashMap<>(); + embed.put("title", title); + embed.put("description", description); + embed.put("color", 0x3498DB); + embed.put("fields", embedFields); + return embed; + } } \ No newline at end of file diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java index fce1dd4..9d43212 100644 --- a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -45,20 +45,12 @@ public void importAllData() throws IOException { geoIPMapper.truncateAll(); // Download zips from MaxMind - Path dataDir = Files.createTempDirectory("geoip-"); - Path cityDir = dataDir.resolve("city"); - Path asnDir = dataDir.resolve("asn"); - - String url = String.format(MAXMIND_DOWNLOAD_URL, "City", maxMindLicenseKey); - FileUtils.downloadAndExtractZip(url, cityDir); - url = String.format(MAXMIND_DOWNLOAD_URL, "ASN", maxMindLicenseKey); - FileUtils.downloadAndExtractZip(url, asnDir); + Path dataDir = downloadAllData(); // Process and import data from MaxMind importAllData(dataDir); } - public void importAllData(Path dataDir) { try { // Import locations first - use Path instead of String @@ -89,13 +81,25 @@ public void importAllData(Path dataDir) { // Wait for both to complete CompletableFuture.allOf(asnFuture, cityFuture).join(); - log.info("All GeoIP data imported successfully from: " + dataDir); + log.info("All GeoIP data imported successfully from: {}", dataDir); } catch (Exception e) { - log.error("Failed to import GeoIP data from: " + dataDir, e); + log.error("Failed to import GeoIP data from: {}", dataDir, e); } } + private Path downloadAllData() throws IOException { + Path dataDir = Files.createTempDirectory("geoip-"); + Path cityDir = dataDir.resolve("city"); + Path asnDir = dataDir.resolve("asn"); + + String url = String.format(MAXMIND_DOWNLOAD_URL, "City", maxMindLicenseKey); + FileUtils.downloadAndExtractZip(url, cityDir); + url = String.format(MAXMIND_DOWNLOAD_URL, "ASN", maxMindLicenseKey); + FileUtils.downloadAndExtractZip(url, asnDir); + return dataDir; + } + // Update method signatures to accept Path instead of String private void importLocations(Path csvPath) throws IOException, CsvValidationException { List locations = new ArrayList<>(); @@ -126,7 +130,7 @@ private void importLocations(Path csvPath) throws IOException, CsvValidationExce locations.add(loc); } } - log.info("Imported {} locations", locations.size()); + log.debug("Imported {} locations", locations.size()); batchInsert(locations, geoIPMapper::insertLocations); } @@ -169,7 +173,7 @@ private void importASNsAndBlocks(Path... csvPaths) throws IOException, CsvValida } } - log.info("Imported {} unique ASNs and {} ASN blocks", uniqueAsns.size(), asnBlocks.size()); + log.debug("Imported {} unique ASNs and {} ASN blocks", uniqueAsns.size(), asnBlocks.size()); // Insert ASNs first (due to foreign key constraint) batchInsert(new ArrayList<>(uniqueAsns.values()), geoIPMapper::insertASNs); @@ -209,7 +213,7 @@ private void importCityBlocks(Path... csvPaths) throws IOException, CsvValidatio } } - log.info("Imported {} city blocks", cityBlocks.size()); + log.debug("Imported {} city blocks", cityBlocks.size()); batchInsert(cityBlocks, geoIPMapper::insertCityBlocks); } @@ -220,7 +224,7 @@ private void batchInsert(List list, Consumer> insertFunction) { try { insertFunction.accept(batch); if (i % (BATCH_SIZE * 10) == 0) { - log.info("Processed {} records", i + batch.size()); + log.debug("Processed {} records", i + batch.size()); } } catch (Exception e) { log.error("Failed to insert batch starting at index {}", i, e); From 4d815684c65691fa321c5cc2d2d51c280ab6e880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 22:23:22 +0100 Subject: [PATCH 10/17] Lookup API / ratelimit fine tuning --- .../api/controllers/GeoIPController.java | 10 ++++-- .../io/paradaux/api/jobs/MaxMindSyncJob.java | 2 -- .../api/services/GeoIPInformationService.java | 9 +++++ .../impl/GeoIPInformationServiceImpl.java | 33 +++++++++++++++++++ .../impl/RateLimitingServiceImpl.java | 2 +- src/main/resources/application-dev.yml | 7 ++++ src/main/resources/application-prod.yml | 6 ++++ 7 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java index 4727ba2..3596fd1 100644 --- a/src/main/java/io/paradaux/api/controllers/GeoIPController.java +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -5,11 +5,10 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.io.IOException; +import java.util.Map; @RestController @RequestMapping("/api/geoip") @@ -20,6 +19,11 @@ public class GeoIPController { private final GeoIPInformationService geoIPInformationService; + @GetMapping("/lookup/{ipAddress}") + public Map lookup(@PathVariable String ipAddress) { + return geoIPInformationService.lookupIP(ipAddress); + } + @PostMapping("/sync") @ProtectedRoute public void syncGeoIPData() { diff --git a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java index 41b2037..9b2fe75 100644 --- a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java +++ b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java @@ -1,6 +1,5 @@ package io.paradaux.api.jobs; -import io.paradaux.api.mappers.GeoIPMapper; import io.paradaux.api.services.DiscordService; import io.paradaux.api.services.GeoIPInformationService; import lombok.RequiredArgsConstructor; @@ -17,7 +16,6 @@ public class MaxMindSyncJob { private final GeoIPInformationService geoIPInformationService; - private final GeoIPMapper geoIPMapper; private final DiscordService discordService; @Scheduled(cron = "0 0 3 * * WED") diff --git a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java index ddf5367..cc4a776 100644 --- a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java +++ b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java @@ -1,8 +1,17 @@ package io.paradaux.api.services; +import io.paradaux.api.models.geoip.CityBlock; +import io.paradaux.api.models.geoip.IPLocation; + import java.io.IOException; +import java.util.Map; public interface GeoIPInformationService { void importAllData() throws IOException; + Map getIPDetails(String ipAddress); + CityBlock getCityBlock(String ipAddress); + Map getASNDetails(String ipAddress); + IPLocation getLocationByGeoNameId(Integer geonameId); + Map lookupIP(String ipAddress); } diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java index 9d43212..8f80a0b 100644 --- a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -39,6 +39,39 @@ public class GeoIPInformationServiceImpl implements GeoIPInformationService { private static final String MAXMIND_DOWNLOAD_URL = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-%s-CSV&license_key=%s&suffix=zip"; + @Override + public Map lookupIP(String ipAddress) { + Map result = new HashMap<>(); + + CityBlock cityBlock = getCityBlock(ipAddress); + result.put("ip", ipAddress); + result.put("details", getIPDetails(ipAddress)); + result.put("asn", getASNDetails(ipAddress)); + result.put("city", cityBlock); + result.put("location", cityBlock != null ? geoIPMapper.getLocationById(cityBlock.getGeonameId()) : null); + result.put("attribution", "This database incorporates GeoNames [https://www.geonames.org] geographical data, which is made available under the Creative Commons Attribution 4.0 License. To view a copy of this license, visit https://creativecommons.org/licenses/by/4.0"); + return result; + } + + @Override + public Map getIPDetails(String ipAddress) { + return geoIPMapper.getIPInfo(ipAddress); + } + + @Override + public CityBlock getCityBlock(String ipAddress) { + return geoIPMapper.getCityBlockByIP(ipAddress); + } + + @Override + public Map getASNDetails(String ipAddress) { + return geoIPMapper.getASNByIP(ipAddress); + } + + @Override + public IPLocation getLocationByGeoNameId(Integer geonameId) { + return geoIPMapper.getLocationById(geonameId); + } public void importAllData() throws IOException { // Clear existing data diff --git a/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java index 1ebdb2f..84a44aa 100644 --- a/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java @@ -17,7 +17,7 @@ public class RateLimitingServiceImpl implements RateLimitingService { private final RedisTemplate redisTemplate; private static final long TIME_WINDOW = 60_000; // 1 min - private static final int MAX_REQUESTS = 10; + private static final int MAX_REQUESTS = 75; @Override public boolean isAllowed(String key) { diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 73f690a..d7744d7 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -4,6 +4,13 @@ spring: username: ${SPRING_DATASOURCE_USERNAME} password: ${SPRING_DATASOURCE_PASSWORD} + data: + redis: + host: ${REDIS_HOST:redis} + port: ${REDIS_PORT:6379} + password: ${REDIS_PASSWORD:} + + api: secret: ${API_SECRET} diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 7e1e77b..3c785b3 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -4,5 +4,11 @@ spring: username: ${SPRING_DATASOURCE_USERNAME} password: ${SPRING_DATASOURCE_PASSWORD} + data: + redis: + host: ${REDIS_HOST:redis} + port: ${REDIS_PORT:6379} + password: ${REDIS_PASSWORD:} + api: secret: ${API_SECRET} \ No newline at end of file From fcd12d560a7b74cfc007e110b8217fe44fb50918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 22:29:24 +0100 Subject: [PATCH 11/17] Update yaml --- src/main/resources/application-dev.yml | 6 +++--- src/main/resources/application-prod.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index d7744d7..c28094e 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -6,9 +6,9 @@ spring: data: redis: - host: ${REDIS_HOST:redis} - port: ${REDIS_PORT:6379} - password: ${REDIS_PASSWORD:} + host: redis + port: 6379 + password: ${REDIS_PASSWORD} api: diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 3c785b3..dd06507 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -6,9 +6,9 @@ spring: data: redis: - host: ${REDIS_HOST:redis} - port: ${REDIS_PORT:6379} - password: ${REDIS_PASSWORD:} + host: redis + port: 6379 + password: ${REDIS_PASSWORD} api: secret: ${API_SECRET} \ No newline at end of file From 0c2f34d27a2ddfe3c5a2f8d2fdbfe081f0ca218e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 22:37:11 +0100 Subject: [PATCH 12/17] Update yaml --- src/main/resources/application-dev.yml | 3 +-- src/main/resources/application-local.yml | 3 +-- src/main/resources/application-prod.yml | 5 ++++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index c28094e..73f866e 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -15,5 +15,4 @@ api: secret: ${API_SECRET} maxmind: - license-key: ${MAXMIND_LICENSE_KEY} - user-id: ${MAXMIND_USER_ID} \ No newline at end of file + license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index d78eaf7..e1384fc 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -15,5 +15,4 @@ api: secret: test maxmind: - license-key: ${MAXMIND_LICENSE_KEY} - user-id: ${MAXMIND_USER_ID} \ No newline at end of file + license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index dd06507..eed0c68 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -11,4 +11,7 @@ spring: password: ${REDIS_PASSWORD} api: - secret: ${API_SECRET} \ No newline at end of file + secret: ${API_SECRET} + +maxmind: + license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file From 24b014607c8fcb8dcf2772902e3bcb204b914328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 22:48:51 +0100 Subject: [PATCH 13/17] Make manual invocation of sync job to run async --- .../paradaux/api/controllers/GeoIPController.java | 13 ++++++------- .../java/io/paradaux/api/jobs/MaxMindSyncJob.java | 9 +++++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java index 3596fd1..1c59c9f 100644 --- a/src/main/java/io/paradaux/api/controllers/GeoIPController.java +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -1,9 +1,11 @@ package io.paradaux.api.controllers; +import io.paradaux.api.jobs.MaxMindSyncJob; import io.paradaux.api.models.annotations.ProtectedRoute; import io.paradaux.api.services.GeoIPInformationService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.http.ResponseEntity; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; @@ -18,6 +20,7 @@ public class GeoIPController { private final GeoIPInformationService geoIPInformationService; + private final MaxMindSyncJob maxMindSyncJob; @GetMapping("/lookup/{ipAddress}") public Map lookup(@PathVariable String ipAddress) { @@ -26,12 +29,8 @@ public Map lookup(@PathVariable String ipAddress) { @PostMapping("/sync") @ProtectedRoute - public void syncGeoIPData() { - try { - log.info("Starting GeoIP data synchronization..."); - geoIPInformationService.importAllData(); - } catch (IOException e) { - log.error("Failed to import GeoIP data", e); - } + public ResponseEntity syncGeoIPData() { + maxMindSyncJob.runSync(); + return ResponseEntity.accepted().body("MaxMind sync started"); } } diff --git a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java index 9b2fe75..aef844f 100644 --- a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java +++ b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java @@ -4,6 +4,7 @@ import io.paradaux.api.services.GeoIPInformationService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -19,8 +20,12 @@ public class MaxMindSyncJob { private final DiscordService discordService; @Scheduled(cron = "0 0 3 * * WED") - public void refreshMaxMindDb() { - // Download all GeoIP data from MaxMind + public void refreshMaxMindDbScheduled() { + runSync(); + } + + @Async + public void runSync() { log.info("Starting MaxMind GeoIP data synchronization..."); discordService.sendMessage("Starting MaxMind GeoIP data synchronization...", "", new HashMap<>()); From b09ce18630bdff0f422359787d727a450284e06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 22:51:40 +0100 Subject: [PATCH 14/17] Enable async --- src/main/java/io/paradaux/api/ParadauxApiApplication.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/io/paradaux/api/ParadauxApiApplication.java b/src/main/java/io/paradaux/api/ParadauxApiApplication.java index ca3e724..fa7f192 100644 --- a/src/main/java/io/paradaux/api/ParadauxApiApplication.java +++ b/src/main/java/io/paradaux/api/ParadauxApiApplication.java @@ -3,11 +3,13 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cache.annotation.EnableCaching; +import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication @EnableScheduling @EnableCaching +@EnableAsync public class ParadauxApiApplication { public static void main(String[] args) { From 49384fa3eae59ebe58018aeca6ac18a8a2b50374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 22:57:07 +0100 Subject: [PATCH 15/17] Add failure message --- src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java | 2 ++ src/main/resources/db/3.sql | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java index aef844f..0111979 100644 --- a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java +++ b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java @@ -33,6 +33,8 @@ public void runSync() { geoIPInformationService.importAllData(); } catch (IOException e) { log.error("Failed to import MaxMind GeoIP data", e); + discordService.sendMessage("MaxMind GeoIP data synchronization failure", e.getMessage(), new HashMap<>()); + return; } discordService.sendMessage("MaxMind GeoIP data synchronization completed successfully", "", new HashMap<>()); diff --git a/src/main/resources/db/3.sql b/src/main/resources/db/3.sql index 19b2296..9a15e14 100644 --- a/src/main/resources/db/3.sql +++ b/src/main/resources/db/3.sql @@ -1,3 +1,4 @@ +-- Went to production on 20/7/2025 CREATE SCHEMA IF NOT EXISTS geoip; CREATE TABLE geoip.location From db4154bfce61cd2bb95ac2c886f91916dda8441c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 23:09:07 +0100 Subject: [PATCH 16/17] Add upload zip --- .../api/controllers/GeoIPController.java | 36 +++++++++++ .../api/services/GeoIPInformationService.java | 2 + .../impl/GeoIPInformationServiceImpl.java | 5 +- .../java/io/paradaux/api/utils/FileUtils.java | 63 +++++++++++++++++++ 4 files changed, 103 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java index 1c59c9f..a63cf12 100644 --- a/src/main/java/io/paradaux/api/controllers/GeoIPController.java +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -8,10 +8,16 @@ import org.springframework.http.ResponseEntity; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; +import static io.paradaux.api.utils.FileUtils.extractZip; + @RestController @RequestMapping("/api/geoip") @Validated @@ -33,4 +39,34 @@ public ResponseEntity syncGeoIPData() { maxMindSyncJob.runSync(); return ResponseEntity.accepted().body("MaxMind sync started"); } + + @PostMapping("/upload-zips") + @ProtectedRoute + public ResponseEntity uploadZips( + @RequestParam("city") MultipartFile cityZip, + @RequestParam("asn") MultipartFile asnZip) { + + try { + Path dataDir = Files.createTempDirectory("geoip-"); + Path cityDir = dataDir.resolve("city"); + Path asnDir = dataDir.resolve("asn"); + + Files.createDirectories(cityDir); + Files.createDirectories(asnDir); + + extractZip(cityZip.getInputStream(), cityDir); + extractZip(asnZip.getInputStream(), asnDir); + + return ResponseEntity.ok(dataDir.toAbsolutePath().toString()); + } catch (IOException e) { + log.error("Failed to extract uploaded ZIPs", e); + return ResponseEntity.internalServerError().body("Extraction failed: " + e.getMessage()); + } + } + + @PostMapping("/process-uploaded-data") + @ProtectedRoute + public void importAllData(@RequestBody String path) { + geoIPInformationService.importAllData(Paths.get(path)); + } } diff --git a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java index cc4a776..aa9a7e4 100644 --- a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java +++ b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java @@ -4,11 +4,13 @@ import io.paradaux.api.models.geoip.IPLocation; import java.io.IOException; +import java.nio.file.Path; import java.util.Map; public interface GeoIPInformationService { void importAllData() throws IOException; + void importAllData(Path dataDir); Map getIPDetails(String ipAddress); CityBlock getCityBlock(String ipAddress); Map getASNDetails(String ipAddress); diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java index 8f80a0b..989c74a 100644 --- a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -74,9 +74,6 @@ public IPLocation getLocationByGeoNameId(Integer geonameId) { } public void importAllData() throws IOException { - // Clear existing data - geoIPMapper.truncateAll(); - // Download zips from MaxMind Path dataDir = downloadAllData(); @@ -85,6 +82,8 @@ public void importAllData() throws IOException { } public void importAllData(Path dataDir) { + geoIPMapper.truncateAll(); + try { // Import locations first - use Path instead of String importLocations(dataDir.resolve("city/GeoLite2-City-Locations-en.csv")); diff --git a/src/main/java/io/paradaux/api/utils/FileUtils.java b/src/main/java/io/paradaux/api/utils/FileUtils.java index 1ee1e41..ac8849b 100644 --- a/src/main/java/io/paradaux/api/utils/FileUtils.java +++ b/src/main/java/io/paradaux/api/utils/FileUtils.java @@ -4,6 +4,7 @@ import okhttp3.Request; import okhttp3.Response; import okhttp3.ResponseBody; +import org.springframework.util.StreamUtils; import java.io.*; import java.nio.file.Files; @@ -118,4 +119,66 @@ public static void downloadAndExtractZip(String url, Path extractDir) throws IOE } } } + + public static void extractZip(InputStream inputStream, Path extractDir) throws IOException { + List entries = new ArrayList<>(); + String commonRoot = null; + + try (BufferedInputStream bis = new BufferedInputStream(inputStream); + ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + StreamUtils.copy(bis, baos); + byte[] zipBytes = baos.toByteArray(); + + try (ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zipStream.getNextEntry()) != null) { + entries.add(new ZipEntry(entry)); + zipStream.closeEntry(); + } + } + + if (!entries.isEmpty()) { + String firstName = entries.get(0).getName(); + int slash = firstName.indexOf('/'); + if (slash > 0) { + String potentialRoot = firstName.substring(0, slash + 1); + boolean allShareRoot = entries.stream() + .allMatch(e -> e.getName().startsWith(potentialRoot)); + if (allShareRoot) commonRoot = potentialRoot; + } + } + + try (ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zipStream.getNextEntry()) != null) { + String name = entry.getName(); + if (commonRoot != null && name.startsWith(commonRoot)) { + name = name.substring(commonRoot.length()); + } + + if (name.isEmpty()) { + zipStream.closeEntry(); + continue; + } + + Path targetPath = extractDir.resolve(name); + if (entry.isDirectory()) { + Files.createDirectories(targetPath); + } else { + Files.createDirectories(targetPath.getParent()); + try (OutputStream os = Files.newOutputStream(targetPath); + BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE)) { + byte[] buffer = new byte[BUFFER_SIZE]; + int len; + while ((len = zipStream.read(buffer)) > 0) { + bos.write(buffer, 0, len); + } + } + } + + zipStream.closeEntry(); + } + } + } + } } From f5e8c09f2be2a1401db3295c3d233adf1daf3281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=ADan=20Errity?= Date: Sun, 20 Jul 2025 23:26:12 +0100 Subject: [PATCH 17/17] Change over to using proper download API --- .../api/controllers/GeoIPController.java | 30 ------------------- .../impl/GeoIPInformationServiceImpl.java | 13 ++++---- .../java/io/paradaux/api/utils/FileUtils.java | 8 ++++- src/main/resources/application-dev.yml | 1 + src/main/resources/application-local.yml | 1 + src/main/resources/application-prod.yml | 1 + 6 files changed, 18 insertions(+), 36 deletions(-) diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java index a63cf12..4b0d5ce 100644 --- a/src/main/java/io/paradaux/api/controllers/GeoIPController.java +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -39,34 +39,4 @@ public ResponseEntity syncGeoIPData() { maxMindSyncJob.runSync(); return ResponseEntity.accepted().body("MaxMind sync started"); } - - @PostMapping("/upload-zips") - @ProtectedRoute - public ResponseEntity uploadZips( - @RequestParam("city") MultipartFile cityZip, - @RequestParam("asn") MultipartFile asnZip) { - - try { - Path dataDir = Files.createTempDirectory("geoip-"); - Path cityDir = dataDir.resolve("city"); - Path asnDir = dataDir.resolve("asn"); - - Files.createDirectories(cityDir); - Files.createDirectories(asnDir); - - extractZip(cityZip.getInputStream(), cityDir); - extractZip(asnZip.getInputStream(), asnDir); - - return ResponseEntity.ok(dataDir.toAbsolutePath().toString()); - } catch (IOException e) { - log.error("Failed to extract uploaded ZIPs", e); - return ResponseEntity.internalServerError().body("Extraction failed: " + e.getMessage()); - } - } - - @PostMapping("/process-uploaded-data") - @ProtectedRoute - public void importAllData(@RequestBody String path) { - geoIPInformationService.importAllData(Paths.get(path)); - } } diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java index 989c74a..7641ad3 100644 --- a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -37,7 +37,10 @@ public class GeoIPInformationServiceImpl implements GeoIPInformationService { @Value("${maxmind.license-key}") private String maxMindLicenseKey; - private static final String MAXMIND_DOWNLOAD_URL = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-%s-CSV&license_key=%s&suffix=zip"; + @Value("${maxmind.user-id}") + private String maxMindUserId; + + private static final String MAXMIND_DOWNLOAD_URL = "https://download.maxmind.com/geoip/databases/GeoLite2-%s-CSV/download?suffix=zip"; @Override public Map lookupIP(String ipAddress) { @@ -125,10 +128,10 @@ private Path downloadAllData() throws IOException { Path cityDir = dataDir.resolve("city"); Path asnDir = dataDir.resolve("asn"); - String url = String.format(MAXMIND_DOWNLOAD_URL, "City", maxMindLicenseKey); - FileUtils.downloadAndExtractZip(url, cityDir); - url = String.format(MAXMIND_DOWNLOAD_URL, "ASN", maxMindLicenseKey); - FileUtils.downloadAndExtractZip(url, asnDir); + String url = String.format(MAXMIND_DOWNLOAD_URL, "City"); + FileUtils.downloadAndExtractZip(url, cityDir, maxMindUserId, maxMindLicenseKey); + url = String.format(MAXMIND_DOWNLOAD_URL, "ASN"); + FileUtils.downloadAndExtractZip(url, asnDir, maxMindUserId, maxMindLicenseKey); return dataDir; } diff --git a/src/main/java/io/paradaux/api/utils/FileUtils.java b/src/main/java/io/paradaux/api/utils/FileUtils.java index ac8849b..4c71715 100644 --- a/src/main/java/io/paradaux/api/utils/FileUtils.java +++ b/src/main/java/io/paradaux/api/utils/FileUtils.java @@ -1,5 +1,6 @@ package io.paradaux.api.utils; +import lombok.extern.slf4j.Slf4j; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -10,10 +11,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +@Slf4j public class FileUtils { private static final int BUFFER_SIZE = 8192; // 8KB buffer private static final OkHttpClient client = new OkHttpClient.Builder() @@ -29,9 +32,12 @@ public class FileUtils { * @param extractDir The directory path where files should be extracted * @throws IOException if download or extraction fails */ - public static void downloadAndExtractZip(String url, Path extractDir) throws IOException { + public static void downloadAndExtractZip(String url, Path extractDir, String username, String password) throws IOException { + String basicAuth = "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()); + log.info("Downloading ZIP from URL: {}", url); Request request = new Request.Builder() .url(url) + .header("Authorization", basicAuth) .build(); try (Response response = client.newCall(request).execute()) { diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 73f866e..f36c514 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -15,4 +15,5 @@ api: secret: ${API_SECRET} maxmind: + user-id: ${MAXMIND_USER_ID} license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index e1384fc..1301674 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -15,4 +15,5 @@ api: secret: test maxmind: + user-id: ${MAXMIND_USER_ID} license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index eed0c68..3b0232b 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -14,4 +14,5 @@ api: secret: ${API_SECRET} maxmind: + user-id: ${MAXMIND_USER_ID} license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file