diff --git a/build.gradle.kts b/build.gradle.kts index 61ae7b8..fc29cd4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,8 +30,17 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-jdbc") implementation("org.mybatis.spring.boot:mybatis-spring-boot-starter:3.0.3") + implementation("org.springframework.boot:spring-boot-starter-data-redis") implementation("org.postgresql:postgresql:42.7.3") + implementation("com.maxmind.geoip2:geoip2:4.2.0") + implementation("com.opencsv:opencsv:5.8") + + // Utilities + implementation("org.apache.commons:commons-compress:1.24.0") + implementation("org.apache.commons:commons-lang3:3.13.0") + implementation("com.squareup.okhttp3:okhttp:4.12.0") + // Caching implementation("org.springframework.boot:spring-boot-starter-cache") implementation("com.github.ben-manes.caffeine:caffeine") @@ -43,14 +52,16 @@ dependencies { compileOnly("org.projectlombok:lombok") annotationProcessor("org.projectlombok:lombok") - - // Test testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } +configurations.all { + exclude(group = "commons-logging", module = "commons-logging") +} + tasks.withType { useJUnitPlatform() } diff --git a/src/main/java/io/paradaux/api/ParadauxApiApplication.java b/src/main/java/io/paradaux/api/ParadauxApiApplication.java index ca3e724..fa7f192 100644 --- a/src/main/java/io/paradaux/api/ParadauxApiApplication.java +++ b/src/main/java/io/paradaux/api/ParadauxApiApplication.java @@ -3,11 +3,13 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cache.annotation.EnableCaching; +import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication @EnableScheduling @EnableCaching +@EnableAsync public class ParadauxApiApplication { public static void main(String[] args) { diff --git a/src/main/java/io/paradaux/api/config/RedisConfig.java b/src/main/java/io/paradaux/api/config/RedisConfig.java new file mode 100644 index 0000000..67da12b --- /dev/null +++ b/src/main/java/io/paradaux/api/config/RedisConfig.java @@ -0,0 +1,16 @@ +package io.paradaux.api.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; + +@Configuration +public class RedisConfig { + @Bean + public RedisTemplate redisTemplate(RedisConnectionFactory factory) { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(factory); + return template; + } +} diff --git a/src/main/java/io/paradaux/api/config/WebConfig.java b/src/main/java/io/paradaux/api/config/WebConfig.java index 55ac869..881fcd2 100644 --- a/src/main/java/io/paradaux/api/config/WebConfig.java +++ b/src/main/java/io/paradaux/api/config/WebConfig.java @@ -1,19 +1,22 @@ package io.paradaux.api.config; import io.paradaux.api.interceptors.ProtectedRouteInterceptor; -import org.springframework.beans.factory.annotation.Autowired; +import io.paradaux.api.interceptors.RateLimitingInterceptor; +import lombok.RequiredArgsConstructor; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration +@RequiredArgsConstructor public class WebConfig implements WebMvcConfigurer { - @Autowired - private ProtectedRouteInterceptor protectedRouteInterceptor; + private final ProtectedRouteInterceptor protectedRouteInterceptor; + private final RateLimitingInterceptor rateLimitingInterceptor; @Override public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(protectedRouteInterceptor); + registry.addInterceptor(rateLimitingInterceptor).addPathPatterns("/**"); } } diff --git a/src/main/java/io/paradaux/api/controllers/ContactController.java b/src/main/java/io/paradaux/api/controllers/ContactController.java index e41145e..a0d05ef 100644 --- a/src/main/java/io/paradaux/api/controllers/ContactController.java +++ b/src/main/java/io/paradaux/api/controllers/ContactController.java @@ -2,7 +2,7 @@ import io.paradaux.api.models.ContactFormRequest; import io.paradaux.api.services.ContactService; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -12,10 +12,10 @@ @RestController @RequestMapping("/api/contact") +@RequiredArgsConstructor public class ContactController { - @Autowired - private ContactService contactService; + private final ContactService contactService; @PostMapping public Mono> handleContactForm(@RequestBody ContactFormRequest request) { diff --git a/src/main/java/io/paradaux/api/controllers/GeoIPController.java b/src/main/java/io/paradaux/api/controllers/GeoIPController.java new file mode 100644 index 0000000..4b0d5ce --- /dev/null +++ b/src/main/java/io/paradaux/api/controllers/GeoIPController.java @@ -0,0 +1,42 @@ +package io.paradaux.api.controllers; + +import io.paradaux.api.jobs.MaxMindSyncJob; +import io.paradaux.api.models.annotations.ProtectedRoute; +import io.paradaux.api.services.GeoIPInformationService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.ResponseEntity; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +import static io.paradaux.api.utils.FileUtils.extractZip; + +@RestController +@RequestMapping("/api/geoip") +@Validated +@Slf4j +@RequiredArgsConstructor +public class GeoIPController { + + private final GeoIPInformationService geoIPInformationService; + private final MaxMindSyncJob maxMindSyncJob; + + @GetMapping("/lookup/{ipAddress}") + public Map lookup(@PathVariable String ipAddress) { + return geoIPInformationService.lookupIP(ipAddress); + } + + @PostMapping("/sync") + @ProtectedRoute + public ResponseEntity syncGeoIPData() { + maxMindSyncJob.runSync(); + return ResponseEntity.accepted().body("MaxMind sync started"); + } +} diff --git a/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java b/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java index 02bd441..4296457 100644 --- a/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java +++ b/src/main/java/io/paradaux/api/interceptors/ProtectedRouteInterceptor.java @@ -3,36 +3,56 @@ import io.paradaux.api.models.annotations.ProtectedRoute; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.method.HandlerMethod; import org.springframework.web.servlet.HandlerInterceptor; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + @Component public class ProtectedRouteInterceptor implements HandlerInterceptor { + private final ConcurrentMap protectedRouteCache = new ConcurrentHashMap<>(); + @Value("${api.secret}") private String secret; + /** + * Intercepts requests to check if they are protected routes. + * If the route is protected, it checks for a valid secret token in the request header "X-SECRET". + * If the secret is invalid or missing, it responds with HTTP 401 Unauthorized. + */ @Override - public boolean preHandle(HttpServletRequest request, - HttpServletResponse response, - Object handler) throws Exception { - if (handler instanceof HandlerMethod) { - HandlerMethod method = (HandlerMethod) handler; - - boolean methodProtected = method.hasMethodAnnotation(ProtectedRoute.class); - boolean classProtected = method.getBeanType().isAnnotationPresent(ProtectedRoute.class); - - if (methodProtected || classProtected) { - String secretHeader = request.getHeader("X-SECRET"); - if (secretHeader == null || !secretHeader.equals(secret)) { - response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); - response.getWriter().write("Unauthorized: invalid or missing secret"); - return false; - } + public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) throws Exception { + if (handler instanceof HandlerMethod method) { + if (!isProtectedRoute(method)) { + return true; + } + if (!hasValidSecret(request)) { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.getWriter().write("Unauthorized: invalid or missing secret"); + return false; } } return true; } + + /** + * A route is protected if the method or its class is annotated with @ProtectedRoute. + * This requires a secret token in the request header "X-SECRET". + */ + private boolean isProtectedRoute(HandlerMethod method) { + return protectedRouteCache.computeIfAbsent(method, hm -> hm.hasMethodAnnotation(ProtectedRoute.class) || hm.getBeanType().isAnnotationPresent(ProtectedRoute.class)); + } + + /** + * Checks if the request has a valid secret token. + */ + public boolean hasValidSecret(HttpServletRequest request) { + String secretHeader = request.getHeader("X-SECRET"); + return secretHeader != null && secretHeader.equals(secret); + } } diff --git a/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java b/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java new file mode 100644 index 0000000..395e8c9 --- /dev/null +++ b/src/main/java/io/paradaux/api/interceptors/RateLimitingInterceptor.java @@ -0,0 +1,39 @@ +package io.paradaux.api.interceptors; + +import io.paradaux.api.services.RateLimitingService; +import io.paradaux.api.utils.IPUtils; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.HandlerInterceptor; + +import java.io.IOException; + +@Component +@RequiredArgsConstructor +public class RateLimitingInterceptor implements HandlerInterceptor { + + private final RateLimitingService rateLimiterService; + private final ProtectedRouteInterceptor protectedRouteInterceptor; + + @Override + public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) throws IOException { + String ip = IPUtils.getClientIp(request); + + // Allow requests that contain a valid secret token + if (protectedRouteInterceptor.hasValidSecret(request)) { + return true; + } + + if (!rateLimiterService.isAllowed(ip)) { + response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + response.getWriter().write("Rate limit exceeded"); + return false; + } + + return true; + } +} diff --git a/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java new file mode 100644 index 0000000..0111979 --- /dev/null +++ b/src/main/java/io/paradaux/api/jobs/MaxMindSyncJob.java @@ -0,0 +1,43 @@ +package io.paradaux.api.jobs; + +import io.paradaux.api.services.DiscordService; +import io.paradaux.api.services.GeoIPInformationService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.HashMap; + +@Service +@Slf4j +@RequiredArgsConstructor +public class MaxMindSyncJob { + + private final GeoIPInformationService geoIPInformationService; + private final DiscordService discordService; + + @Scheduled(cron = "0 0 3 * * WED") + public void refreshMaxMindDbScheduled() { + runSync(); + } + + @Async + public void runSync() { + log.info("Starting MaxMind GeoIP data synchronization..."); + discordService.sendMessage("Starting MaxMind GeoIP data synchronization...", "", new HashMap<>()); + + try { + geoIPInformationService.importAllData(); + } catch (IOException e) { + log.error("Failed to import MaxMind GeoIP data", e); + discordService.sendMessage("MaxMind GeoIP data synchronization failure", e.getMessage(), new HashMap<>()); + return; + } + + discordService.sendMessage("MaxMind GeoIP data synchronization completed successfully", "", new HashMap<>()); + log.info("MaxMind GeoIP data synchronization completed successfully."); + } +} diff --git a/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java b/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java index a92c0ee..9910c2a 100644 --- a/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java +++ b/src/main/java/io/paradaux/api/jobs/VisitCacheRefreshJob.java @@ -1,23 +1,23 @@ package io.paradaux.api.jobs; import io.paradaux.api.mappers.VisitsMapper; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @Service @Slf4j +@RequiredArgsConstructor public class VisitCacheRefreshJob { private final VisitsMapper visitsMapper; - public VisitCacheRefreshJob(VisitsMapper visitsMapper) { - this.visitsMapper = visitsMapper; - } - @Scheduled(fixedRate = 5 * 60 * 1000) // 5 mins public void refreshMaterializedView() { + log.info("Starting visit cache refresh..."); visitsMapper.refreshVisitCache(); log.info("Visit cache refreshed successfully."); + } } diff --git a/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java new file mode 100644 index 0000000..6eda650 --- /dev/null +++ b/src/main/java/io/paradaux/api/mappers/GeoIPMapper.java @@ -0,0 +1,24 @@ +package io.paradaux.api.mappers; + +import io.paradaux.api.models.geoip.ASN; +import io.paradaux.api.models.geoip.ASNBlock; +import io.paradaux.api.models.geoip.CityBlock; +import io.paradaux.api.models.geoip.IPLocation; +import org.apache.ibatis.annotations.Mapper; +import java.util.List; +import java.util.Map; + +@Mapper +public interface GeoIPMapper { + void insertLocations(List locations); + void insertASNs(List asns); + void insertCityBlocks(List cityBlocks); + void insertASNBlocks(List asnBlocks); + + Map getIPInfo(String ipAddress); + CityBlock getCityBlockByIP(String ipAddress); + Map getASNByIP(String ipAddress); + IPLocation getLocationById(Integer geonameId); + + void truncateAll(); +} \ No newline at end of file diff --git a/src/main/java/io/paradaux/api/mappers/typehandlers/UUIDTypeHandler.java b/src/main/java/io/paradaux/api/mappers/typehandlers/UUIDTypeHandler.java new file mode 100644 index 0000000..758bc41 --- /dev/null +++ b/src/main/java/io/paradaux/api/mappers/typehandlers/UUIDTypeHandler.java @@ -0,0 +1,31 @@ +package io.paradaux.api.mappers.typehandlers; + +import org.apache.ibatis.type.BaseTypeHandler; +import org.apache.ibatis.type.JdbcType; + +import java.sql.*; +import java.util.UUID; + +public class UUIDTypeHandler extends BaseTypeHandler { + + @Override + public void setNonNullParameter(PreparedStatement ps, int i, UUID parameter, JdbcType jdbcType) + throws SQLException { + ps.setObject(i, parameter, Types.OTHER); + } + + @Override + public UUID getNullableResult(ResultSet rs, String columnName) throws SQLException { + return (UUID) rs.getObject(columnName); + } + + @Override + public UUID getNullableResult(ResultSet rs, int columnIndex) throws SQLException { + return (UUID) rs.getObject(columnIndex); + } + + @Override + public UUID getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { + return (UUID) cs.getObject(columnIndex); + } +} diff --git a/src/main/java/io/paradaux/api/models/geoip/ASN.java b/src/main/java/io/paradaux/api/models/geoip/ASN.java new file mode 100644 index 0000000..830e61e --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/ASN.java @@ -0,0 +1,9 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class ASN { + private Integer autonomousSystemNumber; + private String autonomousSystemOrganization; +} diff --git a/src/main/java/io/paradaux/api/models/geoip/ASNBlock.java b/src/main/java/io/paradaux/api/models/geoip/ASNBlock.java new file mode 100644 index 0000000..3654898 --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/ASNBlock.java @@ -0,0 +1,9 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class ASNBlock { + private String network; + private Integer autonomousSystemNumber; +} diff --git a/src/main/java/io/paradaux/api/models/geoip/CityBlock.java b/src/main/java/io/paradaux/api/models/geoip/CityBlock.java new file mode 100644 index 0000000..15e39ab --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/CityBlock.java @@ -0,0 +1,18 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class CityBlock { + private String network; + private Integer geonameId; + private Integer registeredCountryGeonameId; + private Integer representedCountryGeonameId; + private Boolean isAnonymousProxy; + private Boolean isSatelliteProvider; + private String postalCode; + private Double latitude; + private Double longitude; + private Integer accuracyRadius; + private Boolean isAnycast; +} diff --git a/src/main/java/io/paradaux/api/models/geoip/IPLocation.java b/src/main/java/io/paradaux/api/models/geoip/IPLocation.java new file mode 100644 index 0000000..8ecd8e0 --- /dev/null +++ b/src/main/java/io/paradaux/api/models/geoip/IPLocation.java @@ -0,0 +1,21 @@ +package io.paradaux.api.models.geoip; + +import lombok.Data; + +@Data +public class IPLocation { + private Integer geonameId; + private String localeCode; + private String continentCode; + private String continentName; + private String countryIsoCode; + private String countryName; + private String subdivision1IsoCode; + private String subdivision1Name; + private String subdivision2IsoCode; + private String subdivision2Name; + private String cityName; + private String metroCode; + private String timeZone; + private Boolean isInEuropeanUnion; +} diff --git a/src/main/java/io/paradaux/api/services/DiscordService.java b/src/main/java/io/paradaux/api/services/DiscordService.java index 4de476d..a17c797 100644 --- a/src/main/java/io/paradaux/api/services/DiscordService.java +++ b/src/main/java/io/paradaux/api/services/DiscordService.java @@ -3,6 +3,9 @@ import io.paradaux.api.models.ContactFormRequest; import reactor.core.publisher.Mono; +import java.util.Map; + public interface DiscordService { Mono sendContactForm(ContactFormRequest request); + void sendMessage(String title, String description, Map fields); } diff --git a/src/main/java/io/paradaux/api/services/GeoIPInformationService.java b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java new file mode 100644 index 0000000..aa9a7e4 --- /dev/null +++ b/src/main/java/io/paradaux/api/services/GeoIPInformationService.java @@ -0,0 +1,19 @@ +package io.paradaux.api.services; + +import io.paradaux.api.models.geoip.CityBlock; +import io.paradaux.api.models.geoip.IPLocation; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +public interface GeoIPInformationService { + + void importAllData() throws IOException; + void importAllData(Path dataDir); + Map getIPDetails(String ipAddress); + CityBlock getCityBlock(String ipAddress); + Map getASNDetails(String ipAddress); + IPLocation getLocationByGeoNameId(Integer geonameId); + Map lookupIP(String ipAddress); +} diff --git a/src/main/java/io/paradaux/api/services/RateLimitingService.java b/src/main/java/io/paradaux/api/services/RateLimitingService.java new file mode 100644 index 0000000..a56518d --- /dev/null +++ b/src/main/java/io/paradaux/api/services/RateLimitingService.java @@ -0,0 +1,5 @@ +package io.paradaux.api.services; + +public interface RateLimitingService { + boolean isAllowed(String key); +} diff --git a/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java index f37489d..0422009 100644 --- a/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/DiscordServiceImpl.java @@ -2,12 +2,15 @@ import io.paradaux.api.models.ContactFormRequest; import io.paradaux.api.services.DiscordService; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; @Service @@ -48,4 +51,39 @@ public Mono sendContactForm(ContactFormRequest request) { .retrieve() .bodyToMono(Void.class); } + + @Override + public void sendMessage(String title, String description, Map fields) { + Map embed = getEmbed(title, description, fields); + Map body = new HashMap<>(); + body.put("embeds", List.of(embed)); + + String webhookPath = webhookUrl.replace("https://discord.com/api/webhooks", ""); + + webClient.post() + .uri(webhookPath) + .bodyValue(body) + .retrieve() + .toBodilessEntity() + .block(); // blocking call + } + + @NotNull + private static Map getEmbed(String title, String description, Map fields) { + List> embedFields = new ArrayList<>(); + for (Map.Entry entry : fields.entrySet()) { + Map field = new HashMap<>(); + field.put("name", entry.getKey()); + field.put("value", entry.getValue()); + field.put("inline", false); + embedFields.add(field); + } + + Map embed = new HashMap<>(); + embed.put("title", title); + embed.put("description", description); + embed.put("color", 0x3498DB); + embed.put("fields", embedFields); + return embed; + } } \ No newline at end of file diff --git a/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java new file mode 100644 index 0000000..7641ad3 --- /dev/null +++ b/src/main/java/io/paradaux/api/services/impl/GeoIPInformationServiceImpl.java @@ -0,0 +1,292 @@ +package io.paradaux.api.services.impl; + +import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvValidationException; +import io.paradaux.api.mappers.GeoIPMapper; +import io.paradaux.api.models.geoip.ASN; +import io.paradaux.api.models.geoip.ASNBlock; +import io.paradaux.api.models.geoip.CityBlock; +import io.paradaux.api.models.geoip.IPLocation; +import io.paradaux.api.services.GeoIPInformationService; +import io.paradaux.api.utils.FileUtils; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +@Slf4j +@Service +@RequiredArgsConstructor +public class GeoIPInformationServiceImpl implements GeoIPInformationService { + + private static final int BATCH_SIZE = 1000; + + private final GeoIPMapper geoIPMapper; + + @Value("${maxmind.license-key}") + private String maxMindLicenseKey; + + @Value("${maxmind.user-id}") + private String maxMindUserId; + + private static final String MAXMIND_DOWNLOAD_URL = "https://download.maxmind.com/geoip/databases/GeoLite2-%s-CSV/download?suffix=zip"; + + @Override + public Map lookupIP(String ipAddress) { + Map result = new HashMap<>(); + + CityBlock cityBlock = getCityBlock(ipAddress); + result.put("ip", ipAddress); + result.put("details", getIPDetails(ipAddress)); + result.put("asn", getASNDetails(ipAddress)); + result.put("city", cityBlock); + result.put("location", cityBlock != null ? geoIPMapper.getLocationById(cityBlock.getGeonameId()) : null); + result.put("attribution", "This database incorporates GeoNames [https://www.geonames.org] geographical data, which is made available under the Creative Commons Attribution 4.0 License. To view a copy of this license, visit https://creativecommons.org/licenses/by/4.0"); + return result; + } + + @Override + public Map getIPDetails(String ipAddress) { + return geoIPMapper.getIPInfo(ipAddress); + } + + @Override + public CityBlock getCityBlock(String ipAddress) { + return geoIPMapper.getCityBlockByIP(ipAddress); + } + + @Override + public Map getASNDetails(String ipAddress) { + return geoIPMapper.getASNByIP(ipAddress); + } + + @Override + public IPLocation getLocationByGeoNameId(Integer geonameId) { + return geoIPMapper.getLocationById(geonameId); + } + + public void importAllData() throws IOException { + // Download zips from MaxMind + Path dataDir = downloadAllData(); + + // Process and import data from MaxMind + importAllData(dataDir); + } + + public void importAllData(Path dataDir) { + geoIPMapper.truncateAll(); + + try { + // Import locations first - use Path instead of String + importLocations(dataDir.resolve("city/GeoLite2-City-Locations-en.csv")); + + // Import ASNs and ASN blocks in parallel with city blocks + CompletableFuture asnFuture = CompletableFuture.runAsync(() -> { + try { + importASNsAndBlocks( + dataDir.resolve("asn/GeoLite2-ASN-Blocks-IPv4.csv"), + dataDir.resolve("asn/GeoLite2-ASN-Blocks-IPv6.csv") + ); + } catch (Exception e) { + log.error("Failed to import ASN data", e); + } + }); + + CompletableFuture cityFuture = CompletableFuture.runAsync(() -> { + try { + importCityBlocks( + dataDir.resolve("city/GeoLite2-City-Blocks-IPv4.csv"), + dataDir.resolve("city/GeoLite2-City-Blocks-IPv6.csv") + ); + } catch (Exception e) { + log.error("Failed to import city blocks", e); + } + }); + + // Wait for both to complete + CompletableFuture.allOf(asnFuture, cityFuture).join(); + log.info("All GeoIP data imported successfully from: {}", dataDir); + + } catch (Exception e) { + log.error("Failed to import GeoIP data from: {}", dataDir, e); + } + } + + private Path downloadAllData() throws IOException { + Path dataDir = Files.createTempDirectory("geoip-"); + Path cityDir = dataDir.resolve("city"); + Path asnDir = dataDir.resolve("asn"); + + String url = String.format(MAXMIND_DOWNLOAD_URL, "City"); + FileUtils.downloadAndExtractZip(url, cityDir, maxMindUserId, maxMindLicenseKey); + url = String.format(MAXMIND_DOWNLOAD_URL, "ASN"); + FileUtils.downloadAndExtractZip(url, asnDir, maxMindUserId, maxMindLicenseKey); + return dataDir; + } + + // Update method signatures to accept Path instead of String + private void importLocations(Path csvPath) throws IOException, CsvValidationException { + List locations = new ArrayList<>(); + try (CSVReader reader = new CSVReader(new FileReader(csvPath.toFile()))) { + reader.readNext(); // skip header + String[] line; + while ((line = reader.readNext()) != null) { + if (line.length < 14) { + log.warn("Skipping location row with insufficient columns: {}", line.length); + continue; + } + + IPLocation loc = new IPLocation(); + loc.setGeonameId(parseIntOrNull(line[0])); + loc.setLocaleCode(parseStringOrNull(line[1])); + loc.setContinentCode(parseStringOrNull(line[2])); + loc.setContinentName(parseStringOrNull(line[3])); + loc.setCountryIsoCode(parseStringOrNull(line[4])); + loc.setCountryName(parseStringOrNull(line[5])); + loc.setSubdivision1IsoCode(parseStringOrNull(line[6])); + loc.setSubdivision1Name(parseStringOrNull(line[7])); + loc.setSubdivision2IsoCode(parseStringOrNull(line[8])); + loc.setSubdivision2Name(parseStringOrNull(line[9])); + loc.setCityName(parseStringOrNull(line[10])); + loc.setMetroCode(parseStringOrNull(line[11])); + loc.setTimeZone(parseStringOrNull(line[12])); + loc.setIsInEuropeanUnion("1".equals(line[13])); + locations.add(loc); + } + } + log.debug("Imported {} locations", locations.size()); + batchInsert(locations, geoIPMapper::insertLocations); + } + + private void importASNsAndBlocks(Path... csvPaths) throws IOException, CsvValidationException { + Map uniqueAsns = new HashMap<>(); + List asnBlocks = new ArrayList<>(); + + for (Path csvPath : csvPaths) { + try (CSVReader reader = new CSVReader(new FileReader(csvPath.toFile()))) { + reader.readNext(); // skip header + String[] line; + while ((line = reader.readNext()) != null) { + if (line.length < 3) { + log.warn("Skipping ASN row with insufficient columns: {}", line.length); + continue; + } + + String network = parseStringOrNull(line[0]); + Integer asnNumber = parseIntOrNull(line[1]); + String asnOrg = parseStringOrNull(line[2]); + + if (network == null || asnNumber == null) { + continue; + } + + // Store unique ASN + if (!uniqueAsns.containsKey(asnNumber)) { + ASN asn = new ASN(); + asn.setAutonomousSystemNumber(asnNumber); + asn.setAutonomousSystemOrganization(asnOrg); + uniqueAsns.put(asnNumber, asn); + } + + // Store ASN block + ASNBlock asnBlock = new ASNBlock(); + asnBlock.setNetwork(network); + asnBlock.setAutonomousSystemNumber(asnNumber); + asnBlocks.add(asnBlock); + } + } + } + + log.debug("Imported {} unique ASNs and {} ASN blocks", uniqueAsns.size(), asnBlocks.size()); + + // Insert ASNs first (due to foreign key constraint) + batchInsert(new ArrayList<>(uniqueAsns.values()), geoIPMapper::insertASNs); + + // Then insert ASN blocks + batchInsert(asnBlocks, geoIPMapper::insertASNBlocks); + } + + private void importCityBlocks(Path... csvPaths) throws IOException, CsvValidationException { + List cityBlocks = new ArrayList<>(); + + for (Path csvPath : csvPaths) { + try (CSVReader reader = new CSVReader(new FileReader(csvPath.toFile()))) { + reader.readNext(); // skip header + String[] line; + while ((line = reader.readNext()) != null) { + if (line.length < 11) { + log.warn("Skipping city block row with insufficient columns: {}", line.length); + continue; + } + + CityBlock block = new CityBlock(); + block.setNetwork(parseStringOrNull(line[0])); + block.setGeonameId(parseIntOrNull(line[1])); + block.setRegisteredCountryGeonameId(parseIntOrNull(line[2])); + block.setRepresentedCountryGeonameId(parseIntOrNull(line[3])); + block.setIsAnonymousProxy("1".equals(line[4])); + block.setIsSatelliteProvider("1".equals(line[5])); + block.setPostalCode(parseStringOrNull(line[6])); + block.setLatitude(parseDoubleOrNull(line[7])); + block.setLongitude(parseDoubleOrNull(line[8])); + block.setAccuracyRadius(parseIntOrNull(line[9])); + block.setIsAnycast("1".equals(line[10])); + + cityBlocks.add(block); + } + } + } + + log.debug("Imported {} city blocks", cityBlocks.size()); + batchInsert(cityBlocks, geoIPMapper::insertCityBlocks); + } + + private void batchInsert(List list, Consumer> insertFunction) { + for (int i = 0; i < list.size(); i += BATCH_SIZE) { + int end = Math.min(i + BATCH_SIZE, list.size()); + List batch = list.subList(i, end); + try { + insertFunction.accept(batch); + if (i % (BATCH_SIZE * 10) == 0) { + log.debug("Processed {} records", i + batch.size()); + } + } catch (Exception e) { + log.error("Failed to insert batch starting at index {}", i, e); + } + } + } + + private Integer parseIntOrNull(String s) { + if (s == null || s.trim().isEmpty()) return null; + try { + return Integer.parseInt(s.trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private Double parseDoubleOrNull(String s) { + if (s == null || s.trim().isEmpty()) return null; + try { + return Double.parseDouble(s.trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private String parseStringOrNull(String s) { + if (s == null || s.trim().isEmpty()) return null; + return s.trim(); + } +} \ No newline at end of file diff --git a/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java new file mode 100644 index 0000000..84a44aa --- /dev/null +++ b/src/main/java/io/paradaux/api/services/impl/RateLimitingServiceImpl.java @@ -0,0 +1,33 @@ +package io.paradaux.api.services.impl; + +import io.paradaux.api.services.RateLimitingService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.UUID; + +@Service +@RequiredArgsConstructor +@Slf4j +public class RateLimitingServiceImpl implements RateLimitingService { + + private final RedisTemplate redisTemplate; + + private static final long TIME_WINDOW = 60_000; // 1 min + private static final int MAX_REQUESTS = 75; + + @Override + public boolean isAllowed(String key) { + long now = System.currentTimeMillis(); + String redisKey = "rate:" + key; + redisTemplate.opsForZSet().removeRangeByScore(redisKey, 0, now - TIME_WINDOW); + redisTemplate.opsForZSet().add(redisKey, UUID.randomUUID().toString(), now); + redisTemplate.expire(redisKey, Duration.ofMinutes(1)); + Long count = redisTemplate.opsForZSet().zCard(redisKey); + log.info("Rate limiter check for key: {}, count: {}", key, count); + return count != null && count <= MAX_REQUESTS; + } +} diff --git a/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java b/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java index 8755335..c226699 100644 --- a/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java +++ b/src/main/java/io/paradaux/api/services/impl/VisitsServiceImpl.java @@ -22,7 +22,7 @@ public class VisitsServiceImpl implements VisitsService { "bot", "crawler", "spider", "scanner", "curl", "wget", "python", "java", "php", "unknown", "discord", "(compatible;", "zgrab", "scrapy", "censys", "okhttp", "axios", "go-http-client", - "google", "bing", "yahoo", "WhatCMS" + "google", "bing", "yahoo", "WhatCMS", "Palo Alto" )); private final VisitsMapper visitsMapper; diff --git a/src/main/java/io/paradaux/api/utils/FileUtils.java b/src/main/java/io/paradaux/api/utils/FileUtils.java new file mode 100644 index 0000000..4c71715 --- /dev/null +++ b/src/main/java/io/paradaux/api/utils/FileUtils.java @@ -0,0 +1,190 @@ +package io.paradaux.api.utils; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.springframework.util.StreamUtils; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +@Slf4j +public class FileUtils { + private static final int BUFFER_SIZE = 8192; // 8KB buffer + private static final OkHttpClient client = new OkHttpClient.Builder() + .followRedirects(true) + .followSslRedirects(true) + .build(); + + /** + * Downloads a ZIP file from the given URL and extracts it to the specified directory. + * If the ZIP contains a single root folder, its contents are flattened to the extract directory. + * + * @param url The URL to download from + * @param extractDir The directory path where files should be extracted + * @throws IOException if download or extraction fails + */ + public static void downloadAndExtractZip(String url, Path extractDir, String username, String password) throws IOException { + String basicAuth = "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes()); + log.info("Downloading ZIP from URL: {}", url); + Request request = new Request.Builder() + .url(url) + .header("Authorization", basicAuth) + .build(); + + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Download failed: " + response.code() + " " + response.message()); + } + + ResponseBody body = response.body(); + if (body == null) { + throw new IOException("Response body is null"); + } + + // First pass: collect all entries to determine structure + List entries = new ArrayList<>(); + String commonRoot = null; + + try (InputStream inputStream = body.byteStream(); + BufferedInputStream bufferedInput = new BufferedInputStream(inputStream, BUFFER_SIZE); + ZipInputStream zipStream = new ZipInputStream(bufferedInput)) { + + ZipEntry entry; + while ((entry = zipStream.getNextEntry()) != null) { + entries.add(new ZipEntry(entry)); + zipStream.closeEntry(); + } + } + + // Determine if there's a single root folder + if (!entries.isEmpty()) { + String firstEntryName = entries.get(0).getName(); + int firstSlash = firstEntryName.indexOf('/'); + + if (firstSlash > 0) { + String potentialRoot = firstEntryName.substring(0, firstSlash + 1); + boolean allEntriesShareRoot = entries.stream() + .allMatch(e -> e.getName().startsWith(potentialRoot)); + + if (allEntriesShareRoot) { + commonRoot = potentialRoot; + } + } + } + + // Second pass: extract files + try (InputStream inputStream2 = client.newCall(request).execute().body().byteStream(); + BufferedInputStream bufferedInput2 = new BufferedInputStream(inputStream2, BUFFER_SIZE); + ZipInputStream zipStream2 = new ZipInputStream(bufferedInput2)) { + + ZipEntry entry; + while ((entry = zipStream2.getNextEntry()) != null) { + String entryName = entry.getName(); + + // Remove common root if present + if (commonRoot != null && entryName.startsWith(commonRoot)) { + entryName = entryName.substring(commonRoot.length()); + } + + // Skip empty paths after root removal + if (entryName.isEmpty()) { + zipStream2.closeEntry(); + continue; + } + + if (entry.isDirectory()) { + // Create directory + Path dirPath = extractDir.resolve(entryName); + Files.createDirectories(dirPath); + } else { + // Extract file + Path filePath = extractDir.resolve(entryName); + Files.createDirectories(filePath.getParent()); + + try (FileOutputStream fos = new FileOutputStream(filePath.toFile()); + BufferedOutputStream bos = new BufferedOutputStream(fos, BUFFER_SIZE)) { + + byte[] buffer = new byte[BUFFER_SIZE]; + int bytesRead; + while ((bytesRead = zipStream2.read(buffer)) != -1) { + bos.write(buffer, 0, bytesRead); + } + } + } + zipStream2.closeEntry(); + } + } + } + } + + public static void extractZip(InputStream inputStream, Path extractDir) throws IOException { + List entries = new ArrayList<>(); + String commonRoot = null; + + try (BufferedInputStream bis = new BufferedInputStream(inputStream); + ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + StreamUtils.copy(bis, baos); + byte[] zipBytes = baos.toByteArray(); + + try (ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zipStream.getNextEntry()) != null) { + entries.add(new ZipEntry(entry)); + zipStream.closeEntry(); + } + } + + if (!entries.isEmpty()) { + String firstName = entries.get(0).getName(); + int slash = firstName.indexOf('/'); + if (slash > 0) { + String potentialRoot = firstName.substring(0, slash + 1); + boolean allShareRoot = entries.stream() + .allMatch(e -> e.getName().startsWith(potentialRoot)); + if (allShareRoot) commonRoot = potentialRoot; + } + } + + try (ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zipStream.getNextEntry()) != null) { + String name = entry.getName(); + if (commonRoot != null && name.startsWith(commonRoot)) { + name = name.substring(commonRoot.length()); + } + + if (name.isEmpty()) { + zipStream.closeEntry(); + continue; + } + + Path targetPath = extractDir.resolve(name); + if (entry.isDirectory()) { + Files.createDirectories(targetPath); + } else { + Files.createDirectories(targetPath.getParent()); + try (OutputStream os = Files.newOutputStream(targetPath); + BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE)) { + byte[] buffer = new byte[BUFFER_SIZE]; + int len; + while ((len = zipStream.read(buffer)) > 0) { + bos.write(buffer, 0, len); + } + } + } + + zipStream.closeEntry(); + } + } + } + } +} diff --git a/src/main/java/io/paradaux/api/utils/IPUtils.java b/src/main/java/io/paradaux/api/utils/IPUtils.java new file mode 100644 index 0000000..ff80cc6 --- /dev/null +++ b/src/main/java/io/paradaux/api/utils/IPUtils.java @@ -0,0 +1,14 @@ +package io.paradaux.api.utils; + +import jakarta.servlet.http.HttpServletRequest; + +public class IPUtils { + + public static String getClientIp(HttpServletRequest request) { + String xfHeader = request.getHeader("X-Forwarded-For"); + if (xfHeader != null && !xfHeader.isEmpty()) { + return xfHeader.split(",")[0].trim(); // real client IP + } + return request.getRemoteAddr(); // fallback + } +} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 7e1e77b..f36c514 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -4,5 +4,16 @@ spring: username: ${SPRING_DATASOURCE_USERNAME} password: ${SPRING_DATASOURCE_PASSWORD} + data: + redis: + host: redis + port: 6379 + password: ${REDIS_PASSWORD} + + api: - secret: ${API_SECRET} \ No newline at end of file + secret: ${API_SECRET} + +maxmind: + user-id: ${MAXMIND_USER_ID} + license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index a6e8645..1301674 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -5,5 +5,15 @@ spring: password: example driver-class-name: org.postgresql.Driver + data: + redis: + host: ${REDIS_HOST:localhost} + port: ${REDIS_PORT:6379} + password: ${REDIS_PASSWORD:} + api: - secret: test \ No newline at end of file + secret: test + +maxmind: + user-id: ${MAXMIND_USER_ID} + license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 7e1e77b..3b0232b 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -4,5 +4,15 @@ spring: username: ${SPRING_DATASOURCE_USERNAME} password: ${SPRING_DATASOURCE_PASSWORD} + data: + redis: + host: redis + port: 6379 + password: ${REDIS_PASSWORD} + api: - secret: ${API_SECRET} \ No newline at end of file + secret: ${API_SECRET} + +maxmind: + user-id: ${MAXMIND_USER_ID} + license-key: ${MAXMIND_LICENSE_KEY} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index fffb3a9..c2df1b8 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -6,6 +6,16 @@ spring: caffeine: spec: maximumSize=1000,expireAfterWrite=10m + data: + redis: + timeout: 2000ms + lettuce: + pool: + max-active: 8 + max-wait: -1ms + max-idle: 8 + min-idle: 0 + cloudflare: turnstile: secret: ${CLOUDFLARE_TURNSTILE_SECRET} @@ -16,3 +26,7 @@ discord: mybatis: type-aliases-package: io.paradaux.api.models mapper-locations: classpath:mappers/*.xml + type-handlers-package: io.paradaux.api.mappers.typehandlers + +maxmind: + base-url: https://download.maxmind.com/app/geoip_download \ No newline at end of file diff --git a/src/main/resources/db/3.sql b/src/main/resources/db/3.sql new file mode 100644 index 0000000..9a15e14 --- /dev/null +++ b/src/main/resources/db/3.sql @@ -0,0 +1,101 @@ +-- Went to production on 20/7/2025 +CREATE SCHEMA IF NOT EXISTS geoip; + +CREATE TABLE geoip.location +( + geoname_id INTEGER PRIMARY KEY, + locale_code TEXT, + continent_code TEXT, + continent_name TEXT, + country_iso_code TEXT, + country_name TEXT, + subdivision_1_iso_code TEXT, + subdivision_1_name TEXT, + subdivision_2_iso_code TEXT, + subdivision_2_name TEXT, + city_name TEXT, + metro_code TEXT, + time_zone TEXT, + is_in_european_union BOOLEAN +); + +CREATE TABLE geoip.autonomous_system +( + autonomous_system_number INTEGER PRIMARY KEY, + autonomous_system_organization TEXT +); + +-- Separate tables for different data sources since they have different network ranges +CREATE TABLE geoip.city_block +( + network CIDR PRIMARY KEY, + geoname_id INTEGER REFERENCES geoip.location (geoname_id), + registered_country_geoname_id INTEGER, + represented_country_geoname_id INTEGER, + is_anonymous_proxy BOOLEAN, + is_satellite_provider BOOLEAN, + postal_code TEXT, + latitude DOUBLE PRECISION, + longitude DOUBLE PRECISION, + accuracy_radius INTEGER, + is_anycast BOOLEAN +); + +CREATE TABLE geoip.asn_block +( + network CIDR PRIMARY KEY, + autonomous_system_number INTEGER REFERENCES geoip.autonomous_system (autonomous_system_number) +); + +-- Indexes for performance +CREATE INDEX idx_city_block_network ON geoip.city_block USING GIST (network inet_ops); +CREATE INDEX idx_asn_block_network ON geoip.asn_block USING GIST (network inet_ops); +CREATE INDEX idx_city_block_geoname ON geoip.city_block (geoname_id); + +-- Function to get complete IP information +CREATE OR REPLACE FUNCTION geoip.get_ip_info(ip_address INET) + RETURNS TABLE + ( + network CIDR, + geoname_id INTEGER, + city_name TEXT, + country_name TEXT, + latitude DOUBLE PRECISION, + longitude DOUBLE PRECISION, + asn_number INTEGER, + asn_organization TEXT + ) +AS +$$ +BEGIN + RETURN QUERY + WITH city_data AS (SELECT cb.network, + cb.geoname_id, + cb.latitude, + cb.longitude, + l.city_name, + l.country_name + FROM geoip.city_block cb + LEFT JOIN geoip.location l ON cb.geoname_id = l.geoname_id + WHERE cb.network >>= ip_address + ORDER BY masklen(cb.network) DESC + LIMIT 1), + asn_data AS (SELECT ab.autonomous_system_number, aut.autonomous_system_organization + FROM geoip.asn_block ab + LEFT JOIN geoip.autonomous_system aut + ON ab.autonomous_system_number = aut.autonomous_system_number + WHERE ab.network >>= ip_address + ORDER BY masklen(ab.network) DESC + LIMIT 1) + SELECT city_data.network, + city_data.geoname_id, + city_data.city_name, + city_data.country_name, + city_data.latitude, + city_data.longitude, + asn_data.autonomous_system_number, + asn_data.autonomous_system_organization + FROM city_data + FULL OUTER JOIN asn_data ON true; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/src/main/resources/db/runbook_queries.sql b/src/main/resources/db/runbook_queries.sql new file mode 100644 index 0000000..cbdf547 --- /dev/null +++ b/src/main/resources/db/runbook_queries.sql @@ -0,0 +1,57 @@ +SELECT + os, + COUNT(*) AS count, + ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (), 2) AS percentage_share +FROM ( + SELECT + CASE + + WHEN user_agent ~* 'Windows NT 10.0' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.3' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.2' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.1' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 6.0' THEN 'Windows' + WHEN user_agent ~* 'Windows NT 5.1' THEN 'Windows' + WHEN user_agent ~* 'iPhone|iPad' THEN 'iOS' + WHEN user_agent ~* 'Macintosh|Mac OS X' THEN 'macOS' + WHEN user_agent ~* 'Android' THEN 'Android' + WHEN user_agent ~* 'Linux' THEN 'Linux' + ELSE 'Other/Unknown' + END AS os + FROM analytics.visits + ) AS derived +GROUP BY os +ORDER BY count DESC; + +REFRESH MATERIALIZED VIEW CONCURRENTLY analytics.visit_count_view; + +-- Get the IP/User-Agent pairs for the project 'ifuckedur.mom' with their viewer number +SELECT ROW_NUMBER() OVER (ORDER BY id) AS row_number, ip_address, user_agent +FROM analytics.visits +WHERE project = 'ifuckedur.mom'; + +-- Get visitor counts for each project in the last 24 hours +SELECT + project, + COUNT(*) AS total_count +FROM + analytics.visits +WHERE created_at > CURRENT_TIMESTAMP - INTERVAL '1 day' +GROUP BY + project; + +-- Get visitor counts for each project in the last 7 days +SELECT + TO_CHAR(created_at, 'FMDay') AS day_of_week, + COUNT(*) FILTER (WHERE project = 'cans.ie') AS cans_ie, + COUNT(*) FILTER (WHERE project = 'ifuckedur.mom') AS ifuckedur_mom, + COUNT(*) FILTER (WHERE project = 'isbetterthandubl.in') AS isbetterthandublin +FROM + analytics.visits +WHERE + created_at >= CURRENT_DATE - INTERVAL '6 days' +GROUP BY + day_of_week, + EXTRACT(DOW FROM created_at) +ORDER BY + EXTRACT(DOW FROM created_at); \ No newline at end of file diff --git a/src/main/resources/mappers/GeoIPMapper.xml b/src/main/resources/mappers/GeoIPMapper.xml new file mode 100644 index 0000000..a75ddb9 --- /dev/null +++ b/src/main/resources/mappers/GeoIPMapper.xml @@ -0,0 +1,95 @@ + + + + + + + INSERT INTO geoip.location ( + geoname_id, locale_code, continent_code, continent_name, + country_iso_code, country_name, subdivision_1_iso_code, + subdivision_1_name, subdivision_2_iso_code, subdivision_2_name, + city_name, metro_code, time_zone, is_in_european_union + ) VALUES + + ( + #{loc.geonameId}, #{loc.localeCode}, #{loc.continentCode}, #{loc.continentName}, + #{loc.countryIsoCode}, #{loc.countryName}, #{loc.subdivision1IsoCode}, + #{loc.subdivision1Name}, #{loc.subdivision2IsoCode}, #{loc.subdivision2Name}, + #{loc.cityName}, #{loc.metroCode}, #{loc.timeZone}, #{loc.isInEuropeanUnion} + ) + + ON CONFLICT (geoname_id) DO NOTHING + + + + INSERT INTO geoip.autonomous_system ( + autonomous_system_number, autonomous_system_organization + ) VALUES + + ( + #{asn.autonomousSystemNumber}, #{asn.autonomousSystemOrganization} + ) + + ON CONFLICT (autonomous_system_number) DO NOTHING + + + + INSERT INTO geoip.city_block ( + network, geoname_id, registered_country_geoname_id, represented_country_geoname_id, + is_anonymous_proxy, is_satellite_provider, postal_code, + latitude, longitude, accuracy_radius, is_anycast + ) VALUES + + ( + CAST(#{block.network} AS cidr), #{block.geonameId}, #{block.registeredCountryGeonameId}, + #{block.representedCountryGeonameId}, #{block.isAnonymousProxy}, #{block.isSatelliteProvider}, + #{block.postalCode}, #{block.latitude}, #{block.longitude}, #{block.accuracyRadius}, #{block.isAnycast} + ) + + ON CONFLICT (network) DO NOTHING + + + + INSERT INTO geoip.asn_block ( + network, autonomous_system_number + ) VALUES + + ( + CAST(#{asnBlock.network} AS cidr), #{asnBlock.autonomousSystemNumber} + ) + + ON CONFLICT (network) DO NOTHING + + + + + + + + + + + + + TRUNCATE TABLE geoip.location, geoip.autonomous_system, geoip.city_block, geoip.asn_block CASCADE + + \ No newline at end of file diff --git a/test-api.py b/test-api.py index f03a6d8..b6f596b 100644 --- a/test-api.py +++ b/test-api.py @@ -6,7 +6,7 @@ # Config # URL = "https://api.paradaux.io/api/ifum/visits" -URL = "http://127.0.0.1:8080/api/visits/project/ifuckedur.mom" +URL = "http://127.0.0.1:8080/api/analytics/visits/project/ifuckedur.mom" TOTAL_REQUESTS = 1500 MAX_CONCURRENCY = 50