Feat/add snyk client

InsightsBackend/src/main/java/org/frankframework/insights/common/client/ApiClient.java

@@ -0,0 +1,42 @@
+package org.frankframework.insights.common.client;
+
+import java.util.function.Consumer;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.web.reactive.function.client.WebClient;
+
+/**
+ * The abstract base class for all external API clients.
+ * It centralizes the creation and configuration of the Spring WebClient.
+ */
+@Slf4j
+public abstract class ApiClient {
+
+    protected final WebClient webClient;
+
+    /**
+     * Flexible constructor for any WebClient-based client.
+     * @param baseUrl the base URL of the external server (mandatory).
+     * @param configurer a Consumer that configures the WebClient.Builder, e.g., for auth headers.
+     */
+    public ApiClient(String baseUrl, Consumer<WebClient.Builder> configurer) {
+        if (baseUrl == null || baseUrl.trim().isEmpty()) {
+            throw new IllegalArgumentException("Base URL cannot be null or empty.");
+        }
+
+        WebClient.Builder builder = WebClient.builder()
+                .baseUrl(baseUrl)
+                .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
+
+        if (configurer != null) {
+            configurer.accept(builder);
+        }
+
+        this.webClient = builder.build();
+        log.info(
+                "WebClient initialized successfully for {} with base URL: {}",
+                this.getClass().getSimpleName(),
+                baseUrl);
+    }
+}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/ApiClientException.java

@@ -0,0 +1,10 @@
+package org.frankframework.insights.common.client;
+
+import org.frankframework.insights.common.exception.ApiException;
+import org.springframework.http.HttpStatus;
+
+public class ApiClientException extends ApiException {
+    public ApiClientException(String message, HttpStatus status, Throwable cause) {
+        super(message, status, cause);
+    }
+}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/graphql/GraphQLClient.java

@@ -0,0 +1,130 @@
+package org.frankframework.insights.common.client.graphql;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.frankframework.insights.common.client.ApiClient;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.graphql.client.HttpGraphQlClient;
+import org.springframework.web.reactive.function.client.WebClient;
+
+@Slf4j
+public abstract class GraphQLClient extends ApiClient {
+    private final HttpGraphQlClient graphQlClient;
+    private final ObjectMapper objectMapper;
+
+    public GraphQLClient(String baseUrl, Consumer<WebClient.Builder> configurer, ObjectMapper objectMapper) {
+        super(baseUrl, configurer);
+        this.graphQlClient = HttpGraphQlClient.builder(this.webClient).build();
+        this.objectMapper = objectMapper;
+    }
+
+    protected <T> T fetchSingleEntity(GraphQLQuery query, Map<String, Object> queryVariables, Class<T> entityType)
+            throws GraphQLClientException {
+        try {
+            return getGraphQlClient()
+                    .documentName(query.getDocumentName())
+                    .variables(queryVariables)
+                    .retrieve(query.getRetrievePath())
+                    .toEntity(entityType)
+                    .block();
+        } catch (Exception e) {
+            throw new GraphQLClientException("Failed GraphQL request for document: " + query.getDocumentName(), e);
+        }
+    }
+
+    /**
+     * A flexible method to fetch paginated data, allowing the caller to define how collections are extracted.
+     * This can handle various GraphQL response structures, such as those with 'edges' or 'nodes'.
+     *
+     * @param query the GraphQL query constant.
+     * @param queryVariables the variables for the query.
+     * @param entityType the class type of the final entities.
+     * @param responseType the type reference for deserializing the raw GraphQL response.
+     * @param collectionExtractor a function to extract the collection of raw data maps from the response.
+     * @param pageInfoExtractor a function to extract pagination info from the response.
+     * @return a set of all fetched entities across all pages.
+     * @throws GraphQLClientException if the request fails.
+     */
+    protected <RAW, T> Set<T> fetchPaginatedCollection(
+            GraphQLQuery query,
+            Map<String, Object> queryVariables,
+            Class<T> entityType,
+            ParameterizedTypeReference<RAW> responseType,
+            Function<RAW, Collection<Map<String, Object>>> collectionExtractor,
+            Function<RAW, GraphQLPageInfoDTO> pageInfoExtractor)
+            throws GraphQLClientException {
+
+        Function<RAW, Collection<T>> entityExtractor = response -> {
+            Collection<Map<String, Object>> rawNodes = collectionExtractor.apply(response);
+            if (rawNodes == null) {
+                return Set.of();
+            }
+            return rawNodes.stream()
+                    .map(node -> objectMapper.convertValue(node, entityType))
+                    .collect(Collectors.toList());
+        };
+
+        return fetchPaginated(query, queryVariables, responseType, entityExtractor, pageInfoExtractor);
+    }
+
+    private <RAW, T> Set<T> fetchPaginated(
+            GraphQLQuery query,
+            Map<String, Object> queryVariables,
+            ParameterizedTypeReference<RAW> responseType,
+            Function<RAW, Collection<T>> entityExtractor,
+            Function<RAW, GraphQLPageInfoDTO> pageInfoExtractor)
+            throws GraphQLClientException {
+        try {
+            Set<T> allEntities = new HashSet<>();
+            String cursor = null;
+            boolean hasNextPage = true;
+
+            while (hasNextPage) {
+                queryVariables.put("after", cursor);
+                RAW response = getGraphQlClient()
+                        .documentName(query.getDocumentName())
+                        .variables(queryVariables)
+                        .retrieve(query.getRetrievePath())
+                        .toEntity(responseType)
+                        .block();
+
+                if (response == null) {
+                    log.warn("Received null response for query: {}", query);
+                    break;
+                }
+
+                Collection<T> entities = entityExtractor.apply(response);
+                if (entities == null || entities.isEmpty()) {
+                    log.warn("Received empty entities for query: {}", query);
+                    break;
+                }
+
+                allEntities.addAll(entities);
+                log.info("Fetched {} entities with query: {}", entities.size(), query);
+
+                GraphQLPageInfoDTO pageInfo = pageInfoExtractor.apply(response);
+                hasNextPage = pageInfo != null && pageInfo.hasNextPage();
+                cursor = (pageInfo != null) ? pageInfo.endCursor() : null;
+            }
+            log.info(
+                    "Completed paginated fetch for [{}], total entities found: {}",
+                    query.getDocumentName(),
+                    allEntities.size());
+            return allEntities;
+        } catch (Exception e) {
+            throw new GraphQLClientException(
+                    "Failed paginated GraphQL request for document: " + query.getDocumentName(), e);
+        }
+    }
+
+    protected HttpGraphQlClient getGraphQlClient() {
+        return graphQlClient;
+    }
+}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/graphql/GraphQLClientException.java

@@ -0,0 +1,10 @@
+package org.frankframework.insights.common.client.graphql;
+
+import org.frankframework.insights.common.client.ApiClientException;
+import org.springframework.http.HttpStatus;
+
+public class GraphQLClientException extends ApiClientException {
+    public GraphQLClientException(String message, Throwable cause) {
+        super(message, HttpStatus.INTERNAL_SERVER_ERROR, cause);
+    }
+}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/graphql/GraphQLConnectionDTO.java

@@ -0,0 +1,10 @@
+package org.frankframework.insights.common.client.graphql;
+
+import java.util.List;
+
+/**
+ * Represents a generic GraphQL "Connection" for cursor-based pagination.
+ * It holds a list of edges and the pagination information.
+ * @param <T> the type of the entity within the connection's nodes.
+ */
+public record GraphQLConnectionDTO<T>(List<GraphQLNodeDTO<T>> edges, GraphQLPageInfoDTO pageInfo) {}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/graphql/GraphQLNodeDTO.java

@@ -0,0 +1,7 @@
+package org.frankframework.insights.common.client.graphql;
+
+/**
+ * A generic container for a "node" in a GraphQL edge, part of the Relay cursor-based pagination spec.
+ * @param <T> the type of the entity contained within the node.
+ */
+public record GraphQLNodeDTO<T>(T node) {}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/graphql/GraphQLPageInfoDTO.java

@@ -0,0 +1,6 @@
+package org.frankframework.insights.common.client.graphql;
+
+/**
+ * A concrete DTO for GraphQL PageInfo.
+ */
+public record GraphQLPageInfoDTO(boolean hasNextPage, String endCursor) {}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/graphql/GraphQLQuery.java

@@ -0,0 +1,7 @@
+package org.frankframework.insights.common.client.graphql;
+
+public interface GraphQLQuery {
+    String getDocumentName();
+
+    String getRetrievePath();
+}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/rest/RestClient.java

@@ -0,0 +1,48 @@
+package org.frankframework.insights.common.client.rest;
+
+import java.util.function.Consumer;
+import lombok.extern.slf4j.Slf4j;
+import org.frankframework.insights.common.client.ApiClient;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.web.reactive.function.client.WebClient;
+
+@Slf4j
+public abstract class RestClient extends ApiClient {
+    /**
+     * Flexible constructor for RestClient.
+     * @param baseUrl the base URL of the external REST server (mandatory).
+     * @param configurer a Consumer that receives the WebClient.Builder for further configuration, such as adding authentication headers. Can be null.
+     */
+    public RestClient(String baseUrl, Consumer<WebClient.Builder> configurer) {
+        super(baseUrl, configurer);
+    }
+
+    /**
+     * Executes a GET request to a specified path.
+     * @param path The endpoint path to request.
+     * @param responseType The class or type reference of the expected response.
+     * @param <T> The type of the response body.
+     * @return The deserialized response body.
+     * @throws RestClientException if the request fails.
+     */
+    protected <T> T get(String path, ParameterizedTypeReference<T> responseType) throws RestClientException {
+        try {
+            return getRestClient()
+                    .get()
+                    .uri(path)
+                    .retrieve()
+                    .bodyToMono(responseType)
+                    .block();
+        } catch (Exception e) {
+            throw new RestClientException(String.format("Failed GET request to path: %s", path), e);
+        }
+    }
+
+    /**
+     * Provides access to the configured WebClient instance.
+     * @return the WebClient instance.
+     */
+    protected WebClient getRestClient() {
+        return this.webClient;
+    }
+}

InsightsBackend/src/main/java/org/frankframework/insights/common/client/rest/RestClientException.java

@@ -0,0 +1,10 @@
+package org.frankframework.insights.common.client.rest;
+
+import org.frankframework.insights.common.client.ApiClientException;
+import org.springframework.http.HttpStatus;
+
+public class RestClientException extends ApiClientException {
+    public RestClientException(String message, Throwable cause) {
+        super(message, HttpStatus.INTERNAL_SERVER_ERROR, cause);
+    }
+}

InsightsBackend/src/main/java/org/frankframework/insights/common/configuration/SystemDataInitializer.java → InsightsBackend/src/main/java/org/frankframework/insights/common/configuration/GitHubConfiguration.java

@@ -3,7 +3,7 @@
 import lombok.extern.slf4j.Slf4j;
 import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
 import org.frankframework.insights.branch.BranchService;
-import org.frankframework.insights.common.configuration.properties.GitHubProperties;
+import org.frankframework.insights.common.configuration.properties.FetchProperties;
 import org.frankframework.insights.github.GitHubClientException;
 import org.frankframework.insights.github.GitHubRepositoryStatisticsService;
 import org.frankframework.insights.issue.IssueService;
@@ -19,7 +19,7 @@
 
 @Configuration
 @Slf4j
-public class SystemDataInitializer implements CommandLineRunner {
+public class GitHubConfiguration implements CommandLineRunner {
     private final GitHubRepositoryStatisticsService gitHubRepositoryStatisticsService;
     private final LabelService labelService;
     private final MilestoneService milestoneService;
@@ -29,9 +29,9 @@ public class SystemDataInitializer implements CommandLineRunner {
     private final IssueService issueService;
     private final PullRequestService pullRequestService;
     private final ReleaseService releaseService;
-    private final Boolean gitHubFetchEnabled;
+    private final Boolean fetchEnabled;
 
-    public SystemDataInitializer(
+    public GitHubConfiguration(
             GitHubRepositoryStatisticsService gitHubRepositoryStatisticsService,
             LabelService labelService,
             MilestoneService milestoneService,
@@ -41,7 +41,7 @@ public SystemDataInitializer(
             IssueService issueService,
             PullRequestService pullRequestService,
             ReleaseService releaseService,
-            GitHubProperties gitHubProperties) {
+            FetchProperties fetchProperties) {
         this.gitHubRepositoryStatisticsService = gitHubRepositoryStatisticsService;
         this.labelService = labelService;
         this.milestoneService = milestoneService;
@@ -51,7 +51,7 @@ public SystemDataInitializer(
         this.issueService = issueService;
         this.pullRequestService = pullRequestService;
         this.releaseService = releaseService;
-        this.gitHubFetchEnabled = gitHubProperties.getFetch();
+        this.fetchEnabled = fetchProperties.getEnabled();
     }
 
     /**
@@ -63,8 +63,8 @@ public SystemDataInitializer(
     public void run(String... args) {
         log.info("Startup: Fetching GitHub statistics");
         fetchGitHubStatistics();
-        log.info("Startup: Fetching full system data");
-        initializeSystemData();
+        log.info("Startup: Fetching all GitHub data");
+        initializeGitHubData();
     }
 
     /**
@@ -75,7 +75,7 @@ public void run(String... args) {
     public void dailyJob() {
         log.info("Daily fetch job started");
         fetchGitHubStatistics();
-        initializeSystemData();
+        initializeGitHubData();
     }
 
     /**
@@ -84,7 +84,7 @@ public void dailyJob() {
     @SchedulerLock(name = "fetchGitHubStatistics", lockAtMostFor = "PT10M")
     public void fetchGitHubStatistics() {
         try {
-            if (!gitHubFetchEnabled) {
+            if (!fetchEnabled) {
                 log.info("Skipping GitHub fetch: skipping due to build/test configuration.");
                 return;
             }
@@ -96,12 +96,12 @@ public void fetchGitHubStatistics() {
     }
 
     /**
-     * Initializes system data by fetching labels, milestones, branches, issues, pull requests, and releases from GitHub.
+     * Initializes data by fetching labels, milestones, branches, issues, pull requests, and releases from GitHub.
      */
-    @SchedulerLock(name = "initializeSystemData", lockAtMostFor = "PT2H")
-    public void initializeSystemData() {
+    @SchedulerLock(name = "initializeGitHubData", lockAtMostFor = "PT2H")
+    public void initializeGitHubData() {
         try {
-            if (!gitHubFetchEnabled) {
+            if (!fetchEnabled) {
                 log.info("Skipping GitHub fetch: skipping due to build/test configuration.");
                 return;
             }
@@ -118,7 +118,7 @@ public void initializeSystemData() {
 
             log.info("Done fetching all GitHub data");
         } catch (Exception e) {
-            log.error("Error initializing system data", e);
+            log.error("Error initializing GitHub data", e);
         }
     }
 }