HTTP175 Remove headers and responses from logs

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+- Amend to not log HTTP request response and header values by default.
+
 ## [0.22.0] - 2025-10-03
 
 -   Added ability to continue on error, introducing new metadata columns and new configuration option

src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java

@@ -210,11 +210,10 @@ private HttpRowDataWrapper processHttpResponse(
             boolean isError) throws IOException {
 
         this.httpPostRequestCallback.call(response, request, "endpoint", Collections.emptyMap());
-
+        request.getHttpRequest().uri();
         var responseBody = response.body();
 
-        log.debug("Received status code [{}] for RestTableSource request with Server response body [{}] ",
-                response.statusCode(), responseBody);
+        log.debug("Received status code [{}] for RestTableSource request", response.statusCode());
         if (!isError && (StringUtils.isNullOrWhitespaceOnly(responseBody) || ignoreResponse(response))) {
             return HttpRowDataWrapper.builder()
                     .data(Collections.emptyList())

src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java

@@ -4,7 +4,6 @@
 import java.net.http.HttpRequest.Builder;
 import java.util.Arrays;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.flink.annotation.VisibleForTesting;
@@ -60,11 +59,6 @@ public RequestFactoryBase(
             );
 
         this.headersAndValues = HttpHeaderUtils.toHeaderAndValueArray(headerMap);
-
-        log.debug("RequestFactoryBase headersAndValues: " +
-                Arrays.stream(headersAndValues)
-                        .map(Object::toString)
-                        .collect(Collectors.joining(",")));
         this.httpRequestTimeOutSeconds = Integer.parseInt(
             options.getProperties().getProperty(
                 HttpConnectorConfigConstants.LOOKUP_HTTP_TIMEOUT_SECONDS,

src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4JHttpLookupPostRequestCallback.java

@@ -2,15 +2,11 @@
 
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
-import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
-import java.util.StringJoiner;
 
 import lombok.extern.slf4j.Slf4j;
 
 import com.getindata.connectors.http.HttpPostRequestCallback;
-import com.getindata.connectors.http.internal.utils.ConfigUtils;
 
 /**
  * A {@link HttpPostRequestCallback} that logs pairs of request and response as <i>INFO</i> level
@@ -31,40 +27,26 @@ public void call(
             Map<String, String> headerMap) {
 
         HttpRequest httpRequest = requestEntry.getHttpRequest();
-        StringJoiner headers = new StringJoiner(";");
-
-        for (Entry<String, List<String>> reqHeaders : httpRequest.headers().map().entrySet()) {
-            StringJoiner values = new StringJoiner(";");
-            for (String value : reqHeaders.getValue()) {
-                values.add(value);
-            }
-            String header = reqHeaders.getKey() + ": [" + values + "]";
-            headers.add(header);
-        }
 
         if (response == null) {
             log.warn("Null Http response for request " + httpRequest.uri().toString());
 
             log.info(
                 "Got response for a request.\n  Request:\n    URL: {}\n    " +
-                    "Method: {}\n    Headers: {}\n    Params/Body: {}\nResponse: null",
+                    "Method: {}\n Params/Body: {}\nResponse: null",
                 httpRequest.uri().toString(),
                 httpRequest.method(),
-                headers,
                 requestEntry.getLookupQueryInfo()
             );
         } else {
             log.info(
                 "Got response for a request.\n  Request:\n    URL: {}\n    " +
-                    "Method: {}\n    Headers: {}\n    Params/Body: {}\nResponse: {}\n    Body: {}",
+                    "Method: {}\n Params/Body: {}\nResponse status code: {}\n",
                 httpRequest.uri().toString(),
                 httpRequest.method(),
-                headers,
                 requestEntry.getLookupQueryInfo(),
-                response,
-                response.body().replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "")
+                response.statusCode()
             );
         }
-
     }
 }

src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java

@@ -1,19 +1,17 @@
 package com.getindata.connectors.http.internal.table.sink;
 
 import java.net.http.HttpResponse;
-import java.nio.charset.StandardCharsets;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import lombok.extern.slf4j.Slf4j;
 
 import com.getindata.connectors.http.HttpPostRequestCallback;
 import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest;
-import com.getindata.connectors.http.internal.utils.ConfigUtils;
 
 /**
  * A {@link HttpPostRequestCallback} that logs pairs of request and response as <i>INFO</i> level
- * logs using <i>Slf4j</i>.
+ * logs using <i>Slf4j</i>. As the request/response body or header might contain sensitive information,
+ * we do not log those values.
  *
  * <p>Serving as a default implementation of {@link HttpPostRequestCallback} for
  * the {@link HttpDynamicSink}.
@@ -28,25 +26,23 @@ public void call(
         String endpointUrl,
         Map<String, String> headerMap) {
 
-        String requestBody = requestEntry.getElements().stream()
-            .map(element -> new String(element, StandardCharsets.UTF_8))
-            .collect(Collectors.joining());
+        // Uncomment if you want to see the requestBody in the log
+        //String requestBody = requestEntry.getElements().stream()
+        //    .map(element -> new String(element, StandardCharsets.UTF_8))
+        //    .collect(Collectors.joining());
 
         if (response == null) {
             log.info(
                 "Got response for a request.\n  Request:\n    " +
-                "Method: {}\n    Body: {}\n  Response: null",
-                requestEntry.getMethod(),
-                requestBody
+                "Method: {}\n   Response: null",
+                requestEntry.getMethod()
             );
         } else {
             log.info(
                 "Got response for a request.\n  Request:\n    " +
-                "Method: {}\n    Body: {}\n  Response: {}\n    Body: {}",
+                "Method: {}\n  Response status code: {}\n ",
                 requestEntry.method,
-                requestBody,
-                response,
-                response.body().replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "")
+                response.statusCode()
             );
         }
     }

src/test/resources/simpleLogger.properties

@@ -0,0 +1,2 @@
+org.slf4j.simpleLogger.defaultLogLevel=INFO
+org.slf4j.simpleLogger.log.com.getindata.connectors.http.internal.table.lookup.RequestAndResponseLogger=DEBUG