cloudflare: move HTTP request debugging to httputil

Within the library, we provide a way to output the HTTP interactions
that the library makes. We previously used a custom approach and format
the output. Even though it works, there are better approaches using
`httputil.DumpRequestOut` and `httputil.DumpResponseOut` for dumping the
HTTP interactions.

Alongside this work, we're adding support for redacting sensitive values
in the HTTP interactions. This is useful for both this library and the
consumers of this library (like the Terraform Provider) to prevent
leaking sensitive information in logs.

Closes cloudflare#1143

.changelog/1164.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+cloudflare: automatically redact sensitive values from HTTP interactions
+```

cloudflare.go

@@ -11,7 +11,9 @@ import (
 	"log"
 	"math"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -250,20 +252,6 @@ func (api *API) makeRequestWithAuthTypeAndHeadersComplete(ctx context.Context, m
 			return nil, fmt.Errorf("error caused by request rate limiting: %w", err)
 		}
 
-		if api.Debug {
-			if method == http.MethodPost || method == http.MethodPut || method == http.MethodPatch {
-				buf := &bytes.Buffer{}
-				tee := io.TeeReader(reqBody, buf)
-				debugBody, _ := io.ReadAll(tee)
-				payloadBody, _ := io.ReadAll(buf)
-				fmt.Printf("cloudflare-go [DEBUG] REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, api.BaseURL+uri, headers, string(debugBody))
-				// ensure we recreate the io.Reader for use
-				reqBody = bytes.NewReader(payloadBody)
-			} else {
-				fmt.Printf("cloudflare-go [DEBUG] REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, api.BaseURL+uri, headers, nil)
-			}
-		}
-
 		resp, respErr = api.request(ctx, method, uri, reqBody, authType, headers)
 
 		// short circuit processing on context timeouts
@@ -307,10 +295,6 @@ func (api *API) makeRequestWithAuthTypeAndHeadersComplete(ctx context.Context, m
 		return nil, respErr
 	}
 
-	if api.Debug {
-		fmt.Printf("cloudflare-go [DEBUG] RESPONSE StatusCode:%d RayID:%s ContentType:%s Body:%#v\n", resp.StatusCode, resp.Header.Get("cf-ray"), resp.Header.Get("content-type"), string(respBody))
-	}
-
 	if resp.StatusCode >= http.StatusBadRequest {
 		if strings.HasSuffix(resp.Request.URL.Path, "/filters/validate-expr") {
 			return nil, fmt.Errorf("%s", respBody)
@@ -379,6 +363,9 @@ func (api *API) makeRequestWithAuthTypeAndHeadersComplete(ctx context.Context, m
 // *http.Response, or an error if one occurred. The caller is responsible for
 // closing the response body.
 func (api *API) request(ctx context.Context, method, uri string, reqBody io.Reader, authType int, headers http.Header) (*http.Response, error) {
+	log.SetPrefix(time.Now().Format(time.RFC3339Nano) + " [DEBUG] cloudflare")
+	log.SetFlags(0)
+
 	req, err := http.NewRequestWithContext(ctx, method, api.BaseURL+uri, reqBody)
 	if err != nil {
 		return nil, fmt.Errorf("HTTP request creation failed: %w", err)
@@ -408,11 +395,36 @@ func (api *API) request(ctx context.Context, method, uri string, reqBody io.Read
 		req.Header.Set("Content-Type", "application/json")
 	}
 
+	if api.Debug {
+		dump, err := httputil.DumpRequestOut(req, true)
+		if err != nil {
+			return nil, err
+		}
+
+		// Strip out any sensitive information from the request payload.
+		sensitiveKeys := []string{api.APIKey, api.APIEmail, api.APIToken, api.APIUserServiceKey}
+		for _, key := range sensitiveKeys {
+			if key != "" {
+				valueRegex := regexp.MustCompile(fmt.Sprintf("(?m)%s", key))
+				dump = valueRegex.ReplaceAll(dump, []byte("[redacted]"))
+			}
+		}
+		log.Printf("\n%s", string(dump))
+	}
+
 	resp, err := api.httpClient.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("HTTP request failed: %w", err)
 	}
 
+	if api.Debug {
+		dump, err := httputil.DumpResponse(resp, true)
+		if err != nil {
+			return resp, err
+		}
+		log.Printf("\n%s", string(dump))
+	}
+
 	return resp, nil
 }
 

cloudflare_experimental.go

@@ -6,8 +6,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -158,6 +161,9 @@ func NewExperimental(config *ClientParams) (*Client, error) {
 // *http.Response, or an error if one occurred. The caller is responsible for
 // closing the response body.
 func (c *Client) request(ctx context.Context, method, uri string, reqBody io.Reader, headers http.Header) (*http.Response, error) {
+	log.SetPrefix(time.Now().Format(time.RFC3339Nano) + " [DEBUG] cloudflare")
+	log.SetFlags(0)
+
 	req, err := http.NewRequestWithContext(ctx, method, c.BaseURL.String()+uri, reqBody)
 	if err != nil {
 		return nil, fmt.Errorf("HTTP request creation failed: %w", err)
@@ -193,11 +199,36 @@ func (c *Client) request(ctx context.Context, method, uri string, reqBody io.Rea
 		req.Header.Set("Content-Type", "application/json")
 	}
 
+	if c.Debug {
+		dump, err := httputil.DumpRequestOut(req, true)
+		if err != nil {
+			return nil, err
+		}
+
+		// Strip out any sensitive information from the request payload.
+		sensitiveKeys := []string{c.Key, c.Email, c.Token, c.UserServiceKey}
+		for _, key := range sensitiveKeys {
+			if key != "" {
+				valueRegex := regexp.MustCompile(fmt.Sprintf("(?m)%s", key))
+				dump = valueRegex.ReplaceAll(dump, []byte("[redacted]"))
+			}
+		}
+		log.Printf("\n%s", string(dump))
+	}
+
 	resp, err := c.HTTPClient.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("HTTP request failed: %w", err)
 	}
 
+	if c.Debug {
+		dump, err := httputil.DumpResponse(resp, true)
+		if err != nil {
+			return resp, err
+		}
+		log.Printf("\n%s", string(dump))
+	}
+
 	return resp, nil
 }
 
@@ -224,25 +255,11 @@ func (c *Client) makeRequest(ctx context.Context, method, uri string, params int
 	var respErr error
 	var respBody []byte
 
-	if method == http.MethodPost || method == http.MethodPut || method == http.MethodPatch {
-		buf := &bytes.Buffer{}
-		tee := io.TeeReader(reqBody, buf)
-		debugBody, _ := io.ReadAll(tee)
-		payloadBody, _ := io.ReadAll(buf)
-		c.Logger.Debugf("REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, c.BaseURL.String()+uri, headers, string(debugBody))
-		// ensure we recreate the io.Reader for use
-		reqBody = bytes.NewReader(payloadBody)
-	} else {
-		c.Logger.Debugf("REQUEST Method:%v URI:%s Headers:%#v Body:%v\n", method, c.BaseURL.String()+uri, headers, nil) //)
-	}
-
 	resp, respErr = c.request(ctx, method, uri, reqBody, headers)
 	if respErr != nil {
 		return nil, respErr
 	}
 
-	c.Logger.Debugf("RESPONSE URI:%s StatusCode:%d Body:%#v RayID:%s\n", c.BaseURL.String()+uri, resp.StatusCode, string(respBody), resp.Header.Get("cf-ray"))
-
 	respBody, err = io.ReadAll(resp.Body)
 	resp.Body.Close()
 	if err != nil {