Updated Data Scrubbing

client.go

@@ -240,14 +240,25 @@ func (packet *Packet) JSON() []byte {
 	return packetJSON
 }
 
+type Processor func(packet *Packet) *Packet
+
+type ClientConfig struct {
+	Processors *[]Processor
+	Tags       map[string]string
+}
+
 // The maximum number of packets that will be buffered waiting to be delivered.
 // Packets will be dropped if the buffer is full. Used by NewClient.
 var MaxQueueBuffer = 100
 
 // NewClient constructs a Sentry client and spawns a background goroutine to
 // handle packets sent by Client.Report.
-func NewClient(dsn string, tags map[string]string) (*Client, error) {
-	client := &Client{Transport: &HTTPTransport{}, Tags: tags, queue: make(chan *outgoingPacket, MaxQueueBuffer)}
+func NewClient(dsn string, config *ClientConfig) (*Client, error) {
+	client := &Client{
+		Config:    config,
+		Transport: &HTTPTransport{},
+		queue:     make(chan *outgoingPacket, MaxQueueBuffer),
+	}
 	go client.worker()
 	return client, client.SetDSN(dsn)
 }
@@ -256,7 +267,7 @@ func NewClient(dsn string, tags map[string]string) (*Client, error) {
 // by calling NewClient. Modification of fields concurrently with Send or after
 // calling Report for the first time is not thread-safe.
 type Client struct {
-	Tags map[string]string
+	Config *ClientConfig
 
 	Transport Transport
 
@@ -332,7 +343,7 @@ func (client *Client) Capture(packet *Packet, captureTags map[string]string) (ev
 
 	// Merge capture tags and client tags
 	packet.AddTags(captureTags)
-	packet.AddTags(client.Tags)
+	packet.AddTags(client.Config.Tags)
 
 	// Initialize any required packet fields
 	client.mu.RLock()
@@ -345,7 +356,9 @@ func (client *Client) Capture(packet *Packet, captureTags map[string]string) (ev
 		return
 	}
 
-	outgoingPacket := &outgoingPacket{packet, ch}
+	// Call all processors and sanitizers for a packet
+	scrubbedPacket := client.Scrub(packet)
+	outgoingPacket := &outgoingPacket{scrubbedPacket, ch}
 
 	select {
 	case client.queue <- outgoingPacket:

examples_test.go

@@ -2,8 +2,8 @@ package raven
 
 import (
 	"fmt"
-	"net/http"
 	"log"
+	"net/http"
 )
 
 func Example() {
@@ -13,7 +13,8 @@ func Example() {
 	var sentryDSN string
 	// r is a request performed when error occured
 	var r *http.Request
-	client, err := NewClient(sentryDSN, nil)
+	config := &ClientConfig{nil, nil}
+	client, err := NewClient(sentryDSN, config)
 	if err != nil {
 		log.Fatal(err)
 	}

http.go

@@ -2,7 +2,6 @@ package raven
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 )
 
@@ -13,8 +12,7 @@ func NewHttp(req *http.Request) *Http {
 	}
 	h := &Http{
 		Method:  req.Method,
-		Cookies: req.Header.Get("Cookie"),
-		Query:   sanitizeQuery(req.URL.Query()).Encode(),
+		Query:   scrubQuery(req.URL.Query()).Encode(),
 		URL:     proto + "://" + req.Host + req.URL.Path,
 		Headers: make(map[string]string, len(req.Header)),
 	}
@@ -27,19 +25,6 @@ func NewHttp(req *http.Request) *Http {
 	return h
 }
 
-var querySecretFields = []string{"password", "passphrase", "passwd", "secret"}
-
-func sanitizeQuery(query url.Values) url.Values {
-	for _, keyword := range querySecretFields {
-		for field := range query {
-			if strings.Contains(field, keyword) {
-				query[field] = []string{"********"}
-			}
-		}
-	}
-	return query
-}
-
 // http://sentry.readthedocs.org/en/latest/developer/interfaces/index.html#sentry.interfaces.Http
 type Http struct {
 	// Required
@@ -48,7 +33,6 @@ type Http struct {
 	Query  string `json:"query_string,omitempty"`
 
 	// Optional
-	Cookies string            `json:"cookies,omitempty"`
 	Headers map[string]string `json:"headers,omitempty"`
 	Env     map[string]string `json:"env,omitempty"`
 

processors.go

@@ -0,0 +1,102 @@
+package raven
+
+import (
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+const Mask = "********"
+
+var querySecretKeys = []string{"api_key", "apikey", "authorization", "passwd", "password", "secret"}
+var querySecretValues = []string{`/^(?:\d[ -]*?){13,16}$/`}
+
+// Scrub all data for a packet
+func (client *Client) Scrub(packet *Packet) *Packet {
+
+	packet = defaultProcessor(packet)
+	for _, processor := range *client.Config.Processors {
+		packet = processor(packet)
+	}
+	return packet
+}
+
+// Default processor for a packet
+func defaultProcessor(packet *Packet) *Packet {
+	for _, packetInterface := range packet.Interfaces {
+		switch typedInterface := packetInterface.(type) {
+		case *Http:
+			scrubStringMap(typedInterface.Headers)
+		default:
+			continue
+		}
+	}
+	return packet
+}
+
+// Scrubs map of string -> string
+func scrubStringMap(stringMap map[string]string) map[string]string {
+	// Loops through the map and scrubs and sensitive data
+	for key, val := range stringMap {
+		stringMap[key] = scrubKeyValuePair(key, val)
+	}
+	return stringMap
+}
+
+// Check key/value pair for sensitive data
+func scrubKeyValuePair(key, val string) string {
+
+	if keyIsSensitive(key) {
+		return Mask
+	}
+
+	if valIsSensitive(val) {
+		return Mask
+	}
+
+	return val
+}
+
+// Check keys for sensitive data, matches list of substrings
+func keyIsSensitive(key string) (sensitive bool) {
+	for _, secretKey := range querySecretKeys {
+		// Make lower for case insensitive compare
+		key = strings.ToLower(key)
+		if strings.Contains(key, secretKey) {
+			return true
+		}
+	}
+	return false
+}
+
+// Check values for sensitive data, matches regex list
+func valIsSensitive(val string) (sensitive bool) {
+	for _, regex := range querySecretValues {
+		// Note: This will panic if querySecretValues has a bad regex
+		regexMatcher := regexp.MustCompile(regex)
+		if regexMatcher.MatchString(val) {
+			return true
+		}
+	}
+	return false
+}
+
+// Sanitize the query before sending it
+func scrubQuery(query url.Values) url.Values {
+
+	for key, values := range query {
+		for index, val := range values {
+			// Check key
+			if keyIsSensitive(key) {
+				query[key] = []string{Mask}
+			}
+
+			// Check value
+			if valIsSensitive(val) {
+				query[key][index] = Mask
+			}
+		}
+	}
+
+	return query
+}

processors_test.go

@@ -0,0 +1,52 @@
+package raven
+
+import (
+	"errors"
+	"log"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func customProcessor(packet *Packet) *Packet {
+	packet.Message = strings.Replace(packet.Message, "Password", "********", -1)
+	return packet
+}
+
+func TestProcessor(t *testing.T) {
+	// ... i.e. raisedErr is incoming error
+	raisedErr := errors.New("Test Password Error")
+	// sentry DSN generated by Sentry server
+	var sentryDSN string
+	// r is a request performed when error occured
+
+	r, _ := http.NewRequest("GET", "http://example.com/", nil)
+
+	r.Header.Set("api_key", "SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15")
+	r.Header.Set("apikey", "SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15")
+	r.Header.Set("Authorization", "Basic SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15")
+	r.Header.Set("X-Passwd", "62442")
+	r.Header.Set("X-Password", "62442")
+	r.Header.Set("X-Secret-Key", "SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15")
+	r.Header.Set("X-PASS", "Harry Potter")
+
+	r.URL.RawQuery = "api_key=SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15&apikey=SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15&auth=SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15&passwd=62442&userpassword=62442&secret-keeper=Sirius&pass=62442"
+
+	config := &ClientConfig{&[]Processor{customProcessor}, nil}
+	client, err := NewClient(sentryDSN, config)
+	if err != nil {
+		log.Fatal(err)
+	}
+	trace := NewStacktrace(0, 2, nil)
+	packet := NewPacket(raisedErr.Error(), NewException(raisedErr, trace), NewHttp(r))
+
+	scrubbedPacket := client.Scrub(packet)
+
+	scrubbedJSON := string(scrubbedPacket.JSON())
+	expectedJSON := `{"message":"Test ******** Error","event_id":"","project":"","timestamp":"0001-01-01T00:00:00","level":"","logger":"","extra":{"runtime.GOMAXPROCS":1,"runtime.NumCPU":8,"runtime.NumGoroutine":6,"runtime.Version":"go1.3.1"},"sentry.interfaces.Exception":{"value":"Test Password Error","type":"*errors.errorString","stacktrace":{"frames":[{"filename":"runtime/proc.c","function":"goexit","module":"runtime","lineno":1445,"abs_path":"/usr/local/Cellar/go/1.3.1/libexec/src/pkg/runtime/proc.c","context_line":"runtime·goexit(void)","pre_context":["#pragma textflag NOSPLIT","void"],"post_context":["{","\u0009if(g-\u003estatus != Grunning)"],"in_app":false},{"filename":"testing/testing.go","function":"tRunner","module":"testing","lineno":422,"abs_path":"/usr/local/Cellar/go/1.3.1/libexec/src/pkg/testing/testing.go","context_line":"\u0009test.F(t)","pre_context":["","\u0009t.start = time.Now()"],"post_context":["\u0009t.finished = true","}"],"in_app":false},{"filename":"Github.com/mattgerstman/raven-go/processors_test.go","function":"TestProcessor","module":"Github.com/mattgerstman/raven-go","lineno":40,"abs_path":"/Users/Matthew/Documents/Go/src/Github.com/mattgerstman/raven-go/processors_test.go","context_line":"\u0009trace := NewStacktrace(0, 2, nil)","pre_context":["\u0009\u0009log.Fatal(err)","\u0009}"],"post_context":["\u0009packet := NewPacket(raisedErr.Error(), NewException(raisedErr, trace), NewHttp(r))",""],"in_app":false}]}},"sentry.interfaces.Http":{"url":"http://example.com/","method":"GET","query_string":"api_key=%2A%2A%2A%2A%2A%2A%2A%2A\u0026apikey=%2A%2A%2A%2A%2A%2A%2A%2A\u0026auth=SGFycnlQb3R0ZXI6RHVtYmxlZG9yZXNBcm15\u0026pass=62442\u0026passwd=%2A%2A%2A%2A%2A%2A%2A%2A\u0026secret-keeper=%2A%2A%2A%2A%2A%2A%2A%2A\u0026userpassword=%2A%2A%2A%2A%2A%2A%2A%2A","headers":{"Api_key":"********","Apikey":"********","Authorization":"********","X-Pass":"Harry Potter","X-Passwd":"********","X-Password":"********","X-Secret-Key":"********"}}}`
+
+	if scrubbedJSON != expectedJSON {
+		t.Errorf("incorrect Value: got %s, want %s", scrubbedJSON, expectedJSON)
+	}
+
+}