feat: support to mask multiple values in query and json (#91)

Co-authored-by: Ward Miao <[email protected]>

Author: wardab0104

pkg/logger/log/maskutil.go

@@ -42,7 +42,7 @@ type FieldRegex struct {
 func (f *FieldRegex) InitFieldRegex(fieldName string) {
 	f.FieldName = fieldName
 	// "fieldName":"(.*?)"
-	f.JsonPattern = regexp.MustCompile(fmt.Sprintf("\"%s\":\"(.*?)\"", fieldName))
+	f.JsonPattern = regexp.MustCompile(fmt.Sprintf("\"%s\":(\\[.*?\\]|\".*?\")", fieldName))
 	// fieldName=(.*?[^&]*)|fieldName=(.*?)$
 	f.QueryStringPattern = regexp.MustCompile(fmt.Sprintf("%s=(.*?[^&]*)|%s=(.*?)$", fieldName, fieldName))
 }
@@ -126,7 +126,21 @@ func MaskPIIFields(contentType, content, fields string) string {
 		if strings.Contains(contentType, "application/json") {
 			content = fieldRegex.JsonPattern.ReplaceAllStringFunc(content, func(m string) string {
 				parts := fieldRegex.JsonPattern.FindStringSubmatch(m)
-				return fmt.Sprintf(`"%s":"%s"`, fieldName, maskLastNChars(parts[1], defaultMaskCharsCount))
+				if len(parts) < 2 {
+					return m
+				}
+				matched := parts[1]
+				if strings.HasPrefix(matched, "[") {
+					matched = strings.Trim(matched, "[]")
+					items := strings.Split(matched, ",")
+					for i, item := range items {
+						item = strings.TrimSpace(item)
+						item = strings.Trim(item, "\"")
+						items[i] = fmt.Sprintf("\"%s\"", maskLastNChars(item, defaultMaskCharsCount))
+					}
+					return fmt.Sprintf(`"%s":[%s]`, fieldName, strings.Join(items, ","))
+				}
+				return fmt.Sprintf(`"%s":"%s"`, fieldName, maskLastNChars(matched, defaultMaskCharsCount))
 			})
 		} else if strings.Contains(contentType, "application/x-www-form-urlencoded") {
 			content = fieldRegex.QueryStringPattern.ReplaceAllStringFunc(content, func(m string) string {
@@ -141,7 +155,21 @@ func MaskPIIFields(contentType, content, fields string) string {
 			if fieldRegex.JsonPattern.MatchString(content) {
 				content = fieldRegex.JsonPattern.ReplaceAllStringFunc(content, func(m string) string {
 					parts := fieldRegex.JsonPattern.FindStringSubmatch(m)
-					return fmt.Sprintf(`"%s":"%s"`, fieldName, maskLastNChars(parts[1], defaultMaskCharsCount))
+					if len(parts) < 2 {
+						return m
+					}
+					matched := parts[1]
+					if strings.HasPrefix(matched, "[") {
+						matched = strings.Trim(matched, "[]")
+						items := strings.Split(matched, ",")
+						for i, item := range items {
+							item = strings.TrimSpace(item)
+							item = strings.Trim(item, "\"")
+							items[i] = fmt.Sprintf("\"%s\"", maskLastNChars(item, defaultMaskCharsCount))
+						}
+						return fmt.Sprintf(`"%s":[%s]`, fieldName, strings.Join(items, ","))
+					}
+					return fmt.Sprintf(`"%s":"%s"`, fieldName, maskLastNChars(matched, defaultMaskCharsCount))
 				})
 			} else if fieldRegex.QueryStringPattern.MatchString(content) {
 				content = fieldRegex.QueryStringPattern.ReplaceAllStringFunc(content, func(m string) string {
@@ -199,6 +227,31 @@ func maskLastNChars(value string, n int) string {
 		return ""
 	}
 
+	if strings.HasPrefix(value, `"`) {
+		value = strings.Trim(value, `""`)
+	}
+	decoded, err := url.QueryUnescape(value)
+	if err != nil {
+		logrus.Warn("failed to unescape url parameter for masking sensitive data: ", err)
+	} else {
+		value = decoded
+	}
+	if strings.Contains(value, ",") {
+		maskedValues := make([]string, 0)
+		values := strings.Split(value, ",")
+		for _, v := range values {
+			maskedValues = append(maskedValues, mask(v, n))
+		}
+		return strings.Join(maskedValues, ",")
+	} else {
+		return mask(value, n)
+	}
+}
+
+func mask(value string, n int) string {
+	if value == "" {
+		return ""
+	}
 	if strings.Contains(value, "@") {
 		parts := strings.Split(value, "@")
 		if len(parts[0]) <= n {

pkg/logger/log/maskutil_test.go

@@ -401,9 +401,9 @@ func TestMaskPIIFieldsInQuery_MaskMultipleFields(t *testing.T) {
 	inputAndExpected := [][]string{
 		// Content-Type = application/json
 		{
-			"application/json",                // content-type
-			"{\"username\":\"username12\"}",   // input
-			"{\"username\":\"username****\"}", // expected
+			"application/json", // content-type
+			"{\"username\":\"username12\", \"emails\":[\"[email protected]\",\"[email protected]\"]}",       // input
+			"{\"username\":\"username****\", \"emails\":[\"tes****@accelbyte.net\",\"tes****@accelbyte.net\"]}", // expected
 		},
 		{
 			"application/json",
@@ -459,8 +459,8 @@ func TestMaskPIIFieldsInQuery_MaskMultipleFields(t *testing.T) {
 		// Content-Type = plain/text
 		{
 			"plain/text",
-			"{\"username\":\"username12\",\"emailAddress\":\"[email protected]\",\"displayName\":\"My Display Name\"}",
-			"{\"username\":\"username****\",\"emailAddress\":\"te****@accelbyte.net\",\"displayName\":\"My Display Name\"}",
+			"{\"username\":\"username12\",\"emailAddress\":\"[email protected]\",\"displayName\":\"My Display Name\", \"emails\":[\"[email protected]\",\"[email protected]\"]}",
+			"{\"username\":\"username****\",\"emailAddress\":\"te****@accelbyte.net\",\"displayName\":\"My Display Name\", \"emails\":[\"tes****@accelbyte.net\",\"tes****@accelbyte.net\"]}",
 		},
 		{
 			"plain/text",
@@ -490,7 +490,7 @@ func TestMaskPIIFieldsInQuery_MaskMultipleFields(t *testing.T) {
 	}
 
 	for _, val := range inputAndExpected {
-		assert.Equal(t, val[2], MaskPIIFields(val[0], val[1], "username,emailAddress"))
+		assert.Equal(t, val[2], MaskPIIFields(val[0], val[1], "username,emailAddress,emails"))
 	}
 }
 
@@ -742,11 +742,11 @@ func TestMaskPIIFields_ConcurrentCall(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(1)
 	go func() {
-		input := "{\"username\":\"username12\"}"
-		expected := "{\"username\":\"username****\"}"
+		input := "{\"username\":\"username12\", \"emails\":[\"[email protected]\",\"[email protected]\"]}"
+		expected := "{\"username\":\"username****\", \"emails\":[\"tes****@accelbyte.net\",\"tes****@accelbyte.net\"]}"
 		i := 0
 		for i < 1000 {
-			assert.Equal(t, expected, MaskPIIFields("application/json", input, "username"))
+			assert.Equal(t, expected, MaskPIIFields("application/json", input, "username,emails"))
 			i++
 		}
 		wg.Done()
@@ -801,10 +801,14 @@ func TestMaskMultiplePIIQueryParams(t *testing.T) {
 			"https://example.net?username=user name12&password=mypassword 123&displayName=My Display Name",
 			"https://example.net?username=user name****&password=mypassword 123&displayName=My Display Name",
 		},
+		{
+			"https://[email protected],[email protected]&password=mypassword 123&displayName=My Display Name",
+			"https://example.net?loginIds=tes****@accelbyte.net,tes****@accelbyte.net&password=mypassword 123&displayName=My Display Name",
+		},
 	}
 
 	for _, val := range inputAndExpected {
-		assert.Equal(t, val[1], MaskPIIQueryParams(val[0], "username,emailAddress"))
+		assert.Equal(t, val[1], MaskPIIQueryParams(val[0], "username,emailAddress,loginIds"))
 	}
 }
 
@@ -824,10 +828,14 @@ func TestMaskPIIQueryParamOfEncodedURLs(t *testing.T) {
 			"https://example.net?openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&openid.mode=id_res&openid.identity=https%3A%2F%2Fsteamcommunity.com%2Fopenid%2Fid%2F123456789&openid.signed=signed%2Cop_endpoint%2Cclaimed_id%2Cidentity&openid.sig=dGhpc19pc19zaWc%3D",
 			"https://example.net?openid.ns=http://specs.openid.net/auth/2.0&openid.mode=id_res&openid.identity=https://steamcommunity.com/openid/id/123456789&openid.signed=signed,op_endpoint,claimed_id,identity&openid.sig=dGhpc19pc19zaWc=",
 		},
+		{
+			"https://example.net?loginIds=test1%40accelbyte.net,test2%40accelbyte.net&displayName=My%20Display%20Name",
+			"https://example.net?loginIds=tes****@accelbyte.net,tes****@accelbyte.net&displayName=My Display Name",
+		},
 	}
 
 	for _, val := range inputAndExpected {
-		assert.Equal(t, val[1], MaskPIIQueryParams(val[0], "username,emailAddress"))
+		assert.Equal(t, val[1], MaskPIIQueryParams(val[0], "username,emailAddress,loginIds"))
 	}
 }
 
@@ -883,5 +891,17 @@ func TestMaskPIIQueryParam_ConcurrentCall(t *testing.T) {
 		wg.Done()
 	}()
 
+	wg.Add(1)
+	go func() {
+		input := "https://[email protected],[email protected]&key=12"
+		expected := "https://example.net?loginIds=tes****@accelbyte.net,tes****@accelbyte.net&key=12"
+		i := 0
+		for i < 1000 {
+			assert.Equal(t, expected, MaskPIIQueryParams(input, "loginIds"))
+			i++
+		}
+		wg.Done()
+	}()
+
 	wg.Wait()
 }