Preserve log keys order

log_entry.go

@@ -4,7 +4,9 @@
 package plugin
 
 import (
+	"bytes"
 	"encoding/json"
+	"sort"
 	"time"
 )
 
@@ -22,6 +24,12 @@ type logEntryKV struct {
 	Value interface{} `json:"value"`
 }
 
+// jsonKey has the log key and its position in the raw log. It's used for ordering
+type jsonKey struct {
+	key      string
+	position int
+}
+
 // flattenKVPairs is used to flatten KVPair slice into []interface{}
 // for hclog consumption.
 func flattenKVPairs(kvs []*logEntryKV) []interface{} {
@@ -64,13 +72,34 @@ func parseJSON(input []byte) (*logEntry, error) {
 		delete(raw, "@timestamp")
 	}
 
-	// Parse dynamic KV args from the hclog payload.
-	for k, v := range raw {
+	orderedKeys := getOrderedKeys(input, raw)
+
+	// Parse dynamic KV args from the hclog payload in order
+	for _, k := range orderedKeys {
 		entry.KVPairs = append(entry.KVPairs, &logEntryKV{
-			Key:   k,
-			Value: v,
+			Key:   k.key,
+			Value: raw[k.key],
 		})
 	}
 
 	return entry, nil
 }
+
+// getOrderedKeys returns the log keys ordered according to their original order of appearance
+func getOrderedKeys(input []byte, raw map[string]interface{}) []jsonKey {
+	var orderedKeys []jsonKey
+
+	for key := range raw {
+		position := bytes.Index(input, []byte("\""+key+"\":"))
+		orderedKeys = append(orderedKeys, jsonKey{
+			key,
+			position,
+		})
+	}
+
+	sort.Slice(orderedKeys, func(i, j int) bool {
+		return orderedKeys[i].position < orderedKeys[j].position
+	})
+
+	return orderedKeys
+}

log_entry_test.go

@@ -0,0 +1,45 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package plugin
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestParseJson(t *testing.T) {
+	keys := []string{"firstKey", "secondKey", "thirdKey"}
+	raw := map[string]interface{}{
+		keys[0]: "thirdKey", // we use keys as values to test correct key matching
+		keys[1]: "secondKey",
+		keys[2]: "firstKey",
+	}
+
+	input := []byte(
+		fmt.Sprintf(
+			`{"@level":"info","@message":"msg","@timestamp":"2023-07-28T17:50:47.333365+02:00","%s":"%s","%s":"%s","%s":"%s"}`,
+			keys[0], raw[keys[0]],
+			keys[1], raw[keys[1]],
+			keys[2], raw[keys[2]],
+		),
+	)
+
+	// the behavior is non deterministic, that's why this test is repeated multiple times
+	for i := 0; i < 100; i++ {
+		entry, err := parseJSON(input)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		for i := 0; i < len(keys); i++ {
+			if keys[i] != entry.KVPairs[i].Key {
+				t.Fatalf("expected key: %v\ngot key: %v", keys[i], entry.KVPairs[i].Key)
+			}
+			if raw[keys[i]] != entry.KVPairs[i].Value {
+				t.Fatalf("expected value: %v\ngot value: %v", keys[i], entry.KVPairs[i].Key)
+			}
+		}
+	}
+
+}