Skip to content

Commit

Permalink
add time ms and txn to encoded json (#132)
Browse files Browse the repository at this point in the history 
* add time ms and txn to encoded json

* Fix jq query
  • Loading branch information
@stlava
stlava authored Jul 26, 2024
1 parent 9dc2aed commit c355207
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 8 deletions.
4 changes: 2 additions & 2 deletions itests/common.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,9 @@ _verify() {
log "Checking ./tests/$BATS_TEST_DESCRIPTION/golden/$file against ./tests/$BATS_TEST_DESCRIPTION/output/$file"

if [ "$SORT" == "false" ]; then
output=$(diff <(cat "./tests/$BATS_TEST_DESCRIPTION/golden/$file") <(cat "./tests/$BATS_TEST_DESCRIPTION/output/$file" | jq 'del(.lsn, .time)' -c -M) || true)
output=$(diff <(cat "./tests/$BATS_TEST_DESCRIPTION/golden/$file") <(cat "./tests/$BATS_TEST_DESCRIPTION/output/$file" | jq 'del(.lsn, .time, .time_ms, .txn)' -c -M) || true)
else
output=$(diff <(cat "./tests/$BATS_TEST_DESCRIPTION/golden/$file") <(sort "./tests/$BATS_TEST_DESCRIPTION/output/$file" | jq 'del(.lsn, .time)' -c -M) || true)
output=$(diff <(cat "./tests/$BATS_TEST_DESCRIPTION/golden/$file") <(sort "./tests/$BATS_TEST_DESCRIPTION/output/$file" | jq 'del(.lsn, .time, .time_ms, .txn)' -c -M) || true)
fi

if ! $output; then
Expand Down
6 changes: 5 additions & 1 deletion marshaller/marshaller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ func New(shutdownHandler shutdown.ShutdownHandler,
// jsonWalEntry is a helper struct which has json field tags
type jsonWalEntry struct {
Time string `json:"time"`
Lsn string `json:"lsn"` // Log Sequence Number that determines position in WAL
TimeMs int64 `json:"time_ms"` // epoch ms
Txn string `json:"txn"` // txn id + time key
Lsn string `json:"lsn"` // Log Sequence Number that determines position in WAL
Table string `json:"table"`
Operation string `json:"operation"`
Columns map[string]map[string]map[string]string `json:"columns"`
Expand Down Expand Up @@ -307,6 +309,8 @@ func marshalWalToJson(msg *replication.WalMessage, noMarshalOldValue bool) ([]by

// Construct WalEntry
reusedWalEntry.Time = t
reusedWalEntry.TimeMs = msg.ServerTime
reusedWalEntry.Txn = msg.TimeBasedKey
reusedWalEntry.Lsn = *(*string)(unsafe.Pointer(&lsnBytes))
reusedWalEntry.Table = msg.Pr.Relation
reusedWalEntry.Operation = msg.Pr.Operation
Expand Down
10 changes: 5 additions & 5 deletions marshaller/marshaller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ func TestBasicInsertMessage(t *testing.T) {
assert.Equal(t, "test.users", marshalled.Table)
assert.Equal(t, "0", marshalled.Transaction)

expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Foo\"}}}}"
expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"time_ms\":0,\"txn\":\"0-0\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Foo\"}}}}"
assert.Equal(t, expectedJson, string(marshalled.Json))
}

Expand Down Expand Up @@ -208,7 +208,7 @@ func TestBasicUpdateMessage(t *testing.T) {
assert.Equal(t, "test.users", marshalled.Table)
assert.Equal(t, "0", marshalled.Transaction)

expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Foo\"},\"old\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"}}}}"
expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"time_ms\":0,\"txn\":\"0-0\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Foo\"},\"old\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"}}}}"
assert.Equal(t, expectedJson, string(marshalled.Json))
}

Expand Down Expand Up @@ -244,7 +244,7 @@ func TestBasicUpdateNoOldMessage(t *testing.T) {
assert.Equal(t, "test.users", marshalled.Table)
assert.Equal(t, "0", marshalled.Transaction)

expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Foo\"}}}}"
expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"time_ms\":0,\"txn\":\"0-0\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Foo\"}}}}"
assert.Equal(t, expectedJson, string(marshalled.Json))
}

Expand Down Expand Up @@ -289,7 +289,7 @@ func TestToastValue(t *testing.T) {
assert.Equal(t, "test.users", marshalled.Table)
assert.Equal(t, "0", marshalled.Transaction)

expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"},\"old\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"}}}}"
expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"time_ms\":0,\"txn\":\"0-0\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"},\"old\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"}}}}"
assert.Equal(t, expectedJson, string(marshalled.Json))
}

Expand Down Expand Up @@ -334,7 +334,7 @@ func TestToastNoOldValue(t *testing.T) {
assert.Equal(t, "test.users", marshalled.Table)
assert.Equal(t, "0", marshalled.Transaction)

expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"}}}}"
expectedJson := "{\"time\":\"1970-01-01T00:00:00Z\",\"time_ms\":0,\"txn\":\"0-0\",\"lsn\":\"0/1\",\"table\":\"test.users\",\"operation\":\"INSERT\",\"columns\":{\"first_name\":{\"new\":{\"q\":\"true\",\"t\":\"string\",\"v\":\"Bar\"}}}}"
assert.Equal(t, expectedJson, string(marshalled.Json))
}

Expand Down

0 comments on commit c355207

Please sign in to comment.