Skip to content

Commit 62f383c

Browse files
stewartboyd119stewartbzillow
stewartboyd119
and
stewartbzillow
authored
Bugfix: compatible schemas not properly handled during deserialization (#25)
Forward/Backwards compatible schemas used to produce/consume messages were not being properly deserialized. `Test_SchemaRegistryReal_Avro_AutoRegisterSchemas_OldProducerCanBeConsumedByNewConsumer` and `Test_SchemaRegistryReal_Avro_AutoRegisterSchemas_NewProducerCanBeConsumedByOldConsumer` demonstrated bug. Schema registry aware avro formatter was updated to reconcile the bug. --------- Co-authored-by: stewartboyd119 <[email protected]>
1 parent 753a8aa commit 62f383c

15 files changed

+294
-48
lines changed

‎Makefile

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
# Directories containing independent Go modules.
22
MODULE_DIRS = .
3-
GOLANGCI_VERSION=1.61.0
4-
AVRO_CMD_PATH=github.com/hamba/avro/v2/cmd/[email protected]
3+
GOLANGCI_VERSION=1.64.5
4+
AVRO_CMD_PATH=github.com/hamba/avro/v2/cmd/[email protected]
5+
SCHEMA_REGISTRY_DOMAIN=schema-registry.shared.zg-int.net:443
56

67

78
# Sets up kafka broker using docker compose

‎changelog.md

+6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
44

55
This project adheres to Semantic Versioning.
66

7+
## 2.1.1 (Feb 13, 2025)
8+
9+
1. Forward/Backwards compatible schemas used to produce/consume messages were not being properly deserialized.
10+
`Test_SchemaRegistryReal_Avro_AutoRegisterSchemas_OldProducerCanBeConsumedByNewConsumer` and `Test_SchemaRegistryReal_Avro_AutoRegisterSchemas_NewProducerCanBeConsumedByOldConsumer`
11+
demonstrated bug. Schema registry aware avro formatter was updated to reconcile the bug.
12+
713
## 2.1.0 (Jan 23, 2025)
814

915
1. Include `DisableTracePropagation` as a WriterOption

‎example/producer_avro/event_gen.go

+1-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎example/worker_avro/event_gen.go

+1-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎formatter.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -163,13 +163,14 @@ func (f avroSchemaRegistryFormatter) unmarshal(req unmarshReq) error {
163163
return fmt.Errorf("failed to parse schema : %w", err)
164164
}
165165
sc := avro.NewSchemaCompatibility()
166-
resolvedSchema, err := sc.Resolve(dataSchema, targetSchema)
166+
167+
resolvedSchema, err := sc.Resolve(targetSchema, dataSchema)
167168
if err != nil {
168-
return fmt.Errorf("failed to get schema from payload: %w", err)
169+
return fmt.Errorf("failed to reconcile producer/consumer schemas: %w", err)
169170
}
170-
171171
err = avro.Unmarshal(resolvedSchema, req.data[5:], req.target)
172172
if err != nil {
173+
173174
return fmt.Errorf("failed to deserialize to confluent schema registry avro type: %w", err)
174175
}
175176
return nil

‎go.mod

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
module github.com/zillow/zkafka/v2
22

3-
go 1.23
3+
go 1.23.0
4+
5+
toolchain go1.23.1
46

57
require (
68
github.com/confluentinc/confluent-kafka-go/v2 v2.8.0
79
github.com/google/go-cmp v0.6.0
810
github.com/google/uuid v1.6.0
9-
github.com/hamba/avro/v2 v2.27.0
11+
github.com/hamba/avro/v2 v2.28.0
1012
github.com/heetch/avro v0.4.6
1113
github.com/sony/gobreaker v1.0.0
1214
github.com/stretchr/testify v1.10.0
1315
github.com/zillow/zfmt v1.0.1
14-
go.opentelemetry.io/otel v1.33.0
15-
go.opentelemetry.io/otel/trace v1.33.0
16+
go.opentelemetry.io/otel v1.34.0
17+
go.opentelemetry.io/otel/trace v1.34.0
1618
go.uber.org/mock v0.5.0
17-
golang.org/x/sync v0.10.0
18-
google.golang.org/protobuf v1.36.2
19+
golang.org/x/sync v0.11.0
20+
google.golang.org/protobuf v1.36.5
1921
)
2022

2123
require (
@@ -38,7 +40,7 @@ require (
3840
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
3941
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
4042
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
41-
go.opentelemetry.io/otel/metric v1.33.0 // indirect
43+
go.opentelemetry.io/otel/metric v1.34.0 // indirect
4244
google.golang.org/genproto v0.0.0-20250106144421-5f5ef82da422 // indirect
4345
gopkg.in/yaml.v3 v3.0.1 // indirect
4446
)

‎go.sum

+14-14
Original file line numberDiff line numberDiff line change
@@ -194,8 +194,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
194194
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
195195
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
196196
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
197-
github.com/hamba/avro/v2 v2.27.0 h1:IAM4lQ0VzUIKBuo4qlAiLKfqALSrFC+zi1iseTtbBKU=
198-
github.com/hamba/avro/v2 v2.27.0/go.mod h1:jN209lopfllfrz7IGoZErlDz+AyUJ3vrBePQFZwYf5I=
197+
github.com/hamba/avro/v2 v2.28.0 h1:E8J5D27biyAulWKNiEBhV85QPc9xRMCUCGJewS0KYCE=
198+
github.com/hamba/avro/v2 v2.28.0/go.mod h1:9TVrlt1cG1kkTUtm9u2eO5Qb7rZXlYzoKqPt8TSH+TA=
199199
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
200200
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
201201
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -238,8 +238,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
238238
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
239239
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
240240
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
241-
github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
242-
github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
241+
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
242+
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
243243
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
244244
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
245245
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -411,8 +411,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
411411
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM=
412412
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
413413
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
414-
go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
415-
go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
414+
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
415+
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
416416
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU=
417417
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU=
418418
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM=
@@ -425,14 +425,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqhe
425425
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
426426
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM=
427427
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I=
428-
go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ=
429-
go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
428+
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
429+
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
430430
go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
431431
go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
432432
go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0=
433433
go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q=
434-
go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s=
435-
go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
434+
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
435+
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
436436
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
437437
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
438438
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
@@ -445,8 +445,8 @@ golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
445445
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
446446
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
447447
golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
448-
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
449-
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
448+
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
449+
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
450450
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
451451
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
452452
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
@@ -467,8 +467,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:
467467
google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4=
468468
google.golang.org/grpc v1.67.3 h1:OgPcDAFKHnH8X3O4WcO4XUc8GRDeKsKReqbQtiCj7N8=
469469
google.golang.org/grpc v1.67.3/go.mod h1:YGaHCc6Oap+FzBJTZLBzkGSYt/cvGPFTPxkn7QfSU8s=
470-
google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU=
471-
google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
470+
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
471+
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
472472
gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
473473
gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI=
474474
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎heap.go

+2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ func (h *offsetHeap) Pop() kafka.TopicPartition {
2121
if len(h.data) == 0 {
2222
panic("popped empty heap")
2323
}
24+
//nolint:errcheck // access control guarantees type is TopicPartition
2425
return heap.Pop(&h.data).(kafka.TopicPartition)
2526
}
2627

@@ -56,6 +57,7 @@ func (h _offsetHeap) Less(i, j int) bool { return h[i].Offset < h[j].Offset }
5657
func (h _offsetHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
5758

5859
func (h *_offsetHeap) Push(x any) {
60+
//nolint:errcheck // access control guarantees type is TopicPartition
5961
*h = append(*h, x.(kafka.TopicPartition))
6062
}
6163

‎test/evolution/avro1/schema_1_gen.go

+10-5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/evolution/avro1x/schema_1_gen.go

+20-6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/evolution/avro2/schema_2_gen.go

+3-3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/evolution/schema_1.avsc

+20
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,26 @@
2525
],
2626
"default": "created"
2727
}
28+
},
29+
{
30+
"name": "interactiveContent",
31+
"type": [
32+
"null",
33+
{
34+
"type": "array",
35+
"items": {
36+
"type": "record",
37+
"name": "InteractiveContentRecord",
38+
"fields": [
39+
{
40+
"name": "url",
41+
"type": "string"
42+
}
43+
]
44+
}
45+
}
46+
],
47+
"default": null
2848
}
2949
]
3050
}

‎test/evolution/schema_2.avsc

+9
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,21 @@
3939
{
4040
"name": "url",
4141
"type": "string"
42+
},
43+
{
44+
"name": "isImx",
45+
"type": [
46+
"null",
47+
"boolean"
48+
],
49+
"default": null
4250
}
4351
]
4452
}
4553
}
4654
],
4755
"default": null
4856
}
57+
4958
]
5059
}

0 commit comments

Comments
 (0)