feat: get config struct as a parameter (#48)

* feat: get config struct as a parameter

---------

Co-authored-by: mhmtszr <[email protected]>

config/config.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"time"
+
+	"github.com/Trendyol/go-dcp-client/config"
 )
 
 type Kafka struct {
@@ -28,11 +30,12 @@ func (k *Kafka) GetCompression() int8 {
 	return k.Compression
 }
 
-type Config struct {
-	Kafka Kafka `yaml:"kafka"`
+type Connector struct {
+	Kafka Kafka      `yaml:"kafka"`
+	Dcp   config.Dcp `yaml:",inline"`
 }
 
-func (c *Config) ApplyDefaults() {
+func (c *Connector) ApplyDefaults() {
 	if c.Kafka.ReadTimeout == 0 {
 		c.Kafka.ReadTimeout = 30 * time.Second
 	}

connector.go

@@ -1,9 +1,11 @@
 package gokafkaconnectcouchbase
 
 import (
+	"errors"
 	"os"
 
 	"github.com/Trendyol/go-dcp-client"
+	dcpClientConfig "github.com/Trendyol/go-dcp-client/config"
 	"github.com/Trendyol/go-dcp-client/logger"
 	"github.com/Trendyol/go-dcp-client/models"
 	"github.com/Trendyol/go-kafka-connect-couchbase/config"
@@ -27,7 +29,7 @@ type connector struct {
 	dcp         godcpclient.Dcp
 	mapper      Mapper
 	producer    producer.Producer
-	config      *config.Config
+	config      *config.Connector
 	logger      logger.Logger
 	errorLogger logger.Logger
 }
@@ -84,56 +86,26 @@ func (c *connector) produce(ctx *models.ListenerContext) {
 	c.producer.Produce(ctx, e.EventTime, messages)
 }
 
-func NewConnector(configPath string, mapper Mapper) (Connector, error) {
-	return newConnector(configPath, mapper, logger.Log, logger.Log)
-}
-
-func NewConnectorWithLoggers(configPath string, mapper Mapper, logger logger.Logger, errorLogger logger.Logger) (Connector, error) {
-	return newConnector(configPath, mapper, logger, errorLogger)
-}
-
-func newConnectorConfig(path string) (*config.Config, error) {
-	file, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	var c config.Config
-	err = yaml.Unmarshal(file, &c)
+func NewConnector(cfg any, mapper Mapper) (Connector, error) {
+	c, err := newConfig(cfg)
 	if err != nil {
 		return nil, err
 	}
 	c.ApplyDefaults()
-	return &c, nil
-}
-
-func newConnector(configPath string, mapper Mapper, logger logger.Logger, errorLogger logger.Logger) (Connector, error) {
-	c, err := newConnectorConfig(configPath)
-	if err != nil {
-		return nil, err
-	}
 
 	connector := &connector{
 		mapper:      mapper,
 		config:      c,
-		logger:      logger,
-		errorLogger: errorLogger,
+		logger:      logger.Log,
+		errorLogger: logger.Log,
 	}
 
-	kafkaClient := kafka.NewClient(c, connector.logger, connector.errorLogger)
-
-	var topics []string
-
-	for _, topic := range c.Kafka.CollectionTopicMapping {
-		topics = append(topics, topic)
-	}
-
-	err = kafkaClient.CheckTopics(topics)
+	kafkaClient, err := createKafkaClient(c, connector)
 	if err != nil {
-		connector.errorLogger.Printf("collection topic mapping error: %v", err)
 		return nil, err
 	}
 
-	dcp, err := godcpclient.NewDcpWithLoggers(configPath, connector.produce, logger, errorLogger)
+	dcp, err := createDcp(cfg, connector.produce, connector.logger, connector.errorLogger)
 	if err != nil {
 		connector.errorLogger.Printf("dcp error: %v", err)
 		return nil, err
@@ -143,8 +115,7 @@ func newConnector(configPath string, mapper Mapper, logger logger.Logger, errorL
 	dcpConfig.Checkpoint.Type = "manual"
 
 	if dcpConfig.Metadata.Type == MetadataTypeKafka {
-		kafkaMetadata := metadata.NewKafkaMetadata(kafkaClient, dcpConfig.Metadata.Config, connector.logger, connector.errorLogger)
-		dcp.SetMetadata(kafkaMetadata)
+		setKafkaMetadata(kafkaClient, dcpConfig, connector, dcp)
 	}
 
 	connector.dcp = dcp
@@ -155,8 +126,78 @@ func newConnector(configPath string, mapper Mapper, logger logger.Logger, errorL
 		return nil, err
 	}
 
+	initializeMetricCollector(connector, dcp)
+
+	return connector, nil
+}
+
+func newConfig(cf any) (*config.Connector, error) {
+	switch v := cf.(type) {
+	case *config.Connector:
+		return v, nil
+	case config.Connector:
+		return &v, nil
+	case string:
+		return newConnectorConfigFromPath(v)
+	default:
+		return nil, errors.New("invalid config")
+	}
+}
+
+func NewConnectorWithLoggers(configPath string, mapper Mapper, infoLogger logger.Logger, errorLogger logger.Logger) (Connector, error) {
+	logger.SetLogger(infoLogger)
+	logger.SetErrorLogger(errorLogger)
+
+	return NewConnector(configPath, mapper)
+}
+
+func createKafkaClient(cc *config.Connector, connector *connector) (kafka.Client, error) {
+	kafkaClient := kafka.NewClient(cc, connector.logger, connector.errorLogger)
+
+	var topics []string
+
+	for _, topic := range cc.Kafka.CollectionTopicMapping {
+		topics = append(topics, topic)
+	}
+
+	if err := kafkaClient.CheckTopics(topics); err != nil {
+		connector.errorLogger.Printf("collection topic mapping error: %v", err)
+		return nil, err
+	}
+	return kafkaClient, nil
+}
+
+func createDcp(cfg any, listener models.Listener, logger logger.Logger, errorLogger logger.Logger) (godcpclient.Dcp, error) {
+	switch v := cfg.(type) {
+	case dcpClientConfig.Dcp:
+		return godcpclient.NewDcpWithLoggers(v.Dcp, listener, logger, errorLogger)
+	case string:
+		return godcpclient.NewDcpWithLoggers(v, listener, logger, errorLogger)
+	default:
+		return nil, errors.New("invalid config")
+	}
+}
+
+func setKafkaMetadata(kafkaClient kafka.Client, dcpConfig *dcpClientConfig.Dcp, connector *connector, dcp godcpclient.Dcp) {
+	kafkaMetadata := metadata.NewKafkaMetadata(kafkaClient, dcpConfig.Metadata.Config, connector.logger, connector.errorLogger)
+	dcp.SetMetadata(kafkaMetadata)
+}
+
+func initializeMetricCollector(connector *connector, dcp godcpclient.Dcp) {
 	metricCollector := metric.NewMetricCollector(connector.producer)
 	dcp.SetMetricCollectors(metricCollector)
+}
 
-	return connector, nil
+func newConnectorConfigFromPath(path string) (*config.Connector, error) {
+	file, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var c config.Connector
+	err = yaml.Unmarshal(file, &c)
+	if err != nil {
+		return nil, err
+	}
+	c.ApplyDefaults()
+	return &c, nil
 }

example/go.mod → example/simple/go.mod

@@ -2,7 +2,7 @@ module example
 
 go 1.19
 
-replace github.com/Trendyol/go-kafka-connect-couchbase => ../.
+replace github.com/Trendyol/go-kafka-connect-couchbase => ./../..
 
 require github.com/Trendyol/go-kafka-connect-couchbase v0.0.0
 

example/main.go → example/simple/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	gokafkaconnectcouchbase "github.com/Trendyol/go-kafka-connect-couchbase"
+	dcpkafka "github.com/Trendyol/go-kafka-connect-couchbase"
 	"github.com/Trendyol/go-kafka-connect-couchbase/couchbase"
 	"github.com/Trendyol/go-kafka-connect-couchbase/kafka/message"
 )
@@ -18,7 +18,7 @@ func mapper(event couchbase.Event) []message.KafkaMessage {
 }
 
 func main() {
-	connector, err := gokafkaconnectcouchbase.NewConnector("config.yml", mapper)
+	connector, err := dcpkafka.NewConnector("config.yml", mapper)
 	if err != nil {
 		panic(err)
 	}

example/struct-config/Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:1.19-alpine as builder
+
+WORKDIR /project
+
+COPY go.mod go.sum ./
+COPY main.go ./
+COPY config.yml ./config.yml
+
+RUN go mod download
+RUN CGO_ENABLED=0 go build -a -o example main.go
+
+FROM alpine:3.17.0
+
+WORKDIR /app
+
+RUN apk --no-cache add ca-certificates
+
+USER nobody
+COPY --from=builder --chown=nobody:nobody /project/example .
+COPY --from=builder --chown=nobody:nobody /project/config.yml ./config.yml
+
+ENTRYPOINT ["./example"]

example/struct-config/go.mod

@@ -0,0 +1,81 @@
+module example
+
+go 1.19
+
+replace github.com/Trendyol/go-kafka-connect-couchbase => ./../..
+
+require github.com/Trendyol/go-kafka-connect-couchbase v0.0.0
+
+require (
+	github.com/Trendyol/go-dcp-client v0.0.58 // indirect
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/ansrivas/fiberprometheus/v2 v2.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gofiber/adaptor/v2 v2.1.32 // indirect
+	github.com/gofiber/fiber/v2 v2.45.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/gnostic v0.6.9 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/prometheus/client_golang v1.15.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
+	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
+	github.com/segmentio/kafka-go v0.4.39 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.47.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	github.com/xdg/scram v1.0.5 // indirect
+	github.com/xdg/stringprep v1.0.3 // indirect
+	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/oauth2 v0.6.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.27.1 // indirect
+	k8s.io/apimachinery v0.27.1 // indirect
+	k8s.io/client-go v0.27.1 // indirect
+	k8s.io/klog/v2 v2.90.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect
+	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)