Init (#1)

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+target/

Makefile

@@ -0,0 +1,31 @@
+.SILENT: ;               # No need for @
+.ONESHELL: ;             # Single shell for a target (required to properly use all of our local variables)
+.EXPORT_ALL_VARIABLES: ; # Send all vars to shell
+.DEFAULT: help # Running Make without target will run the help target
+
+.PHONY: help clean build-vp
+
+help: ## Show Help
+	grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+clean:
+	rm -f src/target/*
+
+# run 'make build-vp OFFLINE=true' to build from vendor folder
+build-vp: clean ## Build binary
+	set -e
+	cd src
+	BUILD_FROM=mod
+	if [ "$$OFFLINE" == "true" ]; then \
+		echo "Building using local vendor folder (ie offline build) ..."; \
+		BUILD_FROM=vendor; \
+	else \
+		echo "Building ..."; \
+	fi
+	ARCH=("linux/amd64" "darwin/arm64")
+	for platform in "$${ARCH[@]}"; do \
+		IFS='/' read -r GOOS GOARCH <<< "$$platform"; \
+		output="vectp-$$GOOS-$$GOARCH"; \
+		GOOS=$$GOOS GOARCH=$$GOARCH go build -mod=$$BUILD_FROM -a -o target/$$output; \
+	done
+	cd -

README.md

@@ -0,0 +1,70 @@
+# Vector Proxy (VP)
+
+VP is a service that acts as a proxy between an ingest script/pipeline and the vector store. It is responsible for computing or delegating vectorization and then sending the vectorized data to the vector store.
+
+> [!NOTE]
+> VP proxies all calls to the vector store. Only indexing calls are modified to include the vectorized data in the payload, other calls being directly forwarded to the vector store.
+
+> [!IMPORTANT]
+> VP will enrich your source document with embeddings for the fields you specify. Each embedding will be added to the document using the name of the original field with the suffix `_embedding`.
+>
+> For example, if you ask VP to embed the `content` field, the resulting vector will be added to the payload as `content_embedding`.
+
+## Features
+
+Currently supported vector stores and vectorization methods:
+
+| Proxy Vector Store API | Delegated Vectorization | Local Vectorization |
+|------------------------|-------------------------|---------------------|
+| OpenSearch/Elasticsearch Bulk API | Amazon Bedrock | - |
+
+## Build
+
+```bash
+make build-vp
+```
+
+Binaries are generated in the `src/target/` directory.
+
+## Usage
+
+```bash
+vectp -h
+```
+
+## Examples
+
+### Elasticsearch/OpenSearch as Vector Store
+
+To compute vectors from a [sample NDJSON file](./samples/opensearch/test.ndjson):
+
+- Start proxy first (in dry mode here just to see the computed vectors):
+
+    ```bash
+    vectp -embeddings-dimension 256 -gjson-paths content,metadata.#.author -dry-run
+    ```
+
+    > [!NOTE]
+    > Here we ask the proxy to vectorize the `content` and `metadata.author` fields. The vectors are computed using the default embedding model and have a dimension of 256.
+
+- Then send the NDJSON data to the proxy, the same way as you would send it to Elasticsearch/OpenSearch (via Bulk API):
+
+    ```bash
+    curl -v -k -u "<USER>:<PWD>" -XPOST "http://localhost:8080/opensearch/_bulk" -H 'Content-Type: application/x-ndjson' --data-binary "@./samples/opensearch/test.ndjson"
+    ```
+
+The proxy output will look like this:
+
+![](./_img/vectorproxy-output.png)
+
+While the enriched payload with vectors will look like:
+
+![](./_img/curl-opensearch-ingestion-dryrun.png)
+
+**As the vector proxy is running in dry-run mode, the data is not actually sent to the vector store**: the output is just a preview of the enriched payload that would have been sent to the vector store. To send the data to the vector store, stop vector proxy and restart it without the `-dry-run` flag.
+
+## GJSON Path Syntax
+
+VP uses the GJSON Path syntax to extract the data from the JSON payload. The syntax is similar to the JSON Path syntax but with some differences.
+
+See the [GJSON Path syntax documentation](https://github.com/tidwall/gjson/blob/master/SYNTAX.md) for more information and [GJSON playground](https://gjson.dev/) to experiment with the syntax.

samples/opensearch/test.ndjson

@@ -0,0 +1,10 @@
+{"index":{"_index":"my_index","_id":"1"}}
+{"id":1,"name":"Document 1","content":"This is the content of document 1.","metadata":[{"author":"Author 1","length":100},{"author":"Author 2","length":100}]}
+{"index":{"_index":"my_index","_id":"2"}}
+{"id":2,"name":"Document 2","content":"This is the content of document 2.","metadata":[{"author":"Author 2","length":200}]}
+{"index":{"_index":"my_index","_id":"3"}}
+{"id":3,"name":"Document 3","content":"This is the content of document 3.","metadata":[{"author":"Author 3","length":300},{"author":"Author 4","length":300},{"author":"Author 5","length":300}]}
+{"index":{"_index":"my_index","_id":"4"}}
+{"id":4,"name":"Document 4","content":"This is the content of document 4.","metadata":[{"author":"Author 4","length":400},{"author":"Author 1","length":300}]}
+{"index":{"_index":"my_index","_id":"5"}}
+{"id":5,"name":"Document 5","content":"This is the content of document 5.","metadata":[{"author":"Author 5","length":500}]}

src/config/config.go

@@ -0,0 +1,22 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package config
+
+var (
+	VectorStoreURL      string
+	GjsonPaths          []string
+	DryRun              bool
+	ProxyPort           string
+	EmbeddingModelID    string
+	EmbeddingsDimension int
+	MaxParallel         int
+)

src/go.mod

@@ -0,0 +1,30 @@
+module asaintsever/vectorproxy
+
+go 1.23.3
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.32.4
+	github.com/aws/aws-sdk-go-v2/config v1.28.4
+	github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.20.0
+	github.com/tidwall/gjson v1.18.0
+	github.com/tidwall/sjson v1.2.5
+	golang.org/x/net v0.31.0
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.45 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.0 // indirect
+	github.com/aws/smithy-go v1.22.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	golang.org/x/text v0.20.0 // indirect
+)

src/go.sum

@@ -0,0 +1,44 @@
+github.com/aws/aws-sdk-go-v2 v1.32.4 h1:S13INUiTxgrPueTmrm5DZ+MiAo99zYzHEFh1UNkOxNE=
+github.com/aws/aws-sdk-go-v2 v1.32.4/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
+github.com/aws/aws-sdk-go-v2/config v1.28.4 h1:qgD0MKmkIzZR2DrAjWJcI9UkndjR+8f6sjUQvXh0mb0=
+github.com/aws/aws-sdk-go-v2/config v1.28.4/go.mod h1:LgnWnNzHZw4MLplSyEGia0WgJ/kCGD86zGCjvNpehJs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.45 h1:DUgm5lFso57E7150RBgu1JpVQoF8fAPretiDStIuVjg=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.45/go.mod h1:dnBpENcPC1ekZrGpSWspX+ZRGzhkvqngT2Qp5xBR1dY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 h1:woXadbf0c7enQ2UGCi8gW/WuKmE0xIzxBF/eD94jMKQ=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19/go.mod h1:zminj5ucw7w0r65bP6nhyOd3xL6veAUMc3ElGMoLVb4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23/go.mod h1:c48kLgzO19wAu3CPkDWC28JbaJ+hfQlsdl7I2+oqIbk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.20.0 h1:c/2Lv0Nq/I+UeWKqUKR/LS9rO8McuXc5CzIfK2aBlhg=
+github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.20.0/go.mod h1:Kh/nzScDldU7Ti7MyFMCA+0Po+LZ4iNjWwl7H1DWYtU=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 h1:tHxQi/XHPK0ctd/wdOw0t7Xrc2OxcRCnVzv8lwWPu0c=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4/go.mod h1:4GQbF1vJzG60poZqWatZlhP31y8PGCCVTvIGPdaaYJ0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 h1:HJwZwRt2Z2Tdec+m+fPjvdmkq2s9Ra+VR0hjF7V2o40=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.5/go.mod h1:wrMCEwjFPms+V86TCQQeOxQF/If4vT44FGIOFiMC2ck=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 h1:zcx9LiGWZ6i6pjdcoE9oXAB6mUdeyC36Ia/QEiIvYdg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4/go.mod h1:Tp/ly1cTjRLGBBmNccFumbZ8oqpZlpdhFf80SrRh4is=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.0 h1:s7LRgBqhwLaxcocnAniBJp7gaAB+4I4vHzqUqjH18yc=
+github.com/aws/aws-sdk-go-v2/service/sts v1.33.0/go.mod h1:9XEUty5v5UAsMiFOBJrNibZgwCeOma73jgGwwhgffa8=
+github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
+github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=

src/main.go

@@ -0,0 +1,43 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package main
+
+import (
+	"asaintsever/vectorproxy/config"
+	"asaintsever/vectorproxy/stores"
+	"flag"
+	"log"
+	"net/http"
+	"strings"
+)
+
+func init() {
+	flag.StringVar(&config.VectorStoreURL, "url", "https://localhost:9200", "Vector Store URL")
+	paths := flag.String("gjson-paths", "", "Comma-separated list of GJSON Path strings")
+	flag.BoolVar(&config.DryRun, "dry-run", false, "If set, do not send the request to Vector Store")
+	flag.StringVar(&config.ProxyPort, "port", "8080", "Port for the proxy server")
+	flag.StringVar(&config.EmbeddingModelID, "embedding-model", "amazon.titan-embed-text-v2:0", "Text Embedding Model ID")
+	flag.IntVar(&config.EmbeddingsDimension, "embeddings-dimension", 1024, "Embeddings Dimension")
+	flag.IntVar(&config.MaxParallel, "parallel", 10, "Maximum number of parallel processing")
+	flag.Parse()
+
+	if *paths != "" {
+		config.GjsonPaths = strings.Split(*paths, ",")
+	}
+}
+
+func main() {
+	http.HandleFunc("/opensearch/_bulk", stores.OpenSearchBulkHandler) // Intercept bulk requests
+	http.HandleFunc("/opensearch/", stores.ProxyHandler)               // Forward all other requests
+	log.Printf("Starting vector proxy on :%s", config.ProxyPort)
+	log.Fatal(http.ListenAndServe(":"+config.ProxyPort, nil))
+}

src/stores/common.go

@@ -0,0 +1,94 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package stores
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"log"
+	"net/http"
+
+	"asaintsever/vectorproxy/config"
+
+	"golang.org/x/net/http2"
+)
+
+func ProxyHandler(w http.ResponseWriter, r *http.Request) {
+	proxyURL := config.VectorStoreURL + r.RequestURI
+
+	// Read the request body
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "Failed to read request body", http.StatusInternalServerError)
+		return
+	}
+	defer r.Body.Close()
+
+	// Create the request
+	req, err := http.NewRequest(r.Method, proxyURL, bytes.NewReader(body))
+	if err != nil {
+		http.Error(w, "Failed to create request to vector store", http.StatusInternalServerError)
+		return
+	}
+
+	// Copy headers from the original request
+	for name, values := range r.Header {
+		for _, value := range values {
+			req.Header.Add(name, value)
+
+			if config.DryRun {
+				if name == "Content-Type" || name == "Authorization" {
+					w.Header().Add(name, value)
+				}
+			}
+		}
+	}
+
+	if config.DryRun {
+		// Log the request being forwarded
+		log.Printf("Forwarding request to: %s", proxyURL)
+
+		// If dry-run is enabled, return the request details without forwarding
+		w.WriteHeader(http.StatusOK)
+		w.Write(body)
+		return
+	}
+
+	// Configure HTTP client to use HTTP/2 and skip TLS certificate validation
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+	http2.ConfigureTransport(client.Transport.(*http.Transport))
+
+	// Perform the request
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Print(err.Error())
+		http.Error(w, "Failed to forward request to vector store", http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	// Copy headers from the response
+	for name, values := range resp.Header {
+		for _, value := range values {
+			w.Header().Add(name, value)
+		}
+	}
+
+	// Write the response back to the client
+	w.WriteHeader(resp.StatusCode)
+	io.Copy(w, resp.Body)
+}