Refactor model registry openapi definition. (#66)

* Refactor openapi definition

* Add name query parameter for model_version url, fix Artifact oneOf definition

* Added sortOrder query param

* Fix allOf composites for BaseArtifactUpdate and ModelArtifactUpdate

* Add model-serving metadata resources, openapi-generator-cli, and generated model, server, and proxy cmd

* Fix allOf types for InferenceServiceCreate and InferenceServiceUpdate

.openapi-generator-ignore

@@ -0,0 +1,47 @@
+# OpenAPI Generator Ignore
+# Generated by openapi-generator https://github.com/openapitools/openapi-generator
+
+# Use this file to prevent files from being overwritten by the generator.
+# The patterns follow closely to .gitignore or .dockerignore.
+
+# As an example, the C# client generator defines ApiClient.cs.
+# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
+#ApiClient.cs
+
+# You can match any string of characters against a directory, file or extension with a single asterisk (*):
+#foo/*/qux
+# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+
+# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
+#foo/**/qux
+# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+
+# You can also negate patterns with an exclamation (!).
+# For example, you can ignore all files in a docs folder with the file extension .md:
+#docs/*.md
+# Then explicitly reverse the ignore rule for a single file:
+#!docs/README.md
+
+# model and client files
+internal/model/openapi/api
+internal/model/openapi/api/**
+internal/model/openapi/git_push.sh
+internal/model/openapi/.gitignore
+internal/model/openapi/.travis.yml
+internal/model/openapi/.openapi-generator-ignore
+
+internal/model/openapi/README.md
+internal/model/openapi/docs/**.md
+internal/model/openapi/test
+internal/model/openapi/test/**
+internal/model/openapi/**all_of.go
+internal/model/openapi/go.mod
+internal/model/openapi/go.sum
+
+# server files to ignore
+internal/server/openapi/api
+internal/server/openapi/api/**
+internal/server/openapi/.openapi-generator-ignore
+internal/server/openapi/README.md
+internal/server/openapi/main.go
+internal/server/openapi/model_*.go

Dockerfile

@@ -8,9 +8,12 @@ COPY ["go.mod", "go.sum", "./"]
 # and so that source changes don't invalidate our downloaded layer
 RUN go mod download
 
-# Copy the go source
 USER root
-COPY ["Makefile", "main.go", "gqlgen.yml", "./"]
+# install npm and java for openapi-generator-cli
+RUN yum install -y nodejs npm java-11
+
+# Copy the go source
+COPY ["Makefile", "main.go", "gqlgen.yml", ".openapi-generator-ignore", "openapitools.json", "./"]
 
 # Download protoc compiler v24.3
 RUN wget -q https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip -O protoc.zip && \

Makefile

@@ -26,13 +26,38 @@ gen/graph: internal/model/graph/models_gen.go
 internal/model/graph/models_gen.go: api/graphql/*.graphqls gqlgen.yml
 	gqlgen generate
 
+# validate the openapi schema
+.PHONY: openapi/validate
+openapi/validate: bin/openapi-generator-cli
+	openapi-generator-cli validate -i api/openapi/model-registry.yaml
+
+# generate the openapi server implementation
+# note: run manually only when model-registry.yaml api changes, for model changes gen/openapi is run automatically
+.PHONY: gen/openapi-server
+gen/openapi-server: bin/openapi-generator-cli openapi/validate
+	openapi-generator-cli generate \
+		-i api/openapi/model-registry.yaml -g go-server -o internal/server/openapi --package-name openapi \
+		--ignore-file-override ./.openapi-generator-ignore --additional-properties=outputAsLibrary=true,enumClassPrefix=true,router=chi,sourceFolder=,onlyInterfaces=true
+	gofmt -w internal/server/openapi
+
+# generate the openapi schema model and client
+.PHONY: gen/openapi
+gen/openapi: bin/openapi-generator-cli openapi/validate internal/model/openapi/client.go
+
+internal/model/openapi/client.go: api/openapi/model-registry.yaml
+	rm -rf internal/model/openapi
+	openapi-generator-cli generate \
+		-i api/openapi/model-registry.yaml -g go -o internal/model/openapi --package-name openapi \
+		--ignore-file-override ./.openapi-generator-ignore --additional-properties=isGoSubmodule=true,enumClassPrefix=true,useOneOfDiscriminatorLookup=true
+	gofmt -w internal/model/openapi
+
 .PHONY: vet
 vet:
 	go vet ./...
 
 .PHONY: clean
 clean:
-	rm -Rf ./model-registry internal/ml_metadata/proto/*.go internal/model/graph/models_gen.go internal/converter/generated/converter.go
+	rm -Rf ./model-registry internal/ml_metadata/proto/*.go internal/model/graph/models_gen.go internal/converter/generated/converter.go internal/model/openapi
 
 bin/go-enum:
 	GOBIN=$(PROJECT_BIN) go install github.com/searKing/golang/tools/[email protected]
@@ -52,8 +77,26 @@ bin/golangci-lint:
 bin/goverter:
 	GOBIN=$(PROJECT_PATH)/bin go install github.com/jmattheis/goverter/cmd/[email protected]
 
+OPENAPI_GENERATOR ?= ${PROJECT_BIN}/openapi-generator-cli
+NPM ?= "$(shell which npm)"
+bin/openapi-generator-cli:
+ifeq (, $(shell which ${NPM} 2> /dev/null))
+	@echo "npm is not available please install it to be able to install openapi-generator"
+	exit 1
+endif
+ifeq (, $(shell which ${PROJECT_BIN}/openapi-generator-cli 2> /dev/null))
+	@{ \
+	set -e ;\
+	mkdir -p ${PROJECT_BIN} ;\
+	mkdir -p ${PROJECT_BIN}/openapi-generator-installation ;\
+	cd ${PROJECT_BIN} ;\
+	${NPM} install -g --prefix ${PROJECT_BIN}/openapi-generator-installation @openapitools/openapi-generator-cli ;\
+	ln -s openapi-generator-installation/bin/openapi-generator-cli openapi-generator-cli ;\
+	}
+endif
+
 .PHONY: deps
-deps: bin/go-enum bin/protoc-gen-go bin/protoc-gen-go-grpc bin/gqlgen bin/golangci-lint bin/goverter
+deps: bin/go-enum bin/protoc-gen-go bin/protoc-gen-go-grpc bin/gqlgen bin/golangci-lint bin/goverter bin/openapi-generator-cli
 
 .PHONY: vendor
 vendor:
@@ -64,7 +107,7 @@ build: gen vet lint
 	go build
 
 .PHONY: gen
-gen: deps gen/grpc gen/graph gen/converter
+gen: deps gen/grpc gen/openapi gen/graph gen/converter
 	go generate ./...
 
 .PHONY: lint
@@ -86,6 +129,10 @@ metadata.sqlite.db: run/migrate
 run/server: gen metadata.sqlite.db
 	go run main.go serve --logtostderr=true
 
+.PHONY: run/proxy
+run/proxy: gen
+	go run main.go proxy --logtostderr=true
+
 .PHONY: run/client
 run/client: gen
 	python test/python/test_mlmetadata.py

README.md

@@ -7,6 +7,8 @@ It adds other features on top of the functionality offered by the gRPC interface
 ## Pre-requisites:
 - go >= 1.19
 - protoc v24.3 - [Protocol Buffers v24.3 Release](https://github.com/protocolbuffers/protobuf/releases/tag/v24.3)
+- npm >= 10.2.0 - [Installing Node.js and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)
+- Java >= 11.0
 - python 3.9
 ## Building
 Run the following command to build the server binary:
@@ -17,6 +19,13 @@ The generated binary uses spf13 cmdline args. More information on using the serv
 ```
 ./model-registry --help
 ```
+## OpenAPI Proxy Server
+### Starting the OpenAPI Proxy Server
+Run the following command to start the OpenAPI proxy server:
+```
+make run/proxy &
+```
+The proxy service implements the OpenAPI defined in [model-registry.yaml](api/openapi/model-registry.yaml) to create an Open Data Hub specific REST API that stores metadata in an ml-metadata CPP server. 
 ## Server
 ### Creating/Migrating Server DB
 The server uses a local SQLite DB file (`metadata.sqlite.db` by default), which can be configured using the `-d` cmdline option.