Skip to content

Commit

Permalink
Fixes kubeflow#35, refactor DB schema
Browse files Browse the repository at this point in the history
Add missing indexes
Constraints for keys
Associations
Use appropriate column sizes for numeric types
Set creation and modification timestamps automatically using gorm tags

Clean up Makefile to install pinned deps in ./bin
  • Loading branch information
dhirajsb committed Sep 29, 2023
1 parent 7293d51 commit b844d91
Show file tree
Hide file tree
Showing 27 changed files with 515 additions and 254 deletions.
21 changes: 14 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
# useful paths
MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
PROJECT_PATH := $(patsubst %/,%,$(dir $(MKFILE_PATH)))
PROJECT_BIN := $(PROJECT_PATH)/bin

# add tools bin directory
PATH := $(PROJECT_BIN):$(PATH)
model-registry: build

internal/ml_metadata/proto/%.pb.go: api/grpc/ml_metadata/proto/%.proto
Expand All @@ -19,15 +26,15 @@ vet:

.PHONY: clean
clean:
rm -Rf ./model-registry internal/ml_metadata/proto/*.go internal/model/graph/*.go
rm -Rf ./model-registry internal/ml_metadata/proto/*.go internal/model/graph/models_gen.go

.PHONY: deps
deps:
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.54.2
go install github.com/99designs/gqlgen@latest
go install github.com/searKing/golang/tools/go-enum@latest
go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(PROJECT_BIN) v1.54.2
GOBIN=$(PROJECT_BIN) go install github.com/99designs/gqlgen@v0.17.36
GOBIN=$(PROJECT_BIN) go install github.com/searKing/golang/tools/go-enum@v1.2.97
GOBIN=$(PROJECT_BIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0
GOBIN=$(PROJECT_BIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0

.PHONY: vendor
vendor:
Expand All @@ -48,7 +55,7 @@ lint: gen

.PHONY: run/migrate
run/migrate: gen
go run main.go migrate --logtostderr=true
go run main.go migrate --logtostderr=true -m config/metadata-library

metadata.sqlite.db: run/migrate

Expand Down
4 changes: 4 additions & 0 deletions bin/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore
5 changes: 2 additions & 3 deletions cmd/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
"github.com/opendatahub-io/model-registry/internal/ml_metadata/proto"
"github.com/opendatahub-io/model-registry/internal/model/db"
"github.com/opendatahub-io/model-registry/internal/model/library"
"github.com/opendatahub-io/model-registry/internal/server"
"github.com/opendatahub-io/model-registry/internal/server/grpc"
"github.com/spf13/cobra"
"gorm.io/gorm"
)
Expand Down Expand Up @@ -96,7 +96,7 @@ func loadLibraries(dbConn *gorm.DB) error {
return fmt.Errorf("failed to read library directories %s: %w", libraryDirs, err)
}
for path, lib := range libs {
grpcServer := server.NewGrpcServer(dbConn)
grpcServer := grpc.NewGrpcServer(dbConn)
typesRequest := proto.PutTypesRequest{}
for _, at := range lib.ArtifactTypes {
typesRequest.ArtifactTypes = append(typesRequest.ArtifactTypes, &proto.ArtifactType{
Expand Down Expand Up @@ -131,7 +131,6 @@ func loadLibraries(dbConn *gorm.DB) error {
}
glog.Infof("created/updated %d ArtifactTypes, %d ContextTypes and %d ExecutionTypes from library file %s",
len(response.ArtifactTypeIds), len(response.ContextTypeIds), len(response.ExecutionTypeIds), path)
return nil
}
return nil
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package cmd
import (
"context"
"fmt"
grpc2 "github.com/opendatahub-io/model-registry/internal/server/grpc"
"log"
"net"
"net/http"
Expand All @@ -30,7 +31,6 @@ import (
"github.com/golang/glog"
"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging"
"github.com/opendatahub-io/model-registry/internal/ml_metadata/proto"
"github.com/opendatahub-io/model-registry/internal/server"
"github.com/opendatahub-io/model-registry/internal/server/graph"
"github.com/soheilhy/cmux"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -103,7 +103,7 @@ func runServer(cmd *cobra.Command, args []string) error {
grpcListener := m.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"))

// gRPC server
grpcServer := grpcListenerServer(server.NewGrpcServer(db))
grpcServer := grpcListenerServer(grpc2.NewGrpcServer(db))
// GraphQL server
gqlServer := graphQlListenerServer(db)

Expand Down
30 changes: 0 additions & 30 deletions internal/model/db/artifact.gen.go

This file was deleted.

52 changes: 52 additions & 0 deletions internal/model/db/artifact.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package db

const TableNameArtifact = "Artifact"

// Artifact mapped from table <Artifact>
type Artifact struct {
ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"`
TypeID int64 `gorm:"column:type_id;not null;uniqueIndex:UniqueArtifactTypeName,priority:1" json:"-"`
URI *string `gorm:"column:uri;type:text;index:idx_artifact_uri,priority:1" json:"-"`
State *int8 `gorm:"column:state" json:"-"`
Name *string `gorm:"column:name;type:varchar(255);uniqueIndex:UniqueArtifactTypeName,priority:2" json:"-"`
ExternalID *string `gorm:"column:external_id;type:varchar(255);uniqueIndex:idx_artifact_external_id,priority:1" json:"-"`
CreateTimeSinceEpoch int64 `gorm:"autoCreateTime:milli;column:create_time_since_epoch;not null;index:idx_artifact_create_time_since_epoch,priority:1" json:"-"`
LastUpdateTimeSinceEpoch int64 `gorm:"autoUpdateTime:milli;column:last_update_time_since_epoch;not null;index:idx_artifact_last_update_time_since_epoch,priority:1" json:"-"`

// relationships
Properties []ArtifactProperty
ArtifactType Type `gorm:"foreignKey:TypeID;references:ID"`
Attributions []Attribution `gorm:"foreignKey:ArtifactID;references:ID"`
Events []Event
}

// TableName Artifact's table name
func (*Artifact) TableName() string {
return TableNameArtifact
}

type ArtifactState int

const (
UNKNOWN ArtifactState = iota
// PENDING A state indicating that the artifact may exist.
PENDING
// LIVE A state indicating that the artifact should exist, unless something
// external to the system deletes it.
LIVE
// MARKED_FOR_DELETION A state indicating that the artifact should be deleted.
MARKED_FOR_DELETION
// DELETED A state indicating that the artifact has been deleted.
DELETED
// ABANDONED A state indicating that the artifact has been abandoned, which may be
// due to a failed or cancelled execution.
ABANDONED
// REFERENCE A state indicating that the artifact is a reference artifact. At
// execution start time, the orchestrator produces an output artifact for
// each output key with state PENDING. However, for an intermediate
// artifact, this first artifact's state will be REFERENCE. Intermediate
// artifacts emitted during a component's execution will copy the REFERENCE
// artifact's attributes. At the end of an execution, the artifact state
// should remain REFERENCE instead of being changed to LIVE.
REFERENCE
)
25 changes: 0 additions & 25 deletions internal/model/db/artifactproperty.gen.go

This file was deleted.

102 changes: 102 additions & 0 deletions internal/model/db/artifactproperty.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package db

const TableNameArtifactProperty = "ArtifactProperty"

// ArtifactProperty mapped from table <ArtifactProperty>
type ArtifactProperty struct {
ArtifactID int64 `gorm:"column:artifact_id;primaryKey" json:"-"`
Name string `gorm:"column:name;primaryKey;index:idx_artifact_property_double,priority:1;index:idx_artifact_property_string,priority:1;index:idx_artifact_property_int,priority:1" json:"-"`
IsCustomProperty bool `gorm:"column:is_custom_property;primaryKey;index:idx_artifact_property_double,priority:2;index:idx_artifact_property_string,priority:2;index:idx_artifact_property_int,priority:2" json:"-"`
IntValue *int64 `gorm:"column:int_value;index:idx_artifact_property_int,priority:3" json:"-"`
DoubleValue *float64 `gorm:"column:double_value;index:idx_artifact_property_double,priority:3" json:"-"`
StringValue *string `gorm:"column:string_value;index:idx_artifact_property_string,priority:3" json:"-"`
ByteValue *[]byte `gorm:"column:byte_value" json:"-"`
ProtoValue *[]byte `gorm:"column:proto_value" json:"-"`
BoolValue *bool `gorm:"column:bool_value" json:"-"`
TypeURL *string `gorm:"column:type_url" json:"-"`
}

// TableName ArtifactProperty's table name
func (*ArtifactProperty) TableName() string {
return TableNameArtifactProperty
}

func (p *ArtifactProperty) GetID() int64 {
return p.ArtifactID
}

func (p *ArtifactProperty) SetID(i int64) {
p.ArtifactID = i
}

func (p *ArtifactProperty) GetName() string {
return p.Name
}

func (p *ArtifactProperty) SetName(s string) {
p.Name = s
}

func (p *ArtifactProperty) GetIsCustomProperty() bool {
return p.IsCustomProperty
}

func (p *ArtifactProperty) SetIsCustomProperty(b bool) {
p.IsCustomProperty = b
}

func (p *ArtifactProperty) GetIntValue() *int64 {
return p.IntValue
}

func (p *ArtifactProperty) SetIntValue(i *int64) {
p.IntValue = i
}

func (p *ArtifactProperty) GetDoubleValue() *float64 {
return p.DoubleValue
}

func (p *ArtifactProperty) SetDoubleValue(f *float64) {
p.DoubleValue = f
}

func (p *ArtifactProperty) GetStringValue() *string {
return p.StringValue
}

func (p *ArtifactProperty) SetStringValue(s *string) {
p.StringValue = s
}

func (p *ArtifactProperty) GetByteValue() *[]byte {
return p.ByteValue
}

func (p *ArtifactProperty) SetByteValue(b *[]byte) {
p.ByteValue = b
}

func (p *ArtifactProperty) GetProtoValue() *[]byte {
return p.ProtoValue
}

func (p *ArtifactProperty) SetProtoValue(b *[]byte) {
p.ProtoValue = b
}

func (p *ArtifactProperty) GetBoolValue() *bool {
return p.BoolValue
}

func (p *ArtifactProperty) SetBoolValue(b *bool) {
p.BoolValue = b
}

func (p *ArtifactProperty) GetTypeURL() *string {
return p.TypeURL
}

func (p *ArtifactProperty) SetTypeURL(s *string) {
p.TypeURL = s
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
// Code generated by gorm.io/gen. DO NOT EDIT.
// Code generated by gorm.io/gen. DO NOT EDIT.
// Code generated by gorm.io/gen. DO NOT EDIT.

package db

const TableNameAssociation = "Association"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
// Code generated by gorm.io/gen. DO NOT EDIT.
// Code generated by gorm.io/gen. DO NOT EDIT.
// Code generated by gorm.io/gen. DO NOT EDIT.

package db

const TableNameAttribution = "Attribution"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
// Code generated by gorm.io/gen. DO NOT EDIT.
// Code generated by gorm.io/gen. DO NOT EDIT.
// Code generated by gorm.io/gen. DO NOT EDIT.

package db

const TableNameContext = "Context"
Expand All @@ -11,7 +7,7 @@ type Context struct {
ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"`
TypeID int64 `gorm:"column:type_id;not null;uniqueIndex:UniqueContextName,priority:1" json:"-"`
Name string `gorm:"column:name;not null;uniqueIndex:UniqueContextName,priority:2" json:"-"`
ExternalID *string `gorm:"column:external_id;uniqueIndex:idx_context_external_id,priority:1" json:"-"`
ExternalID *string `gorm:"column:external_id;type:varchar(255);uniqueIndex:idx_context_external_id,priority:1" json:"-"`
CreateTimeSinceEpoch int64 `gorm:"autoCreateTime:milli;column:create_time_since_epoch;not null;index:idx_context_create_time_since_epoch,priority:1" json:"-"`
LastUpdateTimeSinceEpoch int64 `gorm:"autoUpdateTime:milli;column:last_update_time_since_epoch;not null;index:idx_context_last_update_time_since_epoch,priority:1" json:"-"`

Expand All @@ -21,7 +17,7 @@ type Context struct {
Attributions []Attribution `gorm:"foreignKey:ContextID;references:ID"`
Associations []Association `gorm:"foreignKey:ContextID;references:ID"`

// self-reference for context graphs
// many to many self-reference for context graphs
Parents []ParentContext `gorm:"foreignKey:ContextID;references:ID"`
Children []ParentContext `gorm:"foreignKey:ParentContextID;references:ID"`
}
Expand Down
25 changes: 0 additions & 25 deletions internal/model/db/contextproperty.gen.go

This file was deleted.

Loading

0 comments on commit b844d91

Please sign in to comment.