diff --git a/Makefile b/Makefile index 881ce1fc0..907397751 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,10 @@ +# useful paths +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +PROJECT_PATH := $(patsubst %/,%,$(dir $(MKFILE_PATH))) +PROJECT_BIN := $(PROJECT_PATH)/bin + +# add tools bin directory +PATH := $(PROJECT_BIN):$(PATH) model-registry: build internal/ml_metadata/proto/%.pb.go: api/grpc/ml_metadata/proto/%.proto @@ -19,15 +26,15 @@ vet: .PHONY: clean clean: - rm -Rf ./model-registry internal/ml_metadata/proto/*.go internal/model/graph/*.go + rm -Rf ./model-registry internal/ml_metadata/proto/*.go internal/model/graph/models_gen.go .PHONY: deps deps: - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.54.2 - go install github.com/99designs/gqlgen@latest - go install github.com/searKing/golang/tools/go-enum@latest - go install google.golang.org/protobuf/cmd/protoc-gen-go@latest - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(PROJECT_BIN) v1.54.2 + GOBIN=$(PROJECT_BIN) go install github.com/99designs/gqlgen@v0.17.36 + GOBIN=$(PROJECT_BIN) go install github.com/searKing/golang/tools/go-enum@v1.2.97 + GOBIN=$(PROJECT_BIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 + GOBIN=$(PROJECT_BIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 .PHONY: vendor vendor: @@ -48,7 +55,7 @@ lint: gen .PHONY: run/migrate run/migrate: gen - go run main.go migrate --logtostderr=true + go run main.go migrate --logtostderr=true -m config/metadata-library metadata.sqlite.db: run/migrate diff --git a/bin/.gitignore b/bin/.gitignore new file mode 100644 index 000000000..5e7d2734c --- /dev/null +++ b/bin/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/cmd/migrate.go b/cmd/migrate.go index ed6b195ea..85106f1e3 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -22,7 +22,7 @@ import ( "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" "github.com/opendatahub-io/model-registry/internal/model/db" "github.com/opendatahub-io/model-registry/internal/model/library" - "github.com/opendatahub-io/model-registry/internal/server" + "github.com/opendatahub-io/model-registry/internal/server/grpc" "github.com/spf13/cobra" "gorm.io/gorm" ) @@ -96,7 +96,7 @@ func loadLibraries(dbConn *gorm.DB) error { return fmt.Errorf("failed to read library directories %s: %w", libraryDirs, err) } for path, lib := range libs { - grpcServer := server.NewGrpcServer(dbConn) + grpcServer := grpc.NewGrpcServer(dbConn) typesRequest := proto.PutTypesRequest{} for _, at := range lib.ArtifactTypes { typesRequest.ArtifactTypes = append(typesRequest.ArtifactTypes, &proto.ArtifactType{ @@ -131,7 +131,6 @@ func loadLibraries(dbConn *gorm.DB) error { } glog.Infof("created/updated %d ArtifactTypes, %d ContextTypes and %d ExecutionTypes from library file %s", len(response.ArtifactTypeIds), len(response.ContextTypeIds), len(response.ExecutionTypeIds), path) - return nil } return nil } diff --git a/cmd/serve.go b/cmd/serve.go index cde075892..bb5e0ee4e 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -18,6 +18,7 @@ package cmd import ( "context" "fmt" + grpc2 "github.com/opendatahub-io/model-registry/internal/server/grpc" "log" "net" "net/http" @@ -30,7 +31,6 @@ import ( "github.com/golang/glog" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" - "github.com/opendatahub-io/model-registry/internal/server" "github.com/opendatahub-io/model-registry/internal/server/graph" "github.com/soheilhy/cmux" "github.com/spf13/cobra" @@ -103,7 +103,7 @@ func runServer(cmd *cobra.Command, args []string) error { grpcListener := m.MatchWithWriters(cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc")) // gRPC server - grpcServer := grpcListenerServer(server.NewGrpcServer(db)) + grpcServer := grpcListenerServer(grpc2.NewGrpcServer(db)) // GraphQL server gqlServer := graphQlListenerServer(db) diff --git a/internal/model/db/artifact.gen.go b/internal/model/db/artifact.gen.go deleted file mode 100644 index 7545c2771..000000000 --- a/internal/model/db/artifact.gen.go +++ /dev/null @@ -1,30 +0,0 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - -package db - -const TableNameArtifact = "Artifact" - -// Artifact mapped from table -type Artifact struct { - ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"` - TypeID int64 `gorm:"column:type_id;not null;uniqueIndex:UniqueArtifactTypeName,priority:1" json:"-"` - URI *string `gorm:"column:uri;index:idx_artifact_uri,priority:1" json:"-"` - State *int64 `gorm:"column:state" json:"-"` - Name *string `gorm:"column:name;uniqueIndex:UniqueArtifactTypeName,priority:2" json:"-"` - ExternalID *string `gorm:"column:external_id;uniqueIndex:idx_artifact_external_id,priority:1" json:"-"` - CreateTimeSinceEpoch int64 `gorm:"autoCreateTime:milli;column:create_time_since_epoch;not null;index:idx_artifact_create_time_since_epoch,priority:1" json:"-"` - LastUpdateTimeSinceEpoch int64 `gorm:"autoUpdateTime:milli;column:last_update_time_since_epoch;not null;index:idx_artifact_last_update_time_since_epoch,priority:1" json:"-"` - - // relationships - Properties []ArtifactProperty - ArtifactType Type `gorm:"foreignKey:TypeID;references:ID"` - Attributions []Attribution `gorm:"foreignKey:ArtifactID;references:ID"` - Events []Event -} - -// TableName Artifact's table name -func (*Artifact) TableName() string { - return TableNameArtifact -} diff --git a/internal/model/db/artifact.go b/internal/model/db/artifact.go new file mode 100644 index 000000000..7efefd58f --- /dev/null +++ b/internal/model/db/artifact.go @@ -0,0 +1,52 @@ +package db + +const TableNameArtifact = "Artifact" + +// Artifact mapped from table +type Artifact struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"` + TypeID int64 `gorm:"column:type_id;not null;uniqueIndex:UniqueArtifactTypeName,priority:1" json:"-"` + URI *string `gorm:"column:uri;type:text;index:idx_artifact_uri,priority:1" json:"-"` + State *int8 `gorm:"column:state" json:"-"` + Name *string `gorm:"column:name;type:varchar(255);uniqueIndex:UniqueArtifactTypeName,priority:2" json:"-"` + ExternalID *string `gorm:"column:external_id;type:varchar(255);uniqueIndex:idx_artifact_external_id,priority:1" json:"-"` + CreateTimeSinceEpoch int64 `gorm:"autoCreateTime:milli;column:create_time_since_epoch;not null;index:idx_artifact_create_time_since_epoch,priority:1" json:"-"` + LastUpdateTimeSinceEpoch int64 `gorm:"autoUpdateTime:milli;column:last_update_time_since_epoch;not null;index:idx_artifact_last_update_time_since_epoch,priority:1" json:"-"` + + // relationships + Properties []ArtifactProperty + ArtifactType Type `gorm:"foreignKey:TypeID;references:ID"` + Attributions []Attribution `gorm:"foreignKey:ArtifactID;references:ID"` + Events []Event +} + +// TableName Artifact's table name +func (*Artifact) TableName() string { + return TableNameArtifact +} + +type ArtifactState int + +const ( + UNKNOWN ArtifactState = iota + // PENDING A state indicating that the artifact may exist. + PENDING + // LIVE A state indicating that the artifact should exist, unless something + // external to the system deletes it. + LIVE + // MARKED_FOR_DELETION A state indicating that the artifact should be deleted. + MARKED_FOR_DELETION + // DELETED A state indicating that the artifact has been deleted. + DELETED + // ABANDONED A state indicating that the artifact has been abandoned, which may be + // due to a failed or cancelled execution. + ABANDONED + // REFERENCE A state indicating that the artifact is a reference artifact. At + // execution start time, the orchestrator produces an output artifact for + // each output key with state PENDING. However, for an intermediate + // artifact, this first artifact's state will be REFERENCE. Intermediate + // artifacts emitted during a component's execution will copy the REFERENCE + // artifact's attributes. At the end of an execution, the artifact state + // should remain REFERENCE instead of being changed to LIVE. + REFERENCE +) diff --git a/internal/model/db/artifactproperty.gen.go b/internal/model/db/artifactproperty.gen.go deleted file mode 100644 index cb615844d..000000000 --- a/internal/model/db/artifactproperty.gen.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - -package db - -const TableNameArtifactProperty = "ArtifactProperty" - -// ArtifactProperty mapped from table -type ArtifactProperty struct { - ArtifactID int64 `gorm:"column:artifact_id;primaryKey" json:"-"` - Name string `gorm:"column:name;primaryKey;index:idx_artifact_property_double,priority:1;index:idx_artifact_property_string,priority:1;index:idx_artifact_property_int,priority:1" json:"-"` - IsCustomProperty bool `gorm:"column:is_custom_property;primaryKey;index:idx_artifact_property_double,priority:2;index:idx_artifact_property_string,priority:2;index:idx_artifact_property_int,priority:2" json:"-"` - IntValue *int64 `gorm:"column:int_value;index:idx_artifact_property_int,priority:3" json:"-"` - DoubleValue *float64 `gorm:"column:double_value;index:idx_artifact_property_double,priority:3" json:"-"` - StringValue *string `gorm:"column:string_value;index:idx_artifact_property_string,priority:3" json:"-"` - ByteValue *[]byte `gorm:"column:byte_value" json:"-"` - ProtoValue *[]byte `gorm:"column:proto_value" json:"-"` - BoolValue *bool `gorm:"column:bool_value" json:"-"` -} - -// TableName ArtifactProperty's table name -func (*ArtifactProperty) TableName() string { - return TableNameArtifactProperty -} diff --git a/internal/model/db/artifactproperty.go b/internal/model/db/artifactproperty.go new file mode 100644 index 000000000..2a8aa1e8c --- /dev/null +++ b/internal/model/db/artifactproperty.go @@ -0,0 +1,102 @@ +package db + +const TableNameArtifactProperty = "ArtifactProperty" + +// ArtifactProperty mapped from table +type ArtifactProperty struct { + ArtifactID int64 `gorm:"column:artifact_id;primaryKey" json:"-"` + Name string `gorm:"column:name;primaryKey;index:idx_artifact_property_double,priority:1;index:idx_artifact_property_string,priority:1;index:idx_artifact_property_int,priority:1" json:"-"` + IsCustomProperty bool `gorm:"column:is_custom_property;primaryKey;index:idx_artifact_property_double,priority:2;index:idx_artifact_property_string,priority:2;index:idx_artifact_property_int,priority:2" json:"-"` + IntValue *int64 `gorm:"column:int_value;index:idx_artifact_property_int,priority:3" json:"-"` + DoubleValue *float64 `gorm:"column:double_value;index:idx_artifact_property_double,priority:3" json:"-"` + StringValue *string `gorm:"column:string_value;index:idx_artifact_property_string,priority:3" json:"-"` + ByteValue *[]byte `gorm:"column:byte_value" json:"-"` + ProtoValue *[]byte `gorm:"column:proto_value" json:"-"` + BoolValue *bool `gorm:"column:bool_value" json:"-"` + TypeURL *string `gorm:"column:type_url" json:"-"` +} + +// TableName ArtifactProperty's table name +func (*ArtifactProperty) TableName() string { + return TableNameArtifactProperty +} + +func (p *ArtifactProperty) GetID() int64 { + return p.ArtifactID +} + +func (p *ArtifactProperty) SetID(i int64) { + p.ArtifactID = i +} + +func (p *ArtifactProperty) GetName() string { + return p.Name +} + +func (p *ArtifactProperty) SetName(s string) { + p.Name = s +} + +func (p *ArtifactProperty) GetIsCustomProperty() bool { + return p.IsCustomProperty +} + +func (p *ArtifactProperty) SetIsCustomProperty(b bool) { + p.IsCustomProperty = b +} + +func (p *ArtifactProperty) GetIntValue() *int64 { + return p.IntValue +} + +func (p *ArtifactProperty) SetIntValue(i *int64) { + p.IntValue = i +} + +func (p *ArtifactProperty) GetDoubleValue() *float64 { + return p.DoubleValue +} + +func (p *ArtifactProperty) SetDoubleValue(f *float64) { + p.DoubleValue = f +} + +func (p *ArtifactProperty) GetStringValue() *string { + return p.StringValue +} + +func (p *ArtifactProperty) SetStringValue(s *string) { + p.StringValue = s +} + +func (p *ArtifactProperty) GetByteValue() *[]byte { + return p.ByteValue +} + +func (p *ArtifactProperty) SetByteValue(b *[]byte) { + p.ByteValue = b +} + +func (p *ArtifactProperty) GetProtoValue() *[]byte { + return p.ProtoValue +} + +func (p *ArtifactProperty) SetProtoValue(b *[]byte) { + p.ProtoValue = b +} + +func (p *ArtifactProperty) GetBoolValue() *bool { + return p.BoolValue +} + +func (p *ArtifactProperty) SetBoolValue(b *bool) { + p.BoolValue = b +} + +func (p *ArtifactProperty) GetTypeURL() *string { + return p.TypeURL +} + +func (p *ArtifactProperty) SetTypeURL(s *string) { + p.TypeURL = s +} diff --git a/internal/model/db/association.gen.go b/internal/model/db/association.go similarity index 80% rename from internal/model/db/association.gen.go rename to internal/model/db/association.go index cb108c822..662fcd8f5 100644 --- a/internal/model/db/association.gen.go +++ b/internal/model/db/association.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameAssociation = "Association" diff --git a/internal/model/db/attribution.gen.go b/internal/model/db/attribution.go similarity index 80% rename from internal/model/db/attribution.gen.go rename to internal/model/db/attribution.go index f169e201f..3de48eff8 100644 --- a/internal/model/db/attribution.gen.go +++ b/internal/model/db/attribution.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameAttribution = "Attribution" diff --git a/internal/model/db/context.gen.go b/internal/model/db/context.go similarity index 81% rename from internal/model/db/context.gen.go rename to internal/model/db/context.go index a1bf76739..991b6e4b2 100644 --- a/internal/model/db/context.gen.go +++ b/internal/model/db/context.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameContext = "Context" @@ -11,7 +7,7 @@ type Context struct { ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"` TypeID int64 `gorm:"column:type_id;not null;uniqueIndex:UniqueContextName,priority:1" json:"-"` Name string `gorm:"column:name;not null;uniqueIndex:UniqueContextName,priority:2" json:"-"` - ExternalID *string `gorm:"column:external_id;uniqueIndex:idx_context_external_id,priority:1" json:"-"` + ExternalID *string `gorm:"column:external_id;type:varchar(255);uniqueIndex:idx_context_external_id,priority:1" json:"-"` CreateTimeSinceEpoch int64 `gorm:"autoCreateTime:milli;column:create_time_since_epoch;not null;index:idx_context_create_time_since_epoch,priority:1" json:"-"` LastUpdateTimeSinceEpoch int64 `gorm:"autoUpdateTime:milli;column:last_update_time_since_epoch;not null;index:idx_context_last_update_time_since_epoch,priority:1" json:"-"` @@ -21,7 +17,7 @@ type Context struct { Attributions []Attribution `gorm:"foreignKey:ContextID;references:ID"` Associations []Association `gorm:"foreignKey:ContextID;references:ID"` - // self-reference for context graphs + // many to many self-reference for context graphs Parents []ParentContext `gorm:"foreignKey:ContextID;references:ID"` Children []ParentContext `gorm:"foreignKey:ParentContextID;references:ID"` } diff --git a/internal/model/db/contextproperty.gen.go b/internal/model/db/contextproperty.gen.go deleted file mode 100644 index 64cb841fa..000000000 --- a/internal/model/db/contextproperty.gen.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - -package db - -const TableNameContextProperty = "ContextProperty" - -// ContextProperty mapped from table -type ContextProperty struct { - ContextID int64 `gorm:"column:context_id;primaryKey" json:"-"` - Name string `gorm:"column:name;primaryKey;index:idx_context_property_int,priority:1;index:idx_context_property_string,priority:1;index:idx_context_property_double,priority:1" json:"-"` - IsCustomProperty bool `gorm:"column:is_custom_property;primaryKey;index:idx_context_property_int,priority:2;index:idx_context_property_string,priority:2;index:idx_context_property_double,priority:2" json:"-"` - IntValue *int64 `gorm:"column:int_value;index:idx_context_property_int,priority:3" json:"-"` - DoubleValue *float64 `gorm:"column:double_value;index:idx_context_property_double,priority:3" json:"-"` - StringValue *string `gorm:"column:string_value;index:idx_context_property_string,priority:3" json:"-"` - ByteValue *[]byte `gorm:"column:byte_value" json:"-"` - ProtoValue *[]byte `gorm:"column:proto_value" json:"-"` - BoolValue *bool `gorm:"column:bool_value" json:"-"` -} - -// TableName ContextProperty's table name -func (*ContextProperty) TableName() string { - return TableNameContextProperty -} diff --git a/internal/model/db/contextproperty.go b/internal/model/db/contextproperty.go new file mode 100644 index 000000000..0c6b16096 --- /dev/null +++ b/internal/model/db/contextproperty.go @@ -0,0 +1,102 @@ +package db + +const TableNameContextProperty = "ContextProperty" + +// ContextProperty mapped from table +type ContextProperty struct { + ContextID int64 `gorm:"column:context_id;primaryKey" json:"-"` + Name string `gorm:"column:name;primaryKey;index:idx_context_property_int,priority:1;index:idx_context_property_string,priority:1;index:idx_context_property_double,priority:1" json:"-"` + IsCustomProperty bool `gorm:"column:is_custom_property;primaryKey;index:idx_context_property_int,priority:2;index:idx_context_property_string,priority:2;index:idx_context_property_double,priority:2" json:"-"` + IntValue *int64 `gorm:"column:int_value;index:idx_context_property_int,priority:3" json:"-"` + DoubleValue *float64 `gorm:"column:double_value;index:idx_context_property_double,priority:3" json:"-"` + StringValue *string `gorm:"column:string_value;index:idx_context_property_string,priority:3" json:"-"` + ByteValue *[]byte `gorm:"column:byte_value" json:"-"` + ProtoValue *[]byte `gorm:"column:proto_value" json:"-"` + BoolValue *bool `gorm:"column:bool_value" json:"-"` + TypeURL *string `gorm:"column:type_url" json:"-"` +} + +// TableName ContextProperty's table name +func (*ContextProperty) TableName() string { + return TableNameContextProperty +} + +func (p *ContextProperty) GetID() int64 { + return p.ContextID +} + +func (p *ContextProperty) SetID(i int64) { + p.ContextID = i +} + +func (p *ContextProperty) GetName() string { + return p.Name +} + +func (p *ContextProperty) SetName(s string) { + p.Name = s +} + +func (p *ContextProperty) GetIsCustomProperty() bool { + return p.IsCustomProperty +} + +func (p *ContextProperty) SetIsCustomProperty(b bool) { + p.IsCustomProperty = b +} + +func (p *ContextProperty) GetIntValue() *int64 { + return p.IntValue +} + +func (p *ContextProperty) SetIntValue(i *int64) { + p.IntValue = i +} + +func (p *ContextProperty) GetDoubleValue() *float64 { + return p.DoubleValue +} + +func (p *ContextProperty) SetDoubleValue(f *float64) { + p.DoubleValue = f +} + +func (p *ContextProperty) GetStringValue() *string { + return p.StringValue +} + +func (p *ContextProperty) SetStringValue(s *string) { + p.StringValue = s +} + +func (p *ContextProperty) GetByteValue() *[]byte { + return p.ByteValue +} + +func (p *ContextProperty) SetByteValue(b *[]byte) { + p.ByteValue = b +} + +func (p *ContextProperty) GetProtoValue() *[]byte { + return p.ProtoValue +} + +func (p *ContextProperty) SetProtoValue(b *[]byte) { + p.ProtoValue = b +} + +func (p *ContextProperty) GetBoolValue() *bool { + return p.BoolValue +} + +func (p *ContextProperty) SetBoolValue(b *bool) { + p.BoolValue = b +} + +func (p *ContextProperty) GetTypeURL() *string { + return p.TypeURL +} + +func (p *ContextProperty) SetTypeURL(s *string) { + p.TypeURL = s +} diff --git a/internal/model/db/event.gen.go b/internal/model/db/event.gen.go deleted file mode 100644 index 8a113f333..000000000 --- a/internal/model/db/event.gen.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - -package db - -const TableNameEvent = "Event" - -// Event mapped from table -type Event struct { - ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"` - ArtifactID int64 `gorm:"column:artifact_id;not null;uniqueIndex:UniqueEvent,priority:1" json:"-"` - ExecutionID int64 `gorm:"column:execution_id;not null;uniqueIndex:UniqueEvent,priority:2;index:idx_event_execution_id,priority:1" json:"-"` - Type int64 `gorm:"column:type;not null;uniqueIndex:UniqueEvent,priority:3" json:"-"` - MillisecondsSinceEpoch *int64 `gorm:"autoCreateTime:milli;column:milliseconds_since_epoch;not null" json:"-"` - - // relationships - Artifact Artifact - Execution Execution -} - -// TableName Event's table name -func (*Event) TableName() string { - return TableNameEvent -} diff --git a/internal/model/db/event.go b/internal/model/db/event.go new file mode 100644 index 000000000..414607ddb --- /dev/null +++ b/internal/model/db/event.go @@ -0,0 +1,39 @@ +package db + +const TableNameEvent = "Event" + +// Event mapped from table +type Event struct { + ID int64 `gorm:"column:id;not null;primaryKey;autoIncrement:true" json:"-"` + ArtifactID int64 `gorm:"column:artifact_id;not null;uniqueIndex:UniqueEvent,priority:1" json:"-"` + ExecutionID int64 `gorm:"column:execution_id;not null;uniqueIndex:UniqueEvent,priority:2;index:idx_event_execution_id,priority:1" json:"-"` + Type int8 `gorm:"column:type;not null;uniqueIndex:UniqueEvent,priority:3" json:"-"` + MillisecondsSinceEpoch int64 `gorm:"autoCreateTime:milli;column:milliseconds_since_epoch;not null" json:"-"` + + // relationships + PathSteps []EventPath + Artifact Artifact + Execution Execution +} + +// TableName Event's table name +func (*Event) TableName() string { + return TableNameEvent +} + +type EventType int + +// Events distinguish between an artifact that is written by the execution +// (possibly as a cache), versus artifacts that are part of the declared +// output of the Execution. For more information on what DECLARED_ means, +// see the comment on the message. +const ( + EVENT_TYPE_UNKNOWN EventType = iota + DECLARED_OUTPUT + DECLARED_INPUT + INPUT + OUTPUT + INTERNAL_INPUT + INTERNAL_OUTPUT + PENDING_OUTPUT +) diff --git a/internal/model/db/eventpath.gen.go b/internal/model/db/eventpath.go similarity index 69% rename from internal/model/db/eventpath.gen.go rename to internal/model/db/eventpath.go index 1007865d5..600b10023 100644 --- a/internal/model/db/eventpath.gen.go +++ b/internal/model/db/eventpath.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameEventPath = "EventPath" @@ -10,7 +6,7 @@ const TableNameEventPath = "EventPath" type EventPath struct { EventID int64 `gorm:"column:event_id;not null;index:idx_eventpath_event_id,priority:1" json:"-"` IsIndexStep bool `gorm:"column:is_index_step;not null" json:"-"` - StepIndex *int64 `gorm:"column:step_index" json:"-"` + StepIndex *int `gorm:"column:step_index" json:"-"` StepKey *string `gorm:"column:step_key" json:"-"` } diff --git a/internal/model/db/execution.gen.go b/internal/model/db/execution.go similarity index 68% rename from internal/model/db/execution.gen.go rename to internal/model/db/execution.go index f12c1bfea..fe0674170 100644 --- a/internal/model/db/execution.gen.go +++ b/internal/model/db/execution.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameExecution = "Execution" @@ -10,9 +6,9 @@ const TableNameExecution = "Execution" type Execution struct { ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"` TypeID int64 `gorm:"column:type_id;not null;uniqueIndex:UniqueExecutionTypeName,priority:1" json:"-"` - LastKnownState *int64 `gorm:"column:last_known_state" json:"-"` - Name *string `gorm:"column:name;uniqueIndex:UniqueExecutionTypeName,priority:2" json:"-"` - ExternalID *string `gorm:"column:external_id;uniqueIndex:idx_execution_external_id,priority:1" json:"-"` + LastKnownState *int8 `gorm:"column:last_known_state" json:"-"` + Name *string `gorm:"column:name;type:varchar(255);uniqueIndex:UniqueExecutionTypeName,priority:2" json:"-"` + ExternalID *string `gorm:"column:external_id;type:varchar(255);uniqueIndex:idx_execution_external_id,priority:1" json:"-"` CreateTimeSinceEpoch int64 `gorm:"autoCreateTime:milli;column:create_time_since_epoch;not null;index:idx_execution_create_time_since_epoch,priority:1" json:"-"` LastUpdateTimeSinceEpoch int64 `gorm:"autoUpdateTime:milli;column:last_update_time_since_epoch;not null;index:idx_execution_last_update_time_since_epoch,priority:1" json:"-"` @@ -27,3 +23,15 @@ type Execution struct { func (*Execution) TableName() string { return TableNameExecution } + +type ExecutionState int + +const ( + EXECUTION_STATE_UNKNOWN ExecutionState = iota + NEW + RUNNING + COMPLETE + FAILED + CACHED + CANCELED +) diff --git a/internal/model/db/executionproperty.gen.go b/internal/model/db/executionproperty.gen.go deleted file mode 100644 index 597354889..000000000 --- a/internal/model/db/executionproperty.gen.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - -package db - -const TableNameExecutionProperty = "ExecutionProperty" - -// ExecutionProperty mapped from table -type ExecutionProperty struct { - ExecutionID int64 `gorm:"column:execution_id;primaryKey" json:"-"` - Name string `gorm:"column:name;primaryKey;index:idx_execution_property_string,priority:1;index:idx_execution_property_int,priority:1;index:idx_execution_property_double,priority:1" json:"-"` - IsCustomProperty bool `gorm:"column:is_custom_property;primaryKey;index:idx_execution_property_string,priority:2;index:idx_execution_property_int,priority:2;index:idx_execution_property_double,priority:2" json:"-"` - IntValue *int64 `gorm:"column:int_value;index:idx_execution_property_int,priority:3" json:"-"` - DoubleValue *float64 `gorm:"column:double_value;index:idx_execution_property_double,priority:3" json:"-"` - StringValue *string `gorm:"column:string_value;index:idx_execution_property_string,priority:3" json:"-"` - ByteValue *[]byte `gorm:"column:byte_value" json:"-"` - ProtoValue *[]byte `gorm:"column:proto_value" json:"-"` - BoolValue *bool `gorm:"column:bool_value" json:"-"` -} - -// TableName ExecutionProperty's table name -func (*ExecutionProperty) TableName() string { - return TableNameExecutionProperty -} diff --git a/internal/model/db/executionproperty.go b/internal/model/db/executionproperty.go new file mode 100644 index 000000000..e40619a36 --- /dev/null +++ b/internal/model/db/executionproperty.go @@ -0,0 +1,102 @@ +package db + +const TableNameExecutionProperty = "ExecutionProperty" + +// ExecutionProperty mapped from table +type ExecutionProperty struct { + ExecutionID int64 `gorm:"column:execution_id;primaryKey" json:"-"` + Name string `gorm:"column:name;primaryKey;index:idx_execution_property_string,priority:1;index:idx_execution_property_int,priority:1;index:idx_execution_property_double,priority:1" json:"-"` + IsCustomProperty bool `gorm:"column:is_custom_property;primaryKey;index:idx_execution_property_string,priority:2;index:idx_execution_property_int,priority:2;index:idx_execution_property_double,priority:2" json:"-"` + IntValue *int64 `gorm:"column:int_value;index:idx_execution_property_int,priority:3" json:"-"` + DoubleValue *float64 `gorm:"column:double_value;index:idx_execution_property_double,priority:3" json:"-"` + StringValue *string `gorm:"column:string_value;index:idx_execution_property_string,priority:3" json:"-"` + ByteValue *[]byte `gorm:"column:byte_value" json:"-"` + ProtoValue *[]byte `gorm:"column:proto_value" json:"-"` + BoolValue *bool `gorm:"column:bool_value" json:"-"` + TypeURL *string `gorm:"column:type_url" json:"-"` +} + +// TableName ExecutionProperty's table name +func (*ExecutionProperty) TableName() string { + return TableNameExecutionProperty +} + +func (p *ExecutionProperty) GetID() int64 { + return p.ExecutionID +} + +func (p *ExecutionProperty) SetID(i int64) { + p.ExecutionID = i +} + +func (p *ExecutionProperty) GetName() string { + return p.Name +} + +func (p *ExecutionProperty) SetName(s string) { + p.Name = s +} + +func (p *ExecutionProperty) GetIsCustomProperty() bool { + return p.IsCustomProperty +} + +func (p *ExecutionProperty) SetIsCustomProperty(b bool) { + p.IsCustomProperty = b +} + +func (p *ExecutionProperty) GetIntValue() *int64 { + return p.IntValue +} + +func (p *ExecutionProperty) SetIntValue(i *int64) { + p.IntValue = i +} + +func (p *ExecutionProperty) GetDoubleValue() *float64 { + return p.DoubleValue +} + +func (p *ExecutionProperty) SetDoubleValue(f *float64) { + p.DoubleValue = f +} + +func (p *ExecutionProperty) GetStringValue() *string { + return p.StringValue +} + +func (p *ExecutionProperty) SetStringValue(s *string) { + p.StringValue = s +} + +func (p *ExecutionProperty) GetByteValue() *[]byte { + return p.ByteValue +} + +func (p *ExecutionProperty) SetByteValue(b *[]byte) { + p.ByteValue = b +} + +func (p *ExecutionProperty) GetProtoValue() *[]byte { + return p.ProtoValue +} + +func (p *ExecutionProperty) SetProtoValue(b *[]byte) { + p.ProtoValue = b +} + +func (p *ExecutionProperty) GetBoolValue() *bool { + return p.BoolValue +} + +func (p *ExecutionProperty) SetBoolValue(b *bool) { + p.BoolValue = b +} + +func (p *ExecutionProperty) GetTypeURL() *string { + return p.TypeURL +} + +func (p *ExecutionProperty) SetTypeURL(s *string) { + p.TypeURL = s +} diff --git a/internal/model/db/mlmdenv.gen.go b/internal/model/db/mlmdenv.go similarity index 66% rename from internal/model/db/mlmdenv.gen.go rename to internal/model/db/mlmdenv.go index c12c60d1c..e0b85cd30 100644 --- a/internal/model/db/mlmdenv.gen.go +++ b/internal/model/db/mlmdenv.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameMLMDEnv = "MLMDEnv" diff --git a/internal/model/db/parentcontext.gen.go b/internal/model/db/parentcontext.go similarity index 76% rename from internal/model/db/parentcontext.gen.go rename to internal/model/db/parentcontext.go index 8e9b2e9f5..5b0c4824f 100644 --- a/internal/model/db/parentcontext.gen.go +++ b/internal/model/db/parentcontext.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameParentContext = "ParentContext" diff --git a/internal/model/db/parenttype.gen.go b/internal/model/db/parenttype.go similarity index 72% rename from internal/model/db/parenttype.gen.go rename to internal/model/db/parenttype.go index f4863b9dc..5fc0b3643 100644 --- a/internal/model/db/parenttype.gen.go +++ b/internal/model/db/parenttype.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameParentType = "ParentType" diff --git a/internal/model/db/property_interface.go b/internal/model/db/property_interface.go new file mode 100644 index 000000000..18e512acd --- /dev/null +++ b/internal/model/db/property_interface.go @@ -0,0 +1,24 @@ +package db + +type MetadataProperty interface { + GetID() int64 + SetID(int64) + GetName() string + SetName(string) + GetIsCustomProperty() bool + SetIsCustomProperty(bool) + GetIntValue() *int64 + SetIntValue(*int64) + GetDoubleValue() *float64 + SetDoubleValue(*float64) + GetStringValue() *string + SetStringValue(*string) + GetByteValue() *[]byte + SetByteValue(*[]byte) + GetProtoValue() *[]byte + SetProtoValue(*[]byte) + GetBoolValue() *bool + SetBoolValue(*bool) + GetTypeURL() *string + SetTypeURL(*string) +} diff --git a/internal/model/db/type.gen.go b/internal/model/db/type.gen.go deleted file mode 100644 index 7b626e281..000000000 --- a/internal/model/db/type.gen.go +++ /dev/null @@ -1,27 +0,0 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - -package db - -const TableNameType = "Type" - -// Type mapped from table -type Type struct { - ID int64 `gorm:"column:id;primaryKey;autoIncrement:true" json:"-"` - Name string `gorm:"column:name;not null;uniqueIndex:idx_type_name,priority:1" json:"-"` - Version *string `gorm:"column:version" json:"-"` - TypeKind int32 `gorm:"column:type_kind;not null" json:"-"` - Description *string `gorm:"column:description" json:"-"` - InputType *string `gorm:"column:input_type" json:"-"` - OutputType *string `gorm:"column:output_type" json:"-"` - ExternalID *string `gorm:"column:external_id;uniqueIndex:idx_type_external_id,priority:1" json:"-"` - - // relationships - Properties []TypeProperty -} - -// TableName Type's table name -func (*Type) TableName() string { - return TableNameType -} diff --git a/internal/model/db/type.go b/internal/model/db/type.go new file mode 100644 index 000000000..4b30d57a3 --- /dev/null +++ b/internal/model/db/type.go @@ -0,0 +1,32 @@ +package db + +const TableNameType = "Type" + +// Type mapped from table +type Type struct { + ID int64 `gorm:"column:id;not null;primaryKey;autoIncrement:true" json:"-"` + Name string `gorm:"column:name;type:varchar(255);not null;uniqueIndex:idx_type_name,priority:1" json:"-"` + Version *string `gorm:"column:version;type:varchar(255)" json:"-"` + TypeKind int8 `gorm:"column:type_kind;not null" json:"-"` + Description *string `gorm:"column:description;type:text" json:"-"` + InputType *string `gorm:"column:input_type;type:text" json:"-"` + OutputType *string `gorm:"column:output_type;type:text" json:"-"` + ExternalID *string `gorm:"column:external_id;type:varchar(255);uniqueIndex:idx_type_external_id,priority:1" json:"-"` + + // relationships + Properties []TypeProperty +} + +// TableName Type's table name +func (*Type) TableName() string { + return TableNameType +} + +type TypeKind int + +// artifact type values from ml-metadata table values +const ( + EXECUTION_TYPE TypeKind = iota + ARTIFACT_TYPE + CONTEXT_TYPE +) diff --git a/internal/model/db/typeproperty.gen.go b/internal/model/db/typeproperty.go similarity index 75% rename from internal/model/db/typeproperty.gen.go rename to internal/model/db/typeproperty.go index c60ebbea1..0e9c237c6 100644 --- a/internal/model/db/typeproperty.gen.go +++ b/internal/model/db/typeproperty.go @@ -1,7 +1,3 @@ -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. -// Code generated by gorm.io/gen. DO NOT EDIT. - package db const TableNameTypeProperty = "TypeProperty" diff --git a/internal/server/grpc_server.go b/internal/server/grpc/grpc_server.go similarity index 94% rename from internal/server/grpc_server.go rename to internal/server/grpc/grpc_server.go index 00b401b21..c4d4fec3c 100644 --- a/internal/server/grpc_server.go +++ b/internal/server/grpc/grpc_server.go @@ -1,4 +1,4 @@ -package server +package grpc import ( "context" @@ -6,32 +6,12 @@ import ( "github.com/opendatahub-io/model-registry/internal/ml_metadata/proto" "github.com/opendatahub-io/model-registry/internal/model/db" + "github.com/opendatahub-io/model-registry/internal/server" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "gorm.io/gorm" ) -type TypeKind int32 - -// artifact type values from ml-metadata table values -const ( - EXECUTION_TYPE TypeKind = iota - ARTIFACT_TYPE - CONTEXT_TYPE -) - -func (tk TypeKind) String() string { - switch tk { - case EXECUTION_TYPE: - return "Execution" - case ARTIFACT_TYPE: - return "Artifact" - case CONTEXT_TYPE: - return "Context" - } - return "unknown" -} - type grpcServer struct { proto.UnimplementedMetadataStoreServiceServer dbConnection *gorm.DB @@ -47,7 +27,7 @@ func NewGrpcServer(dbConnection *gorm.DB) proto.MetadataStoreServiceServer { var REQUIRED_TYPE_FIELDS = []string{"name"} func (g grpcServer) PutArtifactType(ctx context.Context, request *proto.PutArtifactTypeRequest) (resp *proto.PutArtifactTypeResponse, err error) { - ctx, _ = Begin(ctx, g.dbConnection) + ctx, _ = server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) artifactType := request.GetArtifactType() @@ -59,7 +39,7 @@ func (g grpcServer) PutArtifactType(ctx context.Context, request *proto.PutArtif value := &db.Type{ Name: *artifactType.Name, Version: artifactType.Version, - TypeKind: int32(ARTIFACT_TYPE), + TypeKind: int8(db.ARTIFACT_TYPE), Description: artifactType.Description, ExternalID: artifactType.ExternalId, } @@ -76,7 +56,7 @@ func (g grpcServer) PutArtifactType(ctx context.Context, request *proto.PutArtif func (g grpcServer) createOrUpdateType(ctx context.Context, value *db.Type, properties map[string]proto.PropertyType) error { // TODO handle CanAdd, CanOmit properties from type request - dbConn, _ := FromContext(ctx) + dbConn, _ := server.FromContext(ctx) if err := dbConn.Where("name = ?", value.Name).Assign(value).FirstOrCreate(value).Error; err != nil { err = fmt.Errorf("error creating type %s: %v", value.Name, err) @@ -90,7 +70,7 @@ func (g grpcServer) createOrUpdateType(ctx context.Context, value *db.Type, } func (g grpcServer) PutExecutionType(ctx context.Context, request *proto.PutExecutionTypeRequest) (resp *proto.PutExecutionTypeResponse, err error) { - ctx, _ = Begin(ctx, g.dbConnection) + ctx, _ = server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) executionType := request.GetExecutionType() @@ -101,7 +81,7 @@ func (g grpcServer) PutExecutionType(ctx context.Context, request *proto.PutExec value := &db.Type{ Name: *executionType.Name, Version: executionType.Version, - TypeKind: int32(EXECUTION_TYPE), + TypeKind: int8(db.EXECUTION_TYPE), Description: executionType.Description, ExternalID: executionType.ExternalId, } @@ -116,7 +96,7 @@ func (g grpcServer) PutExecutionType(ctx context.Context, request *proto.PutExec } func (g grpcServer) PutContextType(ctx context.Context, request *proto.PutContextTypeRequest) (resp *proto.PutContextTypeResponse, err error) { - ctx, _ = Begin(ctx, g.dbConnection) + ctx, _ = server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) contextType := request.GetContextType() @@ -127,7 +107,7 @@ func (g grpcServer) PutContextType(ctx context.Context, request *proto.PutContex value := &db.Type{ Name: *contextType.Name, Version: contextType.Version, - TypeKind: int32(CONTEXT_TYPE), + TypeKind: int8(db.CONTEXT_TYPE), Description: contextType.Description, ExternalID: contextType.ExternalId, } @@ -142,7 +122,7 @@ func (g grpcServer) PutContextType(ctx context.Context, request *proto.PutContex } func (g grpcServer) PutTypes(ctx context.Context, request *proto.PutTypesRequest) (resp *proto.PutTypesResponse, err error) { - ctx, _ = Begin(ctx, g.dbConnection) + ctx, _ = server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) response := &proto.PutTypesResponse{} @@ -192,7 +172,7 @@ func (g grpcServer) PutTypes(ctx context.Context, request *proto.PutTypesRequest var REQUIRED_ARTIFACT_FIELDS = []string{"type_id", "uri"} func (g grpcServer) PutArtifacts(ctx context.Context, request *proto.PutArtifactsRequest) (resp *proto.PutArtifactsResponse, err error) { - ctx, dbConn := Begin(ctx, g.dbConnection) + ctx, dbConn := server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) var artifactIds []int64 @@ -208,7 +188,10 @@ func (g grpcServer) PutArtifacts(ctx context.Context, request *proto.PutArtifact ExternalID: artifact.ExternalId, } nilSafeCopy(&value.ID, artifact.Id, identity[int64]) - nilSafeCopy(&value.State, artifact.State, artifactStateToInt64) + if artifact.State != nil { + state := int8(*artifact.State) + value.State = &state + } // create in DB if err = dbConn.Create(value).Error; err != nil { err = fmt.Errorf("error creating artifact with type_id[%d], name[%s]: %w", value.TypeID, *value.Name, err) @@ -267,14 +250,14 @@ func (g grpcServer) PutParentContexts(ctx context.Context, request *proto.PutPar } func (g grpcServer) GetArtifactType(ctx context.Context, request *proto.GetArtifactTypeRequest) (resp *proto.GetArtifactTypeResponse, err error) { - ctx, dbConn := Begin(ctx, g.dbConnection) + ctx, dbConn := server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) err = requiredFields(REQUIRED_TYPE_FIELDS, request.TypeName) response := &proto.GetArtifactTypeResponse{} var results []db.Type - rx := dbConn.Find(&results, db.Type{Name: *request.TypeName, TypeKind: int32(ARTIFACT_TYPE), Version: request.TypeVersion}) + rx := dbConn.Find(&results, db.Type{Name: *request.TypeName, TypeKind: int8(db.ARTIFACT_TYPE), Version: request.TypeVersion}) if rx.Error != nil { return nil, rx.Error } @@ -501,7 +484,7 @@ func (g grpcServer) mustEmbedUnimplementedMetadataStoreServiceServer() { } func (g grpcServer) createTypeProperties(ctx context.Context, properties map[string]proto.PropertyType, typeId int64) (err error) { - ctx, dbConn := Begin(ctx, g.dbConnection) + ctx, dbConn := server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) for propName, prop := range properties { @@ -521,7 +504,7 @@ func (g grpcServer) createTypeProperties(ctx context.Context, properties map[str } func (g grpcServer) createArtifactProperties(ctx context.Context, artifactId int64, properties map[string]*proto.Value, isCustomProperty bool) (err error) { - ctx, dbConn := Begin(ctx, g.dbConnection) + ctx, dbConn := server.Begin(ctx, g.dbConnection) defer handleTransaction(ctx, &err) for propName, prop := range properties { @@ -571,10 +554,6 @@ func (g grpcServer) createArtifactProperties(ctx context.Context, artifactId int } func identity[T int64 | string](i T) T { return i } -func artifactStateToInt64(i proto.Artifact_State) *int64 { - var result = int64(i) - return &result -} func requiredFields(names []string, args ...interface{}) error { var missing []string @@ -597,14 +576,14 @@ func nilSafeCopy[D int32 | int64 | *int64 | string, S int64 | proto.Artifact_Sta func handleTransaction(ctx context.Context, err *error) { // handle panic if perr := recover(); perr != nil { - _ = Rollback(ctx) + _ = server.Rollback(ctx) *err = status.Errorf(codes.Internal, "server panic: %v", perr) return } if err == nil || *err == nil { - *err = Commit(ctx) + *err = server.Commit(ctx) } else { - _ = Rollback(ctx) + _ = server.Rollback(ctx) if _, ok := status.FromError(*err); !ok { *err = status.Errorf(codes.Internal, "internal error: %v", *err) }