Skip to content

Commit 629da61

Browse files
congqixiacongqixia
authored
enhance: [2.5][GoSDK] sync milvusclient patches for 2.5.1 (#40410)
Cherry pick from master pr: #40268 #40284 #40328 #40373 #40381 ------------------------------------------ #### fix: [GoSDK] Pass base64 passwd content instead of raw data (#40268) Related to #40261 Also add some options for create collection options and refine some behavior ------------------------------------------ #### fix: [GoSDK] Return role without grants (#40284) Related to #40274 Previousy DescribeRole returns only roles with grants, this PR add select role action to check role existence. Also added database properties related option ----------------------------------------- #### fix: [GoSDK] Pass only valid data for nullable column (#40328) Related to #40327 ----------------------------------------- #### enhance: [GoSDK] Add DescribeReplica API & sync rbac v2 (#40373) Related to #31293 #37031 This PR: - Add DescribeReplica API - Add unified RBAC v2 API names(AddPrivilegesToGroup, RemovePrivilegesFromGroup, GrantPrivilegeV2, RevokePrivilegeV2) - Mark old ones deprecated ----------------------------------------- #### enhance: [GoSDK] support update ts caching policy(#40381) Related to #39093 This PR add update timestamp check and retry policy according to the design of the related issue ----------------------------------------- --------- Signed-off-by: Congqi Xia <[email protected]>
1 parent f83567c commit 629da61

25 files changed

+803
-168
lines changed

‎client/column/array.go

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func (c *columnArrayBase[T]) FieldData() *schemapb.FieldData {
3232
fd := &schemapb.FieldData{
3333
Type: schemapb.DataType_Array,
3434
FieldName: c.name,
35+
ValidData: c.validData,
3536
}
3637

3738
data := make([]*schemapb.ScalarField, 0, c.Len())

‎client/column/generic_base.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ func (c *genericColumnBase[T]) Type() entity.FieldType {
4949
}
5050

5151
func (c *genericColumnBase[T]) Len() int {
52+
if c.validData != nil {
53+
return len(c.validData)
54+
}
5255
return len(c.values)
5356
}
5457

@@ -166,9 +169,9 @@ func (c *genericColumnBase[T]) AppendNull() error {
166169
if !c.nullable {
167170
return errors.New("append null to not nullable column")
168171
}
169-
var v T
172+
// var v T
170173
c.validData = append(c.validData, true)
171-
c.values = append(c.values, v)
174+
// c.values = append(c.values, v)
172175
return nil
173176
}
174177

‎client/column/nullable.go

+10-3
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616

1717
package column
1818

19-
import "github.com/cockroachdb/errors"
19+
import (
20+
"github.com/cockroachdb/errors"
21+
"github.com/samber/lo"
22+
)
2023

2124
var (
2225
// scalars
@@ -55,9 +58,13 @@ type NullableColumnCreator[col interface {
5558

5659
func (c NullableColumnCreator[col, T]) New(name string, values []T, validData []bool) (col, error) {
5760
var result col
58-
if len(values) != len(validData) {
59-
return result, errors.New("values & validData slice has different length")
61+
validCnt := lo.CountBy(validData, func(v bool) bool {
62+
return v
63+
})
64+
if validCnt != len(values) {
65+
return result, errors.Newf("values number(%d) does not match valid count(%d)", len(values), validCnt)
6066
}
67+
6168
result = c.base(name, values)
6269
result.withValidData(validData)
6370

‎client/column/nullable_test.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ type NullableScalarSuite struct {
3333
func (s *NullableScalarSuite) TestBasic() {
3434
s.Run("nullable_bool", func() {
3535
name := fmt.Sprintf("field_%d", rand.Intn(1000))
36-
data := []bool{true, false}
36+
data := []bool{false}
3737
validData := []bool{true, false}
3838
column, err := NewNullableColumnBool(name, data, validData)
3939
s.NoError(err)
@@ -63,7 +63,7 @@ func (s *NullableScalarSuite) TestBasic() {
6363

6464
s.Run("nullable_int8", func() {
6565
name := fmt.Sprintf("field_%d", rand.Intn(1000))
66-
data := []int8{1, 2, 3}
66+
data := []int8{1, 3}
6767
validData := []bool{true, false, true}
6868
column, err := NewNullableColumnInt8(name, data, validData)
6969
s.NoError(err)
@@ -93,7 +93,7 @@ func (s *NullableScalarSuite) TestBasic() {
9393

9494
s.Run("nullable_int16", func() {
9595
name := fmt.Sprintf("field_%d", rand.Intn(1000))
96-
data := []int16{1, 2, 3}
96+
data := []int16{1, 3}
9797
validData := []bool{true, false, true}
9898
column, err := NewNullableColumnInt16(name, data, validData)
9999
s.NoError(err)
@@ -123,7 +123,7 @@ func (s *NullableScalarSuite) TestBasic() {
123123

124124
s.Run("nullable_int32", func() {
125125
name := fmt.Sprintf("field_%d", rand.Intn(1000))
126-
data := []int32{1, 2, 3}
126+
data := []int32{1, 3}
127127
validData := []bool{true, false, true}
128128
column, err := NewNullableColumnInt32(name, data, validData)
129129
s.NoError(err)
@@ -153,7 +153,7 @@ func (s *NullableScalarSuite) TestBasic() {
153153

154154
s.Run("nullable_int64", func() {
155155
name := fmt.Sprintf("field_%d", rand.Intn(1000))
156-
data := []int64{1, 2, 3}
156+
data := []int64{1, 3}
157157
validData := []bool{true, false, true}
158158
column, err := NewNullableColumnInt64(name, data, validData)
159159
s.NoError(err)
@@ -183,7 +183,7 @@ func (s *NullableScalarSuite) TestBasic() {
183183

184184
s.Run("nullable_float", func() {
185185
name := fmt.Sprintf("field_%d", rand.Intn(1000))
186-
data := []float32{0.1, 0.2, 0.3}
186+
data := []float32{0.1, 0.3}
187187
validData := []bool{true, false, true}
188188
column, err := NewNullableColumnFloat(name, data, validData)
189189
s.NoError(err)
@@ -213,7 +213,7 @@ func (s *NullableScalarSuite) TestBasic() {
213213

214214
s.Run("nullable_double", func() {
215215
name := fmt.Sprintf("field_%d", rand.Intn(1000))
216-
data := []float64{0.1, 0.2, 0.3}
216+
data := []float64{0.1, 0.3}
217217
validData := []bool{true, false, true}
218218
column, err := NewNullableColumnDouble(name, data, validData)
219219
s.NoError(err)

‎client/entity/collection.go

+3
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ type Collection struct {
3333
ConsistencyLevel ConsistencyLevel
3434
ShardNum int32
3535
Properties map[string]string
36+
37+
// collection update timestamp, usually used for internal change detection
38+
UpdateTimestamp uint64
3639
}
3740

3841
// Partition represent partition meta in Milvus

‎client/entity/field.go

+1
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ func (f *Field) ProtoMessage() *schemapb.FieldSchema {
212212
IsPartitionKey: f.IsPartitionKey,
213213
IsClusteringKey: f.IsClusteringKey,
214214
ElementType: schemapb.DataType(f.ElementType),
215+
Nullable: f.Nullable,
215216
}
216217
}
217218

‎client/entity/resource_group.go

+14
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,17 @@ type ResourceGroupConfig struct {
5252
TransferTo []*ResourceGroupTransfer
5353
NodeFilter ResourceGroupNodeFilter
5454
}
55+
56+
type ReplicaInfo struct {
57+
ReplicaID int64
58+
Shards []*Shard
59+
Nodes []int64
60+
ResourceGroupName string
61+
NumOutboundNode map[string]int32
62+
}
63+
64+
type Shard struct {
65+
ChannelName string
66+
ShardNodes []int64
67+
ShardLeader int64
68+
}

‎client/index/index.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,12 @@ func (gi GenericIndex) Params() map[string]string {
6868
return m
6969
}
7070

71+
func (gi GenericIndex) WithMetricType(metricType MetricType) {
72+
gi.baseIndex.metricType = metricType
73+
}
74+
7175
// NewGenericIndex create generic index instance
72-
func NewGenericIndex(name string, params map[string]string) Index {
76+
func NewGenericIndex(name string, params map[string]string) GenericIndex {
7377
return GenericIndex{
7478
baseIndex: baseIndex{
7579
name: name,

‎client/milvusclient/collection.go

+1
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ func (c *Client) DescribeCollection(ctx context.Context, option DescribeCollecti
9595
ConsistencyLevel: entity.ConsistencyLevel(resp.ConsistencyLevel),
9696
ShardNum: resp.GetShardsNum(),
9797
Properties: entity.KvPairsMap(resp.GetProperties()),
98+
UpdateTimestamp: resp.GetUpdateTimestamp(),
9899
}
99100
collection.Name = collection.Schema.CollectionName
100101
return nil

‎client/milvusclient/collection_example_test.go

+62-1
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,67 @@ func ExampleClient_CreateCollection_ttl() {
154154
}
155155
}
156156

157+
func ExampleClient_CreateCollection_quickSetup() {
158+
ctx, cancel := context.WithCancel(context.Background())
159+
defer cancel()
160+
161+
collectionName := `quick_setup_1`
162+
cli, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
163+
Address: milvusAddr,
164+
})
165+
if err != nil {
166+
// handle err
167+
}
168+
169+
err = cli.CreateCollection(ctx, milvusclient.SimpleCreateCollectionOptions(collectionName, 512))
170+
if err != nil {
171+
// handle error
172+
}
173+
}
174+
175+
func ExampleClient_CreateCollection_quickSetupWithIndexParams() {
176+
ctx, cancel := context.WithCancel(context.Background())
177+
defer cancel()
178+
179+
collectionName := `quick_setup_2`
180+
cli, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
181+
Address: milvusAddr,
182+
})
183+
if err != nil {
184+
// handle err
185+
}
186+
187+
err = cli.CreateCollection(ctx, milvusclient.SimpleCreateCollectionOptions(collectionName, 512).WithIndexOptions(
188+
milvusclient.NewCreateIndexOption(collectionName, "vector", index.NewHNSWIndex(entity.L2, 64, 128)),
189+
))
190+
if err != nil {
191+
log.Println(err.Error())
192+
// handle error
193+
}
194+
}
195+
196+
func ExampleClient_CreateCollection_quickSetupCustomize() {
197+
ctx, cancel := context.WithCancel(context.Background())
198+
defer cancel()
199+
200+
collectionName := `quick_setup_3`
201+
cli, err := milvusclient.New(ctx, &milvusclient.ClientConfig{
202+
Address: milvusAddr,
203+
})
204+
if err != nil {
205+
// handle err
206+
}
207+
208+
err = cli.CreateCollection(ctx, milvusclient.SimpleCreateCollectionOptions(collectionName, 512).
209+
WithVarcharPK(true, 64).
210+
WithShardNum(1),
211+
)
212+
if err != nil {
213+
log.Println(err.Error())
214+
// handle error
215+
}
216+
}
217+
157218
func ExampleClient_CreateCollection_consistencyLevel() {
158219
ctx, cancel := context.WithCancel(context.Background())
159220
defer cancel()
@@ -245,7 +306,7 @@ func ExampleClient_RenameCollection() {
245306
}
246307
}
247308

248-
func ExampleClient_AlterCollection_setTTL() {
309+
func ExampleClient_AlterCollectionProperties_setTTL() {
249310
ctx, cancel := context.WithCancel(context.Background())
250311
defer cancel()
251312

‎client/milvusclient/collection_options.go

+23-3
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func (opt *createCollectionOption) WithVarcharPK(varcharPK bool, maxLen int) *cr
8888
}
8989

9090
func (opt *createCollectionOption) WithIndexOptions(indexOpts ...CreateIndexOption) *createCollectionOption {
91-
opt.indexOptions = append(opt.indexOptions, indexOpts...)
91+
opt.indexOptions = indexOpts
9292
return opt
9393
}
9494

@@ -102,6 +102,26 @@ func (opt *createCollectionOption) WithConsistencyLevel(cl entity.ConsistencyLev
102102
return opt
103103
}
104104

105+
func (opt *createCollectionOption) WithMetricType(metricType entity.MetricType) *createCollectionOption {
106+
opt.metricType = metricType
107+
return opt
108+
}
109+
110+
func (opt *createCollectionOption) WithPKFieldName(name string) *createCollectionOption {
111+
opt.pkFieldName = name
112+
return opt
113+
}
114+
115+
func (opt *createCollectionOption) WithVectorFieldName(name string) *createCollectionOption {
116+
opt.vectorFieldName = name
117+
return opt
118+
}
119+
120+
func (opt *createCollectionOption) WithNumPartitions(numPartitions int64) *createCollectionOption {
121+
opt.numPartitions = numPartitions
122+
return opt
123+
}
124+
105125
func (opt *createCollectionOption) Request() *milvuspb.CreateCollectionRequest {
106126
// fast create collection
107127
if opt.isFast {
@@ -140,12 +160,12 @@ func (opt *createCollectionOption) Request() *milvuspb.CreateCollectionRequest {
140160

141161
func (opt *createCollectionOption) Indexes() []CreateIndexOption {
142162
// fast create
143-
if opt.isFast {
163+
if opt.isFast && opt.indexOptions == nil {
144164
return []CreateIndexOption{
145165
NewCreateIndexOption(opt.name, opt.vectorFieldName, index.NewGenericIndex("", map[string]string{})),
146166
}
147167
}
148-
return nil
168+
return opt.indexOptions
149169
}
150170

151171
func (opt *createCollectionOption) IsFast() bool {

‎client/milvusclient/collection_test.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,9 @@ func (s *CollectionSuite) TestCreateCollectionOptions() {
104104
s.True(collSchema.GetEnableDynamicField())
105105

106106
collectionName = fmt.Sprintf("test_collection_%s", s.randString(6))
107-
opt = SimpleCreateCollectionOptions(collectionName, 128).WithVarcharPK(true, 64).WithAutoID(false).WithDynamicSchema(false)
107+
opt = SimpleCreateCollectionOptions(collectionName, 128).WithVarcharPK(true, 64).WithAutoID(false).
108+
WithPKFieldName("pk").WithVectorFieldName("embedding").WithMetricType(entity.L2).
109+
WithDynamicSchema(false)
108110
req = opt.Request()
109111
s.Equal(collectionName, req.GetCollectionName())
110112
s.EqualValues(1, req.GetShardsNum())

‎client/milvusclient/common.go

+31
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@ package milvusclient
22

33
import (
44
"context"
5+
"math"
6+
7+
"github.com/cockroachdb/errors"
58

69
"github.com/milvus-io/milvus/client/v2/entity"
710
"github.com/milvus-io/milvus/pkg/v2/util/conc"
11+
"github.com/milvus-io/milvus/pkg/v2/util/merr"
12+
"github.com/milvus-io/milvus/pkg/v2/util/retry"
813
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
914
)
1015

@@ -32,6 +37,11 @@ func (c *CollectionCache) GetCollection(ctx context.Context, collName string) (*
3237
return coll, err
3338
}
3439

40+
// Evict removes the collection cache related to the provided collection name.
41+
func (c *CollectionCache) Evict(collName string) {
42+
c.collections.Remove(collName)
43+
}
44+
3545
// Reset clears all cached info, used when client switching env.
3646
func (c *CollectionCache) Reset() {
3747
c.collections = typeutil.NewConcurrentMap[string, *entity.Collection]()
@@ -47,3 +57,24 @@ func NewCollectionCache(fetcher func(context.Context, string) (*entity.Collectio
4757
func (c *Client) getCollection(ctx context.Context, collName string) (*entity.Collection, error) {
4858
return c.collCache.GetCollection(ctx, collName)
4959
}
60+
61+
func (c *Client) retryIfSchemaError(ctx context.Context, collName string, work func(ctx context.Context) (uint64, error)) error {
62+
var lastTs uint64 = math.MaxUint64
63+
return retry.Handle(ctx, func() (bool, error) {
64+
ts, err := work(ctx)
65+
if err != nil {
66+
// if schema error
67+
if errors.Is(err, merr.ErrCollectionSchemaMismatch) {
68+
sameTs := ts == lastTs
69+
lastTs = ts
70+
if !sameTs {
71+
c.collCache.Evict(collName)
72+
}
73+
// retry if not same ts
74+
return !sameTs, err
75+
}
76+
return false, err
77+
}
78+
return false, nil
79+
})
80+
}

0 commit comments

Comments
 (0)