repo-sync-2024-07-17T16:34:05+0800 (#384)

.circleci/config.yml

@@ -51,7 +51,7 @@ commands:
           command: |
             git config --global --add safe.directory ./
             make build
-            
+
             DIR="/tmp/build/linux"
             mkdir -p $DIR
             cp -rf ./build/linux/* $DIR

Makefile

@@ -15,8 +15,8 @@ IMG := secretflow/kuscia:${TAG}
 # TEST_SUITE used by integration test
 TEST_SUITE ?= all
 
-ENVOY_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240407
-DEPS_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0
+ENVOY_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.6.0b0
+DEPS_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.6.0b0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))

build/dockerfile/kuscia-anolis.Dockerfile

@@ -1,13 +1,14 @@
-ARG DEPS_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0"
-ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.6.0.dev20240611"
-ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0"
+ARG DEPS_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.6.0b0"
+ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.6.0b0"
+ARG PROM_NODE_EXPORTER="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/node-exporter:v1.7.0"
+ARG BASE_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/anolisos:23"
 
 FROM ${DEPS_IMAGE} as deps
 
 FROM ${PROM_NODE_EXPORTER} as node_exporter
 FROM ${KUSCIA_ENVOY_IMAGE} as kuscia_envoy
 
-FROM openanolis/anolisos:23
+FROM ${BASE_IMAGE}
 
 ENV TZ=Asia/Shanghai
 ARG TARGETPLATFORM
@@ -19,6 +20,7 @@ RUN yum install -y openssl net-tools which jq logrotate iproute procps-ng && \
     mkdir -p /bin/aux && \
     mkdir -p ${ROOT_DIR}/scripts && \
     mkdir -p ${ROOT_DIR}/var/storage && \
+    mkdir -p ${ROOT_DIR}/var/k3s/server/db && \
     mkdir -p ${ROOT_DIR}/pause
 
 COPY --from=deps /image/home/kuscia/bin ${ROOT_DIR}/bin

build/dockerfile/kuscia-secretflow.Dockerfile

@@ -1,7 +1,7 @@
-ARG KUSCIA_IMAGE="secretflow/kuscia:latest"
-
-FROM secretflow/anolis8-python:3.10.13 as python
+ARG KUSCIA_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:latest"
+ARG PYTHON_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/anolis8-python:3.10.13"
 
+FROM ${PYTHON_IMAGE} as python
 FROM ${KUSCIA_IMAGE}
 
 COPY --from=python /root/miniconda3/envs/secretflow/bin/ /usr/local/bin/
@@ -12,7 +12,7 @@ RUN yum install -y protobuf libnl3 libgomp && \
     grep -rl '#!/root/miniconda3/envs/secretflow/bin' /usr/local/bin/ | xargs sed -i -e 's/#!\/root\/miniconda3\/envs\/secretflow/#!\/usr\/local/g' && \
     rm /usr/local/bin/openssl
 
-ARG SF_VERSION="1.6.0b0"
+ARG SF_VERSION="1.7.0b0"
 RUN pip install secretflow-lite==${SF_VERSION} --extra-index-url https://mirrors.aliyun.com/pypi/simple/ && rm -rf /root/.cache
 
 RUN kuscia image builtin secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/secretflow-lite-anolis8:${SF_VERSION} --store /home/kuscia/var/images

CHANGELOG.md → changelog/CHANGELOG.md

@@ -15,31 +15,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 `Security` in case of vulnerabilities.
 `Breaking Changed` Breaking for backward-incompatible changes that require user intervention.
 
-## [v0.10.0.dev240531] - 2024-05-31
-
-### Added
-[Feature] domainRoute now supports setting prefix paths
-[Feature] kuscia job now supports multiple parties (1 party , 2 parties , 3 parties)
-[Feature] appImage's mountPath now supports relative paths
-[Feature] Support for setting DomainRoute reverse tunnel mode through the Kuscia API
-[Feature] Support for PKCS 8 format certificates
-[Feature] Added DataSource List API to KusciaAPI. KusciaAPI Master side supports Query and List DataSource but does not display sensitive information.
-
-### Changed
-[Documentation] Correct description of monitoring metrics in the Kuscia metric documentation
-[scripts] During kuscia init, check that the node ID cannot be master
-[Feature] Remove the trailing "/" from the RelatedURI field when creating domainData via KusciaAPI
-[Feature] Added heartbeat time check to domainRoute, setting it to not ready if it exceeds the threshold
-[Feature] Moved error code definitions in the code to Protobuf
-
-### Breaking Changed
-[NA]
-
-### Fixed
-[Bugfix] Fixed the issue of port conflicts in runp serving multi-replica
-[Bugfix] Fixed the issue of state anomalies being caused by multiple executions of kusciaJob stop and restart
-[Unit Test] Improved the stability and concurrency of unit tests
-
 ## [v0.9.0.dev240430] - 2024-04-30
 
 ### Added

changelog/v0.10.0_CHANGELOG.md

@@ -0,0 +1,83 @@
+## 更新日志
+
+本项目的所有显著变更都将记录在此文档中。
+变更记录的格式遵循 [保持变更日志](https://keepachangelog.com/zh-CN/1.0.0/)
+约定，同时本项目遵守 [语义化版本控制](https://semver.org/lang/zh-CN/spec/v2.0.0.html) 规范。
+
+### 变更类型
+- `新增`：引入新功能。
+- `修改`：对现有功能的改进或调整。
+- `废弃`：计划在未来移除的功能。
+- `移除`：已从项目中移除的功能。
+- `修复`：错误或漏洞的修复。
+- `安全`：涉及安全漏洞的修复或更新。
+- `重大变更`：引入了不兼容的更改，可能需要用户做出相应调整。
+
+### [v0.10.0.dev240531] - 2024-05-31
+
+#### 新增
+- **[功能]** DomainRoute 现在支持配置前缀路径。
+- **[功能]** Kuscia 任务现在支持多参与方模式（包括单方、双方、三方）。
+- **[功能]** AppImage 的挂载路径现在接受相对路径。
+- **[功能]** 通过 Kuscia API 添加了设置 DomainRoute 反向隧道模式的支持。
+- **[功能]** 新增对 PKCS 8 格式证书的支持。
+- **[功能]** KusciaAPI 中新增了数据源列表接口，Master 端支持查询和列举数据源，但不展示敏感信息。
+
+#### 修改
+- **[文档]** 更新了 Kuscia 监控指标文档中的描述，确保准确性。
+- **[脚本]** 在执行 Kuscia 初始化时，增加检查以确保节点 ID 不得为 Master。
+- **[功能]** 通过 KusciaAPI 创建 DomainData 时，自动去除 RelatedURI 字段末尾的斜线。
+- **[功能]** DomainRoute 增加了心跳检测，当心跳超时时标记为非就绪状态。
+- **[功能]** 将代码中错误码的定义迁移至 Protobuf 文件中。
+
+#### 重大变更
+- **[无]**
+
+#### 修复
+- **[问题修复]** 解决了 Runp Serving 在多副本部署时可能出现的端口冲突问题。
+- **[问题修复]** 修正了重复执行 KusciaJob 停止与重启命令导致状态异常的问题。
+- **[单元测试]** 优化了单元测试的稳定性和并发处理能力。
+
+---
+
+## Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+### Types of changes
+
+- `Added ` for new features.
+- `Changed` for changes in existing functionality.
+- `Deprecated` for soon-to-be removed features.
+- `Removed` for now removed features.
+- `Fixed` for any bug fixes.
+- `Security` in case of vulnerabilities.
+- `Breaking Changed` Breaking for backward-incompatible changes that require user intervention.
+
+### [v0.10.0.dev240531] - 2024-05-31
+
+#### Added
+- [Feature] domainRoute now supports setting prefix paths
+- [Feature] kuscia job now supports multiple parties (1 party , 2 parties , 3 parties)
+- [Feature] appImage's mountPath now supports relative paths
+- [Feature] Support for setting DomainRoute reverse tunnel mode through the Kuscia API
+- [Feature] Support for PKCS 8 format certificates
+- [Feature] Added DataSource List API to KusciaAPI. KusciaAPI Master side supports Query and List DataSource but does not display sensitive information.
+
+#### Changed
+- [Documentation] Correct description of monitoring metrics in the Kuscia metric documentation
+- [scripts] During kuscia init, check that the node ID cannot be master
+- [Feature] Remove the trailing "/" from the RelatedURI field when creating domainData via KusciaAPI
+- [Feature] Added heartbeat time check to domainRoute, setting it to not ready if it exceeds the threshold
+- [Feature] Moved error code definitions in the code to Protobuf
+
+#### Breaking Changed
+- [NA]
+
+#### Fixed
+- [Bugfix] Fixed the issue of port conflicts in runp serving multi-replica
+- [Bugfix] Fixed the issue of state anomalies being caused by multiple executions of kusciaJob stop and restart
+- [Unit Test] Improved the stability and concurrency of unit tests

cmd/example/datamesh/dm_client.go

@@ -41,7 +41,7 @@ import (
 	"google.golang.org/protobuf/proto"
 
 	"github.com/secretflow/kuscia/pkg/common"
-	"github.com/secretflow/kuscia/pkg/datamesh/dmflight"
+	"github.com/secretflow/kuscia/pkg/datamesh/dataserver/utils"
 	"github.com/secretflow/kuscia/pkg/kusciaapi/service"
 	"github.com/secretflow/kuscia/pkg/utils/nlog"
 	"github.com/secretflow/kuscia/pkg/utils/paths"
@@ -222,7 +222,7 @@ func (m *MockFlightClient) QueryDataSource(datasourceID string) error {
 	}
 
 	dsResp := queryDatasourceResp.Data
-	nlog.Infof("DsType is %s, dsInfo:%v", dsResp.Type, dsResp.Info)
+	nlog.Infof("DataSourceType is %s, dsInfo:%v", dsResp.Type, dsResp.Info)
 	return nil
 }
 
@@ -279,7 +279,7 @@ func (m *MockFlightClient) getData(domainDataID string, useRawData bool) error {
 	if useRawData {
 		cmd.ContentType = datamesh.ContentType_RAW
 	}
-	desc, err := dmflight.DescForCommand(cmd)
+	desc, err := utils.DescForCommand(cmd)
 	if err != nil {
 		nlog.Warnf(err.Error())
 		return err
@@ -331,7 +331,7 @@ func (m *MockFlightClient) putData(domainDataID string) error {
 			},
 		},
 	}
-	desc, err := dmflight.DescForCommand(cmd)
+	desc, err := utils.DescForCommand(cmd)
 	if err != nil {
 		nlog.Warnf(err.Error())
 		return err

cmd/example/datamesh/dm_server.go

@@ -221,7 +221,7 @@ func makeMemConfigurationService() cmservice.IConfigurationService {
 
 func loadConfigFromCmdEnv(conf *config.DataMeshConfig, certsConfig *CertsConfig, o *opts) error {
 	conf.DisableTLS = !o.enableDataMeshTLS
-	conf.ExternalDataProxyList = []config.ExternalDataProxyConfig{
+	conf.DataProxyList = []config.DataProxyConfig{
 		{
 			Endpoint: o.dataProxyEndpoint,
 		},

cmd/kuscia/kernel.go

@@ -0,0 +1,42 @@
+// Copyright 2024 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/secretflow/kuscia/pkg/utils/runtime"
+)
+
+func NewKernelCheckCommand(ctx context.Context) *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use:          "kernel-check",
+		Short:        "Check Kernel Params",
+		Long:         `Check Kernel Params, make sure satisfy the requirements`,
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			for _, param := range runtime.CurrentKernel().List() {
+				println(fmt.Sprintf("Key=%s, Match(%v), Value(%s), Description(%s)", param.Key, param.IsMatch, param.Value, param.Description))
+			}
+			return nil
+		},
+	}
+
+	return cmd
+}