implement user-server(service-proxy), provide user a https server to access managed cluster service on the hub side. (#244)

Signed-off-by: xuezhaojun <[email protected]>

Author: xuezhaojun

.github/workflows/go-presubmit.yml

@@ -90,25 +90,11 @@ jobs:
         run: curl -L https://raw.githubusercontent.com/open-cluster-management-io/clusteradm/main/install.sh | bash
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]
-      - name: Prepare OCM testing environment
-        run: |
-          clusteradm init --output-join-command-file join.sh --wait
-          sh -c "$(cat join.sh) loopback --force-internal-endpoint-lookup"
-          clusteradm accept --clusters loopback --wait 30
-          kubectl wait --for=condition=ManagedClusterConditionAvailable managedcluster/loopback
+        with:
+          cluster_name: e2e
       - name: Build image
-        run: |
-          make images
-          kind load docker-image quay.io/open-cluster-management/cluster-proxy:latest  --name chart-testing
-      - name: Install latest cluster-proxy
-        run: |
-          helm install \
-             -n open-cluster-management-addon --create-namespace \
-             cluster-proxy charts/cluster-proxy/ \
-             --set tag=latest --set installByPlacement.placementName=default
-      - name: Build&Run e2e test
-        run: |
-          kubectl wait --for=condition=ProxyServerDeployed=true managedproxyconfiguration cluster-proxy --timeout=60s
-          kubectl wait --for=condition=Available deployment/cluster-proxy --timeout=60s -n open-cluster-management-addon
-          kubectl port-forward -n open-cluster-management-addon services/proxy-entrypoint 8090:8090 &
-          make test-e2e
+        run: make images
+      - name: Setup environment
+        run: make setup-env-for-e2e && make deploy-cluster-proxy-e2e
+      - name: Run e2e tests
+        run: make test-e2e

Makefile

@@ -3,11 +3,15 @@ IMG ?= controller:latest
 IMAGE_REGISTRY_NAME ?= quay.io/open-cluster-management
 IMAGE_NAME = cluster-proxy
 IMAGE_TAG ?= latest
-E2E_TEST_CLUSTER_NAME ?= loopback
+E2E_TEST_CLUSTER_NAME ?= e2e
 CONTAINER_ENGINE ?= docker
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:crdVersions={v1},allowDangerousTypes=true,generateEmbeddedObjectMeta=true"
 
+# Label filter for e2e tests (Ginkgo v2 label filter expression)
+# Examples: "install", "connectivity", "certificate && !rotation", etc.
+LABEL_FILTER ?=
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -56,11 +60,7 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
-golint:
-	go install github.com/golangci/golangci-lint/cmd/[email protected]
-	golangci-lint run --timeout=3m ./...
-
-verify: fmt vet golint
+verify: fmt vet
 
 test: manifests generate fmt vet ## Run tests.
 	go test ./pkg/... -coverprofile cover.out
@@ -70,6 +70,7 @@ test: manifests generate fmt vet ## Run tests.
 build: generate fmt vet
 	go build -o bin/addon-manager cmd/addon-manager/main.go
 	go build -o bin/addon-agent cmd/addon-agent/main.go
+	go build -o bin/cluster-proxy cmd/cluster-proxy/main.go
 
 docker-build: test ## Build docker image with the manager.
 	$(CONTAINER_ENGINE) build -t ${IMG} .
@@ -81,7 +82,7 @@ docker-push: ## Push docker image with the manager.
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.18.0)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.19.0)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
@@ -117,6 +118,16 @@ images:
 		--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
 		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
 
+images-amd64:
+	$(CONTAINER_ENGINE) buildx build \
+		--platform linux/amd64 \
+		--load \
+		-f cmd/Dockerfile \
+		--build-arg ADDON_AGENT_IMAGE_NAME=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) \
+		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) .
+
+## Integration Testing
+
 ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test-integration: manifests generate fmt vet
 	mkdir -p ${ENVTEST_ASSETS_DIR}
@@ -126,13 +137,124 @@ test-integration: manifests generate fmt vet
 		setup_envtest_env $(ENVTEST_ASSETS_DIR); \
 		go test ./test/integration/... -coverprofile cover.out
 
-e2e-job-image:
+## E2E Testing
+
+# Note: here we use internal service ns as the entrypointAddress. The test cluster should be registered to itself as a managed cluster.
+setup-env-for-e2e:
+	@echo "Setting up environment for e2e tests..."
+	./test/e2e/env/init.sh
+.PHONY: setup-env-for-e2e
+
+# load cluster-proxy image into kind cluster
+load-cluster-proxy-image-kind:
+	@echo "Loading cluster-proxy image into kind cluster..."
+	kind load docker-image $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) --name $(E2E_TEST_CLUSTER_NAME)
+.PHONY: load-cluster-proxy-image-kind
+
+# delete cluster-proxy image from kind cluster nodes
+delete-cluster-proxy-image-from-kind:
+	@echo "Deleting cluster-proxy image from kind cluster nodes..."
+	@for node in $$(kind get nodes --name $(E2E_TEST_CLUSTER_NAME) 2>/dev/null || echo ""); do \
+		if [ -n "$$node" ]; then \
+			docker exec $$node crictl rmi $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) 2>/dev/null || true; \
+		fi; \
+	done
+.PHONY: delete-cluster-proxy-image-from-kind
+
+deploy-cluster-proxy-e2e: delete-cluster-proxy-image-from-kind load-cluster-proxy-image-kind
+	@echo "Deploying cluster-proxy..."
+	helm install \
+	-n open-cluster-management-addon --create-namespace \
+	cluster-proxy charts/cluster-proxy \
+	--set registry=$(IMAGE_REGISTRY_NAME) \
+	--set image=$(IMAGE_NAME) \
+	--set tag=$(IMAGE_TAG) \
+	--set proxyServerImage=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME) \
+	--set proxyAgentImage=$(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME) \
+	--set proxyServer.entrypointAddress="proxy-entrypoint.open-cluster-management-addon.svc" \
+	--set proxyServer.port=8091 \
+	--set enableServiceProxy=true
+	@echo "Cluster-proxy deployed successfully!"
+.PHONY: deploy-cluster-proxy-e2e
+
+# Build e2e test container image
+build-e2e-image:
+	@echo "Building e2e test container image..."
 	$(CONTAINER_ENGINE) build \
-		-f test/e2e/job/Dockerfile \
-		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e-job:$(IMAGE_TAG) .
+		-f test/e2e/Dockerfile \
+		-t $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) .
+.PHONY: build-e2e-image
+
+# Load e2e image into kind cluster (for local testing)
+load-e2e-image-kind:
+	@echo "Loading e2e image into kind cluster..."
+	kind load docker-image $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) --name $(E2E_TEST_CLUSTER_NAME)
+.PHONY: load-e2e-image-kind
+
+# Delete e2e image from kind cluster nodes (for rapid iteration)
+delete-e2e-image-from-kind:
+	@echo "Deleting e2e image from kind cluster nodes..."
+	@for node in $$(kind get nodes --name $(E2E_TEST_CLUSTER_NAME) 2>/dev/null || echo ""); do \
+		if [ -n "$$node" ]; then \
+			docker exec $$node crictl rmi $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG) 2>/dev/null || true; \
+		fi; \
+	done
+.PHONY: delete-e2e-image-from-kind
+
+# Run e2e tests in cluster using container image (Kubernetes-native approach)
+# Use LABEL_FILTER to run specific tests, e.g.: make test-e2e LABEL_FILTER="install"
+test-e2e: delete-e2e-image-from-kind build-e2e-image load-e2e-image-kind
+	@echo "Deleting existing e2e test job if present..."
+	@kubectl delete job cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found
+	@echo "Deploying e2e test job..."
+	@if [ -n "$(LABEL_FILTER)" ]; then \
+		echo "Running tests with label filter: $(LABEL_FILTER)"; \
+	fi
+	@sed -e '/name: LABEL_FILTER/{n;s|value: ""|value: "$(LABEL_FILTER)"|;}' \
+	     -e 's|image: quay.io/open-cluster-management/cluster-proxy-e2e:latest|image: $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG)|g' \
+	     test/e2e/env/job.yaml | kubectl apply -f -
+	@./test/e2e/env/wait-for-job.sh cluster-proxy-e2e open-cluster-management-addon 1200
+.PHONY: test-e2e
+
+# Rapid iteration workflow for e2e tests (cleans up everything first)
+# Use LABEL_FILTER to run specific tests, e.g.: make retest-e2e LABEL_FILTER="connectivity"
+retest-e2e: clean-e2e delete-e2e-image-from-kind build-e2e-image load-e2e-image-kind
+	@echo "Deleting existing e2e test job if present..."
+	@kubectl delete job cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found
+	@echo "Deploying e2e test job..."
+	@if [ -n "$(LABEL_FILTER)" ]; then \
+		echo "Running tests with label filter: $(LABEL_FILTER)"; \
+	fi
+	@sed -e '/name: LABEL_FILTER/{n;s|value: ""|value: "$(LABEL_FILTER)"|;}' \
+	     -e 's|image: quay.io/open-cluster-management/cluster-proxy-e2e:latest|image: $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME)-e2e:$(IMAGE_TAG)|g' \
+	     test/e2e/env/job.yaml | kubectl apply -f -
+	@./test/e2e/env/wait-for-job.sh cluster-proxy-e2e open-cluster-management-addon 1200
+.PHONY: retest-e2e
 
-build-e2e:
-	go test -c -o bin/e2e ./test/e2e/
+# Clean up e2e test job and related resources
+clean-e2e:
+	@echo "Cleaning up e2e test resources..."
+	kubectl delete job/cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found=true
+	kubectl delete serviceaccount/cluster-proxy-e2e -n open-cluster-management-addon --ignore-not-found=true
+	kubectl delete clusterrolebinding/cluster-proxy-e2e --ignore-not-found=true
+	kubectl delete clusterrole/cluster-proxy-e2e --ignore-not-found=true
+.PHONY: clean-e2e
 
-test-e2e: build-e2e
-	./bin/e2e --test-cluster $(E2E_TEST_CLUSTER_NAME)
+# Quick verify of user-server
+# Example result:
+# {
+#   "kind": "APIVersions",
+#   "versions": [
+#     "v1"
+#   ],
+#   "serverAddressByClientCIDRs": [
+#     {
+#       "clientCIDR": "0.0.0.0/0",
+#       "serverAddress": "172.17.0.2:6443"
+#     }
+#   ]
+# }
+verify-user-server:
+	@echo "Verifying user-server..."
+	TOKEN=$$(kubectl create token default -n default) && POD=$$(kubectl get pods -n open-cluster-management-addon -l component=cluster-proxy-addon-user --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && kubectl debug -it $$POD -n open-cluster-management-addon --image=praqma/network-multitool -- sh -c "curl -k -H 'Authorization: Bearer $$TOKEN' https://cluster-proxy-addon-user.open-cluster-management-addon.svc.cluster.local:9092/loopback/api"
+.PHONY: verify-user-server

charts/cluster-proxy/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: cluster-proxy
 description: A Helm chart for Cluster-Proxy OCM Addon
 type: application
-version: 0.8.0
-appVersion: 1.0.0
+version: 0.9.0
+appVersion: 1.1.0

charts/cluster-proxy/README.md

@@ -0,0 +1,162 @@
+# Cluster Proxy Helm Chart
+
+This Helm chart installs the Cluster Proxy addon for Open Cluster Management (OCM), which enables accessing services in isolated managed clusters through reverse proxy tunnels.
+
+## Prerequisites
+
+- Kubernetes cluster
+- Helm 3.x
+- Open Cluster Management (OCM) installed
+
+## Installation
+
+```bash
+helm install cluster-proxy ./charts/cluster-proxy \
+  --namespace open-cluster-management-addon \
+  --create-namespace
+```
+
+## Configuration
+
+### Values
+
+| Parameter                               | Description                        | Default                                         |
+| --------------------------------------- | ---------------------------------- | ----------------------------------------------- |
+| `registry`                              | Image registry                     | `quay.io/open-cluster-management`               |
+| `image`                                 | Image name                         | `cluster-proxy`                                 |
+| `tag`                                   | Image tag                          | Chart version                                   |
+| `replicas`                              | Number of replicas                 | `1`                                             |
+| `spokeAddonNamespace`                   | Namespace for spoke addon          | `open-cluster-management-cluster-proxy`         |
+| `proxyServerImage`                      | Proxy server image                 | `quay.io/open-cluster-management/cluster-proxy` |
+| `proxyAgentImage`                       | Proxy agent image                  | `quay.io/open-cluster-management/cluster-proxy` |
+| `proxyServer.entrypointLoadBalancer`    | Enable LoadBalancer for entrypoint | `false`                                         |
+| `proxyServer.entrypointAddress`         | Custom entrypoint address          | `""`                                            |
+| `proxyServer.port`                      | Proxy server port                  | `8091`                                          |
+| `installByPlacement.placementName`      | Placement name for installation    | `""`                                            |
+| `installByPlacement.placementNamespace` | Placement namespace                | `""`                                            |
+| `enableServiceProxy`                    | Enable user server deployment      | `false`                                         |
+
+### User Server Configuration
+
+The user server provides an API endpoint for managing cluster proxy connections. To enable it:
+
+```bash
+helm install cluster-proxy ./charts/cluster-proxy \
+  --set enableServiceProxy=true
+```
+
+#### Important Prerequisites for User Server
+
+**Before enabling the user server, you MUST create the following secret in the installation namespace:**
+
+**cluster-proxy-user-serving-cert** - TLS certificate for the user server
+
+```yaml
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/tls
+metadata:
+  name: cluster-proxy-user-serving-cert
+  namespace: <release-namespace>
+data:
+  tls.crt: <base64-encoded-certificate>
+  tls.key: <base64-encoded-private-key>
+```
+
+**Automatically Created Secrets:**
+
+The following secrets will be automatically created by the controller and do NOT need to be created manually:
+
+- **proxy-server-ca** - CA certificate for the proxy server
+- **proxy-client** - Client certificate for proxy authentication
+
+**⚠️ Warning:** If the `cluster-proxy-user-serving-cert` secret is not present before installation, the user-server deployment will remain in **Pending** state and pods will fail to start.
+
+To verify the secret is created:
+
+```bash
+kubectl get secret -n <release-namespace> cluster-proxy-user-serving-cert
+```
+
+## Examples
+
+### Basic Installation
+
+```bash
+helm install cluster-proxy ./charts/cluster-proxy
+```
+
+### With User Server Enabled
+
+```bash
+# First, create the required secret
+kubectl create secret tls cluster-proxy-user-serving-cert \
+  --cert=path/to/tls.crt \
+  --key=path/to/tls.key \
+  -n open-cluster-management-addon
+
+# Then install with user server enabled
+# Note: proxy-server-ca and proxy-client secrets will be created automatically by the controller
+helm install cluster-proxy ./charts/cluster-proxy \
+  --namespace open-cluster-management-addon \
+  --set enableServiceProxy=true
+```
+
+### Custom Image and Replicas
+
+```bash
+helm install cluster-proxy ./charts/cluster-proxy \
+  --set image=my-custom-proxy \
+  --set tag=v1.0.0 \
+  --set replicas=3
+```
+
+## Upgrading
+
+```bash
+helm upgrade cluster-proxy ./charts/cluster-proxy
+```
+
+## Uninstallation
+
+```bash
+helm uninstall cluster-proxy
+```
+
+## Troubleshooting
+
+### User Server Pods Stuck in Pending
+
+**Symptom:** After enabling `enableServiceProxy=true`, the deployment pods remain in Pending state.
+
+**Solution:** Verify that the required secret exists in the namespace:
+
+```bash
+kubectl get secret -n <namespace> cluster-proxy-user-serving-cert
+```
+
+If the secret is missing, create it:
+
+```bash
+kubectl create secret tls cluster-proxy-user-serving-cert \
+  --cert=path/to/tls.crt \
+  --key=path/to/tls.key \
+  -n <namespace>
+```
+
+Note: The `proxy-server-ca` and `proxy-client` secrets are created automatically by the controller and do not need manual creation.
+
+### ImagePullBackOff Errors
+
+**Solution:** Verify the image registry and credentials:
+
+```bash
+helm upgrade cluster-proxy ./charts/cluster-proxy \
+  --set registry=<your-registry> \
+  --set image=<your-image> \
+  --set tag=<your-tag>
+```
+
+## More Information
+
+For more details about the Cluster Proxy project, visit the [GitHub repository](https://github.com/open-cluster-management-io/cluster-proxy).