diff --git a/docs/connection-from-host.md b/docs/connection-from-host.md new file mode 100644 index 000000000..036fb9d4d --- /dev/null +++ b/docs/connection-from-host.md @@ -0,0 +1,79 @@ +# Connection from host + +### Using NodePort + +To connect to a node from your host machine, you can use the NodePort service type. This exposes the node's desired ports on the host machine, allowing you to connect to them directly. + +For example to connect to a Bitcoin Core node using the RPC port, add this to the node's configuration in the network graph definition: + +```yaml +nodes: + - name: tank-0001 + service: + type: NodePort + rpcNodePort: 30443 +``` + +To get the IP address of a node in your cluster, execute `warnet host`: + +```shell +# Minikube on Linux +> warnet host +192.168.49.2 + +# Docker Desktop on MacOS +> warnet host +kubernetes.docker.internal + +# Remote cluster +> warnet host +159.223.123.163 +``` +Then you can connect to the NodePort in the cluster `:30443`. + +> [!WARNING] +> If you are using MiniKube on MacOS, you must rely on `minikube service` to get both hostname and port for your tank: + +``` +(.venv) --> minikube service tank-0001 +|-----------|-----------|-------------------------|---------------------------| +| NAMESPACE | NAME | TARGET PORT | URL | +|-----------|-----------|-------------------------|---------------------------| +| default | tank-0001 | rpc/18443 p2p/18444 | http://192.168.49.2:30002 | +| | | zmq-tx/28333 | http://192.168.49.2:30984 | +| | | zmq-block/28332 | http://192.168.49.2:30682 | +| | | prometheus-metrics/9332 | http://192.168.49.2:30230 | +| | | | http://192.168.49.2:30175 | +|-----------|-----------|-------------------------|---------------------------| +🏃 Starting tunnel for service tank-0001. +|-----------|-----------|-------------|------------------------| +| NAMESPACE | NAME | TARGET PORT | URL | +|-----------|-----------|-------------|------------------------| +| default | tank-0001 | | http://127.0.0.1:62254 | +| | | | http://127.0.0.1:62255 | +| | | | http://127.0.0.1:62256 | +| | | | http://127.0.0.1:62257 | +| | | | http://127.0.0.1:62258 | +|-----------|-----------|-------------|------------------------| +[default tank-0001 http://127.0.0.1:62254 +http://127.0.0.1:62255 +http://127.0.0.1:62256 +http://127.0.0.1:62257 +http://127.0.0.1:62258] +❗ Because you are using a Docker driver on darwin, the terminal needs to be open to run it. +``` + + + + +All the different port options can be seen in values.yaml files. The exposed port values must be in the range 30000-32767. If left empty, a random port in that range will be assigned by Kubernetes. + +To check which ports are open on the host machine, use `kubectl get svc -n ` and look for the `PORT(S)` column. + +### Using port-forward + +Alternatively, you can use `kubectl port-forward` command. For example to expose the regtest RPC port of a Bitcoin Core node, run the command below. The first port is the local port on your machine, and the second port is the port inside the cluster. You can choose any available local port. + +```shell +kubectl port-forward pod/tank-0001 18443:18443 +``` \ No newline at end of file diff --git a/resources/charts/bitcoincore/charts/cln/templates/service.yaml b/resources/charts/bitcoincore/charts/cln/templates/service.yaml index 565f50182..2e6a911c0 100644 --- a/resources/charts/bitcoincore/charts/cln/templates/service.yaml +++ b/resources/charts/bitcoincore/charts/cln/templates/service.yaml @@ -12,13 +12,22 @@ spec: targetPort: p2p protocol: TCP name: p2p + {{- if and (eq .Values.service.type "NodePort") (.Values.service.p2pNodePort) }} + nodePort: {{ .Values.service.p2pNodePort }} + {{- end }} - port: {{ .Values.RPCPort }} targetPort: rpc protocol: TCP name: rpc + {{- if and (eq .Values.service.type "NodePort") (.Values.service.rpcNodePort) }} + nodePort: {{ .Values.service.rpcNodePort }} + {{- end }} - port: {{ .Values.RestPort }} targetPort: rest protocol: TCP name: rest + {{- if and (eq .Values.service.type "NodePort") (.Values.service.restNodePort) }} + nodePort: {{ .Values.service.restNodePort }} + {{- end }} selector: {{- include "cln.selectorLabels" . | nindent 4 }} diff --git a/resources/charts/bitcoincore/charts/cln/values.yaml b/resources/charts/bitcoincore/charts/cln/values.yaml index 953280445..ae7d1a510 100644 --- a/resources/charts/bitcoincore/charts/cln/values.yaml +++ b/resources/charts/bitcoincore/charts/cln/values.yaml @@ -22,6 +22,10 @@ securityContext: {} service: type: ClusterIP + # NodePorts are only used if service.type is NodePort, values must be in the range 30000-32767 + p2pNodePort: + rpcNodePort: + restNodePort: P2PPort: 9735 RPCPort: 9736 diff --git a/resources/charts/bitcoincore/charts/lnd/templates/service.yaml b/resources/charts/bitcoincore/charts/lnd/templates/service.yaml index aecf301fe..3a7492ad1 100644 --- a/resources/charts/bitcoincore/charts/lnd/templates/service.yaml +++ b/resources/charts/bitcoincore/charts/lnd/templates/service.yaml @@ -12,19 +12,31 @@ spec: targetPort: rpc protocol: TCP name: rpc + {{- if and (eq .Values.service.type "NodePort") (.Values.service.rpcNodePort) }} + nodePort: {{ .Values.service.rpcNodePort }} + {{- end }} - port: {{ .Values.P2PPort }} targetPort: p2p protocol: TCP name: p2p + {{- if and (eq .Values.service.type "NodePort") (.Values.service.p2pNodePort) }} + nodePort: {{ .Values.service.p2pNodePort }} + {{- end }} - port: {{ .Values.RestPort }} targetPort: rest protocol: TCP name: rest + {{- if and (eq .Values.service.type "NodePort") (.Values.service.restNodePort) }} + nodePort: {{ .Values.service.restNodePort }} + {{- end }} {{- if .Values.metricsExport }} - port: {{ .Values.prometheusMetricsPort }} targetPort: prom-metrics protocol: TCP name: prometheus-metrics + {{- if and (eq .Values.service.type "NodePort") (.Values.service.prometheusNodePort) }} + nodePort: {{ .Values.service.prometheusNodePort }} + {{- end }} {{- end }} selector: {{- include "lnd.selectorLabels" . | nindent 4 }} diff --git a/resources/charts/bitcoincore/charts/lnd/values.yaml b/resources/charts/bitcoincore/charts/lnd/values.yaml index a97e9470d..06de17af4 100644 --- a/resources/charts/bitcoincore/charts/lnd/values.yaml +++ b/resources/charts/bitcoincore/charts/lnd/values.yaml @@ -32,6 +32,11 @@ securityContext: {} service: type: ClusterIP + # NodePorts are only used if service.type is NodePort, values must be in the range 30000-32767 + p2pNodePort: + rpcNodePort: + restNodePort: + prometheusNodePort: RPCPort: 10009 P2PPort: 9735 diff --git a/resources/charts/bitcoincore/templates/service.yaml b/resources/charts/bitcoincore/templates/service.yaml index 8d8fa5324..4ca3db641 100644 --- a/resources/charts/bitcoincore/templates/service.yaml +++ b/resources/charts/bitcoincore/templates/service.yaml @@ -12,21 +12,36 @@ spec: targetPort: rpc protocol: TCP name: rpc + {{- if and (eq .Values.service.type "NodePort") (.Values.service.rpcNodePort) }} + nodePort: {{ .Values.service.rpcNodePort }} + {{- end }} - port: {{ index .Values.global .Values.global.chain "P2PPort" }} targetPort: p2p protocol: TCP name: p2p + {{- if and (eq .Values.service.type "NodePort") (.Values.service.p2pNodePort) }} + nodePort: {{ .Values.service.p2pNodePort }} + {{- end }} - port: {{ .Values.global.ZMQTxPort }} targetPort: zmq-tx protocol: TCP name: zmq-tx + {{- if and (eq .Values.service.type "NodePort") (.Values.service.zmqTxNodePort) }} + nodePort: {{ .Values.service.zmqTxNodePort }} + {{- end }} - port: {{ .Values.global.ZMQBlockPort }} targetPort: zmq-block protocol: TCP name: zmq-block + {{- if and (eq .Values.service.type "NodePort") (.Values.service.zmqBlockNodePort) }} + nodePort: {{ .Values.service.zmqBlockNodePort }} + {{- end }} - port: {{ .Values.prometheusMetricsPort }} targetPort: prom-metrics protocol: TCP name: prometheus-metrics + {{- if and (eq .Values.service.type "NodePort") (.Values.service.prometheusNodePort) }} + nodePort: {{ .Values.service.prometheusNodePort }} + {{- end }} selector: {{- include "bitcoincore.selectorLabels" . | nindent 4 }} diff --git a/resources/charts/bitcoincore/values.yaml b/resources/charts/bitcoincore/values.yaml index 92416f6bb..5b8f486b9 100644 --- a/resources/charts/bitcoincore/values.yaml +++ b/resources/charts/bitcoincore/values.yaml @@ -32,6 +32,12 @@ securityContext: {} service: type: ClusterIP + # NodePorts are only used if service.type is NodePort, values must be in the range 30000-32767 + p2pNodePort: + rpcNodePort: + zmqTxNodePort: + zmqBlockNodePort: + prometheusNodePort: ingress: enabled: false diff --git a/resources/scenarios/commander.py b/resources/scenarios/commander.py index a5129dbd9..0ed8da9b1 100644 --- a/resources/scenarios/commander.py +++ b/resources/scenarios/commander.py @@ -90,6 +90,12 @@ if "mission" not in pod.metadata.labels: continue + pod_ip = pod.status.pod_ip + while pod_ip is None: + sleep(5) + pod = sclient.read_namespaced_pod(pod.metadata.name, pod.metadata.namespace) + pod_ip = pod.status.pod_ip + if pod.metadata.labels["mission"] == "tank": WARNET["tanks"].append( { @@ -97,7 +103,7 @@ "namespace": pod.metadata.namespace, "chain": pod.metadata.labels["chain"], "p2pport": int(pod.metadata.labels["P2PPort"]), - "rpc_host": pod.status.pod_ip, + "rpc_host": pod_ip, "rpc_port": int(pod.metadata.labels["RPCPort"]), "rpc_user": "user", "rpc_password": pod.metadata.labels["rpcpassword"], @@ -110,7 +116,7 @@ lnnode = LND( pod.metadata.name, pod.metadata.namespace, - pod.status.pod_ip, + pod_ip, pod.metadata.annotations["adminMacaroon"], ) if "cln" in pod.metadata.labels["app.kubernetes.io/name"]: @@ -196,17 +202,19 @@ def b64_to_hex(b64, reverse=False): def wait_for_tanks_connected(self): def tank_connected(self, tank): while True: - peers = tank.getpeerinfo() - count = sum( - 1 - for peer in peers - if peer.get("connection_type") == "manual" or peer.get("addnode") is True - ) - self.log.info(f"Tank {tank.tank} connected to {count}/{tank.init_peers} peers") - if count >= tank.init_peers: - break - else: - sleep(5) + try: + peers = tank.getpeerinfo() + count = sum( + 1 + for peer in peers + if peer.get("connection_type") == "manual" or peer.get("addnode") is True + ) + self.log.info(f"Tank {tank.tank} connected to {count}/{tank.init_peers} peers") + if count >= tank.init_peers: + break + except Exception as e: + self.log.warning(f"Couldn't get peer info from {tank.tank} : {e}") + sleep(5) conn_threads = [ threading.Thread(target=tank_connected, args=(self, tank)) for tank in self.nodes diff --git a/resources/scenarios/ln_framework/ln.py b/resources/scenarios/ln_framework/ln.py index 639bb1813..4128733a8 100644 --- a/resources/scenarios/ln_framework/ln.py +++ b/resources/scenarios/ln_framework/ln.py @@ -208,7 +208,11 @@ def post(self, uri, data=None): def createrune(self): while True: - response = requests.get(f"http://{self.ip_address}:8080/rune.json", timeout=5).text + response = None + try: + response = requests.get(f"http://{self.ip_address}:8080/rune.json", timeout=5).text + except Exception as e: + self.log.warning(f"Error requesting /rune.json: {e}") if not response: self.log.warning(f"Unable to fetch rune from {self.name}, retrying in 2 seconds...") sleep(2) diff --git a/src/warnet/dashboard.py b/src/warnet/dashboard.py index 6eb693911..65f63c230 100644 --- a/src/warnet/dashboard.py +++ b/src/warnet/dashboard.py @@ -2,7 +2,7 @@ import click -from .k8s import get_ingress_ip_or_host, wait_for_ingress_endpoint +from .k8s import get_host, get_ingress_ip_or_host, wait_for_ingress_endpoint @click.command() @@ -23,3 +23,9 @@ def dashboard(): webbrowser.open(url) click.echo(f"Warnet dashboard opened in default browser. URL: {url}") + + +@click.command() +def host(): + """Get one cluster node IP, used for accessing NodePorts""" + click.echo(get_host()) diff --git a/src/warnet/k8s.py b/src/warnet/k8s.py index 41981e337..6516c2c4a 100644 --- a/src/warnet/k8s.py +++ b/src/warnet/k8s.py @@ -1,3 +1,4 @@ +import ipaddress import json import os import sys @@ -6,10 +7,11 @@ from pathlib import Path from time import sleep, time from typing import Optional +from urllib.parse import urlparse import yaml from kubernetes import client, config, watch -from kubernetes.client import CoreV1Api +from kubernetes.client import Configuration, CoreV1Api from kubernetes.client.models import ( V1DeleteOptions, V1Namespace, @@ -413,6 +415,32 @@ def get_ingress_ip_or_host(): return None +def get_host(): + # On macos, k8s runs in a VM with IP addresses that are not directly + # accessible from the host. Look for that first and return the provided + # by Docker Desktop or Minikube. + config.load_kube_config() + server = Configuration.get_default_copy().host + hostname = urlparse(server).hostname + try: + ip = ipaddress.ip_address(hostname) + if ip.is_private or ip.is_loopback: + return hostname + except ValueError: + if hostname == "localhost" or hostname.endswith((".local", ".internal")): + return hostname + + # On Linux or when connecting to a remote cluster, we can get the + # actual external IP address of a kubernetes node. A NodePort is exposed + # on every node in the cluster so just return the first IP. + sclient = get_static_client() + addr_list = sclient.list_node().items[0].status.addresses + addresses = {addr.type: addr.address for addr in addr_list} + # Preference order: + # ExternalIP (cloud) > InternalIP (common) > Hostname + return addresses.get("ExternalIP") or addresses.get("InternalIP") or addresses.get("Hostname") + + def pod_log( pod_name, container_name=None, follow=False, namespace: Optional[str] = None, tail_lines=None ): diff --git a/src/warnet/main.py b/src/warnet/main.py index 768a82f96..6b8682ee3 100644 --- a/src/warnet/main.py +++ b/src/warnet/main.py @@ -3,7 +3,7 @@ from .admin import admin from .bitcoin import bitcoin from .control import down, logs, run, snapshot, stop -from .dashboard import dashboard +from .dashboard import dashboard, host from .deploy import deploy from .graph import create, graph, import_network from .image import image @@ -68,6 +68,7 @@ def version() -> None: cli.add_command(down) cli.add_command(dashboard) cli.add_command(graph) +cli.add_command(host) cli.add_command(import_network) cli.add_command(image) cli.add_command(init) diff --git a/test/data/ln/network.yaml b/test/data/ln/network.yaml index 953080d8a..42d7213ef 100644 --- a/test/data/ln/network.yaml +++ b/test/data/ln/network.yaml @@ -10,10 +10,16 @@ nodes: cln: persistence: enabled: true + service: + type: NodePort + restNodePort: 30001 - name: tank-0001 addnode: - tank-0002 + service: + type: NodePort + rpcNodePort: 30002 - name: tank-0002 addnode: @@ -23,6 +29,9 @@ nodes: lnd: persistence: enabled: true + service: + type: NodePort + restNodePort: 30003 - name: tank-0003 addnode: diff --git a/test/ln_basic_test.py b/test/ln_basic_test.py index d42c5aff6..b5976d8f6 100755 --- a/test/ln_basic_test.py +++ b/test/ln_basic_test.py @@ -8,9 +8,11 @@ from time import sleep import pexpect +import requests +from requests.auth import HTTPBasicAuth from test_base import TestBase -from warnet.process import stream_command +from warnet.process import run_command, stream_command class LNBasicTest(TestBase): @@ -56,6 +58,9 @@ def run_test(self): # Test data persistence self.test_data_persistence() + + # Check NodePorts feature + self.test_nodeports() finally: self.cleanup() @@ -261,6 +266,46 @@ def test_data_persistence(self): second = self.get_ln_node_state(ln) assert first[ln] != second, first.items() + def test_nodeports(self): + self.log.info("Testing local access via NodePort") + + if sys.platform.lower() == "darwin": + cmd = "kubectl config current-context" + if run_command(cmd).strip().lower() == "minikube": + self.log.warning( + "Skipping test: MiniKube on MacOS requires external tunnels for NodePort" + ) + return + + host = self.warnet("host") + + tank0000_cln_rest = requests.post( + f"https://{host}:30001/v1/newaddr", + json={"addresstype": "p2tr"}, + verify=False, # equivalent to curl --insecure + ) + assert "code" in tank0000_cln_rest.json() + assert "data" in tank0000_cln_rest.json() + assert "message" in tank0000_cln_rest.json() + + tank0001_bitcoin_rpc = requests.post( + f"http://{host}:30002/", + json={"method": "getblockcount"}, + auth=HTTPBasicAuth("user", "gn0cchi"), + ) + assert "result" in tank0001_bitcoin_rpc.json() + assert "error" in tank0001_bitcoin_rpc.json() + assert "id" in tank0001_bitcoin_rpc.json() + + tank0002_lnd_rest = requests.get( + f"https://{host}:30003/v1/newaddress", + params={"type": "TAPROOT_PUBKEY"}, + verify=False, # equivalent to curl --insecure + ) + assert "code" in tank0002_lnd_rest.json() + assert "message" in tank0002_lnd_rest.json() + assert "details" in tank0002_lnd_rest.json() + if __name__ == "__main__": test = LNBasicTest()