Skip to content

Commit 7724a0a

Browse files
authored
Merge pull request #712 from Mirantis/ivan4th/diag
Virtlet diagnostics
2 parents 2d3ebd0 + a1c5174 commit 7724a0a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+2338
-284
lines changed

cmd/virtlet/virtlet.go

+40-5
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
3131
"github.com/Mirantis/virtlet/pkg/cni"
3232
"github.com/Mirantis/virtlet/pkg/config"
33+
"github.com/Mirantis/virtlet/pkg/diag"
3334
"github.com/Mirantis/virtlet/pkg/libvirttools"
3435
"github.com/Mirantis/virtlet/pkg/manager"
3536
"github.com/Mirantis/virtlet/pkg/nsfix"
@@ -39,12 +40,17 @@ import (
3940
)
4041

4142
const (
42-
wantTapManagerEnv = "WANT_TAP_MANAGER"
43-
nodeNameEnv = "KUBE_NODE_NAME"
43+
wantTapManagerEnv = "WANT_TAP_MANAGER"
44+
nodeNameEnv = "KUBE_NODE_NAME"
45+
diagSocket = "/run/virtlet-diag.sock"
46+
netnsDiagCommand = `if [ -d /var/run/netns ]; then cd /var/run/netns; for ns in *; do echo "*** ${ns} ***"; ip netns exec "${ns}" ip a; ip netns exec "${ns}" ip r; echo; done; fi`
47+
criproxyLogCommand = `nsenter -t 1 -m -u -i journalctl -xe -u criproxy -n 20000 --no-pager || true`
48+
qemuLogDir = "/var/log/libvirt/qemu"
4449
)
4550

4651
var (
4752
dumpConfig = flag.Bool("dump-config", false, "Dump node-specific Virtlet config as a shell script and exit")
53+
dumpDiag = flag.Bool("diag", false, "Dump diagnostics as JSON and exit")
4854
displayVersion = flag.Bool("version", false, "Display version and exit")
4955
versionFormat = flag.String("version-format", "text", "Version format to use (text, short, json, yaml)")
5056
)
@@ -55,8 +61,8 @@ func configWithDefaults(cfg *v1.VirtletConfig) *v1.VirtletConfig {
5561
return r
5662
}
5763

58-
func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig) {
59-
manager := manager.NewVirtletManager(config, nil, clientCfg)
64+
func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig, diagSet *diag.Set) {
65+
manager := manager.NewVirtletManager(config, nil, clientCfg, diagSet)
6066
if err := manager.Run(); err != nil {
6167
glog.Errorf("Error: %v", err)
6268
os.Exit(1)
@@ -106,6 +112,32 @@ func setLogLevel(config *v1.VirtletConfig) {
106112
})
107113
}
108114

115+
func runDiagServer() *diag.Set {
116+
diagSet := diag.NewDiagSet()
117+
diagSet.RegisterDiagSource("ip-a", diag.NewCommandSource("txt", []string{"ip", "a"}))
118+
diagSet.RegisterDiagSource("ip-r", diag.NewCommandSource("txt", []string{"ip", "r"}))
119+
diagSet.RegisterDiagSource("psaux", diag.NewCommandSource("txt", []string{"ps", "aux"}))
120+
diagSet.RegisterDiagSource("netns", diag.NewCommandSource("txt", []string{"/bin/bash", "-c", netnsDiagCommand}))
121+
diagSet.RegisterDiagSource("criproxy", diag.NewCommandSource("log", []string{"/bin/bash", "-c", criproxyLogCommand}))
122+
diagSet.RegisterDiagSource("libvirt-logs", diag.NewLogDirSource(qemuLogDir))
123+
diagSet.RegisterDiagSource("stack", diag.StackDumpSource)
124+
server := diag.NewServer(diagSet)
125+
go func() {
126+
err := server.Serve(diagSocket, nil)
127+
glog.V(1).Infof("Diag server returned: %v", err)
128+
}()
129+
return diagSet
130+
}
131+
132+
func doDiag() {
133+
dr, err := diag.RetrieveDiagnostics(diagSocket)
134+
if err != nil {
135+
glog.Errorf("Failed to retrieve diagnostics: %v", err)
136+
os.Exit(1)
137+
}
138+
os.Stdout.Write(dr.ToJSON())
139+
}
140+
109141
func main() {
110142
nsfix.HandleReexec()
111143
clientCfg := utils.BindFlags(flag.CommandLine)
@@ -131,9 +163,12 @@ func main() {
131163
glog.Errorf("Error writing config: %v", err)
132164
os.Exit(1)
133165
}
166+
case *dumpDiag:
167+
doDiag()
134168
default:
135169
localConfig = configWithDefaults(localConfig)
136170
go runTapManager(localConfig)
137-
runVirtlet(localConfig, clientCfg)
171+
diagSet := runDiagServer()
172+
runVirtlet(localConfig, clientCfg, diagSet)
138173
}
139174
}

cmd/virtletctl/virtletctl.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,14 @@ func newRootCmd() *cobra.Command {
4343
flag.CommandLine.Parse([]string{})
4444

4545
client := tools.NewRealKubeClient(clientCfg)
46-
cmd.AddCommand(tools.NewDumpMetadataCmd(client))
4746
cmd.AddCommand(tools.NewVirshCmd(client, os.Stdout))
4847
cmd.AddCommand(tools.NewSSHCmd(client, os.Stdout, ""))
4948
cmd.AddCommand(tools.NewVNCCmd(client, os.Stdout, true))
5049
cmd.AddCommand(tools.NewInstallCmd(cmd, "", ""))
5150
cmd.AddCommand(tools.NewGenDocCmd(cmd, os.Stdout))
5251
cmd.AddCommand(tools.NewGenCmd(os.Stdout))
5352
cmd.AddCommand(tools.NewVersionCommand(client, os.Stdout, nil))
53+
cmd.AddCommand(tools.NewDiagCommand(client, os.Stdin, os.Stdout))
5454

5555
for _, c := range cmd.Commands() {
5656
c.PreRunE = func(*cobra.Command, []string) error {

deploy/data/virtlet-ds.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,9 @@ spec:
198198
name: image-name-translations
199199
- name: pods-log
200200
mountPath: /var/log/pods
201+
# needed for diagnostic purposes
202+
- name: libvirt-log
203+
mountPath: /var/log/libvirt
201204
securityContext:
202205
privileged: true
203206
readinessProbe:
@@ -275,9 +278,6 @@ spec:
275278
- hostPath:
276279
path: /var/log/pods
277280
name: pods-log
278-
- hostPath:
279-
path: /var/run/netns
280-
name: netns-dir
281281
- configMap:
282282
name: virtlet-image-translations
283283
name: image-name-translations

docs/README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
This directory holds files containing documentation.
1+
This directory contains the Virtlet documentation.
22

33
* For a basic example of VM pod definition, see [examples/cirros-vm.yaml](../examples/cirros-vm.yaml)
44
* [Cloud-init data generation](cloud-init-data-generation.md)
@@ -11,4 +11,5 @@ This directory holds files containing documentation.
1111
* [Environment variables](environment-variables.md) support
1212
* [Image Handling](images.md)
1313
* [Image Name Translation](image-name-translation.md)
14+
* [Diagnostics](diagnostics.md)
1415
* [Update notes](update-notes.md)

docs/diagnostics.md

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Diagnostics
2+
3+
Virtlet provides a set of
4+
[virtletctl diag](virtletctl/virtletctl_diag.md) commands that can
5+
help with troubleshooting. The diagnostics can be invoked either
6+
directly or by means of a
7+
[Sonobuoy](https://github.com/heptio/sonobuoy) plugin.
8+
9+
## Direct invocation
10+
11+
The most basic diagnostics command is [virtletctl diag dump](virtletctl/virtletctl_diag_dump.md):
12+
```
13+
$ virtletctl diag out/
14+
$ ls -lR out
15+
total 0
16+
drwxr-xr-x 3 user wheel 96 Jul 11 01:56 nodes
17+
18+
out/nodes:
19+
total 0
20+
drwxr-xr-x 12 user wheel 384 Jul 11 01:56 kube-node-1
21+
22+
out/nodes/kube-node-1:
23+
total 5352
24+
-rwxr-xr-x 1 user wheel 1276000 Jul 11 01:56 criproxy.log
25+
-rwxr-xr-x 1 user wheel 1787 Jul 11 01:56 ip-a.txt
26+
-rwxr-xr-x 1 user wheel 322 Jul 11 01:56 ip-r.txt
27+
drwxr-xr-x 3 user wheel 96 Jul 11 01:56 libvirt-logs
28+
drwxr-xr-x 5 user wheel 160 Jul 11 01:56 libvirt-xml
29+
-rwxr-xr-x 1 user wheel 9964 Jul 11 01:56 metadata.txt
30+
-rwxr-xr-x 1 user wheel 1443 Jul 11 01:56 netns.txt
31+
-rwxr-xr-x 1 user wheel 9217 Jul 11 02:56 psaux.txt
32+
-rwxr-xr-x 1 user wheel 18214 Jul 11 01:56 stack.log
33+
-rwxr-xr-x 1 user wheel 64314 Jul 11 01:56 virtlet-pod-libvirt.log
34+
-rwxr-xr-x 1 user wheel 1349763 Jul 11 01:56 virtlet-pod-virtlet.log
35+
36+
out/nodes/kube-node-1/libvirt-logs:
37+
total 8
38+
-rwxr-xr-x 1 user wheel 2172 Jul 11 01:56 virtlet-1b2261ca-7ed6-cirros-vm.log
39+
40+
out/nodes/kube-node-1/libvirt-xml:
41+
total 24
42+
-rwxr-xr-x 1 user wheel 3511 Jul 11 01:56 domain-virtlet-1b2261ca-7ed6-cirros-vm.xml
43+
-rwxr-xr-x 1 user wheel 445 Jul 11 01:56 pool-volumes.xml
44+
-rwxr-xr-x 1 user wheel 1041 Jul 11 01:56 volume-virtlet_root_1b2261ca-7ed6-58e7-58de-0eef2c9d5320.xml
45+
```
46+
47+
The following files and directories are produced for each Kubernetes
48+
node that runs Virtlet:
49+
* `criproxy.log` - the logs of CRI Proxy's systemd unit
50+
* `ip-a.txt` - the output of `ip a` on the node
51+
* `ip-r.txt` - the output of `ip r` on the node
52+
* `metadata.txt` - the contents of Virtlet's internal metadata db in a text form
53+
* `netns.txt` - the output of `ip a` and `ip r` for each network
54+
namespace that's managed by Virtlet
55+
* `psaux.txt` - the output of `ps aux` command on the node
56+
* `stack.log` - the dump of Go stack of Virtlet process
57+
* `virtlet-pod-libvirt.log` - the log of Virtlet pod's libvirt container
58+
* `virtlet-pod-virtlet.log` - the log of Virtlet pod's virtlet container
59+
* `livirt-logs` - a directory with libvirt/QEMU logs for each domain
60+
* `libvirt-xml` - the dumps of all the domains, storage pools and storage volumes in libvirt
61+
62+
It's also possible to dump Virtlet diagnostics as JSON to stdout using
63+
`virtletctl diag dump --json`. The JSON file can be subsequently
64+
unpacked into the aforementioned directory structure using
65+
[virtletctl diag unpack](virtletctl/virtletctl_diag_unpack.md).
66+
67+
## Sonobuoy
68+
69+
Virtlet diagnostics can be run as a
70+
[Sonobuoy](https://github.com/heptio/sonobuoy) plugin. Unfortunately,
71+
right now Sonobuoy's plugin support is
72+
[somewhat limited](https://github.com/heptio/sonobuoy/issues/405). Because
73+
of that problem, Sonobuoy run must be done in two phases, first
74+
generating YAML and then using `virtletctl` to patch it (inject
75+
Virtlet sonobuoy plugin):
76+
```
77+
$ cat sonobuoy.json
78+
{
79+
"plugins": [ { "name": "virtlet" } ]
80+
}
81+
$ sonobuoy gen --config sonobuoy.json --e2e-focus nosuchtest |
82+
virtletctl diag sonobuoy |
83+
kubectl apply -f -
84+
$ # wait till sonobuoy run is complete
85+
$ sonobuoy status
86+
PLUGIN STATUS COUNT
87+
virtlet complete 1
88+
89+
Sonobuoy has completed. Use `sonobuoy retrieve` to get results.
90+
$ sonobuoy retrieve
91+
```
92+
93+
The diagnostics results are placed under `plugins/virtlet/results` and
94+
can be unpacked using [virtletctl diag unpack](virtletctl/virtletctl_diag_unpack.md):
95+
```
96+
$ virtletctl diag unpack out/ <sonobuoy_output_dir/plugins/virtlet/results
97+
```

docs/virtletctl/virtletctl.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ Kubernetes cluster.
4040

4141
### SEE ALSO
4242

43-
* [virtletctl dump-metadata](virtletctl_dump-metadata.md) - Dump Virtlet metadata db
43+
* [virtletctl diag](virtletctl_diag.md) - Virtlet diagnostics
4444
* [virtletctl gen](virtletctl_gen.md) - Generate Kubernetes YAML for Virtlet deployment
4545
* [virtletctl gendoc](virtletctl_gendoc.md) - Generate Markdown documentation for the commands
4646
* [virtletctl install](virtletctl_install.md) - Install virtletctl as a kubectl plugin
@@ -49,4 +49,4 @@ Kubernetes cluster.
4949
* [virtletctl virsh](virtletctl_virsh.md) - Execute a virsh command
5050
* [virtletctl vnc](virtletctl_vnc.md) - Provide access to the VNC console of a VM pod
5151

52-
###### Auto generated by spf13/cobra on 11-Jun-2018
52+
###### Auto generated by spf13/cobra on 11-Jul-2018

docs/virtletctl/virtletctl_diag.md

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
## virtletctl diag
2+
3+
Virtlet diagnostics
4+
5+
### Synopsis
6+
7+
Retrieve and unpack Virtlet diagnostics information
8+
9+
### Options
10+
11+
```
12+
-h, --help help for diag
13+
```
14+
15+
### Options inherited from parent commands
16+
17+
```
18+
--alsologtostderr log to standard error as well as files
19+
--as string Username to impersonate for the operation
20+
--as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
21+
--certificate-authority string Path to a cert file for the certificate authority
22+
--client-certificate string Path to a client certificate file for TLS
23+
--client-key string Path to a client key file for TLS
24+
--cluster string The name of the kubeconfig cluster to use
25+
--context string The name of the kubeconfig context to use
26+
--insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
27+
--kubeconfig string Path to the kubeconfig file to use for CLI requests.
28+
--log-backtrace-at traceLocation when logging hits line file:N, emit a stack trace (default :0)
29+
--log-dir string If non-empty, write log files in this directory
30+
--logtostderr log to standard error instead of files
31+
-n, --namespace string If present, the namespace scope for this CLI request
32+
--password string Password for basic authentication to the API server
33+
--request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
34+
-s, --server string The address and port of the Kubernetes API server
35+
--stderrthreshold severity logs at or above this threshold go to stderr (default 2)
36+
--token string Bearer token for authentication to the API server
37+
--user string The name of the kubeconfig user to use
38+
--username string Username for basic authentication to the API server
39+
-v, --v Level log level for V logs
40+
--virtlet-runtime string the name of virtlet runtime used in kubernetes.io/target-runtime annotation (default "virtlet.cloud")
41+
--vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging
42+
```
43+
44+
### SEE ALSO
45+
46+
* [virtletctl](virtletctl.md) - Virtlet control tool
47+
* [virtletctl diag dump](virtletctl_diag_dump.md) - Dump Virtlet diagnostics information
48+
* [virtletctl diag sonobuoy](virtletctl_diag_sonobuoy.md) - Add Virtlet sonobuoy plugin to the sonobuoy output
49+
* [virtletctl diag unpack](virtletctl_diag_unpack.md) - Unpack Virtlet diagnostics information
50+
51+
###### Auto generated by spf13/cobra on 11-Jul-2018
+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
## virtletctl diag dump
2+
3+
Dump Virtlet diagnostics information
4+
5+
### Synopsis
6+
7+
Pull Virtlet diagnostics information from the nodes and dump it as a directory tree or JSON
8+
9+
```
10+
virtletctl diag dump output_dir [flags]
11+
```
12+
13+
### Options
14+
15+
```
16+
-h, --help help for dump
17+
--json Use JSON output
18+
```
19+
20+
### Options inherited from parent commands
21+
22+
```
23+
--alsologtostderr log to standard error as well as files
24+
--as string Username to impersonate for the operation
25+
--as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
26+
--certificate-authority string Path to a cert file for the certificate authority
27+
--client-certificate string Path to a client certificate file for TLS
28+
--client-key string Path to a client key file for TLS
29+
--cluster string The name of the kubeconfig cluster to use
30+
--context string The name of the kubeconfig context to use
31+
--insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure
32+
--kubeconfig string Path to the kubeconfig file to use for CLI requests.
33+
--log-backtrace-at traceLocation when logging hits line file:N, emit a stack trace (default :0)
34+
--log-dir string If non-empty, write log files in this directory
35+
--logtostderr log to standard error instead of files
36+
-n, --namespace string If present, the namespace scope for this CLI request
37+
--password string Password for basic authentication to the API server
38+
--request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0")
39+
-s, --server string The address and port of the Kubernetes API server
40+
--stderrthreshold severity logs at or above this threshold go to stderr (default 2)
41+
--token string Bearer token for authentication to the API server
42+
--user string The name of the kubeconfig user to use
43+
--username string Username for basic authentication to the API server
44+
-v, --v Level log level for V logs
45+
--virtlet-runtime string the name of virtlet runtime used in kubernetes.io/target-runtime annotation (default "virtlet.cloud")
46+
--vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging
47+
```
48+
49+
### SEE ALSO
50+
51+
* [virtletctl diag](virtletctl_diag.md) - Virtlet diagnostics
52+
53+
###### Auto generated by spf13/cobra on 11-Jul-2018

0 commit comments

Comments
 (0)