skstat.in

#!/usr/bin/env ${Python_EXECUTABLE}

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import os
import sys
from datetime import datetime, timezone
from time import ctime, strftime, gmtime

extra_paths = [
    "${CMAKE_INSTALL_PREFIX}/lib/skupper-router/python/"
]


def getpath(env):
    """Return a path environment variable split into a list"""
    path = os.environ.get(env)
    if path:
        return path.split(os.pathsep)
    return []


sys.path = extra_paths + sys.path
python_path = extra_paths + getpath("PYTHONPATH")

env_vars = {
    'PYTHONPATH': os.pathsep.join(python_path),
}
os.environ.update(env_vars)

from skupper_router._skupper_router_site import populate_pythonpath
populate_pythonpath()

from skupper_router.management.client import Node
from skupper_router_internal.management.qdrouter import QdSchema
from skupper_router_internal.tools import Display, Header, Sorter, TimeLong, TimeShort, BodyFormat, PlainNum
from skupper_router_internal.tools import NumKMG
from skupper_router_internal.tools.command import (parse_args_skstat, main,
                                                   opts_ssl_domain, opts_sasl,
                                                   opts_url)
from skupper_router_internal.compat import UNICODE


def get(obj, attr):
    return getattr(obj, attr, None)


class BusManager:

    schema = QdSchema()

    def __init__(self, opts):
        self.opts = opts
        self.node = Node.connect(opts_url(opts), opts.router,
                                 timeout=opts.timeout,
                                 ssl_domain=opts_ssl_domain(opts),
                                 sasl=opts_sasl(opts),
                                 edge_router=opts.edge_router)
        self.show = getattr(self, opts.show)
        self.bodyFormat = BodyFormat.CSV if opts.csv else BodyFormat.CLASSIC

    def query(self, entity_type, attribute_names=None, limit=None):
        if attribute_names:
            unames = []
            for a in attribute_names:
                unames.append(UNICODE(a))
            attribute_names = unames
        return self.node.query(entity_type, attribute_names, count=limit).get_entities()

    def close(self):
        self.node.close()

    def connAuth(self, conn):
        ##
        # Summarize the authentication for a connection:
        # no-auth
        # anonymous-user
        # <user>(PLAIN)
        # <user>(kerberos)
        # <user>(x.509)
        ##
        if not conn.isAuthenticated:
            return "no-auth"
        sasl = conn.sasl
        if sasl == "GSSAPI":
            sasl = "Kerberos"
        elif sasl == "EXTERNAL":
            sasl = "x.509"
        elif sasl == "ANONYMOUS":
            return "anonymous-user"
        if not conn.user:
            return sasl
        return "%s(%s)" % (conn.user, sasl)

    def connSecurity(self, conn):
        ##
        # Summarize the security of a connection:
        # no-security
        ##   SSLv3 (cipher)
        ##   TLS (cipher)
        # Kerberos
        ##
        if not conn.isEncrypted:
            return "no-security"
        if conn.sasl == "GSSAPI":
            return "Kerberos"
        return "%s(%s)" % (conn.sslProto, conn.sslCipher)

    def noTrailingSlash(self, text):
        if text is None:
            return ""
        if text[-1:] == '/':
            return text[:-1]
        return text

    def displayRouterId(self):
        objects = self.query('io.skupper.router.routerMetrics')
        router = objects[0]
        if router:
            print(router.id)

    def display_datetime_router_id(self):
        print(str(datetime.now(timezone.utc)) + " UTC")
        self.displayRouterId()
        print("")

    def displayEdges(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("id"))
        heads.append(Header("host"))
        heads.append(Header("container"))
        heads.append(Header("dir"))
        heads.append(Header("security"))
        heads.append(Header("authentication"))

        rows = []
        objects = self.query('io.skupper.router.connection', limit=self.opts.limit)

        if not objects:
            if show_date_id:
                self.display_datetime_router_id()
            print("No Edge Router Connections")
            return

        if show_date_id:
            self.display_datetime_router_id()

        has_active = False

        first = objects[0]
        try:
            if first:
                active = first.active
                has_active = True
        except:
            pass

        if has_active:
            heads.append(Header("active"))

        for conn in objects:
            if conn.role == "edge":
                row = []
                row.append(conn.identity)
                row.append(conn.host)
                row.append(conn.container)
                row.append(conn.dir)
                row.append(self.connSecurity(conn))
                row.append(self.connAuth(conn))
                if has_active:
                    if conn.active:
                        row.append("yes")
                    else:
                        row.append("no")
                rows.append(row)

        if rows:
            title = "Connections"
        else:
            return

        dispRows = rows
        disp.formattedTable(title, heads, dispRows)

    def displayConnections(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("id"))
        heads.append(Header("host"))
        heads.append(Header("container"))
        heads.append(Header("role"))
        heads.append(Header("proto"))
        heads.append(Header("dir"))
        heads.append(Header("security"))
        heads.append(Header("authentication"))
        heads.append(Header("meshId"))

        rows = []
        objects = self.query('io.skupper.router.connection', limit=self.opts.limit)

        if show_date_id:
            self.display_datetime_router_id()

        has_uptime = False
        has_last_dlv = False

        if objects:
            first_conn = objects[0]
            try:
                lastDlvSeconds = first_conn.lastDlvSeconds
                has_last_dlv = True
            except:
                pass
            try:
                uptime = first_conn.uptimeSeconds
                has_uptime = True
            except:
                pass

        # This is so that, this skstat can be used against older routers without Python keyerrors
        if has_last_dlv:
            heads.append(Header("last dlv"))
        if has_uptime:
            heads.append(Header("uptime"))

        for conn in objects:
            row = []
            row.append(conn.identity)
            row.append(conn.host)
            row.append(conn.container)
            row.append(get(conn, 'role'))
            row.append(conn.protocol)
            row.append(conn.dir)
            row.append(self.connSecurity(conn))
            row.append(self.connAuth(conn))
            row.append(get(conn, 'meshId'))
            if has_last_dlv:
                if conn.lastDlvSeconds is None:
                    row.append('-')
                else:
                    row.append(TimeLong(conn.lastDlvSeconds))
            if has_uptime:
                row.append(TimeLong(conn.uptimeSeconds))
            rows.append(row)
        title = "Connections"
        dispRows = rows
        disp.formattedTable(title, heads, dispRows)

    def _addr_summary(self, addr):
        cls   = self._addr_class(addr)
        text  = self._addr_text(addr)
        if cls == '-':
            return "-"
        if cls == 'M':
            return text
        return "%s:%s" % (cls, text)

    def _addr_class(self, addr):
        if not addr:
            return ""
        if addr[0] == 'M'  :
            return "mobile"
        if addr[0] == 'R'  :
            return "router"
        if addr[0] == 'A'  :
            return "area"
        if addr[0] == 'L'  :
            return "local"
        if addr[0] == 'T'  :
            return "topo"
        if addr[0] in 'CE' :
            return "link-in"
        if addr[0] in 'DF' :
            return "link-out"
        if addr[0] == 'H'  :
            return "edge"
        return "unknown: %s" % addr[0]

    def _addr_text(self, addr):
        if not addr:
            return ""
        else:
            return addr[1:]

    def _identity_clean(self, identity, router_id=None):
        if router_id:
            return router_id
        if not identity:
            return "-"
        pos = identity.find('/')
        if pos >= 0:
            return identity[pos + 1:]
        return identity

    def _list_clean(self, inlist):
        outlist = []
        if not inlist:
            return outlist
        for i in inlist:
            outlist.append(str(i))
        return outlist

    def displayGeneral(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("attr"))
        heads.append(Header("value"))
        rows = []

        router_objects = self.query('io.skupper.router.router')
        router_metrics_objects = self.query('io.skupper.router.routerMetrics')

        router = router_objects[0]
        router_metrics = router_metrics_objects[0]

        if show_date_id:
            self.display_datetime_router_id()

        rows.append(('Version',                   router_metrics.version))
        rows.append(('Mode',                      router.mode))
        rows.append(('Router Id',                 router_metrics.id))
        rows.append(('Host Name',                 router.hostName))
        rows.append(('Hello Interval Seconds',    PlainNum(router.helloIntervalSeconds)))
        rows.append(('Hello Max Age Seconds',     PlainNum(router.helloMaxAgeSeconds)))
        rows.append(('RA Interval Seconds',       PlainNum(router.raIntervalSeconds)))
        rows.append(('RA Interval Flux Seconds',  PlainNum(router.raIntervalFluxSeconds)))
        rows.append(('Remote Ls Max Age Seconds', PlainNum(router.remoteLsMaxAgeSeconds)))
        rows.append(('Debug Dump File',           router.debugDumpFile))
        rows.append(('Sasl Config Dir',           router.saslConfigDir))
        rows.append(('Sasl Config Name',          router.saslConfigName))
        if router.timestampsInUTC:
            rows.append(('Timestamps In UTC',         "yes"))
        else:
            rows.append(('Timestamps In UTC',         "no"))
        rows.append(('Timestamp Format',          router.timestampFormat))
        rows.append(('Default Distribution',      router.defaultDistribution))
        rows.append(('Metadata',                  router.metadata))
        rows.append(('Data Connection Count',     router.dataConnectionCount))
        if router.vanId is not None:
            rows.append(('Van Id', router.vanId))
        try:
            rows.append(('Worker Threads', PlainNum(router.workerThreads)))
        except:
            pass

        try:
            rows.append(('Uptime',        TimeLong(router_metrics.uptimeSeconds)))
        except:
            pass

        try:
            if router_metrics.memoryUsage is not None:
                rows.append(('VmSize', NumKMG(router_metrics.memoryUsage,
                                              base=1024)))
        except Exception:
            pass
        try:
            if router_metrics.residentMemoryUsage is not None:
                rows.append(('RSS', NumKMG(router_metrics.residentMemoryUsage,
                                           base=1024)))
        except Exception:
            pass
        rows.append(('Area',          router.area))
        rows.append(('Auto Links',    PlainNum(router_metrics.autoLinkCount)))
        rows.append(('Links',         PlainNum(router_metrics.linkCount)))
        rows.append(('Nodes',         PlainNum(router_metrics.nodeCount)))
        rows.append(('Addresses',     PlainNum(router_metrics.addrCount)))
        rows.append(('Total Connections',   PlainNum(router_metrics.connectionCount)))

        # Overall delivery related counts.
        # These overall statistics were introduced in 1.1 version.
        # Wrap these in a try except so that newer versions of skstat works with older version of router
        try:
            rows.append(('Presettled Count', PlainNum(router_metrics.presettledDeliveries)))
            rows.append(('Dropped Presettled Count', PlainNum(router_metrics.droppedPresettledDeliveries)))
            rows.append(('Accepted Count', PlainNum(router_metrics.acceptedDeliveries)))
            rows.append(('Rejected Count', PlainNum(router_metrics.rejectedDeliveries)))
            rows.append(('Released Count', PlainNum(router_metrics.releasedDeliveries)))
            rows.append(('Modified Count', PlainNum(router_metrics.modifiedDeliveries)))
            try:
                rows.append(('Deliveries Delayed > 1sec',  PlainNum(router_metrics.deliveriesDelayed1Sec)))
                rows.append(('Deliveries Delayed > 10sec', PlainNum(router_metrics.deliveriesDelayed10Sec)))
                rows.append(('Deliveries Stuck > 10sec',   PlainNum(router_metrics.deliveriesStuck)))
            except:
                pass
            rows.append(('Ingress Count', PlainNum(router_metrics.deliveriesIngress)))
            rows.append(('Egress Count', PlainNum(router_metrics.deliveriesEgress)))
            rows.append(('Transit Count', PlainNum(router_metrics.deliveriesTransit)))
            rows.append(('Deliveries from Route Container', PlainNum(router_metrics.deliveriesIngressRouteContainer)))
            rows.append(('Deliveries to Route Container', PlainNum(router_metrics.deliveriesEgressRouteContainer)))
        except:
            pass

        # per-protocol user connection counts
        try:
            for proto, count in router_metrics.connectionCounters.items():
                rows.append((f"{proto.upper()} Service Connections", PlainNum(count)))
        except:
            pass

        title = "Router Statistics"
        dispRows = rows
        disp.formattedTable(title, heads, dispRows)

    def displayRouterLinks(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("type"))
        heads.append(Header("dir"))
        heads.append(Header("conn id"))
        heads.append(Header("id"))
        heads.append(Header("class"))
        heads.append(Header("addr"))
        heads.append(Header("cap", Header.PLAIN_NUM))

        rows = []
        cols = ('linkType', 'linkDir', 'connectionId', 'identity', 'owningAddr',
                'capacity', 'undeliveredCount', 'unsettledCount', 'deliveryCount',
                'presettledCount', 'droppedPresettledCount', 'acceptedCount', 'rejectedCount', 'releasedCount',
                'modifiedCount', 'deliveriesDelayed1Sec', 'deliveriesDelayed10Sec', 'deliveriesStuck',
                'creditAvailable', 'zeroCreditSeconds',
                'adminStatus', 'operStatus', 'linkName', 'priority', 'settleRate')

        objects = self.query('io.skupper.router.router.link', cols, limit=self.opts.limit)

        has_dropped_presettled_count = False
        has_priority = False
        has_delayed  = False
        has_stuck    = False
        has_credit   = False

        if show_date_id:
            self.display_datetime_router_id()

        if objects:
            first_row = objects[0]
            if first_row:
                if hasattr(first_row, 'droppedPresettledCount'):
                    has_dropped_presettled_count = True
                if hasattr(first_row, 'priority'):
                    has_priority = True
                if hasattr(first_row, 'deliveriesDelayed1Sec'):
                    has_delayed = True
                if hasattr(first_row, 'deliveriesStuck'):
                    has_stuck = True
                if hasattr(first_row, 'creditAvailable'):
                    has_credit = True

        if has_priority:
            heads.append(Header("pri", Header.PLAIN_NUM))
        heads.append(Header("undel", Header.PLAIN_NUM))
        heads.append(Header("unsett", Header.PLAIN_NUM))
        heads.append(Header("deliv", Header.PLAIN_NUM))
        heads.append(Header("presett", Header.PLAIN_NUM))

        if has_dropped_presettled_count:
            heads.append(Header("psdrop", Header.PLAIN_NUM))

        heads.append(Header("acc", Header.PLAIN_NUM))
        heads.append(Header("rej", Header.PLAIN_NUM))
        heads.append(Header("rel", Header.PLAIN_NUM))
        heads.append(Header("mod", Header.PLAIN_NUM))
        if has_delayed:
            heads.append(Header("delay", Header.PLAIN_NUM))
            heads.append(Header("rate", Header.PLAIN_NUM))
        if has_stuck:
            heads.append(Header("stuck", Header.PLAIN_NUM))
        if has_credit:
            heads.append(Header("cred", Header.PLAIN_NUM))
            heads.append(Header("blkd"))
        if self.opts.verbose:
            heads.append(Header("admin"))
            heads.append(Header("oper"))
            heads.append(Header("name"))

        for link in objects:
            row = []
            row.append(link.linkType)
            row.append(link.linkDir)
            row.append(link.connectionId)
            row.append(link.identity)
            row.append(self._addr_class(link.owningAddr))
            row.append(self._addr_text(link.owningAddr))
            row.append(link.capacity)
            if has_priority:
                row.append(link.priority)
            row.append(link.undeliveredCount)
            row.append(link.unsettledCount)
            row.append(link.deliveryCount)
            row.append(link.presettledCount)
            if has_dropped_presettled_count:
                row.append(link.droppedPresettledCount)
            row.append(link.acceptedCount)
            row.append(link.rejectedCount)
            row.append(link.releasedCount)
            row.append(link.modifiedCount)
            if has_delayed:
                row.append(link.deliveriesDelayed1Sec + link.deliveriesDelayed10Sec)
                row.append(link.settleRate)
            if has_stuck:
                row.append(link.deliveriesStuck)
            if has_credit:
                row.append(link.creditAvailable)
                if link.zeroCreditSeconds > 0:
                    row.append(TimeShort(link.zeroCreditSeconds * 1000000000))
                else:
                    row.append('-')
            if self.opts.verbose:
                row.append(link.adminStatus)
                row.append(link.operStatus)
                row.append(link.linkName)
            rows.append(row)
        title = "Router Links"
        dispRows = rows
        disp.formattedTable(title, heads, dispRows)

    def displayRouterNodes(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("router-id"))
        heads.append(Header("next-hop"))
        heads.append(Header("link"))
        if self.opts.verbose:
            heads.append(Header("ver"))
            heads.append(Header("cost"))
            heads.append(Header("neighbors"))
            heads.append(Header("valid-origins"))
        rows = []
        objects = self.query('io.skupper.router.router.node', limit=self.opts.limit)

        # Find the most recent topo change in this neighborhood.
        lastTopoChange = 0.0

        if show_date_id:
            self.display_datetime_router_id()

        for node in objects:
            row = []
            if node.lastTopoChange:
                if float(node.lastTopoChange) > lastTopoChange:
                    lastTopoChange = float(node.lastTopoChange)
            row.append(node.id)
            if node.nextHop is not None:
                row.append(node.nextHop)
                row.append('-')
            else:
                row.append('-')
                row.append(PlainNum(node.routerLink))

            if self.opts.verbose:
                row.append(PlainNum(get(node, 'protocolVersion')))
                cost = get(node, 'cost')
                if cost:
                    row.append(PlainNum(cost))
                else:
                    row.append("")
                row.append('%r' % self._list_clean(node.linkState))
                row.append('%r' % self._list_clean(node.validOrigins))
            rows.append(row)
        if len(rows) > 0:
            title = "Routers in the Network"
            # Use gmtime to make times comparable across large networks.
            if lastTopoChange > 1.0:
                topoLine = "\nLast Topology Change: " + strftime('%A %b %d %H:%M:%S %Y', gmtime(lastTopoChange)) + " GMT"
                title += topoLine
            sort = Sorter(heads, rows, 'router-id')
            dispRows = sort.getSorted()
            disp.formattedTable(title, heads, dispRows)
        else:
            print("Router is Standalone - No Router List")

    def displayAddresses(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("class"))
        heads.append(Header("addr"))
        heads.append(Header("distrib"))

        rows = []
        cols = ('distribution', 'inProcess', 'subscriberCount', 'remoteCount',
                'deliveriesIngress', 'deliveriesEgress',
                'deliveriesTransit', 'deliveriesToContainer', 'deliveriesFromContainer', 'name', 'priority', 'watch')

        objects = self.query('io.skupper.router.router.address', cols, limit=self.opts.limit)

        has_priority = False

        if show_date_id:
            self.display_datetime_router_id()

        if objects:
            first_row = objects[0]
            if first_row:
                if hasattr(first_row, 'priority'):
                    has_priority = True

        if has_priority:
            heads.append(Header("pri"))
        if self.opts.verbose:
            heads.append(Header("in-proc", Header.COMMAS))
        heads.append(Header("local", Header.COMMAS))
        heads.append(Header("remote", Header.COMMAS))
        heads.append(Header("in", Header.COMMAS))
        heads.append(Header("out", Header.COMMAS))
        heads.append(Header("thru", Header.COMMAS))
        if self.opts.verbose:
            heads.append(Header("to-proc", Header.COMMAS))
            heads.append(Header("from-proc", Header.COMMAS))
            heads.append(Header("watch", Header.COMMAS))

        for addr in objects:
            row = []
            row.append(self._addr_class(addr.name))
            row.append(self._addr_text(addr.name))
            row.append(addr.distribution)
            if has_priority:
                row.append(PlainNum(addr.priority) if addr.priority >= 0 else "-")
            if self.opts.verbose:
                row.append(PlainNum(addr.inProcess))
            row.append(PlainNum(addr.subscriberCount))
            row.append(PlainNum(addr.remoteCount))
            row.append(PlainNum(addr.deliveriesIngress))
            row.append(PlainNum(addr.deliveriesEgress))
            row.append(PlainNum(addr.deliveriesTransit))
            if self.opts.verbose:
                row.append(PlainNum(addr.deliveriesToContainer))
                row.append(PlainNum(addr.deliveriesFromContainer))
                row.append("Y" if addr.watch else "")
            rows.append(row)
        title = "Router Addresses"
        sorter = Sorter(heads, rows, 'addr', 0, True)
        dispRows = sorter.getSorted()
        disp.formattedTable(title, heads, dispRows)

    def displayAutolinks(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("addr"))
        heads.append(Header("dir"))
        heads.append(Header("extAddr"))
        heads.append(Header("link"))
        heads.append(Header("status"))
        heads.append(Header("lastErr"))
        rows = []
        cols = ('address', 'direction', 'externalAddress', 'linkRef', 'operStatus', 'lastError')

        objects = self.query('io.skupper.router.router.config.autoLink', cols, limit=self.opts.limit)

        if show_date_id:
            self.display_datetime_router_id()

        if not objects:
            print("AutoLinks")
            print("No AutoLinks found")
            return

        for al in objects:
            row = []
            row.append(al.address)
            row.append(al.direction)
            row.append(al.externalAddress)
            row.append(al.linkRef)
            row.append(al.operStatus)
            row.append(al.lastError)
            rows.append(row)

        title = "AutoLinks"
        sorter = Sorter(heads, rows, 'addr', 0, True)
        dispRows = sorter.getSorted()
        disp.formattedTable(title, heads, dispRows)

    def displayMemory(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("type"))
        heads.append(Header("size", Header.COMMAS))
        heads.append(Header("batch"))
        heads.append(Header("thread-max", Header.COMMAS))
        heads.append(Header("total", Header.COMMAS))
        heads.append(Header("in-threads", Header.COMMAS))
        heads.append(Header("rebal-in", Header.COMMAS))
        heads.append(Header("rebal-out", Header.COMMAS))
        rows = []
        cols = ('identity', 'typeSize', 'transferBatchSize', 'localFreeListMax',
                'totalAllocFromHeap', 'totalFreeToHeap', 'heldByThreads',
                'batchesRebalancedToThreads', 'batchesRebalancedToGlobal')

        objects = self.query('io.skupper.router.allocator', cols)

        if show_date_id:
            self.display_datetime_router_id()

        pooled_total = 0
        memory_stats_enabled = True
        for t in objects:
            # t.totalAllocFromHeap is None *only* if memory stats is disabled.
            if t.totalAllocFromHeap is None:
                memory_stats_enabled = False
                break
            heap_count = int(t.totalAllocFromHeap)
            if heap_count == 0:
                # ignore any unused objects
                continue
            heap_count -= int(t.totalFreeToHeap)
            row = []
            row.append(self._identity_clean(t.identity))
            row.append(PlainNum(t.typeSize))
            row.append(PlainNum(t.transferBatchSize))
            row.append(PlainNum(t.localFreeListMax))
            row.append(PlainNum(heap_count))
            row.append(PlainNum(t.heldByThreads))
            row.append(PlainNum(t.batchesRebalancedToThreads))
            row.append(PlainNum(t.batchesRebalancedToGlobal))
            rows.append(row)
            pooled_total += (int(t.typeSize) * heap_count)

        if not rows or not memory_stats_enabled:
            # router built w/o memory pools:
            print("No memory statistics available")
            return

        title = "Memory Pools"
        sorter = Sorter(heads, rows, 'type', 0, True)
        dispRows = sorter.getSorted()
        disp.formattedTable(title, heads, dispRows)

        # attempt to get the qdrouterd process memory usage
        # this may not be present on all platforms
        router_metrics = self.query('io.skupper.router.routerMetrics')[0]

        try:
            vm_size = router_metrics.memoryUsage
        except AttributeError:
            vm_size = None
        try:
            rss_size = router_metrics.residentMemoryUsage
        except AttributeError:
            rss_size = None

        headers = []
        values = []
        if vm_size is not None:
            headers.append(Header("VmSize", Header.KiMiGi))
            values.append(vm_size)

        if rss_size is not None:
            headers.append(Header("RSS", Header.KiMiGi))
            values.append(rss_size)

        headers.append(Header("Pooled", Header.KiMiGi))
        values.append(pooled_total)
        disp.formattedTable("\nMemory Summary", headers, [values])

    def displayPolicy(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)
        heads = []
        heads.append(Header("attr"))
        heads.append(Header("value"))
        rows = []

        objects = self.query('io.skupper.router.policy')

        policy = objects[0]

        if show_date_id:
            self.display_datetime_router_id()

        rows.append(('Maximum Concurrent Connections', PlainNum(policy.maxConnections)))
        if 'maxMessageSize' in policy:
            rows.append(('Maximum Message Size',           PlainNum(policy.maxMessageSize)))
        rows.append(('Enable Vhost Policy',            policy.enableVhostPolicy))
        rows.append(('Enable Vhost Name Patterns',     policy.enableVhostNamePatterns))
        rows.append(('Policy Directory',               policy.policyDir if len(policy.policyDir) > 0 else '(nil)'))
        rows.append(('Default Vhost',                  policy.defaultVhost if len(policy.defaultVhost) > 0 else '(nil)'))
        disp.formattedTable("Policy Configuration", heads, rows)

        heads = []
        heads.append(Header("attr"))
        heads.append(Header("value"))
        rows = []

        rows.append(('Connections Processed',       PlainNum(policy.connectionsProcessed)))
        rows.append(('Connections Denied',          PlainNum(policy.connectionsDenied)))
        rows.append(('Connections Current',         PlainNum(policy.connectionsCurrent)))
        rows.append(('Links Denied',                PlainNum(policy.linksDenied)))
        if 'maxMessageSizeDenied' in policy:
            rows.append(('Maximum Message Size Denied', PlainNum(policy.maxMessageSizeDenied)))
        rows.append(('Total Denials',               PlainNum(policy.totalDenials)))

        disp.formattedTable("\nPolicy Status", heads, rows)

    def showNodefaultString(self, dict, key, schema):
        return dict[key] if key in dict else schema[key]['default'] if 'default' in schema[key] else '(nil)'

    def showNodefaultBool(self, dict, key, schema):
        return dict[key] if key in dict else schema[key]['default'] if 'default' in schema[key] else 'False'

    def showNodefaultInt(self, dict, key, schema):
        return PlainNum(dict[key]) if key in dict else PlainNum(schema[key]['default']) if 'default' in schema[key] else '(nil)'

    def displayVhosts(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)

        vhosts = self.query('io.skupper.router.vhost')

        if len(vhosts) == 0:
            print("No Vhosts")
            return
        heads = []
        heads.append(Header("hostname"))
        heads.append(Header("maxConnections"))

        have_max_message_size = False
        if 'maxMessageSize' in vhosts[0]:
            have_max_message_size = True
            heads.append(Header('maxMessageSize'))

        heads.append(Header("maxConnectionsPerUser"))
        heads.append(Header("maxConnectionsPerHost"))
        heads.append(Header("allowUnknownUser"))
        heads.append(Header("groups"))

        rows = []
        for vhost in vhosts:
            row = []
            row.append(vhost.hostname)
            row.append(PlainNum(vhost.maxConnections))
            if have_max_message_size:
                row.append(PlainNum(vhost.maxMessageSize))
            row.append(PlainNum(vhost.maxConnectionsPerUser))
            row.append(PlainNum(vhost.maxConnectionsPerHost))
            row.append(vhost.allowUnknownUser)
            row.append(PlainNum(len(vhost.groups)))
            rows.append(tuple(row))

        disp.formattedTable("Vhosts", heads, rows)

    def displayVhostgroups(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)

        vhosts = self.query('io.skupper.router.vhost')
        schema = self.node.get_schema()
        vg_schema = schema['entityTypes']['vhostUserGroupSettings']['attributes']

        if len(vhosts) == 0:
            print("No Vhosts")
            return
        # Create table for all vhosts and groups omitting
        # strings that are potentially 'very long'.
        heads = []
        heads.append(Header("vhost"))
        heads.append(Header("group"))
        heads.append(Header("maxConnectionsPerUser"))
        heads.append(Header("maxConnectionsPerHost"))

        have_max_message_size = False
        if 'maxMessageSize' in vhosts[0]:
            have_max_message_size = True
            heads.append(Header('maxMessageSize'))

        heads.append(Header("maxFrameSize"))
        heads.append(Header("maxSessionWindow"))
        heads.append(Header("maxSessions"))
        heads.append(Header("maxSenders"))
        heads.append(Header("maxReceivers"))
        heads.append(Header("allowDynamicSource"))
        heads.append(Header("allowAnonymousSender"))
        heads.append(Header("allowUserIdProxy"))
        heads.append(Header("allowAdminStatusUpdate"))
        rows = []
        for vhost in vhosts:
            groups = vhost["groups"]
            if len(groups) > 0:
                for groupname in groups:
                    group = groups[groupname]
                    row = []
                    row.append(vhost.hostname)
                    row.append(groupname)
                    row.append(self.showNodefaultInt(group, "maxConnectionsPerUser", vg_schema))
                    row.append(self.showNodefaultInt(group, "maxConnectionsPerHost", vg_schema))
                    if have_max_message_size:
                        row.append(self.showNodefaultInt(group, "maxMessageSize", vg_schema))
                    row.append(self.showNodefaultInt(group, "maxFrameSize", vg_schema))
                    row.append(self.showNodefaultInt(group, "maxSessionWindow", vg_schema))
                    row.append(self.showNodefaultInt(group, "maxSessions", vg_schema))
                    row.append(self.showNodefaultInt(group, "maxSenders", vg_schema))
                    row.append(self.showNodefaultInt(group, "maxReceivers", vg_schema))
                    row.append(self.showNodefaultBool(group, "allowDynamicSource", vg_schema))
                    row.append(self.showNodefaultBool(group, "allowAnonymousSender", vg_schema))
                    row.append(self.showNodefaultBool(group, "allowUserIdProxy", vg_schema))
                    row.append(self.showNodefaultBool(group, "allowAdminStatusUpdate", vg_schema))
                    rows.append(tuple(row))
        disp.formattedTable("Vhost Groups", heads, rows)

        # Create tables per vhost and per usergroup
        # showing only potentially lengthy strings.
        heads = []
        heads.append(Header("attr"))
        heads.append(Header("value"))

        for vhost in vhosts:
            groups = vhost["groups"]
            if len(groups) > 0:
                for groupname in groups:
                    group = groups[groupname]
                    rows = []
                    rows.append(("vhost",         vhost.hostname))
                    rows.append(("group",         groupname))
                    rows.append(("users",         self.showNodefaultString(group, "users", vg_schema)))
                    rows.append(("remoteHosts",   self.showNodefaultString(group, "remoteHosts", vg_schema)))
                    rows.append(("sources",       self.showNodefaultString(group, "sources", vg_schema)))
                    rows.append(("targets",       self.showNodefaultString(group, "targets", vg_schema)))
                    rows.append(("sourcePattern", self.showNodefaultString(group, "sourcePattern", vg_schema)))
                    rows.append(("targetPattern", self.showNodefaultString(group, "targetPattern", vg_schema)))
                    disp.formattedTable(("\nVhost '%s' UserGroup '%s'" % (vhost.hostname, groupname)), heads, rows)

    def displayVhoststats(self, show_date_id=True):
        disp = Display(prefix="  ", bodyFormat=self.bodyFormat)

        vstats = self.query('io.skupper.router.vhostStats')

        if len(vstats) == 0:
            print("No Vhost Stats")
            return
        heads = []
        heads.append(Header("hostname"))
        heads.append(Header("connectionsApproved"))
        heads.append(Header("connectionsDenied"))
        heads.append(Header("connectionsCurrent"))
        heads.append(Header("sessionDenied"))
        heads.append(Header("senderDenied"))
        heads.append(Header("receiverDenied"))
        have_max_message_size = False
        if 'maxMessageSizeDenied' in vstats[0]:
            have_max_message_size = True
            heads.append(Header("maxMessageSizeDenied"))
        rows = []

        for vstat in vstats:
            row = []
            row.append(vstat.hostname)
            row.append(PlainNum(vstat.connectionsApproved))
            row.append(PlainNum(vstat.connectionsDenied))
            row.append(PlainNum(vstat.connectionsCurrent))
            row.append(PlainNum(vstat.sessionDenied))
            row.append(PlainNum(vstat.senderDenied))
            row.append(PlainNum(vstat.receiverDenied))
            if have_max_message_size:
                row.append(PlainNum(vstat.maxMessageSizeDenied))
            rows.append(tuple(row))
        disp.formattedTable("Vhost Stats", heads, rows)

        heads = []
        heads.append(Header("vhost"))
        heads.append(Header("user"))
        heads.append(Header("remote hosts"))
        rows = []

        for vstat in vstats:
            ustates = vstat["perUserState"]
            if len(ustates) > 0:
                for ustate in ustates:
                    rows.append((
                        vstat.hostname,
                        ustate,
                        ustates[ustate]
                    ))
        disp.formattedTable("\nVhost User Stats", heads, rows)

    def displayLog(self):
        log = self.node.get_log(limit=self.opts.limit)
        for line in log:
            print("%s %s (%s) %s" % (ctime(line[5]), line[0], line[1], line[2]))

    def show_all(self, show_summary=True):
        if show_summary:
            # print the datetime and router id
            print(str(datetime.now(timezone.utc)) + " UTC")
            self.displayRouterId()
            print("")

        self.displayRouterLinks(show_date_id=False)
        print("")
        self.displayAddresses(show_date_id=False)
        print("")
        self.displayConnections(show_date_id=False)
        print("")
        self.displayAutolinks(show_date_id=False)
        print("")
        self.displayGeneral(show_date_id=False)
        print("")
        self.displayMemory(show_date_id=False)
        print("")
        self.displayPolicy(show_date_id=False)
        print("")
        self.displayVhosts(show_date_id=False)
        print("")
        # Groups is an ugly display. Show it only on demand.
        # self.displayVhostgroups(show_date_id=False)
        # print("")
        self.displayVhoststats(show_date_id=False)
        print("")

    def has_nodes(self):
        all_nodes = self.node.get_mgmt_nodes()
        has_nodes = True
        if all_nodes:
            if len(all_nodes) < 2:
                has_nodes = False
        else:
            has_nodes = False

        return has_nodes, all_nodes

    def display(self):
        has_nodes, nodes = self.has_nodes()
        if self.opts.all_routers and has_nodes:
            print(str(datetime.now(timezone.utc)) + " UTC")
            for node in nodes:
                self.node.set_client(node[6:])
                parts = node.split("/")
                print("Router ", parts[3])
                print("")
                if self.show.__name__ == 'show_all':
                    self.show(show_summary=False)
                else:
                    self.show(show_date_id=False)
                print("")
        else:
            self.show()


def run(argv):
    args = parse_args_skstat(BusManager)
    bm = BusManager(args)
    try:
        bm.display()
    finally:
        bm.close()


if __name__ == "__main__":
    sys.exit(main(run))