pci_device: Add bridge reset when a device is unplugged

python/opae.admin/opae/admin/sysfs.py

@@ -28,6 +28,7 @@
 import glob
 import os
 import re
+import time
 from contextlib import contextmanager
 from pathlib import Path
 from subprocess import CalledProcessError, check_call, DEVNULL
@@ -229,6 +230,8 @@ def __init__(self, pci_address, parent=None, **kwargs):
             pci_address['pci_address'])
         self._aer_cmd2 = 'setpci -s {} ECAP_AER+0x14.L'.format(
             pci_address['pci_address'])
+        self._bridge_ctrl_cmd = 'setpci -s {} BRIDGE_CONTROL'.format(
+            pci_address['pci_address'])
         if self.have_node('driver'):
             self._driver = os.readlink(self.node('driver').sysfs_path)
 
@@ -596,6 +599,26 @@ def aer(self, values):
         except CalledProcessError as err:
             self.log.warn('error setting aer: %s', err)
 
+    def reset_bridge(self):
+        """reset_bridge Reset the devices under root port bridge object.
+
+        Notes:
+            This relies on calling 'setpci' and will log an exception if an
+            error was encountered while calling 'setpci'.
+        """
+        try:
+            # Get current bridge control value
+            bc = int(call_process(self._bridge_ctrl_cmd), 16)
+            # Enable bus reset
+            bc_reset = bc | 0x40
+            call_process(f"{self._bridge_ctrl_cmd}=0x{bc_reset:x}")
+            time.sleep(0.1)
+            # Clear bus reset
+            call_process(f"{self._bridge_ctrl_cmd}=0x{bc:x}")
+            time.sleep(0.25)
+        except CalledProcessError as err:
+            self.log.warn('error resetting bridge: %s', err)
+
     @property
     def sriov_totalvfs(self):
         """sriov_totalvfs Get total number of VFs supported"""

python/opae.admin/opae/admin/tools/pci_device.py

@@ -24,6 +24,7 @@
 # CONTRACT,  STRICT LIABILITY,  OR TORT  (INCLUDING NEGLIGENCE  OR OTHERWISE)
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,  EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
+import logging
 import re
 import sys
 import subprocess
@@ -259,6 +260,10 @@ def unplug_node(root, debug, exclude_node):
             remove = False
 
         if remove:
+            if debug:
+                print(f'Resetting the root bridge {root.pci_address}')
+            root.reset_bridge()
+
             if debug:
                 print(f'Removing the root device {root.pci_address}')
             root.remove()
@@ -348,6 +353,11 @@ def main():
                'plug': (plug.add_subparser, plug())}
 
     parser = ArgumentParser()
+
+    log_levels = ['verbose', 'info']
+    parser.add_argument('--log-level', choices=log_levels,
+                        default='info', help='log level to use')
+
     parser.add_argument('devices', type=pci_devices,
                         metavar='device-filter',
                         help=('PCIe filter of device '
@@ -366,6 +376,11 @@ def main():
 
     args = parser.parse_args()
 
+    if args.log_level == 'info':
+        logging.basicConfig(level=logging.INFO, format='%(message)s')
+    else:
+        logging.basicConfig(level=logging.DEBUG, format='%(message)s')
+
     if hasattr(args, 'which') and args.which and args.which == 'plug':
         # Force a PCI bus rescan.
         with open('/sys/bus/pci/rescan', 'w') as fd:

python/opae.admin/opae/admin/utils/process.py

@@ -36,6 +36,7 @@ def call_process(cmd, no_dry=False):
         print(cmd)
         return '0x0' if 'setpci' in cmd else ''
 
+    LOG('process').error('Calling %s', cmd)
     try:
         output = subprocess.check_output(cmd.split(),
                                          stderr=subprocess.STDOUT)