Cache previously loaded metadata as a pickle file. (#1396)

* fix that applies the gpio_dict data to the ip_dict when constructing the dictionaries from the metadata

* When loading a bitstream a hash of the bitfile is checked and a pickled
cache of the metadata is loaded if there is a hit. For SDbuild pickled
versions of the metadata for base overlay are also preloaded.

boards/Pynq-Z1/Pynq-Z1.spec

@@ -6,4 +6,4 @@ BSP_Pynq-Z1 :=
 BITSTREAM_Pynq-Z1 := base/base.bit
 FPGA_MANAGER_Pynq-Z1 := 1
 
-STAGE4_PACKAGES_Pynq-Z1 := xrt pynq boot_leds ethernet pynq_peripherals sdcardshrink
+STAGE4_PACKAGES_Pynq-Z1 := xrt pynq boot_leds ethernet pynq_peripherals sdcardshrink precached_metadata

boards/Pynq-Z2/Pynq-Z2.spec

@@ -6,4 +6,4 @@ BSP_Pynq-Z2 :=
 BITSTREAM_Pynq-Z2 := base/base.bit
 FPGA_MANAGER_Pynq-Z2 := 1
 
-STAGE4_PACKAGES_Pynq-Z2 := xrt pynq boot_leds ethernet pynq_peripherals sdcardshrink
+STAGE4_PACKAGES_Pynq-Z2 := xrt pynq boot_leds ethernet pynq_peripherals sdcardshrink precached_metadata

pynq/_cli/get_notebooks.py

@@ -6,8 +6,9 @@
 import os
 from shutil import move
 
-from pynqutils.runtime import detect_devices, get_logger
+from pynqutils.runtime import get_logger
 from pynqutils.setup_utils import ExtensionsManager, deliver_notebooks
+from pynq.utils import _detect_devices
 
 
 
@@ -129,7 +130,7 @@ def main():
     if args.device:
         device = args.device
     elif args.interactive and "XILINX_XRT" in os.environ:
-        shells = list(dict.fromkeys(detect_devices()))
+        shells = list(dict.fromkeys(_detect_devices()))
         shells_num = len(shells)
         print("Detected shells:")
         for i in range(shells_num):
@@ -151,7 +152,7 @@ def main():
     elif args.ignore_overlays:  # overlays are ignored, set device to `None`
         device = None
     else:  # default case, detect devices and use default device
-        device = detect_devices(active_only=True)
+        device = _detect_devices(active_only=True)
     if not notebooks_ext_man.list:
         logger.warn("No notebooks available, nothing can be " "delivered")
         return

pynq/bitstream.py

@@ -167,6 +167,10 @@ def download(self, parser=None):
         """
         self.device.download(self, parser)
 
+    def gen_cache(self, parser=None):
+        """ Generates the pickled metadata cache in pl_server/ even if no download has occurred """
+        self.device.gen_cache(self, parser)
+
     def remove_dtbo(self):
         """Remove dtbo file from the system.
 

pynq/overlay.py

@@ -284,7 +284,7 @@ class Overlay(Bitstream):
     """
 
     def __init__(
-        self, bitfile_name, dtbo=None, download=True, ignore_version=False, device=None
+        self, bitfile_name, dtbo=None, download=True, ignore_version=False, device=None, gen_cache=False
     ):
         """Return a new Overlay object.
 
@@ -303,6 +303,8 @@ def __init__(
         device : pynq.Device
             Device on which to load the Overlay. Defaults to
             pynq.Device.active_device
+        gen_cache: bool
+            if true generates a pickeled cache of the metadata
 
         Note
         ----
@@ -344,6 +346,9 @@ def __init__(
 
         if download:
             self.download()
+        else:
+            if gen_cache:
+                self.gen_cache()
 
         self.__doc__ = _build_docstring(
             self._ip_map._description, bitfile_name, "overlay"
@@ -386,6 +391,10 @@ def free(self):
             self.remove_dtbo()
         self.device.close()
 
+    def gen_cache(self):
+        """ Generate a pickled cache of the metadata even if a download has not occurred """
+        super().gen_cache(self.parser)
+
     def download(self, dtbo=None):
         """The method to download a full bitstream onto PL.
 

pynq/pl.py

@@ -83,13 +83,18 @@ def dict_views(cls):
             return cls._dict_view_cache
         else:
             import os
+            import pickle
             from pynqmetadata.frontends import Metadata
             from .metadata.runtime_metadata_parser import RuntimeMetadataParser
-            metadata_state_file = Path(f"{os.path.dirname(__file__)}/pl_server/_current_metadata.json")
+            metadata_state_file = Path(f"{os.path.dirname(__file__)}/pl_server/_current_metadata.pkl")
             if os.path.isfile(metadata_state_file):
                 cls._dict_views_cached = True
-                cls._dict_view_cache = RuntimeMetadataParser(Metadata(input=metadata_state_file))
-                return cls._dict_view_cache 
+                try:
+                    cls._dict_view_cache = pickle.load(open(metadata_state_file, "rb"))
+                    return cls._dict_view_cache 
+                except:
+                    os.remove(metadata_state_file)
+                    return None
             else:
                 return None
 
@@ -175,7 +180,7 @@ def hierarchy_dict(cls):
             if not Device.active_device.hierarchy_dict is None:
                 return Device.active_device.hierarchy_dict
 
-        if not cls.dict_views.hierarchy_dict is None:
+        if not cls.dict_views is None:
             return cls.dict_views.hierarchy_dict
 
     @property
@@ -216,7 +221,7 @@ def mem_dict(self):
             if not Device.active_device.mem_dict is None:
                 return Device.active_device.mem_dict
 
-        if not self.dict_views.mem_dict is None:
+        if not self.dict_views is None:
             return self.dict_views.mem_dict
 
     def shutdown(cls):
@@ -246,10 +251,12 @@ def reset(cls, parser=None):
 
         """
         clear_global_state()
-        for i in cls.mem_dict.values():
-            i['state'] = None
-        for i in cls.ip_dict.values():
-            i['state'] = None
+        if not cls.mem_dict is None:
+            for i in cls.mem_dict.values():
+                i['state'] = None
+        if not cls.ip_dict is None:
+            for i in cls.ip_dict.values():
+                i['state'] = None
 
     def clear_dict(cls):
         """Clear all the dictionaries stored in PL.

pynq/pl_server/device.py

@@ -3,6 +3,7 @@
 
 
 import os
+import pickle
 import warnings
 from copy import deepcopy
 
@@ -420,27 +421,6 @@ def post_download(self, bitstream, parser, name: str = "Unknown"):
             )
             self.reset(parser, bitstream.timestamp, bitstream.bitfile_name)
 
-            if not hasattr(parser, "_from_cache"):
-                from .global_state import GlobalState, save_global_state
-
-                gs = GlobalState(bitfile_name=bitstream.bitfile_name,
-                                 timestamp=bitstream.timestamp,
-                                 active_name=name,
-                                 psddr=self.mem_dict.get("PSDDR", {}))
-                ip = self.ip_dict
-                for sd_name, details in ip.items():
-                    if details["type"] in [
-                        "xilinx.com:ip:pr_axi_shutdown_manager:1.0",
-                        "xilinx.com:ip:dfx_axi_shutdown_manager:1.0",
-                    ]:
-                        gs.add(name=sd_name, addr=details["phys_addr"])
-                save_global_state(gs)
-
-                if hasattr(self, "systemgraph"):
-                    if not self.systemgraph is None:
-                        import os
-                        STATE_DIR = os.path.dirname(__file__)
-                        self.systemgraph.export(path=f"{STATE_DIR}/_current_metadata.json")
 
     def has_capability(self, cap):
         """Test if the device as a desired capability

pynq/pl_server/embedded_device.py

@@ -2,18 +2,25 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import os
+import pickle
 import struct
 from pathlib import Path
+import datetime
 
 import numpy as np
 from pynqmetadata.frontends import Metadata
 
 from ..metadata.runtime_metadata_parser import RuntimeMetadataParser
 from .xrt_device import XrtDevice, XrtMemory
-from .global_state import initial_global_state_file_boot_check, load_global_state, global_state_file_exists
+from .global_state import GlobalState, save_global_state
+from .global_state import initial_global_state_file_boot_check, load_global_state
+from .global_state import bitstream_hash, global_state_file_exists, clear_global_state
 
 DEFAULT_XCLBIN = (Path(__file__).parent / "default.xclbin").read_bytes()
 
+class CacheMetadataError(Exception):
+    """ An exception that is raised when there is no cached metadata """
+    pass
 
 def _unify_dictionaries(hwh_parser, xclbin_parser):
     """Merges the XRT specific info from the xclbin file into
@@ -181,13 +188,45 @@ def is_xsa(self):
     def _cache_exists(self)->bool:
         """ Checks to see if this bitstream is already on the system and
         use the cached metadata """
-        from .global_state import bitstream_hash, load_global_state, global_state_file_exists
         if global_state_file_exists():
             glob_state = load_global_state()
             return glob_state.bitfile_hash == bitstream_hash(self._filepath)
         return False
 
 
+    def _clear_cache(self):
+        """ Clears the cache file """
+        metadata_state_file = Path(f"{os.path.dirname(__file__)}/_current_metadata.pkl")
+        if os.path.isfile(metadata_state_file):
+            os.remove(metadata_state_file)
+
+    def _get_cache(self):
+        """ Tries to return the Cached data """
+        if self._cache_exists():
+            metadata_state_file = Path(f"{os.path.dirname(__file__)}/_current_metadata.pkl")
+            if os.path.isfile(metadata_state_file):
+                try:
+                    parser = pickle.load(open(metadata_state_file, "rb"))
+                    parser._from_cache = True
+
+                    # Removing previous synthetic XRT mem_dict items
+                    mem_dict_to_remove = []
+                    for itemname, item in parser.mem_dict.items():
+                        if not "fullpath" in item:
+                            mem_dict_to_remove.append(itemname)
+                    for i in mem_dict_to_remove:
+                        del parser.mem_dict[i]
+
+                    return parser
+                except:
+                    self._clear_cache()
+                    raise CacheMetadataError(f"Global state file exists, but pickled metadata cannot be found")
+            else:
+                clear_global_state()
+                raise CacheMetadataError(f"Global state file exists, but pickled metadata cannot be found")
+        else:
+            raise CacheMetadataError(f"No cached metadata present")
+
     def get_parser(self, partial:bool=False):
         """Returns a parser object for the bitstream
 
@@ -209,16 +248,12 @@ def get_parser(self, partial:bool=False):
             if partial:
                 parser = HWH(hwh_data=hwh_data)
             else:
-                if self._cache_exists():
-                    metadata_state_file = Path(f"{os.path.dirname(__file__)}/_current_metadata.json")
-                    if os.path.isfile(metadata_state_file):
-                        parser = RuntimeMetadataParser(Metadata(input=metadata_state_file))
-                        parser._from_cache = True
-                    else:
-                        parser = RuntimeMetadataParser(Metadata(input=self._filepath.with_suffix(".hwh")))
-                else:
+                try:
+                    parser = self._get_cache() 
+                except CacheMetadataError:
                     parser = RuntimeMetadataParser(Metadata(input=self._filepath.with_suffix(".hwh")))
-
+                except:
+                    raise RuntimeError(f"Unable to parse metadata")
 
             if xclbin_data is None:
                 xclbin_data = _create_xclbin(parser.mem_dict)
@@ -242,8 +277,35 @@ def get_parser(self, partial:bool=False):
         parser.bin_data = self.get_bin_data()
         parser.xclbin_data = xclbin_data
         parser.dtbo_data = self.get_dtbo_data()
+        self._cache_metadata(parser)
         return parser
 
+    def _cache_metadata(self, parser, name:str="Unknown")->None:
+        """ Caches the metadata and global state """
+        if not hasattr(parser, "_from_cache"):
+
+            t = datetime.datetime.now()
+            ts = "{}/{}/{} {}:{}:{} +{}".format(
+                t.year, t.month, t.day, t.hour, t.minute, t.second, t.microsecond
+            )
+
+            gs = GlobalState(bitfile_name=str(self._filepath),
+                             timestamp=ts,
+                             active_name=name,
+                             psddr=parser.mem_dict.get("PSDDR", {}))
+            ip =parser.ip_dict
+            for sd_name, details in ip.items():
+                if details["type"] in ["xilinx.com:ip:pr_axi_shutdown_manager:1.0",
+                                       "xilinx.com:ip:dfx_axi_shutdown_manager:1.0",]:
+                    gs.add(name=sd_name, addr=details["phys_addr"])
+            save_global_state(gs)
+
+            if hasattr(parser, "systemgraph"):
+                if not parser.systemgraph is None:
+                    STATE_DIR = os.path.dirname(__file__)
+                    pickle.dump(parser, open(f"{STATE_DIR}/_current_metadata.pkl", "wb"))
+
+
 
 class BitfileHandler(BitstreamHandler):
     def get_bin_data(self):
@@ -599,31 +661,34 @@ def set_axi_port_width(self, parser):
                         f = ZU_AXIFM_REG[para][reg_name]["field"]
                         Register(addr)[f[0] : f[1]] = ZU_AXIFM_VALUE[width]
 
+    def gen_cache(self, bitstream, parser=None):
+        """ Generates the cache of the metadata even if no download occurred """
+        super()._cache_metadata(parser, bitstream, self.name)
+
     def download(self, bitstream, parser=None):
 
-        if not hasattr(parser, "_from_cache"):
-            if parser is None:
-                from .xclbin_parser import XclBin
+        if parser is None:
+            from .xclbin_parser import XclBin
 
-                parser = XclBin(xclbin_data=DEFAULT_XCLBIN)
+            parser = XclBin(xclbin_data=DEFAULT_XCLBIN)
 
-            if not bitstream.binfile_name:
-                _preload_binfile(bitstream, parser)
+        if not bitstream.binfile_name:
+            _preload_binfile(bitstream, parser)
 
-            if not bitstream.partial:
-                self.shutdown()
-                flag = 0
-            else:
-                flag = 1
+        if not bitstream.partial:
+            self.shutdown()
+            flag = 0
+        else:
+            flag = 1
 
-            with open(self.BS_FPGA_MAN_FLAGS, "w") as fd:
-                fd.write(str(flag))
-            with open(self.BS_FPGA_MAN, "w") as fd:
-                fd.write(bitstream.binfile_name)
+        with open(self.BS_FPGA_MAN_FLAGS, "w") as fd:
+            fd.write(str(flag))
+        with open(self.BS_FPGA_MAN, "w") as fd:
+            fd.write(bitstream.binfile_name)
 
-            self.set_axi_port_width(parser)
+        self.set_axi_port_width(parser)
 
-            self._xrt_download(parser.xclbin_data)
+        self._xrt_download(parser.xclbin_data)
         super().post_download(bitstream, parser, self.name)
 
     def get_bitfile_metadata(self, bitfile_name:str, partial:bool=False):

pynq/pl_server/xrt_device.py

@@ -532,6 +532,9 @@ def _xrt_download(self, data):
         finally:
             xrt.xclUnlockDevice(self.handle)
 
+    def gen_cache(self, bitstream, parser=None):
+        pass
+
     def download(self, bitstream, parser=None):
         with open(bitstream.bitfile_name, "rb") as f:
             data = f.read()