VITIS-13434 Provide more error details for AIE async errors via xrt::error

src/runtime_src/core/common/api/xrt_error.cpp

@@ -205,6 +205,7 @@ class error_impl
 {
   xrtErrorCode m_errcode = 0;
   xrtErrorTime m_timestamp = 0;
+  std::string  m_ex_error_str;
 public:
   error_impl(const xrt_core::device* device, xrtErrorClass ecl)
   {
@@ -213,13 +214,23 @@ class error_impl
       auto buf = xrt_core::device_query<xrt_core::query::xocl_errors>(device);
       if (buf.empty())
         return;
-      auto ect = xrt_core::query::xocl_errors::to_value(buf, ecl);
-      std::tie(m_errcode, m_timestamp) = ect;
-      return;
+      if (device->get_ex_error_support() == true) {
+        auto ect = xrt_core::query::xocl_errors::to_ex_value(buf, ecl);
+        m_errcode = std::get<0>(ect);
+        m_timestamp = std::get<1>(ect);
+        uint64_t ex_error_code = std::get<2>(ect);
+        m_ex_error_str = xrt_core::device_query<xrt_core::query::xocl_ex_error_code2string>(device, ex_error_code);
+      }
+      else {
+        auto ect = xrt_core::query::xocl_errors::to_value(buf, ecl);
+        std::tie(m_errcode, m_timestamp) = ect;
+        m_ex_error_str = "";
+      }
     } catch (const xrt_core::query::no_such_key&) {
       // Ignoring for now. Check below for edge if not available
       // query table of zocl doesn't have xocl_errors key
     }
+
     //Below code will be removed after zocl changes for new format
     auto errors = xrt_core::device_query<xrt_core::query::error>(device);
     for (auto& line : errors) {
@@ -258,7 +269,8 @@ class error_impl
       ("%s\n"
        "Timestamp: %s")
       % error_code_to_string(m_errcode)
-      % error_time_to_string(m_timestamp);
+      % error_time_to_string(m_timestamp)
+      % m_ex_error_str;
 
     return fmt.str();
   }

src/runtime_src/core/common/device.cpp

@@ -69,6 +69,25 @@ is_nodma() const
   return *m_nodma;
 }
 
+bool
+device::
+get_ex_error_support() const
+{
+  std::lock_guard lk(m_mutex);
+  if (m_ex_error_support != std::nullopt)
+    return *m_ex_error_support;
+
+  try {
+    auto ex_error_support = xrt_core::device_query<xrt_core::query::xocl_errors_ex>(this);
+    m_ex_error_support = xrt_core::query::xocl_errors_ex::to_bool(ex_error_support);
+  }
+  catch (const std::exception&) {
+    m_ex_error_support = false;
+  }
+
+  return *m_ex_error_support;
+}
+
 uuid
 device::
 get_xclbin_uuid() const

src/runtime_src/core/common/device.h

@@ -24,6 +24,7 @@
 #include <string>
 #include <map>
 #include <memory>
+#include <optional>
 #include <boost/property_tree/ptree.hpp>
 #include <boost/optional/optional.hpp>
 
@@ -195,6 +196,16 @@ class device : public ishim
   bool
   is_nodma() const;
 
+  /**
+    * get_ex_error_support() - Does this device support extended error code
+    *
+    * Return: true if device support extended error code
+    *
+    */
+  XRT_CORE_COMMON_EXPORT
+  bool
+  get_ex_error_support() const;
+
  private:
   // Private look up function for concrete query::request
   virtual const query::request&
@@ -476,6 +487,7 @@ class device : public ishim
  private:
   id_type m_device_id;
   mutable boost::optional<bool> m_nodma = boost::none;
+  mutable std::optional<bool> m_ex_error_support = std::nullopt;
 
   using name2idx_type = std::map<std::string, cuidx_type>;
   std::map<slot_id, name2idx_type> m_cu2idx;  // slot -> cu name mapping to cuidx

src/runtime_src/core/common/query_requests.cpp

@@ -132,6 +132,35 @@ to_value(const std::vector<char>& buf, xrtErrorClass ecl)
   return {0, 0};
 }
 
+std::tuple<uint64_t, uint64_t, uint64_t>
+xrt_core::query::xocl_errors::
+to_ex_value(const std::vector<char>& buf, xrtErrorClass ecl)
+{
+  if (buf.empty())
+    return { 0, 0, 0 };
+
+  auto errors_buf = reinterpret_cast<const xcl_errors*>(buf.data());
+  if (errors_buf->num_err <= 0)
+    return { 0, 0, 0 };
+
+  if (errors_buf->num_err > XCL_ERROR_CAPACITY)
+    throw xrt_core::system_error(EINVAL, "Invalid data in sysfs");
+
+  uint64_t error_code = 0;
+  uint64_t time_stamp = 0;
+  uint64_t ex_error_code = 0;
+  for (int i = errors_buf->num_err - 1; i >= 0; i--) {
+    if (XRT_ERROR_CLASS(errors_buf->errors[i].err_code) == ecl) {
+      error_code = errors_buf->errors[i].err_code;
+      time_stamp = errors_buf->errors[i].ts;
+      ex_error_code = errors_buf->errors[i].ex_error_code;
+      break;
+    }
+  }
+
+  return { error_code, time_stamp, ex_error_code };
+}
+
 std::vector<xclErrorLast>
 xrt_core::query::xocl_errors::
 to_errors(const std::vector<char>& buf)

src/runtime_src/core/common/query_requests.h

@@ -330,7 +330,10 @@ enum class key_type
   kernel_max_bandwidth_mbps,
   sub_device_path,
   read_trace_data,
-  noop
+  noop,
+
+  xocl_errors_ex,
+  xocl_ex_error_code2string
 };
 
 struct pcie_vendor : request
@@ -1673,6 +1676,39 @@ struct error : request
   }
 };
 
+// Retrieve support for extended asynchronous xocl errors from xocl driver
+struct xocl_errors_ex : request
+{
+  using result_type = uint32_t;
+  static const key_type key = key_type::xocl_errors_ex;
+
+  virtual std::any
+    get(const device*) const override = 0;
+
+  static bool
+    to_bool(const result_type& value)
+  {
+    return (value == std::numeric_limits<uint32_t>::max())
+      ? false : value;
+  }
+};
+
+// Retrieve extended asynchronous xocl errors string corresponding to the error code from xocl driver
+struct xocl_ex_error_code2string : request
+{
+  using result_type = std::string;  // get value type
+  static const key_type key = key_type::xocl_ex_error_code2string;
+
+  virtual std::any
+    get(const device*) const override = 0;
+
+  static std::string
+    to_string(const std::string& errstr)
+  {
+    return std::string(errstr);
+  }
+};
+
 // Retrieve asynchronous xocl errors from xocl driver
 struct xocl_errors : request
 {
@@ -1687,6 +1723,11 @@ struct xocl_errors : request
   static std::pair<uint64_t, uint64_t>
   to_value(const std::vector<char>& buf, xrtErrorClass ecl);
 
+  // Parse buffer, get error code and timestamp
+  XRT_CORE_COMMON_EXPORT
+  static std::tuple<uint64_t, uint64_t, uint64_t>
+  to_ex_value(const std::vector<char>& buf, xrtErrorClass ecl);
+
   // Parse sysfs raw data and get list of errors
   XRT_CORE_COMMON_EXPORT
   static std::vector<xclErrorLast>

src/runtime_src/core/include/xclerr_int.h

@@ -51,9 +51,10 @@
  * xrtErrorCode (64 bits) = ErrorNum + Driver + Severity + Module + Class
  */
 typedef struct xclErrorLast {
-	xrtErrorCode	err_code;	/* 64 bits; XRT error code */
-	xrtErrorTime	ts;		/* 64 bits; timestamp */
-	unsigned        pid;            /* 32 bits; pid associated with error, if available */
+	xrtErrorCode	 err_code;	/* 64 bits; XRT error code */
+	xrtErrorTime	 ts;		/* 64 bits; timestamp */
+	unsigned       pid;            /* 32 bits; pid associated with error, if available */
+  xrtExErrorCode ex_error_code; /* 64 bits; XRT extra error code*/
 } xclErrorLast;
 
 typedef struct xcl_errors {

src/runtime_src/core/include/xrt/detail/xrt_error_code.h

@@ -152,4 +152,44 @@ enum xrtErrorClass {
   XRT_ERROR_CLASS_LAST_ENTRY = XRT_ERROR_CLASS_UNKNOWN
 };
 
+typedef uint64_t xrtExErrorCode;
+
+/**
+*xrtExErrorCode layout
+*
+* This layout is internal to XRT(akin to a POSIX error code).
+*
+* The error code is populated by driver and consumed by XRT
+* implementation where it is translated into an actual error / info /
+* warning that is propagated to the end user.
+*
+*63 - 48   47 - 32   31 - 16   15 - 0
+* --------------------------------------
+* |    |    |    |    |    |    |----|  ExErrorID
+* |    |    |    |    |----|----------- AIE_LOC_COL
+* |    |    |----|----------------------AIR_LOC_ROW
+* |----|--------------------------------RESERVED
+*
+*/
+
+#define XRT_EX_ERROR_ID_MASK          0xFFFFUL
+#define XRT_EX_ERROR_ID_SHIFT         0
+#define XRT_EX_ERROR_LOC_COL_MASK     0xFFFFUL
+#define XRT_EX_ERROR_LOC_COL_SHIFT    16
+#define XRT_EX_ERROR_LOC_ROW_MASK     0xFFFFUL
+#define XRT_EX_ERROR_LOC_ROW_SHIFT    32
+#define XRT_EX_ERROR_RESERVED_MASK    0xFFFFUL
+#define XRT_EX_ERROR_RESERVED_SHIFT   48
+
+#define  XRT_EX_ERROR_CODE_BUILD(ID, COL, ROW, RESERVED) \
+    ((static_cast<uint64_t>((ID) & XRT_EX_ERROR_ID_MASK) << XRT_ERROR_NUM_SHIFT) | \
+    (static_cast<uint64_t>((COL) & XRT_EX_ERROR_LOC_COL_MASK) << XRT_EX_ERROR_LOC_COL_SHIFT) | \
+    (static_cast<uint64_t>((ROW) & XRT_EX_ERROR_LOC_ROW_MASK) << XRT_EX_ERROR_LOC_ROW_SHIFT) | \
+    (static_cast<uint64_t>((RESERVED) & XRT_EX_ERROR_RESERVED_MASK) << XRT_EX_ERROR_RESERVED_SHIFT))
+
+#define XRT_EX_ERROR_ID(code) (((code) >> XRT_EX_ERROR_ID_SHIFT) & XRT_EX_ERROR_ID_MASK)
+#define XRT_EX_ERROR_LOC_COL(code) (((code) >> XRT_EX_ERROR_LOC_COL_SHIFT) & XRT_EX_ERROR_LOC_COL_MASK)
+#define XRT_EX_ERROR_LOC_ROW(code) (((code) >> XRT_EX_ERROR_LOC_ROW_SHIFT) & XRT_EX_ERROR_LOC_ROW_MASK)
+
+
 #endif