ESP32-S3: Support execute in place from PSRAM

esp-hal/CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - SPI: Added support for 3-wire SPI (#2919)
 - Add separate config for Rx and Tx (UART) #2965
+- ESP32-S3: Support execute in place from PSRAM
 
 ### Changed
 

esp-hal/src/soc/esp32s3/mmu.rs

@@ -0,0 +1,166 @@
+//! Thin MMU bindings
+//!
+//! More general information about the MMU can be found here:
+//! <https://docs.espressif.com/projects/esp-idf/en/stable/esp32s3/api-reference/system/mm.html#introduction>
+
+const DBUS_VADDR_BASE: u32 = 0x3C000000;
+const DR_REG_MMU_TABLE: u32 = 0x600C5000;
+const ENTRY_ACCESS_FLASH: u32 = 0;
+const ENTRY_INVALID: u32 = 1 << 14;
+const ENTRY_TYPE: u32 = 1 << 15;
+const ENTRY_VALID: u32 = 0;
+const ENTRY_VALID_VAL_MASK: u32 = 0x3fff;
+const ICACHE_MMU_SIZE: usize = 0x800;
+
+pub(super) const ENTRY_ACCESS_SPIRAM: u32 = 1 << 15;
+pub(super) const PAGE_SIZE: usize = 0x10000;
+pub(super) const TABLE_SIZE: usize = ICACHE_MMU_SIZE / core::mem::size_of::<u32>();
+
+extern "C" {
+    /// Set DCache mmu mapping.
+    ///
+    /// [`ext_ram`]: u32 ENTRY_ACCESS_FLASH for flash, ENTRY_ACCESS_SPIRAM for spiram, ENTRY_INVALID for invalid.
+    /// [`vaddr`]: u32 Virtual address in CPU address space.
+    /// [`paddr`]: u32 Physical address in external memory. Should be aligned by psize.
+    /// [`psize`]: u32 Page size of DCache, in kilobytes. Should be 64 here.
+    /// [`num`]: u32 Pages to be set.
+    /// [`fixes`]: u32 0 for physical pages grow with virtual pages, other for virtual pages map to same physical page.
+    pub(super) fn cache_dbus_mmu_set(
+        ext_ram: u32,
+        vaddr: u32,
+        paddr: u32,
+        psize: u32,
+        num: u32,
+        fixed: u32,
+    ) -> i32;
+
+    fn Cache_Invalidate_Addr(addr: u32, size: u32);
+    fn Cache_WriteBack_All();
+    fn rom_Cache_WriteBack_Addr(addr: u32, size: u32);
+}
+
+#[procmacros::ram]
+pub(super) fn last_mapped_index() -> Option<usize> {
+    let mmu_table_ptr = DR_REG_MMU_TABLE as *const u32;
+    (0..TABLE_SIZE)
+        .rev()
+        .find(|&i| unsafe { mmu_table_ptr.add(i).read_volatile() } != ENTRY_INVALID)
+}
+
+#[procmacros::ram]
+pub(super) fn index_to_data_address(index: usize) -> u32 {
+    DBUS_VADDR_BASE + (PAGE_SIZE * index) as u32
+}
+
+/// Count flash-mapped pages, de-duplicating mappings which refer to flash page
+/// 0
+#[procmacros::ram]
+pub(super) fn count_effective_flash_pages() -> usize {
+    let mmu_table_ptr = DR_REG_MMU_TABLE as *const u32;
+    let mut page0_seen = false;
+    let mut flash_pages = 0;
+    for i in 0..(TABLE_SIZE - 1) {
+        let mapping = unsafe { mmu_table_ptr.add(i).read_volatile() };
+        if mapping & (ENTRY_INVALID | ENTRY_TYPE) == ENTRY_VALID | ENTRY_ACCESS_FLASH {
+            if mapping & ENTRY_VALID_VAL_MASK == 0 {
+                if page0_seen {
+                    continue;
+                }
+                page0_seen = true;
+            }
+            flash_pages += 1;
+        }
+    }
+    flash_pages
+}
+
+#[procmacros::ram]
+unsafe fn move_flash_to_psram_with_spare(
+    target_entry: usize,
+    psram_page: usize,
+    spare_entry: usize,
+) {
+    let mmu_table_ptr = DR_REG_MMU_TABLE as *mut u32;
+    let target_entry_addr = DBUS_VADDR_BASE + (target_entry * PAGE_SIZE) as u32;
+    let spare_entry_addr = DBUS_VADDR_BASE + (spare_entry * PAGE_SIZE) as u32;
+    unsafe {
+        mmu_table_ptr
+            .add(spare_entry)
+            .write_volatile(psram_page as u32 | ENTRY_ACCESS_SPIRAM);
+        Cache_Invalidate_Addr(spare_entry_addr, PAGE_SIZE as u32);
+        core::ptr::copy_nonoverlapping(
+            target_entry_addr as *const u8,
+            spare_entry_addr as *mut u8,
+            PAGE_SIZE,
+        );
+        rom_Cache_WriteBack_Addr(spare_entry_addr, PAGE_SIZE as u32);
+        mmu_table_ptr
+            .add(target_entry)
+            .write_volatile(psram_page as u32 | ENTRY_ACCESS_SPIRAM);
+    }
+}
+
+/// Copy flash-mapped pages to PSRAM, copying flash-page 0 only once, and re-map
+/// those pages to the PSRAM copies
+#[procmacros::ram]
+pub(super) unsafe fn copy_flash_to_psram_and_remap(free_page: usize) -> usize {
+    let mmu_table_ptr = DR_REG_MMU_TABLE as *mut u32;
+
+    const SPARE_PAGE: usize = TABLE_SIZE - 1;
+    const SPARE_PAGE_DCACHE_ADDR: u32 = DBUS_VADDR_BASE + (SPARE_PAGE * PAGE_SIZE) as u32;
+
+    let spare_page_mapping = unsafe { mmu_table_ptr.add(SPARE_PAGE).read_volatile() };
+    let mut page0_page = None;
+    let mut psram_page = free_page;
+
+    unsafe { Cache_WriteBack_All() };
+    for i in 0..(TABLE_SIZE - 1) {
+        let mapping = unsafe { mmu_table_ptr.add(i).read_volatile() };
+        if mapping & (ENTRY_INVALID | ENTRY_TYPE) != ENTRY_VALID | ENTRY_ACCESS_FLASH {
+            continue;
+        }
+        if mapping & ENTRY_VALID_VAL_MASK == 0 {
+            match page0_page {
+                Some(page) => {
+                    unsafe {
+                        mmu_table_ptr
+                            .add(i)
+                            .write_volatile(page as u32 | ENTRY_ACCESS_SPIRAM)
+                    };
+                    continue;
+                }
+                None => page0_page = Some(psram_page),
+            }
+        }
+        unsafe { move_flash_to_psram_with_spare(i, psram_page, SPARE_PAGE) };
+        psram_page += 1;
+    }
+
+    // Restore spare page mapping
+    unsafe {
+        mmu_table_ptr
+            .add(SPARE_PAGE)
+            .write_volatile(spare_page_mapping);
+        Cache_Invalidate_Addr(SPARE_PAGE_DCACHE_ADDR, PAGE_SIZE as u32);
+    }
+
+    // Special handling if the spare page was mapped to flash
+    if spare_page_mapping & (ENTRY_INVALID | ENTRY_TYPE) == ENTRY_VALID | ENTRY_ACCESS_FLASH {
+        unsafe {
+            // We're running from ram so using the first page should not cause issues
+            const SECOND_SPARE: usize = 0;
+            let second_spare_mapping = mmu_table_ptr.add(SECOND_SPARE).read_volatile();
+
+            move_flash_to_psram_with_spare(SPARE_PAGE, psram_page, SECOND_SPARE);
+
+            // Restore spare page mapping
+            mmu_table_ptr.add(0).write_volatile(second_spare_mapping);
+            Cache_Invalidate_Addr(
+                DBUS_VADDR_BASE + (SECOND_SPARE * PAGE_SIZE) as u32,
+                PAGE_SIZE as u32,
+            );
+        }
+        psram_page += 1;
+    }
+    psram_page - free_page
+}

esp-hal/src/soc/esp32s3/mod.rs

@@ -23,6 +23,8 @@ crate::unstable_module! {
 }
 pub mod cpu_control;
 pub mod gpio;
+#[cfg(feature = "psram")]
+mod mmu;
 pub mod peripherals;
 
 /// The name of the chip ("esp32s3") as `&str`

esp-hal/src/soc/esp32s3/psram.rs

@@ -57,11 +57,10 @@
 //! # }
 //! ```
 
+use super::mmu;
 use crate::peripherals::{EXTMEM, IO_MUX, SPI0, SPI1};
 pub use crate::soc::psram_common::*;
 
-const EXTMEM_ORIGIN: u32 = 0x3C000000;
-
 /// Frequency of flash memory
 #[derive(Copy, Clone, Debug, Default)]
 #[cfg_attr(feature = "defmt", derive(defmt::Format))]
@@ -109,6 +108,13 @@ pub struct PsramConfig {
     pub flash_frequency: FlashFreq,
     /// Frequency of PSRAM memory
     pub ram_frequency: SpiRamFreq,
+    /// Copy code and read-only data from flash to PSRAM and remap the
+    /// respective pages to point to PSRAM
+    ///
+    /// Refer to
+    /// <https://docs.espressif.com/projects/esp-idf/en/stable/esp32s3/api-guides/external-ram.html#execute-in-place-xip-from-psram>
+    /// for more information.
+    pub execute_from_psram: bool,
 }
 
 /// Initialize PSRAM to be used for data.
@@ -125,8 +131,9 @@ pub(crate) fn init_psram(config: PsramConfig) {
     const CONFIG_ESP32S3_DATA_CACHE_SIZE: u32 = 0x8000;
     const CONFIG_ESP32S3_DCACHE_ASSOCIATED_WAYS: u8 = 8;
     const CONFIG_ESP32S3_DATA_CACHE_LINE_SIZE: u8 = 32;
-    const MMU_ACCESS_SPIRAM: u32 = 1 << 15;
-    const START_PAGE: u32 = 0;
+
+    let mut free_page = 0;
+    let mut psram_size = config.size.get();
 
     extern "C" {
         fn rom_config_instruction_cache_mode(
@@ -144,48 +151,34 @@ pub(crate) fn init_psram(config: PsramConfig) {
         );
 
         fn Cache_Resume_DCache(param: u32);
+    }
 
-        /// Set DCache mmu mapping.
-        ///
-        /// [`ext_ram`]: u32 DPORT_MMU_ACCESS_FLASH for flash, DPORT_MMU_ACCESS_SPIRAM for spiram, DPORT_MMU_INVALID for invalid.
-        /// [`vaddr`]: u32 Virtual address in CPU address space.
-        /// [`paddr`]: u32 Physical address in external memory. Should be aligned by psize.
-        /// [`psize`]: u32 Page size of DCache, in kilobytes. Should be 64 here.
-        /// [`num`]: u32 Pages to be set.
-        /// [`fixes`]: u32 0 for physical pages grow with virtual pages, other for virtual pages map to same physical page.
-        fn cache_dbus_mmu_set(
-            ext_ram: u32,
-            vaddr: u32,
-            paddr: u32,
-            psize: u32,
-            num: u32,
-            fixed: u32,
-        ) -> i32;
+    // Vaguely based off of the ESP-IDF equivalent code:
+    // https://github.com/espressif/esp-idf/blob/3c99557eeea4e0945e77aabac672fbef52294d54/components/esp_psram/mmu_psram_flash.c#L46-L134
+    if config.execute_from_psram {
+        let flash_pages = mmu::count_effective_flash_pages();
+        let psram_pages = psram_size / mmu::PAGE_SIZE;
+
+        if flash_pages > psram_pages {
+            panic!("Cannot execute from PSRAM: The number of PSRAM pages ({}) is too small to fit {} flash pages", psram_pages, flash_pages);
+        }
+
+        let psram_pages_used = unsafe { mmu::copy_flash_to_psram_and_remap(free_page) };
+
+        free_page += psram_pages_used;
+        psram_size -= psram_pages_used * mmu::PAGE_SIZE;
     }
 
     let start = unsafe {
-        const MMU_PAGE_SIZE: u32 = 0x10000;
-        const ICACHE_MMU_SIZE: usize = 0x800;
-        const FLASH_MMU_TABLE_SIZE: usize = ICACHE_MMU_SIZE / core::mem::size_of::<u32>();
-        const MMU_INVALID: u32 = 1 << 14;
-        const DR_REG_MMU_TABLE: u32 = 0x600C5000;
-
         // calculate the PSRAM start address to map
         // the linker scripts can produce a gap between mapped IROM and DROM segments
         // bigger than a flash page - i.e. we will see an unmapped memory slot
         // start from the end and find the last mapped flash page
-        //
-        // More general information about the MMU can be found here:
-        // https://docs.espressif.com/projects/esp-idf/en/stable/esp32s3/api-reference/system/mm.html#introduction
-        let mmu_table_ptr = DR_REG_MMU_TABLE as *const u32;
-        let mut mapped_pages = 0;
-        for i in (0..FLASH_MMU_TABLE_SIZE).rev() {
-            if mmu_table_ptr.add(i).read_volatile() != MMU_INVALID {
-                mapped_pages = (i + 1) as u32;
-                break;
-            }
+        let psram_start_index = mmu::last_mapped_index().map(|e| e + 1).unwrap_or(0);
+        if psram_start_index >= mmu::TABLE_SIZE {
+            panic!("PSRAM cannot be mapped as MMU table has no empty trailing entries");
         }
-        let start = EXTMEM_ORIGIN + (MMU_PAGE_SIZE * mapped_pages);
+        let start = mmu::index_to_data_address(psram_start_index);
         debug!("PSRAM start address = {:x}", start);
 
         // Configure the mode of instruction cache : cache size, cache line size.
@@ -205,12 +198,12 @@ pub(crate) fn init_psram(config: PsramConfig) {
             CONFIG_ESP32S3_DATA_CACHE_LINE_SIZE,
         );
 
-        if cache_dbus_mmu_set(
-            MMU_ACCESS_SPIRAM,
+        if mmu::cache_dbus_mmu_set(
+            mmu::ENTRY_ACCESS_SPIRAM,
             start,
-            START_PAGE << 16,
+            (free_page as u32) << 16,
             64,
-            config.size.get() as u32 / 1024 / 64, // number of pages to map
+            (psram_size / mmu::PAGE_SIZE) as u32, // number of pages to map
             0,
         ) != 0
         {