diff --git a/arch/arm64/include/memtag.h b/arch/arm64/include/memtag.h new file mode 100644 index 0000000000000..fa1474c5652b2 --- /dev/null +++ b/arch/arm64/include/memtag.h @@ -0,0 +1,58 @@ +/**************************************************************************** + * arch/arm64/include/memtag.h + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef ___ARCH_ARM64_INCLUDE_MTE_H +#define ___ARCH_ARM64_INCLUDE_MTE_H + +/**************************************************************************** + * Public Function Prototypes + ****************************************************************************/ + +/* Initialize MTE settings and enable memory tagging */ + +void up_memtag_init(void); + +/* Check if MTE is enabled */ + +bool up_memtag_is_enable(void); + +/* Set MTE state */ + +void up_memtag_bypass(bool bypass); + +/* Set memory tags for a given memory range */ + +void up_memtag_set_tag(const void *addr, size_t size); + +/* Get a random label based on the address through the mte register */ + +uint8_t up_memtag_get_random_tag(const void *addr); + +/* Get the address without label */ + +FAR void *up_memtag_get_untagged_addr(const void *addr); + +/* Get the address with label */ + +FAR void *up_memtag_get_tagged_addr(const void *addr, uint8_t tag); + +#endif /* ___ARCH_ARM64_INCLUDE_MTE_H */ diff --git a/arch/arm64/src/common/arm64_arch.h b/arch/arm64/src/common/arm64_arch.h index 3a32703cadb32..349104007c571 100644 --- a/arch/arm64/src/common/arm64_arch.h +++ b/arch/arm64/src/common/arm64_arch.h @@ -506,12 +506,6 @@ uint64_t arm64_get_mpid(int cpu); int arm64_get_cpuid(uint64_t mpid); #endif -#ifdef CONFIG_ARM64_MTE -void arm64_enable_mte(void); -#else -#define arm64_enable_mte() -#endif - #endif /* __ASSEMBLY__ */ #endif /* ___ARCH_ARM64_SRC_COMMON_ARM64_ARCH_H */ diff --git a/arch/arm64/src/common/arm64_mmu.h b/arch/arm64/src/common/arm64_mmu.h index a0677568c530d..345baf46c71af 100644 --- a/arch/arm64/src/common/arm64_mmu.h +++ b/arch/arm64/src/common/arm64_mmu.h @@ -170,8 +170,10 @@ * in the address range [59:55] = 0b00000 are unchecked accesses. */ -#define TCR_TCMA0 (1ULL << 57) -#define TCR_TCMA1 (1ULL << 58) +#define TCR_TCMA0 BIT(57) +#define TCR_TCMA1 BIT(58) +#define TCR_MTX0_SHIFT BIT(60) +#define TCR_MTX1_SHIFT BIT(61) #define TCR_PS_BITS_4GB 0x0ULL #define TCR_PS_BITS_64GB 0x1ULL diff --git a/arch/arm64/src/common/arm64_mte.c b/arch/arm64/src/common/arm64_mte.c index 6a2230f90f085..7432adad1bc0f 100644 --- a/arch/arm64/src/common/arm64_mte.c +++ b/arch/arm64/src/common/arm64_mte.c @@ -29,18 +29,24 @@ #include #include "arm64_arch.h" +#include "arm64_mmu.h" /**************************************************************************** * Pre-processor Definitions ****************************************************************************/ #define GCR_EL1_VAL 0x10001 +#define MTE_TAG_SHIFT 56 + +/* The alignment length of the MTE must be a multiple of sixteen */ + +#define MTE_MM_AILGN 16 /**************************************************************************** * Private Functions ****************************************************************************/ -static int arm64_mte_is_support(void) +static int mte_is_support(void) { int supported; __asm__ volatile ( @@ -53,15 +59,83 @@ static int arm64_mte_is_support(void) return supported != 0; } +static inline uint8_t mte_get_ptr_tag(const void *ptr) +{ + return 0xf0 | (uint8_t)(((uint64_t)(ptr)) >> MTE_TAG_SHIFT); +} + /**************************************************************************** * Public Functions ****************************************************************************/ -void arm64_enable_mte(void) +uint8_t up_memtag_get_random_tag(const void *addr) +{ + asm("irg %0, %0" : "=r" (addr)); + + return mte_get_ptr_tag(addr); +} + +FAR void *up_memtag_get_untagged_addr(const void *addr) +{ + return (FAR void *) + (((uint64_t)(addr)) & ~((uint64_t)0xff << MTE_TAG_SHIFT)); +} + +FAR void *up_memtag_get_tagged_addr(const void *addr, uint8_t tag) +{ + return (FAR void *) + (((uint64_t)(addr)) | ((uint64_t)tag << MTE_TAG_SHIFT)); +} + +/* Read MTE TCF1 */ + +bool up_memtag_is_enable(void) +{ + uint64_t val = read_sysreg(sctlr_el1); + + return (val & SCTLR_TCF1_BIT); +} + +/* Set MTE state */ + +void up_memtag_bypass(bool bypass) +{ + uint64_t val = read_sysreg(sctlr_el1); + + if (bypass) + { + val |= SCTLR_TCF1_BIT; + } + else + { + val &= ~SCTLR_TCF1_BIT; + } + + write_sysreg(val, sctlr_el1); +} + +/* Set memory tags for a given memory range */ + +void up_memtag_set_tag(const void *addr, size_t size) +{ + size_t i; + + DEBUGASSERT((uintptr_t)addr % MTE_MM_AILGN == 0); + DEBUGASSERT(size % MTE_MM_AILGN == 0); + + for (i = 0; i < size; i += MTE_MM_AILGN) + { + asm("stg %0, [%0]" : : "r"(addr + i)); + } +} + +/* Initialize MTE settings and enable memory tagging */ + +void up_memtag_init(void) { uint64_t val; - if (!arm64_mte_is_support()) + if (!mte_is_support()) { return; } @@ -78,6 +152,14 @@ void arm64_enable_mte(void) assert(!(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT)); assert(!(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT)); + /* Controls the default value for skipping high bytes */ + + val = read_sysreg(tcr_el1); + val |= TCR_TCMA1; + write_sysreg(val, tcr_el1); + + /* Enable the MTE function */ + val = read_sysreg(sctlr_el1); val |= SCTLR_ATA_BIT | SCTLR_TCF1_BIT; write_sysreg(val, sctlr_el1); diff --git a/arch/arm64/src/qemu/qemu_boot.c b/arch/arm64/src/qemu/qemu_boot.c index 4686c60b24c0d..6d8715e16e8fb 100644 --- a/arch/arm64/src/qemu/qemu_boot.c +++ b/arch/arm64/src/qemu/qemu_boot.c @@ -36,6 +36,7 @@ # include #endif +#include #include #ifdef CONFIG_SMP @@ -161,7 +162,9 @@ void arm64_chip_boot(void) arm64_mmu_init(true); - arm64_enable_mte(); +#ifdef CONFIG_ARM64_MTE + up_memtag_init(); +#endif #ifdef CONFIG_DEVICE_TREE fdt_register((const char *)0x40000000); diff --git a/include/nuttx/mm/kasan.h b/include/nuttx/mm/kasan.h index a349eb9675bb7..b024e03e166ff 100644 --- a/include/nuttx/mm/kasan.h +++ b/include/nuttx/mm/kasan.h @@ -159,7 +159,11 @@ FAR void *kasan_reset_tag(FAR const void *addr); * ****************************************************************************/ +#ifdef CONFIG_MM_KASAN_INSTRUMENT void kasan_start(void); +#else +# define kasan_start() +#endif /**************************************************************************** * Name: kasan_stop @@ -177,7 +181,11 @@ void kasan_start(void); * ****************************************************************************/ +#ifdef CONFIG_MM_KASAN_INSTRUMENT void kasan_stop(void); +#else +# define kasan_stop() +#endif /**************************************************************************** * Name: kasan_debugpoint @@ -201,6 +209,32 @@ void kasan_stop(void); int kasan_debugpoint(int type, FAR void *addr, size_t size); +#ifndef CONFIG_MM_KASAN_HW_TAGS +# define kasan_bypass(bypass) +# define kasan_bypass_save() +# define kasan_bypass_restore() +#else + +/**************************************************************************** + * Name: kasan_bypass + ****************************************************************************/ + +void kasan_bypass(bool bypass); + +/**************************************************************************** + * Name: kasan_bypass_save + ****************************************************************************/ + +bool kasan_bypass_save(void); + +/**************************************************************************** + * Name: kasan_bypass_restore + ****************************************************************************/ + +void kasan_bypass_restore(bool state); + +#endif + #undef EXTERN #ifdef __cplusplus } diff --git a/mm/kasan/Kconfig b/mm/kasan/Kconfig index f80cfb303e785..d434cd04d5174 100644 --- a/mm/kasan/Kconfig +++ b/mm/kasan/Kconfig @@ -32,13 +32,18 @@ config MM_KASAN_GENERIC KASan generic mode that does not require hardware support at all config MM_KASAN_SW_TAGS - bool "KAsan SW tags" - select ARM64_TBI + bool "KAsan softtags tags" + select ARM64_TBI if ARCH_ARM64 select MM_KASAN_INSTRUMENT - depends on ARCH_ARM64 ---help--- KAsan based on software tags +config MM_KASAN_HW_TAGS + bool "KAsan hardware tags" + select ARM64_MTE if ARCH_ARM64 + ---help--- + KAsan based on hardware tags + endchoice config MM_KASAN_INSTRUMENT_ALL @@ -52,6 +57,8 @@ config MM_KASAN_INSTRUMENT_ALL to check. Enabling this option will get image size increased and performance decreased significantly. +if MM_KASAN_INSTRUMENT + config MM_KASAN_REGIONS int "Kasan region count" default 8 @@ -126,4 +133,5 @@ config MM_KASAN_GLOBAL_ALIGN It is recommended to use 1, 2, 4, 8, 16, 32. The maximum value is 32. +endif # MM_KASAN_INSTRUMENT endif # MM_KASAN diff --git a/mm/kasan/hook.c b/mm/kasan/hook.c index d4dc19c21295d..e67b2cee1c6aa 100644 --- a/mm/kasan/hook.c +++ b/mm/kasan/hook.c @@ -42,6 +42,8 @@ # include "generic.c" #elif defined(CONFIG_MM_KASAN_SW_TAGS) # include "sw_tags.c" +#elif defined(CONFIG_MM_KASAN_HW_TAGS) +# include "hw_tags.c" #else # define kasan_is_poisoned(addr, size) false #endif @@ -96,6 +98,8 @@ #define KASAN_INIT_VALUE 0xcafe +#ifdef CONFIG_MM_KASAN_INSTRUMENT + /**************************************************************************** * Private Types ****************************************************************************/ @@ -405,3 +409,6 @@ DEFINE_ASAN_LOAD_STORE(2) DEFINE_ASAN_LOAD_STORE(4) DEFINE_ASAN_LOAD_STORE(8) DEFINE_ASAN_LOAD_STORE(16) + +#endif /* CONFIG_MM_KASAN_INSTRUMENT */ + diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c new file mode 100644 index 0000000000000..a0eb4f6c0699a --- /dev/null +++ b/mm/kasan/hw_tags.c @@ -0,0 +1,106 @@ +/**************************************************************************** + * mm/kasan/mte_tags.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +static void *kasan_set_poison(FAR const void *addr, + size_t size, + uint8_t tag) +{ + FAR const void *tag_addr; + + /* Get random labels and the addresses after labeling */ + + tag_addr = up_memtag_get_tagged_addr(addr, tag); + + /* Add MTE hardware label to memory block */ + + up_memtag_set_tag(tag_addr, size); + + return (FAR void *)tag_addr; +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +void kasan_bypass(bool bypass) +{ + up_memtag_bypass(bypass); +} + +bool kasan_bypass_save(void) +{ + bool ret = up_memtag_is_enable(); + + kasan_bypass(false); + return ret; +} + +void kasan_bypass_restore(bool state) +{ + if (state) + { + kasan_bypass(true); + } + else + { + kasan_bypass(false); + } +} + +FAR void *kasan_reset_tag(FAR const void *addr) +{ + return up_memtag_get_untagged_addr(addr); +} + +void kasan_poison(FAR const void *addr, size_t size) +{ + uint8_t tag = up_memtag_get_random_tag(addr); + + kasan_set_poison(addr, size, tag); +} + +FAR void *kasan_unpoison(FAR const void *addr, size_t size) +{ + uint8_t tag = up_memtag_get_random_tag(addr); + + return kasan_set_poison(addr, size, tag); +} + + +void kasan_register(FAR void *addr, FAR size_t *size) +{ + uint8_t tag = up_memtag_get_random_tag(addr); + + kasan_set_poison(addr, *size, tag); +} + +void kasan_unregister(FAR void *addr) +{ +} diff --git a/mm/mm_heap/mm_lock.c b/mm/mm_heap/mm_lock.c index eb9d8ddb18b79..b2fb287eeef5e 100644 --- a/mm/mm_heap/mm_lock.c +++ b/mm/mm_heap/mm_lock.c @@ -32,6 +32,7 @@ #include #include +#include #include #include "mm_heap/mm.h" @@ -59,6 +60,10 @@ int mm_lock(FAR struct mm_heap_s *heap) { + int ret; + + kasan_bypass(false); + #if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) /* Check current environment */ @@ -69,12 +74,19 @@ int mm_lock(FAR struct mm_heap_s *heap) * Or, touch the heap internal data directly. */ - return nxmutex_is_locked(&heap->mm_lock) ? -EAGAIN : 0; + ret = nxmutex_is_locked(&heap->mm_lock) ? -EAGAIN : 0; # else /* Can't take mutex in SMP interrupt handler */ - return -EAGAIN; + ret = -EAGAIN; # endif + + if (ret < 0) + { + kasan_bypass(true); + } + + return ret; } #endif @@ -90,6 +102,7 @@ int mm_lock(FAR struct mm_heap_s *heap) if (_SCHED_GETTID() < 0) { + kasan_bypass(true); return -ESRCH; } else @@ -111,11 +124,13 @@ void mm_unlock(FAR struct mm_heap_s *heap) #if defined(CONFIG_BUILD_FLAT) || defined(__KERNEL__) if (up_interrupt_context()) { + kasan_bypass(true); return; } #endif DEBUGVERIFY(nxmutex_unlock(&heap->mm_lock)); + kasan_bypass(true); } /**************************************************************************** @@ -129,6 +144,7 @@ void mm_unlock(FAR struct mm_heap_s *heap) irqstate_t mm_lock_irq(FAR struct mm_heap_s *heap) { UNUSED(heap); + kasan_bypass(false); return up_irq_save(); } @@ -143,5 +159,6 @@ irqstate_t mm_lock_irq(FAR struct mm_heap_s *heap) void mm_unlock_irq(FAR struct mm_heap_s *heap, irqstate_t state) { UNUSED(heap); + kasan_bypass(true); up_irq_restore(state); }