add extern "custom" functions

Author: folkertdev

compiler/rustc_abi/src/extern_abi.rs

@@ -25,6 +25,10 @@ pub enum ExternAbi {
     Cdecl {
         unwind: bool,
     },
+    /// An ABI that rustc does not know how to call or define. Functions with this ABI can
+    /// only be created using `#[naked]` functions or `extern "custom"` blocks, and can only
+    /// be called from inline assembly.
+    Custom,
     Stdcall {
         unwind: bool,
     },
@@ -141,6 +145,7 @@ abi_impls! {
             Win64 { unwind: false } =><= "win64",
             Win64 { unwind: true } =><= "win64-unwind",
             X86Interrupt =><= "x86-interrupt",
+            Custom =><= "custom",
     }
 }
 

compiler/rustc_ast_lowering/src/stability.rs

@@ -134,5 +134,8 @@ pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {
             feature: sym::cmse_nonsecure_entry,
             explain: GateReason::Experimental,
         }),
+        ExternAbi::Custom => {
+            Err(UnstableAbi { abi, feature: sym::abi_custom, explain: GateReason::Experimental })
+        }
     }
 }

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -68,6 +68,11 @@ pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: Call
         Conv::Msp430Intr | Conv::GpuKernel | Conv::AvrInterrupt | Conv::AvrNonBlockingInterrupt => {
             unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");
         }
+
+        // Functions with this calling convention can only be called from assembly, but it is
+        // possible to declare an `extern "custom"` block, so the backend still needs a calling
+        // convention for declaring foreign functions.
+        Conv::Custom => default_call_conv,
     }
 }
 

compiler/rustc_codegen_gcc/src/abi.rs

@@ -272,6 +272,11 @@ pub fn conv_to_fn_attribute<'gcc>(conv: Conv, arch: &str) -> Option<FnAttribute<
         Conv::X86VectorCall => return None,
         Conv::X86_64SysV => FnAttribute::SysvAbi,
         Conv::X86_64Win64 => FnAttribute::MsAbi,
+
+        // Functions with this calling convention can only be called from assembly, but it is
+        // possible to declare an `extern "custom"` block, so the backend still needs a calling
+        // convention for declaring foreign functions.
+        Conv::Custom => return None,
     };
     Some(attribute)
 }

compiler/rustc_codegen_llvm/src/abi.rs

@@ -676,6 +676,11 @@ impl llvm::CallConv {
             Conv::X86VectorCall => llvm::X86_VectorCall,
             Conv::X86_64SysV => llvm::X86_64_SysV,
             Conv::X86_64Win64 => llvm::X86_64_Win64,
+
+            // Functions with this calling convention can only be called from assembly, but it is
+            // possible to declare an `extern "custom"` block, so the backend still needs a calling
+            // convention for declaring foreign functions.
+            Conv::Custom => llvm::CCallConv,
         }
     }
 }

compiler/rustc_feature/src/unstable.rs

@@ -354,6 +354,8 @@ declare_features! (
     (unstable, abi_avr_interrupt, "1.45.0", Some(69664)),
     /// Allows `extern "C-cmse-nonsecure-call" fn()`.
     (unstable, abi_c_cmse_nonsecure_call, "1.51.0", Some(81391)),
+    /// Allows `extern "custom" fn()`.
+    (unstable, abi_custom, "CURRENT_RUSTC_VERSION", Some(140566)),
     /// Allows `extern "gpu-kernel" fn()`.
     (unstable, abi_gpu_kernel, "1.86.0", Some(135467)),
     /// Allows `extern "msp430-interrupt" fn()`.

compiler/rustc_interface/src/passes.rs

@@ -960,6 +960,7 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
                     tcx.ensure_ok().check_mod_loops(module);
                     tcx.ensure_ok().check_mod_attrs(module);
                     tcx.ensure_ok().check_mod_naked_functions(module);
+                    tcx.ensure_ok().check_mod_custom_abi(module);
                     tcx.ensure_ok().check_mod_unstable_api_usage(module);
                 });
             },

compiler/rustc_middle/src/query/mod.rs

@@ -1117,6 +1117,10 @@ rustc_queries! {
         desc { |tcx| "checking naked functions in {}", describe_as_module(key, tcx) }
     }
 
+    query check_mod_custom_abi(key: LocalModDefId) {
+        desc { |tcx| "checking use of `extern \"custom\"` in {}", describe_as_module(key, tcx) }
+    }
+
     query check_mod_privacy(key: LocalModDefId) {
         desc { |tcx| "checking privacy in {}", describe_as_module(key.to_local_def_id(), tcx) }
     }

compiler/rustc_middle/src/ty/layout.rs

@@ -1264,6 +1264,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi)
         | RiscvInterruptS
         | CCmseNonSecureCall
         | CCmseNonSecureEntry
+        | Custom
         | Unadjusted => false,
         Rust | RustCall | RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind,
     }

compiler/rustc_passes/messages.ftl

@@ -4,6 +4,22 @@
 -passes_see_issue =
     see issue #{$issue} <https://github.com/rust-lang/rust/issues/{$issue}> for more information
 
+passes_abi_custom_call =
+    functions with the `"custom"` ABI cannot be called
+    .note = an `extern "custom"` function can only be called from within inline assembly
+
+passes_abi_custom_clothed_function =
+    functions with the `"custom"` ABI must be naked
+    .suggestion = add the `#[unsafe(naked)]` attribute to this function
+
+passes_abi_custom_safe_foreign_function =
+    foreign functions with the `"custom"` ABI cannot be safe
+    .suggestion = remove the `safe` keyword from this definition
+
+passes_abi_custom_safe_function =
+    functions with the `"custom"` ABI must be unsafe
+    .suggestion = add the `unsafe` keyword this function definition
+
 passes_abi_invalid_attribute =
     `#[rustc_abi]` can only be applied to function items, type aliases, and associated functions
 passes_abi_ne =