AArch64: Add libcall impl declarations for __arm_sc* memory functions (#144977)

These were bypassing the ordinary libcall emission mechanism. Make sure
we have entries in RuntimeLibcalls, which should include all possible
calls the compiler could emit.

Fixes not emitting the # prefix in the arm64ec case.

Author: arsenm

llvm/include/llvm/IR/RuntimeLibcalls.td

@@ -357,6 +357,11 @@ multiclass LibmLongDoubleLibCall<string libcall_basename = !toupper(NAME),
                            !strconcat(rtbasename, "l")>;
 }
 
+// AArch64 calls
+def SC_MEMCPY : RuntimeLibcall;
+def SC_MEMMOVE : RuntimeLibcall;
+def SC_MEMSET : RuntimeLibcall;
+
 // ARM EABI calls
 def AEABI_MEMCPY4 : RuntimeLibcall; // Align 4
 def AEABI_MEMCPY8 : RuntimeLibcall; // Align 8
@@ -985,6 +990,10 @@ defset list<RuntimeLibcallImpl> AArch64LibcallImpls = {
     defm __aarch64_ldeor#MemSize
         : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_LDEOR"#MemSize>;
   }
+
+  def __arm_sc_memcpy : RuntimeLibcallImpl<SC_MEMCPY>;
+  def __arm_sc_memmove : RuntimeLibcallImpl<SC_MEMMOVE>;
+  def __arm_sc_memset : RuntimeLibcallImpl<SC_MEMSET>;
 }
 
 foreach libcall = AArch64LibcallImpls in {

llvm/lib/IR/RuntimeLibcalls.cpp

@@ -489,8 +489,17 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
   }
 
   if (TT.isAArch64()) {
-    if (TT.isWindowsArm64EC())
+    if (TT.isWindowsArm64EC()) {
       setWindowsArm64LibCallNameOverrides();
+      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::arm64ec___arm_sc_memcpy);
+      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::arm64ec___arm_sc_memmove);
+      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::arm64ec___arm_sc_memset);
+    } else {
+      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::__arm_sc_memcpy);
+      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::__arm_sc_memmove);
+      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::__arm_sc_memset);
+    }
+
     setAArch64LibcallNames(*this, TT);
   } else if (TT.isARM() || TT.isThumb()) {
     setARMLibcallNames(*this, TT, FloatABI, EABIVersion);

llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp

@@ -164,35 +164,34 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
   const AArch64Subtarget &STI =
       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
   const AArch64TargetLowering *TLI = STI.getTargetLowering();
-  SDValue Symbol;
   TargetLowering::ArgListEntry DstEntry;
   DstEntry.Ty = PointerType::getUnqual(*DAG.getContext());
   DstEntry.Node = Dst;
   TargetLowering::ArgListTy Args;
   Args.push_back(DstEntry);
-  EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());
 
+  RTLIB::Libcall NewLC;
   switch (LC) {
   case RTLIB::MEMCPY: {
+    NewLC = RTLIB::SC_MEMCPY;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memcpy", PointerVT);
     Entry.Node = Src;
     Args.push_back(Entry);
     break;
   }
   case RTLIB::MEMMOVE: {
+    NewLC = RTLIB::SC_MEMMOVE;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memmove", PointerVT);
     Entry.Node = Src;
     Args.push_back(Entry);
     break;
   }
   case RTLIB::MEMSET: {
+    NewLC = RTLIB::SC_MEMSET;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = Type::getInt32Ty(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memset", PointerVT);
     Src = DAG.getZExtOrTrunc(Src, DL, MVT::i32);
     Entry.Node = Src;
     Args.push_back(Entry);
@@ -202,17 +201,17 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
     return SDValue();
   }
 
+  EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());
+  SDValue Symbol = DAG.getExternalSymbol(TLI->getLibcallName(NewLC), PointerVT);
   TargetLowering::ArgListEntry SizeEntry;
   SizeEntry.Node = Size;
   SizeEntry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
   Args.push_back(SizeEntry);
-  assert(Symbol->getOpcode() == ISD::ExternalSymbol &&
-         "Function name is not set");
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   PointerType *RetTy = PointerType::getUnqual(*DAG.getContext());
   CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
-      TLI->getLibcallCallingConv(LC), RetTy, Symbol, std::move(Args));
+      TLI->getLibcallCallingConv(NewLC), RetTy, Symbol, std::move(Args));
   return TLI->LowerCallTo(CLI).second;
 }
 

llvm/test/CodeGen/AArch64/arm64ec-builtins.ll

@@ -46,27 +46,24 @@ define float @f6(float %val, i32 %a) {
 @dst = global [512 x i8] zeroinitializer, align 1
 @src = global [512 x i8] zeroinitializer, align 1
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memcpy(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memcpy":
-; CHECK: bl __arm_sc_memcpy
+; CHECK: bl "#__arm_sc_memcpy"
 
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
   ret void
 }
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memmove(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memmove":
-; CHECK: bl __arm_sc_memmove
+; CHECK: bl "#__arm_sc_memmove"
   tail call void @llvm.memmove.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
   ret void
 }
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memset(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memset":
-; CHECK: bl __arm_sc_memset
+; CHECK: bl "#__arm_sc_memset"
   tail call void @llvm.memset.p0.i64(ptr align 1 @dst, i8 2, i64 %n, i1 false)
   ret void
 }