[AIEX] Delay metalizing of multi-slot until iterative scheduling is converged

llvm/lib/Target/AIE/AIE2MultiSlotPseudoInstrInfo.td

@@ -73,7 +73,7 @@ let isMoveImm = 1, isReMaterializable = 1, isAsCheapAsAMove = 1, Itinerary = II_
 
 // Pseudo VLD
 let hasSideEffects = false, mayLoad = true, mayStore = false in {
-  let Itinerary = II_VLDA_W in {
+  let Itinerary = II_VLDB in {
   def VLD_idx_pseudo : MultiSlot_Pseudo< (outs mWa:$dst), 
                                          (ins eP:$ptr, eDJ:$dj),
       "vld_idx_pseudo", "$dst, [$ptr, $dj]", 
@@ -83,7 +83,7 @@ let hasSideEffects = false, mayLoad = true, mayStore = false in {
       "vld_idx_imm_imm3x32_pseudo", "$dst, [$ptr, $imm]", 
       [VLDB_dmw_ldb_ag_idx_imm, VLDA_dmw_lda_w_ag_idx_imm]>;
   }
-  let Itinerary = II_VLDA_POSTINC_W in
+  let Itinerary = II_VLDB_POSTINC in
   let Constraints = "$ptr_out = $ptr" in {
   def VLD_pstm_pseudo : MultiSlot_Pseudo< (outs mWa:$dst, eP:$ptr_out), 
                                           (ins eP:$ptr, eM:$mod),
@@ -94,14 +94,14 @@ let hasSideEffects = false, mayLoad = true, mayStore = false in {
       "vld_pstm_imm_4x32_pseudo", "$dst, [$ptr], $imm", 
       [VLDB_dmw_ldb_ag_pstm_nrm_imm, VLDA_dmw_lda_w_ag_pstm_nrm_imm]>;
   }
-  let Itinerary = II_VLDA_2D_W in
+  let Itinerary = II_VLDB_2D in
   let Constraints = "$ptr_out = $ptr" in {
   def VLD_2D_pseudo : MultiSlot_Pseudo< (outs mWa:$dst, eP:$ptr_out, eDC:$count_out), 
                                         (ins eP:$ptr, eD:$mod),
       "vld.2d_pseudo", "$dst, [$ptr], $mod", 
       [VLDB_2D, VLDA_2D_dmw_lda_w]>, AIE_HasTiedSubregister;
   }
-  let Itinerary = II_VLDA_3D_W in
+  let Itinerary = II_VLDB_3D in
   let Constraints = "$ptr_out = $ptr" in {
   def VLD_3D_pseudo : MultiSlot_Pseudo< (outs mWa:$dst, eP:$ptr_out, eDC:$count_lo_out, eDC:$count_hi_out), 
                                         (ins eP:$ptr, eDS:$mod),

llvm/lib/Target/AIE/AIEAlternateDescriptors.h

@@ -31,6 +31,9 @@ class AIEAlternateDescriptors {
   AIEAlternateDescriptors() = default;
   ~AIEAlternateDescriptors() = default;
 
+  MIAltDescsMap::const_iterator begin() const { return AlternateDescs.begin(); }
+  MIAltDescsMap::const_iterator end() const { return AlternateDescs.end(); }
+
   // Construct an alternate descriptor with the given alternate descriptors.
   AIEAlternateDescriptors(const MIAltDescsMap &AltDescs)
       : AlternateDescs(AltDescs) {}
@@ -43,6 +46,10 @@ class AIEAlternateDescriptors {
     AlternateDescs[MI] = &TII->get(AltInstOpcode);
   }
 
+  void setAlternateDescriptor(MachineInstr *MI, const MCInstrDesc *AltDesc) {
+    AlternateDescs[MI] = AltDesc;
+  }
+
   // Return the alternate descriptor for the given multi-opcode instruction.
   std::optional<const MCInstrDesc *>
   getSelectedDescriptor(MachineInstr *MI) const {

llvm/lib/Target/AIE/AIEHazardRecognizer.cpp

@@ -473,7 +473,7 @@ ScheduleHazardRecognizer::HazardType AIEHazardRecognizer::getHazardType(
 bool AIEHazardRecognizer::checkConflict(
     const ResourceScoreboard<FuncUnitWrapper> &Scoreboard, MachineInstr &MI,
     int DeltaCycles) const {
-  const MCInstrDesc &Desc = MI.getDesc();
+  const MCInstrDesc &Desc = *SelectedAltDescs.getDesc(&MI);
   const unsigned SchedClass =
       TII->getSchedClass(Desc, MI.operands(), MI.getMF()->getRegInfo());
   const MemoryBankBits MemoryBanks = getMemoryBanks(&MI);

llvm/lib/Target/AIE/AIEInterBlockScheduling.cpp

@@ -300,6 +300,7 @@ namespace {
 /// into the appropriate blockstate region.
 /// TimedRegion is built one bundle at the time
 class PipelineExtractor : public PipelineScheduleVisitor {
+  AIEAlternateDescriptors &AlternateDesc;
   BlockState &Loop;
   BlockState *Prologue = nullptr;
   BlockState *Epilogue = nullptr;
@@ -334,14 +335,20 @@ class PipelineExtractor : public PipelineScheduleVisitor {
     // Prologue and epilogue obtain copies.
     MachineInstr *ToBeEmitted =
         InLoop ? MI : Loop.TheBlock->getParent()->CloneMachineInstr(MI);
-    CurrentBundle.add(ToBeEmitted);
+    // We force the prologue/epilogue copies to have the same descriptor as the
+    // original instruction in the steady state.
+    if (auto AltDesc = AlternateDesc.getSelectedDescriptor(MI))
+      AlternateDesc.setAlternateDescriptor(ToBeEmitted, AltDesc.value());
+
+    CurrentBundle.add(ToBeEmitted, AlternateDesc.getOpcode(MI));
   }
   void endBundle() override { TimedRegion.emplace_back(CurrentBundle); }
 
 public:
   PipelineExtractor(InterBlockScheduling &InterBlock, BlockState &BS,
                     const AIEBaseInstrInfo &TII)
-      : Loop(BS), CurrentBundle(TII.getFormatInterface()) {
+      : AlternateDesc(InterBlock.getSelectedAltDescs()), Loop(BS),
+        CurrentBundle(TII.getFormatInterface()) {
     MachineBasicBlock *LoopBlock = Loop.TheBlock;
     for (auto *P : LoopBlock->predecessors()) {
       if (P == LoopBlock) {

llvm/lib/Target/AIE/AIEMachineScheduler.cpp

@@ -93,6 +93,10 @@ static cl::opt<bool> PreSchedFollowsSkipPipeliner(
     "aie-presched-follows-skip-pipeliner", cl::init(true),
     cl::desc("Don't run the prescheduler if the pipeliner is skipped"));
 
+static cl::opt<bool> ReAssignMultiSlotInstr(
+    "aie-reassign-multislot-instr", cl::init(true),
+    cl::desc("Re-assign multi-slot instructions during iterative scheduling"));
+
 namespace {
 // A sentinel value to represent an unknown SUnit.
 const constexpr unsigned UnknownSUNum = ~0;
@@ -277,6 +281,9 @@ void AIEPostRASchedStrategy::initializeBotScoreBoard(ScoreboardTrust Trust) {
   /// make sure we always have enough lookahead available. We arrange for that
   /// by starting in the earliest possible cycle, -Depth
   auto InsertInCycle = [=](MachineInstr &MI, int Cycle) {
+    assert(BotHazardRec->getSelectedAltDescs().getSelectedDescriptor(&MI) ==
+               std::nullopt &&
+           "Instructions opcode are already materialized");
     BotHazardRec->emitInScoreboard(
         MI.getDesc(), BotHazardRec->getMemoryBanks(&MI), MI.operands(),
         MI.getMF()->getRegInfo(), Cycle - Depth);
@@ -607,6 +614,9 @@ void AIEPostRASchedStrategy::enterMBB(MachineBasicBlock *MBB) {
 void AIEPostRASchedStrategy::commitBlockSchedule(MachineBasicBlock *BB) {
   auto &BS = InterBlock.getBlockState(BB);
 
+  if (ReAssignMultiSlotInstr)
+    materializeMultiSlotInstrs();
+
   // TODO: Update assert when the fixed instructions become part of the
   // scheduling region.
   assert(BS.getRegions().empty() ||
@@ -673,8 +683,8 @@ void AIEPostRASchedStrategy::leaveRegion(const SUnit &ExitSU) {
   if (BS.FixPoint.Stage != SchedulingStage::Scheduling) {
     return;
   }
-  materializeMultiOpcodeInstrs();
-  InterBlock.getSelectedAltDescs().clear();
+  if (!ReAssignMultiSlotInstr)
+    materializeMultiSlotInstrs();
   if (IsBottomRegion) {
     // This is the earliest point where we can destroy the recorded
     // schedule in iterative scheduling. enterMBB and enterRegion are too early,
@@ -700,25 +710,13 @@ void AIEPostRASchedStrategy::leaveRegion(const SUnit &ExitSU) {
   DEBUG_BLOCKS(dbgs() << "    << leaveRegion\n");
 }
 
-void AIEPostRASchedStrategy::materializeMultiOpcodeInstrs() {
-  const TargetInstrInfo *TII = getTII(CurMBB);
-  const AIEHazardRecognizer &TopHazardRec = *getAIEHazardRecognizer(Top);
-  const AIEHazardRecognizer &BotHazardRec = *getAIEHazardRecognizer(Bot);
-
-  auto MaterializePseudo = [&TII](MachineInstr &MI,
-                                  const AIEHazardRecognizer &HazardRec) {
-    // Materialize instructions with multiple opcode options
-    if (std::optional<unsigned> AltOpcode =
-            HazardRec.getSelectedAltDescs().getSelectedOpcode(&MI)) {
-      MI.setDesc(TII->get(*AltOpcode));
-    }
-  };
+void AIEPostRASchedStrategy::materializeMultiSlotInstrs() {
+  for (auto &[MI, Desc] : make_range(InterBlock.getSelectedAltDescs().begin(),
+                                     InterBlock.getSelectedAltDescs().end())) {
+    MI->setDesc(*Desc);
+  }
 
-  assert(DAG->top() == DAG->bottom());
-  for (MachineInstr &MI : make_range(DAG->begin(), DAG->top()))
-    MaterializePseudo(MI, TopHazardRec);
-  for (MachineInstr &MI : make_range(DAG->bottom(), DAG->end()))
-    MaterializePseudo(MI, BotHazardRec);
+  InterBlock.getSelectedAltDescs().clear();
 }
 
 const SUnit &getBundledSUnit(const ScheduleDAGMI *DAG, MachineInstr *MI) {

llvm/lib/Target/AIE/AIEMachineScheduler.h

@@ -152,7 +152,7 @@ class AIEPostRASchedStrategy : public PostGenericScheduler {
 
   /// Materialize "multi-opcode" instructions into the option that was selected
   /// at schedule time. See AIEHazardRecognizer::getSelectedAltOpcode().
-  void materializeMultiOpcodeInstrs();
+  void materializeMultiSlotInstrs();
 
   /// Returns true if, when "concatenated", the Top and Bot zone have resource
   /// conflicts or timing issues.

llvm/lib/Target/AIE/AIEPostPipeliner.cpp

@@ -115,16 +115,18 @@ bool PostPipeliner::canAccept(MachineBasicBlock &LoopBlock) {
   return true;
 }
 
-static SlotCounts getSlotCounts(MachineInstr &MI, const AIEBaseInstrInfo *TII) {
-  auto *SlotInfo = TII->getSlotInfo(TII->getSlotKind(MI.getOpcode()));
+static SlotCounts getSlotCounts(MachineInstr &MI, const AIEBaseInstrInfo *TII,
+                                const AIEHazardRecognizer &HR) {
+  const unsigned Opcode = HR.getSelectedAltDescs().getOpcode(&MI);
+  auto *SlotInfo = TII->getSlotInfo(TII->getSlotKind(Opcode));
   return SlotInfo ? SlotInfo->getSlotSet() : 0;
 }
 
 int PostPipeliner::getResMII(MachineBasicBlock &LoopBlock) {
   // Add up all slot requirements and return the maximum slot count
   SlotCounts Counts;
   for (auto &MI : LoopBlock) {
-    Counts += getSlotCounts(MI, TII);
+    Counts += getSlotCounts(MI, TII, HR);
   }
   int MII = Counts.max();
   LLVM_DEBUG(dbgs() << "PostPipeliner: ResMII=" << MII << "\n");
@@ -221,7 +223,7 @@ void PostPipeliner::computeForward() {
       const int NewEarliest = Me.Earliest + Dep.getSignedLatency();
       SInfo.Earliest = std::max(SInfo.Earliest, NewEarliest);
     }
-    Me.Slots = getSlotCounts(*SU.getInstr(), TII);
+    Me.Slots = getSlotCounts(*SU.getInstr(), TII, HR);
   }
 }
 
@@ -460,9 +462,9 @@ bool PostPipeliner::scheduleFirstIteration(PostPipelinerStrategy &Strategy) {
       if (N > 0 && HR.checkConflict(Scoreboard, *MI, Cycle)) {
         return false;
       }
-
-      HR.emitInScoreboard(Scoreboard, MI->getDesc(), MemoryBanks,
-                          MI->operands(), MI->getMF()->getRegInfo(), Cycle);
+      const MCInstrDesc &Desc = *HR.getSelectedAltDescs().getDesc(MI);
+      HR.emitInScoreboard(Scoreboard, Desc, MemoryBanks, MI->operands(),
+                          MI->getMF()->getRegInfo(), Cycle);
       Cycle += II;
     }