diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h index 6fff7838470e7..d77d17d21d304 100644 --- a/llvm/include/llvm/IR/DebugInfo.h +++ b/llvm/include/llvm/IR/DebugInfo.h @@ -39,6 +39,7 @@ class Module; /// Finds dbg.declare intrinsics declaring local variables as living in the /// memory that 'V' points to. TinyPtrVector FindDbgDeclareUses(Value *V); +TinyPtrVector FindDPDeclareUses(Value *V); /// Finds the llvm.dbg.value intrinsics describing a value. void findDbgValues(SmallVectorImpl &DbgValues, @@ -54,6 +55,7 @@ DISubprogram *getDISubprogram(const MDNode *Scope); /// Produce a DebugLoc to use for each dbg.declare that is promoted to a /// dbg.value. DebugLoc getDebugValueLoc(DbgVariableIntrinsic *DII); +DebugLoc getDebugValueLoc(DPValue *DPV); /// Strip debug info in the module if it exists. /// @@ -187,6 +189,11 @@ AssignmentInstRange getAssignmentInsts(DIAssignID *ID); inline AssignmentInstRange getAssignmentInsts(const DbgAssignIntrinsic *DAI) { return getAssignmentInsts(DAI->getAssignID()); } +inline AssignmentInstRange getAssignmentInsts(const DPValue *DPV) { + assert(DPV->isDbgAssign() && + "Can't get assignment instructions for non-assign DPV!"); + return getAssignmentInsts(DPV->getAssignID()); +} // // Utilities for enumerating llvm.dbg.assign intrinsic from an assignment ID. @@ -210,6 +217,7 @@ using AssignmentMarkerRange = iterator_range; /// Return a range of dbg.assign intrinsics which use \ID as an operand. /// Iterators invalidated by deleting an intrinsic contained in this range. AssignmentMarkerRange getAssignmentMarkers(DIAssignID *ID); +void getDPAssignmentMarkers(DIAssignID *ID, SmallVectorImpl DPAssigns); /// Return a range of dbg.assign intrinsics for which \p Inst performs the /// assignment they encode. /// Iterators invalidated by deleting an intrinsic contained in this range. @@ -219,6 +227,15 @@ inline AssignmentMarkerRange getAssignmentMarkers(const Instruction *Inst) { else return make_range(Value::user_iterator(), Value::user_iterator()); } +inline void getDPAssignmentMarkers(const Instruction *Inst, SmallVectorImpl &DPAssigns) { + if (auto *ID = Inst->getMetadata(LLVMContext::MD_DIAssignID)) + DPAssigns = cast(ID)->getAllDPValueUsers(); +} +inline SmallVector getDPAssignmentMarkers(const Instruction *Inst) { + if (auto *ID = Inst->getMetadata(LLVMContext::MD_DIAssignID)) + return cast(ID)->getAllDPValueUsers(); + return SmallVector(); +} /// Delete the llvm.dbg.assign intrinsics linked to \p Inst. void deleteAssignmentMarkers(const Instruction *Inst); @@ -240,7 +257,8 @@ void deleteAll(Function *F); /// Result contains a zero-sized fragment if there's no intersect. bool calculateFragmentIntersect( const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits, - uint64_t SliceSizeInBits, const DbgAssignIntrinsic *DAI, + uint64_t SliceSizeInBits, + PointerUnion Assign, std::optional &Result); /// Helper struct for trackAssignments, below. We don't use the similar @@ -255,6 +273,8 @@ struct VarRecord { VarRecord(DbgVariableIntrinsic *DVI) : Var(DVI->getVariable()), DL(getDebugValueLoc(DVI)) {} + VarRecord(DPValue *DPV) + : Var(DPV->getVariable()), DL(getDebugValueLoc(DPV)) {} VarRecord(DILocalVariable *Var, DILocation *DL) : Var(Var), DL(DL) {} friend bool operator<(const VarRecord &LHS, const VarRecord &RHS) { return std::tie(LHS.Var, LHS.DL) < std::tie(RHS.Var, RHS.DL); diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h index 5a238c82086fe..48c43875a5e61 100644 --- a/llvm/include/llvm/IR/DebugInfoMetadata.h +++ b/llvm/include/llvm/IR/DebugInfoMetadata.h @@ -318,6 +318,10 @@ class DIAssignID : public MDNode { // This node has no operands to replace. void replaceOperandWith(unsigned I, Metadata *New) = delete; + SmallVector getAllDPValueUsers() { + return Context.getReplaceableUses()->getAllDPValueUsers(); + } + static DIAssignID *getDistinct(LLVMContext &Context) { return getImpl(Context, Distinct); } @@ -3790,6 +3794,7 @@ class DebugVariable { public: DebugVariable(const DbgVariableIntrinsic *DII); + DebugVariable(const DPValue *DPV); DebugVariable(const DILocalVariable *Var, std::optional FragmentInfo, diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index eeee91f155be5..a6ccab99e6a85 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -90,6 +90,8 @@ class DPValue : public ilist_node, private DebugValueUser { DILocalVariable *Variable; DIExpression *Expression; DebugLoc DbgLoc; + DIExpression *AddressExpression; + DIAssignID *AssignID; public: void deleteInstr(); @@ -100,12 +102,21 @@ class DPValue : public ilist_node, private DebugValueUser { void removeFromParent(); void eraseFromParent(); + + DPValue *getNextNode() { + return &*(++getIterator()); + } + DPValue *getPrevNode() { + return &*(--getIterator()); + } + using self_iterator = simple_ilist::iterator; using const_self_iterator = simple_ilist::const_iterator; enum class LocationType { Declare, Value, + Assign, }; /// Classification of the debug-info record that this DPValue represents. /// Essentially, "is this a dbg.value or dbg.declare?". dbg.declares are not @@ -123,6 +134,27 @@ class DPValue : public ilist_node, private DebugValueUser { /// assigning \p Location to the DV / Expr / DI variable. DPValue(Metadata *Location, DILocalVariable *DV, DIExpression *Expr, const DILocation *DI); + DPValue(Metadata *Value, DILocalVariable *Variable, + DIExpression *Expression, DIAssignID *AssignID, Metadata *Address, + DIExpression *AddressExpression, const DILocation *DI); + ~DPValue() { + untrackAssignID(); + } + + static DPValue *createDPValue( + Metadata *Location, DILocalVariable *DV, DIExpression *Expr, + const DILocation *DI, Instruction *InsertBefore = nullptr); + static DPValue *createDPDeclare( + Value *Address, DILocalVariable *DV, DIExpression *Expr, + const DILocation *DI, Instruction *InsertBefore = nullptr); + static DPValue *createDPAssign( + Metadata *Value, DILocalVariable *Variable, DIExpression *Expression, + DIAssignID *AssignID, Metadata *Address, DIExpression *AddressExpression, + const DILocation *DI, Instruction *InsertBefore = nullptr); + static DPValue *createLinkedDPAssign( + Instruction *LinkedInstr, Metadata *ValueMD, DILocalVariable *Variable, + DIExpression *Expression, Value *Address, + DIExpression *AddressExpression, const DILocation *DI); /// Iterator for ValueAsMetadata that internally uses direct pointer iteration /// over either a ValueAsMetadata* or a ValueAsMetadata**, dereferencing to the @@ -172,6 +204,9 @@ class DPValue : public ilist_node, private DebugValueUser { } }; + bool isDbgDeclare() { return Type == LocationType::Declare; } + bool isDbgValue() { return Type == LocationType::Value; } + /// Get the locations corresponding to the variable referenced by the debug /// info intrinsic. Depending on the intrinsic, this could be the /// variable's value or its address. @@ -238,6 +273,10 @@ class DPValue : public ilist_node, private DebugValueUser { /// a DIArgList which is a list of values. Metadata *getRawLocation() const { return DebugValues[0]; } + Value *getValue(unsigned OpIdx = 0) const { + return getVariableLocationOp(OpIdx); + } + /// Use of this should generally be avoided; instead, /// replaceVariableLocationOp and addVariableLocationOps should be used where /// possible to avoid creating invalid state. @@ -253,6 +292,78 @@ class DPValue : public ilist_node, private DebugValueUser { /// is described. std::optional getFragmentSizeInBits() const; + /// Get the FragmentInfo for the variable if it exists, otherwise return a + /// FragmentInfo that covers the entire variable if the variable size is + /// known, otherwise return a zero-sized fragment. + DIExpression::FragmentInfo getFragmentOrEntireVariable() const { + DIExpression::FragmentInfo VariableSlice(0, 0); + // Get the fragment or variable size, or zero. + if (auto Sz = getFragmentSizeInBits()) + VariableSlice.SizeInBits = *Sz; + if (auto Frag = getExpression()->getFragmentInfo()) + VariableSlice.OffsetInBits = Frag->OffsetInBits; + return VariableSlice; + } + + ///////////////////////////////////////////// + /// DbgAssign Methods + + bool isDbgAssign() const { return getType() == LocationType::Assign; } + + Value *getAddress() const; + Metadata *getRawAddress() const { + return DebugValues[1]; + } + Metadata *getRawAssignID() const { + return AssignID; + } + DIAssignID *getAssignID() const { return AssignID; } + Metadata *getRawAddressExpression() const { + return AddressExpression; + } + DIExpression *getAddressExpression() const { + return AddressExpression; + } + void setAddressExpression(DIExpression *NewExpr) { + AddressExpression = NewExpr; + } + void setAssignId(DIAssignID *New) { + untrackAssignID(); + AssignID = New; + trackAssignID(); + } + void setAddress(Value *V) { + resetDebugValue(1, ValueAsMetadata::get(V)); + } + /// Kill the address component. + void setKillAddress() { + resetDebugValue(1, ValueAsMetadata::get(UndefValue::get(getAddress()->getType()))); + } + /// Check whether this kills the address component. This doesn't take into + /// account the position of the intrinsic, therefore a returned value of false + /// does not guarentee the address is a valid location for the variable at the + /// intrinsic's position in IR. + bool isKillAddress() const { + Value *Addr = getAddress(); + return !Addr || isa(Addr); + } + +private: + void trackAssignID() { + if (AssignID) + MetadataTracking::track(&AssignID, *AssignID, *this); + } + void untrackAssignID() { + if (AssignID) + MetadataTracking::untrack(&AssignID, *AssignID); + } +public: + ///////////////////////////////////////////// + + bool isEquivalentTo(const DPValue &Other) { + return std::tie(Type, DebugValues, Variable, Expression, DbgLoc, AddressExpression, AssignID) == std::tie(Other.Type, Other.DebugValues, Other.Variable, Other.Expression, Other.DbgLoc, Other.AddressExpression, Other.AssignID); + } + DPValue *clone() const; /// Convert this DPValue back into a dbg.value intrinsic. /// \p InsertBefore Optional position to insert this intrinsic. @@ -276,6 +387,11 @@ class DPValue : public ilist_node, private DebugValueUser { LLVMContext &getContext(); const LLVMContext &getContext() const; + /// Insert this DPValue prior to \p InsertBefore. Must not be called if this + /// is already contained in a DPMarker. + void insertBefore(DPValue *InsertBefore); + void insertAfter(DPValue *InsertAfter); + void print(raw_ostream &O, bool IsForDebug = false) const; void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const; }; @@ -339,6 +455,10 @@ class DPMarker { /// Insert a DPValue into this DPMarker, at the end of the list. If /// \p InsertAtHead is true, at the start. void insertDPValue(DPValue *New, bool InsertAtHead); + /// Insert a DPValue prior to a DPValue contained within this marker. + void insertDPValue(DPValue *New, DPValue *InsertBefore); + /// Insert a DPValue after a DPValue contained within this marker. + void insertDPValueAfter(DPValue *New, DPValue *InsertAfter); /// Clone all DPMarkers from \p From into this marker. There are numerous /// options to customise the source/destination, due to gnarliness, see class /// comment. diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 62bd833198f02..a4bc762db4bc3 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -535,6 +535,68 @@ class DbgLabelInst : public DbgInfoIntrinsic { /// @} }; +//////////////////////////////////// +// Wrapper functions for opaque interfacing between DbgVariableIntrinsics and +// DPValues. + +inline bool IsaDbgValue(DPValue *DPV) { return DPV->isDbgValue(); } +inline bool IsaDbgDeclare(DPValue *DPV) { return DPV->isDbgDeclare(); } +inline bool IsaDbgAssign(DPValue *DPV) { return DPV->isDbgAssign(); } + +inline bool IsaDbgValue(DbgVariableIntrinsic *DVI) { + return isa(DVI); +} +inline bool IsaDbgDeclare(DbgVariableIntrinsic *DVI) { + return isa(DVI); +} +inline bool IsaDbgAssign(DbgVariableIntrinsic *DVI) { + return isa(DVI); +} + +inline DPValue *CastToDbgValue(DPValue *DPV) { return DPV; } +inline DPValue *CastToDbgDeclare(DPValue *DPV) { return DPV; } +inline DPValue *CastToDbgAssign(DPValue *DPV) { return DPV; } + +inline DbgValueInst *CastToDbgValue(DbgVariableIntrinsic *DVI) { + return cast(DVI); +} +inline DbgDeclareInst *CastToDbgDeclare(DbgVariableIntrinsic *DVI) { + return cast(DVI); +} +inline DbgAssignIntrinsic *CastToDbgAssign(DbgVariableIntrinsic *DVI) { + return cast(DVI); +} + +inline DPValue *DynCastToDbgValue(DPValue *DPV) { + if (!DPV->isDbgValue()) + return nullptr; + return DPV; +} +inline DPValue *DynCastToDbgDeclare(DPValue *DPV) { + if (!DPV->isDbgDeclare()) + return nullptr; + return DPV; +} +inline DPValue *DynCastToDbgAssign(DPValue *DPV) { + if (!DPV->isDbgAssign()) + return nullptr; + return DPV; +} + +inline DbgValueInst *DynCastToDbgValue(DbgVariableIntrinsic *DVI) { + return dyn_cast(DVI); +} +inline DbgDeclareInst *DynCastToDbgDeclare(DbgVariableIntrinsic *DVI) { + return dyn_cast(DVI); +} +inline DbgAssignIntrinsic *DynCastToDbgAssign(DbgVariableIntrinsic *DVI) { + return dyn_cast(DVI); +} + + + +//// + /// This is the common base class for vector predication intrinsics. class VPIntrinsic : public IntrinsicInst { public: diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h index 5f597739d80cf..0c72eff14cdc4 100644 --- a/llvm/include/llvm/IR/Metadata.h +++ b/llvm/include/llvm/IR/Metadata.h @@ -212,26 +212,11 @@ class MetadataAsValue : public Value { /// lookup and callback handling. class DebugValueUser { protected: - std::unique_ptr DebugValues; - size_t Length = 0; // Can we remove this somehow? + // Capacity to store 2 debug values. + std::array DebugValues; - MutableArrayRef getDebugValues() const { - return MutableArrayRef(DebugValues.get(), Length); - } - - void resetArray(const ArrayRef &NewMD) { - if (Length != NewMD.size()) { - DebugValues.reset(new Metadata *[NewMD.size()]); - Length = NewMD.size(); - } - - for (const auto &[NewMD, InitMD] : zip(getDebugValues(), NewMD)) - NewMD = InitMD; - } - - void takeArray(DebugValueUser &From) { - DebugValues = std::move(From.DebugValues); - Length = From.Length; + ArrayRef getDebugValues() const { + return DebugValues; } public: @@ -239,16 +224,15 @@ class DebugValueUser { const DPValue *getUser() const; void handleChangedValue(void *Old, Metadata *NewDebugValue); DebugValueUser() = default; - explicit DebugValueUser(const ArrayRef &MD) { - resetArray(MD); + explicit DebugValueUser(std::array DebugValues) : DebugValues(DebugValues) { trackDebugValues(); } DebugValueUser(DebugValueUser &&X) { - resetArray(X.getDebugValues()); + DebugValues = X.DebugValues; retrackDebugValues(X); } DebugValueUser(const DebugValueUser &X) { - resetArray(X.getDebugValues()); + DebugValues = X.DebugValues; trackDebugValues(); } @@ -257,7 +241,7 @@ class DebugValueUser { return *this; untrackDebugValues(); - takeArray(X); + DebugValues = X.DebugValues; retrackDebugValues(X); return *this; } @@ -267,7 +251,7 @@ class DebugValueUser { return *this; untrackDebugValues(); - resetArray(X.getDebugValues()); + DebugValues = X.DebugValues; trackDebugValues(); return *this; } @@ -276,27 +260,28 @@ class DebugValueUser { void resetDebugValues() { untrackDebugValues(); - DebugValues.reset(); + DebugValues.fill(nullptr); } - void resetDebugValue(int Idx, Metadata *DebugValue) { + void resetDebugValue(size_t Idx, Metadata *DebugValue) { + assert(Idx < 2 && "Invalid debug value index."); untrackDebugValue(Idx); DebugValues[Idx] = DebugValue; trackDebugValue(Idx); } bool operator==(const DebugValueUser &X) const { - return getDebugValues() == X.getDebugValues(); + return DebugValues == X.DebugValues; } bool operator!=(const DebugValueUser &X) const { - return getDebugValues() != X.getDebugValues(); + return DebugValues != X.DebugValues; } private: - void trackDebugValue(int Idx); + void trackDebugValue(size_t Idx); void trackDebugValues(); - void untrackDebugValue(int Idx); + void untrackDebugValue(size_t Idx); void untrackDebugValues(); void retrackDebugValues(DebugValueUser &X); @@ -1066,6 +1051,7 @@ class MDNode : public Metadata { friend class ReplaceableMetadataImpl; friend class LLVMContextImpl; friend class DIArgList; + friend class DIAssignID; /// The header that is coallocated with an MDNode along with its "small" /// operands. It is located immediately before the main body of the node. @@ -1249,7 +1235,10 @@ class MDNode : public Metadata { bool isTemporary() const { return Storage == Temporary; } bool isReplaceable() const { - return isTemporary() || getMetadataID() == DIArgListKind; + return isTemporary() || isAlwaysReplaceable(); + } + bool isAlwaysReplaceable() const { + return getMetadataID() == DIArgListKind || getMetadataID() == DIAssignIDKind; } /// RAUW a temporary. diff --git a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp index 5ef850d09d925..a59066873adef 100644 --- a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp +++ b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp @@ -13,6 +13,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugProgramInstruction.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/IntrinsicInst.h" @@ -290,6 +291,9 @@ using DebugAggregate = std::pair; static DebugAggregate getAggregate(const DbgVariableIntrinsic *DII) { return DebugAggregate(DII->getVariable(), DII->getDebugLoc().getInlinedAt()); } +static DebugAggregate getAggregate(const DPValue *DPV) { + return DebugAggregate(DPV->getVariable(), DPV->getDebugLoc().getInlinedAt()); +} static DebugAggregate getAggregate(const DebugVariable &Var) { return DebugAggregate(Var.getVariable(), Var.getInlinedAt()); } @@ -982,13 +986,14 @@ class AssignmentTrackingLowering { /// i.e. for all values x and y where x != y: /// join(x, x) = x /// join(x, y) = NoneOrPhi + using AssignRecord = PointerUnion; struct Assignment { enum S { Known, NoneOrPhi } Status; /// ID of the assignment. nullptr if Status is not Known. DIAssignID *ID; /// The dbg.assign that marks this dbg-def. Mem-defs don't use this field. /// May be nullptr. - DbgAssignIntrinsic *Source; + AssignRecord Source; bool isSameSourceAssignment(const Assignment &Other) const { // Don't include Source in the equality check. Assignments are @@ -1003,31 +1008,56 @@ class AssignmentTrackingLowering { else OS << "null"; OS << ", s="; - if (Source) - OS << *Source; - else + if (Source.isNull()) OS << "null"; + else if (isa(Source)) + OS << Source.get(); + else + OS << Source.get(); OS << ")"; } static Assignment make(DIAssignID *ID, DbgAssignIntrinsic *Source) { return Assignment(Known, ID, Source); } + static Assignment make(DIAssignID *ID, DPValue *Source) { + assert(Source->isDbgAssign() && + "Cannot make an assignment from a non-assign DPValue"); + return Assignment(Known, ID, Source); + } + static Assignment make(DIAssignID *ID, AssignRecord Source) { + return Assignment(Known, ID, Source); + } static Assignment makeFromMemDef(DIAssignID *ID) { - return Assignment(Known, ID, nullptr); + return Assignment(Known, ID); } static Assignment makeNoneOrPhi() { - return Assignment(NoneOrPhi, nullptr, nullptr); + return Assignment(NoneOrPhi, nullptr); } // Again, need a Top value? Assignment() - : Status(NoneOrPhi), ID(nullptr), Source(nullptr) { + : Status(NoneOrPhi), ID(nullptr) { } // Can we delete this? + Assignment(S Status, DIAssignID *ID) + : Status(Status), ID(ID) { + // If the Status is Known then we expect there to be an assignment ID. + assert(Status == NoneOrPhi || ID); + } Assignment(S Status, DIAssignID *ID, DbgAssignIntrinsic *Source) : Status(Status), ID(ID), Source(Source) { // If the Status is Known then we expect there to be an assignment ID. assert(Status == NoneOrPhi || ID); } + Assignment(S Status, DIAssignID *ID, DPValue *Source) + : Status(Status), ID(ID), Source(Source) { + // If the Status is Known then we expect there to be an assignment ID. + assert(Status == NoneOrPhi || ID); + } + Assignment(S Status, DIAssignID *ID, AssignRecord Source) + : Status(Status), ID(ID), Source(Source) { + // If the Status is Known then we expect there to be an assignment ID. + assert(Status == NoneOrPhi || ID); + } }; using AssignmentMap = SmallVector; @@ -1048,13 +1078,30 @@ class AssignmentTrackingLowering { UntaggedStoreAssignmentMap UntaggedStoreVars; // Machinery to defer inserting dbg.values. - using InsertMap = MapVector>; - InsertMap InsertBeforeMap; + using InstInsertMap = MapVector>; + InstInsertMap InsertBeforeMap; + // InsertBeforeMap must only have VarLocInfo added in order of appearance, in + // order to ensure identical + void InsertBeforePosition(Instruction *Key, VarLocInfo VarLoc) { + InsertBeforeMap[Key].push_back(VarLoc); + } + void InsertBeforePosition(DPValue *Key, VarLocInfo VarLoc) { + InsertBeforeMap[Key->getMarker()->MarkedInstr].push_back(VarLoc); + } + std::optional*> InsertBeforeMapFind(Instruction *Key) { + auto *R = InsertBeforeMap.find(Key); + if (R == InsertBeforeMap.end()) + return {}; + return &R->second; + } /// Clear the location definitions currently cached for insertion after /p /// After. void resetInsertionPoint(Instruction &After); - void emitDbgValue(LocKind Kind, const DbgVariableIntrinsic *Source, - Instruction *After); + + template + void emitDbgValue(LocKind Kind, AssignRecord Source, U After); + template + void emitDbgValue(LocKind Kind, const T Source, U After); static bool mapsAreEqual(const BitVector &Mask, const AssignmentMap &A, const AssignmentMap &B) { @@ -1279,8 +1326,10 @@ class AssignmentTrackingLowering { /// Update \p LiveSet after encountering an instruciton without a DIAssignID /// attachment, \p I. void processUntaggedInstruction(Instruction &I, BlockInfo *LiveSet); - void processDbgAssign(DbgAssignIntrinsic &DAI, BlockInfo *LiveSet); - void processDbgValue(DbgValueInst &DVI, BlockInfo *LiveSet); + void processDbgAssign(AssignRecord Assign, BlockInfo *LiveSet); + void processDPAssign(DPValue &DPV, BlockInfo *LiveSet); + void processDPValue(DPValue &DPV, BlockInfo *LiveSet); + void processDbgValue(PointerUnion DbgValueRecord, BlockInfo *LiveSet); /// Add an assignment to memory for the variable /p Var. void addMemDef(BlockInfo *LiveSet, VariableID Var, const Assignment &AV); /// Add an assignment to the variable /p Var. @@ -1380,6 +1429,11 @@ static DIAssignID *getIDFromMarker(const DbgAssignIntrinsic &DAI) { return cast(DAI.getAssignID()); } +static DIAssignID *getIDFromMarker(const DPValue &DPV) { + assert(DPV.isDbgAssign() && "Cannot get an ID from a non-assign DPValue"); + return cast(DPV.getAssignID()); +} + /// Return true if \p Var has an assignment in \p M matching \p AV. bool AssignmentTrackingLowering::hasVarWithAssignment( BlockInfo *LiveSet, BlockInfo::AssignmentKind Kind, VariableID Var, @@ -1410,9 +1464,22 @@ const char *locStr(AssignmentTrackingLowering::LocKind Loc) { } #endif +Instruction *getNextNode(DPValue *DPV) { return DPV->getMarker()->MarkedInstr; } +Instruction *getNextNode(Instruction *Inst) { return Inst->getNextNode(); } + +template void AssignmentTrackingLowering::emitDbgValue( AssignmentTrackingLowering::LocKind Kind, - const DbgVariableIntrinsic *Source, Instruction *After) { + AssignmentTrackingLowering::AssignRecord Source, U After) { + if (isa(Source)) + emitDbgValue(Kind, cast(Source), After); + else + emitDbgValue(Kind, cast(Source), After); +} +template +void AssignmentTrackingLowering::emitDbgValue( + AssignmentTrackingLowering::LocKind Kind, + const T Source, U After) { DILocation *DL = Source->getDebugLoc(); auto Emit = [this, Source, After, DL](Metadata *Val, DIExpression *Expr) { @@ -1422,7 +1489,7 @@ void AssignmentTrackingLowering::emitDbgValue( PoisonValue::get(Type::getInt1Ty(Source->getContext()))); // Find a suitable insert point. - Instruction *InsertBefore = After->getNextNode(); + auto *InsertBefore = getNextNode(After); assert(InsertBefore && "Shouldn't be inserting after a terminator"); VariableID Var = getVariableID(DebugVariable(Source)); @@ -1432,20 +1499,20 @@ void AssignmentTrackingLowering::emitDbgValue( VarLoc.Values = RawLocationWrapper(Val); VarLoc.DL = DL; // Insert it into the map for later. - InsertBeforeMap[InsertBefore].push_back(VarLoc); + InsertBeforePosition(InsertBefore, VarLoc); }; // NOTE: This block can mutate Kind. if (Kind == LocKind::Mem) { - const auto *DAI = cast(Source); + const auto *Assign = CastToDbgAssign(Source); // Check the address hasn't been dropped (e.g. the debug uses may not have // been replaced before deleting a Value). - if (DAI->isKillAddress()) { + if (Assign->isKillAddress()) { // The address isn't valid so treat this as a non-memory def. Kind = LocKind::Val; } else { - Value *Val = DAI->getAddress(); - DIExpression *Expr = DAI->getAddressExpression(); + Value *Val = Assign->getAddress(); + DIExpression *Expr = Assign->getAddressExpression(); assert(!Expr->getFragmentInfo() && "fragment info should be stored in value-expression only"); // Copy the fragment info over from the value-expression to the new @@ -1546,32 +1613,33 @@ void AssignmentTrackingLowering::processUntaggedInstruction( ValueAsMetadata::get(const_cast(Info.Base))); VarLoc.DL = DILoc; // 3. Insert it into the map for later. - InsertBeforeMap[InsertBefore].push_back(VarLoc); + InsertBeforePosition(InsertBefore, VarLoc); } } void AssignmentTrackingLowering::processTaggedInstruction( Instruction &I, AssignmentTrackingLowering::BlockInfo *LiveSet) { auto Linked = at::getAssignmentMarkers(&I); + auto LinkedDPAssigns = at::getDPAssignmentMarkers(&I); // No dbg.assign intrinsics linked. // FIXME: All vars that have a stack slot this store modifies that don't have // a dbg.assign linked to it should probably treat this like an untagged // store. - if (Linked.empty()) + if (Linked.empty() && LinkedDPAssigns.empty()) return; LLVM_DEBUG(dbgs() << "processTaggedInstruction on " << I << "\n"); - for (DbgAssignIntrinsic *DAI : Linked) { - VariableID Var = getVariableID(DebugVariable(DAI)); + auto ProcessLinkedAssign = [&](auto Assign) { + VariableID Var = getVariableID(DebugVariable(Assign)); // Something has gone wrong if VarsWithStackSlot doesn't contain a variable // that is linked to a store. - assert(VarsWithStackSlot->count(getAggregate(DAI)) && - "expected DAI's variable to have stack slot"); + assert(VarsWithStackSlot->count(getAggregate(Assign)) && + "expected Assign's variable to have stack slot"); + LLVM_DEBUG(dbgs() << " linked to " << *Assign << "\n"); Assignment AV = Assignment::makeFromMemDef(getIDFromInst(I)); addMemDef(LiveSet, Var, AV); - LLVM_DEBUG(dbgs() << " linked to " << *DAI << "\n"); LLVM_DEBUG(dbgs() << " LiveLoc " << locStr(getLocKind(LiveSet, Var)) << " -> "); @@ -1586,8 +1654,8 @@ void AssignmentTrackingLowering::processTaggedInstruction( LiveSet->DebugValue[static_cast(Var)].dump(dbgs()); dbgs() << "\n"); setLocKind(LiveSet, Var, LocKind::Mem); - emitDbgValue(LocKind::Mem, DAI, &I); - continue; + emitDbgValue(LocKind::Mem, Assign, &I); + return; } // The StackHomeValue and DebugValue for this variable do not match. I.e. @@ -1612,7 +1680,7 @@ void AssignmentTrackingLowering::processTaggedInstruction( // We need to terminate any previously open location now. LLVM_DEBUG(dbgs() << "None, No Debug value available\n";); setLocKind(LiveSet, Var, LocKind::None); - emitDbgValue(LocKind::None, DAI, &I); + emitDbgValue(LocKind::None, Assign, &I); } else { // The previous DebugValue Value can be used here. LLVM_DEBUG(dbgs() << "Val, Debug value is Known\n";); @@ -1621,7 +1689,7 @@ void AssignmentTrackingLowering::processTaggedInstruction( emitDbgValue(LocKind::Val, DbgAV.Source, &I); } else { // PrevAV.Source is nullptr so we must emit undef here. - emitDbgValue(LocKind::None, DAI, &I); + emitDbgValue(LocKind::None, Assign, &I); } } } break; @@ -1632,78 +1700,92 @@ void AssignmentTrackingLowering::processTaggedInstruction( setLocKind(LiveSet, Var, LocKind::None); } break; } - } + }; + for (DbgAssignIntrinsic *DAI : Linked) + ProcessLinkedAssign(DAI); + for (DPValue *DPV : LinkedDPAssigns) + ProcessLinkedAssign(DPV); } -void AssignmentTrackingLowering::processDbgAssign(DbgAssignIntrinsic &DAI, +void AssignmentTrackingLowering::processDbgAssign(AssignRecord Assign, BlockInfo *LiveSet) { - // Only bother tracking variables that are at some point stack homed. Other - // variables can be dealt with trivially later. - if (!VarsWithStackSlot->count(getAggregate(&DAI))) - return; + auto ProcessDbgAssignImpl = [&](auto *DbgAssign) { + // Only bother tracking variables that are at some point stack homed. Other + // variables can be dealt with trivially later. + if (!VarsWithStackSlot->count(getAggregate(DbgAssign))) + return; - VariableID Var = getVariableID(DebugVariable(&DAI)); - Assignment AV = Assignment::make(getIDFromMarker(DAI), &DAI); - addDbgDef(LiveSet, Var, AV); - - LLVM_DEBUG(dbgs() << "processDbgAssign on " << DAI << "\n";); - LLVM_DEBUG(dbgs() << " LiveLoc " << locStr(getLocKind(LiveSet, Var)) - << " -> "); - - // Check if the DebugValue and StackHomeValue both hold the same - // Assignment. - if (hasVarWithAssignment(LiveSet, BlockInfo::Stack, Var, AV)) { - // They match. We can use the stack home because the debug intrinsics state - // that an assignment happened here, and we know that specific assignment - // was the last one to take place in memory for this variable. - LocKind Kind; - if (DAI.isKillAddress()) { - LLVM_DEBUG( - dbgs() - << "Val, Stack matches Debug program but address is killed\n";); - Kind = LocKind::Val; + VariableID Var = getVariableID(DebugVariable(DbgAssign)); + Assignment AV = Assignment::make(getIDFromMarker(*DbgAssign), DbgAssign); + addDbgDef(LiveSet, Var, AV); + + LLVM_DEBUG(dbgs() << "processDbgAssign on " << *DbgAssign << "\n";); + LLVM_DEBUG(dbgs() << " LiveLoc " << locStr(getLocKind(LiveSet, Var)) + << " -> "); + + // Check if the DebugValue and StackHomeValue both hold the same + // Assignment. + if (hasVarWithAssignment(LiveSet, BlockInfo::Stack, Var, AV)) { + // They match. We can use the stack home because the debug intrinsics state + // that an assignment happened here, and we know that specific assignment + // was the last one to take place in memory for this variable. + LocKind Kind; + if (DbgAssign->isKillAddress()) { + LLVM_DEBUG( + dbgs() + << "Val, Stack matches Debug program but address is killed\n";); + Kind = LocKind::Val; + } else { + LLVM_DEBUG(dbgs() << "Mem, Stack matches Debug program\n";); + Kind = LocKind::Mem; + }; + setLocKind(LiveSet, Var, Kind); + emitDbgValue(Kind, DbgAssign, DbgAssign); } else { - LLVM_DEBUG(dbgs() << "Mem, Stack matches Debug program\n";); - Kind = LocKind::Mem; - }; - setLocKind(LiveSet, Var, Kind); - emitDbgValue(Kind, &DAI, &DAI); - } else { - // The last assignment to the memory location isn't the one that we want to - // show to the user so emit a dbg.value(Value). Value may be undef. - LLVM_DEBUG(dbgs() << "Val, Stack contents is unknown\n";); - setLocKind(LiveSet, Var, LocKind::Val); - emitDbgValue(LocKind::Val, &DAI, &DAI); - } + // The last assignment to the memory location isn't the one that we want to + // show to the user so emit a dbg.value(Value). Value may be undef. + LLVM_DEBUG(dbgs() << "Val, Stack contents is unknown\n";); + setLocKind(LiveSet, Var, LocKind::Val); + emitDbgValue(LocKind::Val, DbgAssign, DbgAssign); + } + }; + if (isa(Assign)) + return ProcessDbgAssignImpl(cast(Assign)); + return ProcessDbgAssignImpl(cast(Assign)); } -void AssignmentTrackingLowering::processDbgValue(DbgValueInst &DVI, +void AssignmentTrackingLowering::processDbgValue(PointerUnion DbgValueRecord, BlockInfo *LiveSet) { - // Only other tracking variables that are at some point stack homed. - // Other variables can be dealt with trivally later. - if (!VarsWithStackSlot->count(getAggregate(&DVI))) - return; + auto ProcessDbgValueImpl = [&](auto *DbgValue) { + // Only other tracking variables that are at some point stack homed. + // Other variables can be dealt with trivally later. + if (!VarsWithStackSlot->count(getAggregate(DbgValue))) + return; - VariableID Var = getVariableID(DebugVariable(&DVI)); - // We have no ID to create an Assignment with so we mark this assignment as - // NoneOrPhi. Note that the dbg.value still exists, we just cannot determine - // the assignment responsible for setting this value. - // This is fine; dbg.values are essentially interchangable with unlinked - // dbg.assigns, and some passes such as mem2reg and instcombine add them to - // PHIs for promoted variables. - Assignment AV = Assignment::makeNoneOrPhi(); - addDbgDef(LiveSet, Var, AV); - - LLVM_DEBUG(dbgs() << "processDbgValue on " << DVI << "\n";); - LLVM_DEBUG(dbgs() << " LiveLoc " << locStr(getLocKind(LiveSet, Var)) - << " -> Val, dbg.value override"); - - setLocKind(LiveSet, Var, LocKind::Val); - emitDbgValue(LocKind::Val, &DVI, &DVI); + VariableID Var = getVariableID(DebugVariable(DbgValue)); + // We have no ID to create an Assignment with so we mark this assignment as + // NoneOrPhi. Note that the dbg.value still exists, we just cannot determine + // the assignment responsible for setting this value. + // This is fine; dbg.values are essentially interchangable with unlinked + // dbg.assigns, and some passes such as mem2reg and instcombine add them to + // PHIs for promoted variables. + Assignment AV = Assignment::makeNoneOrPhi(); + addDbgDef(LiveSet, Var, AV); + + LLVM_DEBUG(dbgs() << "processDbgValue on " << *DbgValue << "\n";); + LLVM_DEBUG(dbgs() << " LiveLoc " << locStr(getLocKind(LiveSet, Var)) + << " -> Val, dbg.value override"); + + setLocKind(LiveSet, Var, LocKind::Val); + emitDbgValue(LocKind::Val, DbgValue, DbgValue); + }; + if (isa(DbgValueRecord)) + return ProcessDbgValueImpl(cast(DbgValueRecord)); + return ProcessDbgValueImpl(cast(DbgValueRecord)); } -static bool hasZeroSizedFragment(DbgVariableIntrinsic &DVI) { - if (auto F = DVI.getExpression()->getFragmentInfo()) +static bool hasZeroSizedFragment(DIExpression *Expr) { + if (auto F = Expr->getFragmentInfo()) return F->SizeInBits == 0; return false; } @@ -1715,21 +1797,31 @@ void AssignmentTrackingLowering::processDbgInstruction( return; // Ignore assignments to zero bits of the variable. - if (hasZeroSizedFragment(*DVI)) + if (hasZeroSizedFragment(DVI->getExpression())) return; if (auto *DAI = dyn_cast(&I)) - processDbgAssign(*DAI, LiveSet); + processDbgAssign(DAI, LiveSet); else if (auto *DVI = dyn_cast(&I)) - processDbgValue(*DVI, LiveSet); + processDbgValue(DVI, LiveSet); +} +void AssignmentTrackingLowering::processDPValue( + DPValue &DPV, AssignmentTrackingLowering::BlockInfo *LiveSet) { + // Ignore assignments to zero bits of the variable. + if (hasZeroSizedFragment(DPV.getExpression())) + return; + + if (DPV.isDbgAssign()) + processDbgAssign(&DPV, LiveSet); + else if (DPV.isDbgValue()) + processDbgValue(&DPV, LiveSet); } void AssignmentTrackingLowering::resetInsertionPoint(Instruction &After) { assert(!After.isTerminator() && "Can't insert after a terminator"); - auto R = InsertBeforeMap.find(After.getNextNode()); - if (R == InsertBeforeMap.end()) - return; - R->second.clear(); + auto R = InsertBeforeMapFind(After.getNextNode()); + if (R) + (*R)->clear(); } void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { @@ -1738,7 +1830,11 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { // Process the instructions in "frames". A "frame" includes a single // non-debug instruction followed any debug instructions before the // next non-debug instruction. - if (!isa(&*II)) { + + // II is now either a debug intrinsic, a non-debug instruction with no + // attached DPValues, or a non-debug instruction with attached DPValues. + // II has not been processed. + if (!isa(&*II) && !II->hasDbgValues()) { if (II->isTerminator()) break; resetInsertionPoint(*II); @@ -1746,15 +1842,25 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { assert(LiveSet->isValid()); ++II; } - while (II != EI) { - auto *Dbg = dyn_cast(&*II); - if (!Dbg) - break; - resetInsertionPoint(*II); - processDbgInstruction(*Dbg, LiveSet); - assert(LiveSet->isValid()); - ++II; + if (II != EI && II->hasDbgValues()) { + for (DPValue &DPV : II->getDbgValueRange()) { + processDPValue(DPV, LiveSet); + assert(LiveSet->isValid()); + } + } else { + while (II != EI) { + auto *Dbg = dyn_cast(&*II); + if (!Dbg) + break; + resetInsertionPoint(*II); + processDbgInstruction(*Dbg, LiveSet); + assert(LiveSet->isValid()); + ++II; + } } + // II is now a non-debug instruction either with no attached DPValues, or + // with attached processed DPValues. II has not been processed, and all + // debug instructions or DPValues in II's frame have been processed. // We've processed everything in the "frame". Now determine which variables // cannot be represented by a dbg.declare. @@ -1774,6 +1880,17 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) { } } VarsTouchedThisFrame.clear(); + + // In order to prevent the DPValues attached to II (if any) from being + // processed again, we process II now and then move to the next instruction. + if (II->hasDbgValues()) { + if (II->isTerminator()) + break; + resetInsertionPoint(*II); + processNonDbgInstruction(*II, LiveSet); + assert(LiveSet->isValid()); + ++II; + } } } @@ -1811,16 +1928,19 @@ AssignmentTrackingLowering::joinAssignment(const Assignment &A, // Here the same assignment (!1) was performed in both preds in the source, // but we can't use either one unless they are identical (e.g. .we don't // want to arbitrarily pick between constant values). - auto JoinSource = [&]() -> DbgAssignIntrinsic * { + auto JoinSource = [&]() -> AssignRecord { if (A.Source == B.Source) return A.Source; - if (A.Source == nullptr || B.Source == nullptr) - return nullptr; - if (A.Source->isIdenticalTo(B.Source)) + if (!A.Source || !B.Source) + return AssignRecord(); + assert(isa(A.Source) == isa(B.Source)); + if (isa(A.Source) && cast(A.Source)->isEquivalentTo(*cast(B.Source))) return A.Source; - return nullptr; + if (isa(A.Source) && cast(A.Source)->isIdenticalTo(cast(B.Source))) + return A.Source; + return AssignRecord(); }; - DbgAssignIntrinsic *Source = JoinSource(); + AssignRecord Source = JoinSource(); assert(A.Status == B.Status && A.Status == Assignment::Known); assert(A.ID == B.ID); return Assignment::make(A.ID, Source); @@ -1963,42 +2083,50 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares( // UntaggedStoreVars. // We need to add fragments for untagged stores too so that we can correctly // clobber overlapped fragment locations later. - SmallVector Declares; + SmallVector InstDeclares; + SmallVector DPDeclares; + auto ProcessDbgRecord = [&](auto *Record, auto &DeclareList) { + if (auto *Declare = DynCastToDbgDeclare(Record)) { + DeclareList.push_back(Declare); + return; + } + DebugVariable DV = DebugVariable(Record); + DebugAggregate DA = {DV.getVariable(), DV.getInlinedAt()}; + if (!VarsWithStackSlot.contains(DA)) + return; + if (Seen.insert(DV).second) + FragmentMap[DA].push_back(DV); + }; for (auto &BB : Fn) { for (auto &I : BB) { - if (auto *DDI = dyn_cast(&I)) { - Declares.push_back(DDI); - } else if (auto *DII = dyn_cast(&I)) { - DebugVariable DV = DebugVariable(DII); - DebugAggregate DA = {DV.getVariable(), DV.getInlinedAt()}; - if (!VarsWithStackSlot.contains(DA)) - continue; - if (Seen.insert(DV).second) - FragmentMap[DA].push_back(DV); + for (auto &DPV : I.getDbgValueRange()) + ProcessDbgRecord(&DPV, DPDeclares); + if (auto *DII = dyn_cast(&I)) { + ProcessDbgRecord(DII, InstDeclares); } else if (auto Info = getUntaggedStoreAssignmentInfo( I, Fn.getParent()->getDataLayout())) { // Find markers linked to this alloca. - for (DbgAssignIntrinsic *DAI : at::getAssignmentMarkers(Info->Base)) { + auto HandleDbgAssignForStore = [&](auto *Assign) { // Discard the fragment if it covers the entire variable. std::optional FragInfo = - [&Info, DAI]() -> std::optional { + [&Info, Assign]() -> std::optional { DIExpression::FragmentInfo F; F.OffsetInBits = Info->OffsetInBits; F.SizeInBits = Info->SizeInBits; - if (auto ExistingFrag = DAI->getExpression()->getFragmentInfo()) + if (auto ExistingFrag = Assign->getExpression()->getFragmentInfo()) F.OffsetInBits += ExistingFrag->OffsetInBits; - if (auto Sz = DAI->getVariable()->getSizeInBits()) { + if (auto Sz = Assign->getVariable()->getSizeInBits()) { if (F.OffsetInBits == 0 && F.SizeInBits == *Sz) return std::nullopt; } return F; }(); - DebugVariable DV = DebugVariable(DAI->getVariable(), FragInfo, - DAI->getDebugLoc().getInlinedAt()); + DebugVariable DV = DebugVariable(Assign->getVariable(), FragInfo, + Assign->getDebugLoc().getInlinedAt()); DebugAggregate DA = {DV.getVariable(), DV.getInlinedAt()}; if (!VarsWithStackSlot.contains(DA)) - continue; + return; // Cache this info for later. UntaggedStoreVars[&I].push_back( @@ -2006,7 +2134,11 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares( if (Seen.insert(DV).second) FragmentMap[DA].push_back(DV); - } + }; + for (DbgAssignIntrinsic *DAI : at::getAssignmentMarkers(Info->Base)) + HandleDbgAssignForStore(DAI); + for (DPValue *DPV : at::getDPAssignmentMarkers(Info->Base)) + HandleDbgAssignForStore(DPV); } } } @@ -2053,9 +2185,12 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares( // Finally, insert the declares afterwards, so the first IDs are all // partially stack homed vars. - for (auto *DDI : Declares) + for (auto *DDI : InstDeclares) FnVarLocs->addSingleLocVar(DebugVariable(DDI), DDI->getExpression(), DDI->getDebugLoc(), DDI->getWrappedLocation()); + for (auto *DPV : DPDeclares) + FnVarLocs->addSingleLocVar(DebugVariable(DPV), DPV->getExpression(), + DPV->getDebugLoc(), RawLocationWrapper(DPV->getRawLocation())); return Map; } @@ -2168,7 +2303,7 @@ bool AssignmentTrackingLowering::run(FunctionVarLocsBuilder *FnVarLocsBuilder) { // we can identify those uneeded defs later. DenseSet AlwaysStackHomed; for (const auto &Pair : InsertBeforeMap) { - const auto &Vec = Pair.second; + auto &Vec = Pair.second; for (VarLocInfo VarLoc : Vec) { DebugVariable Var = FnVarLocs->getVariable(VarLoc.VariableID); DebugAggregate Aggr{Var.getVariable(), Var.getInlinedAt()}; @@ -2234,22 +2369,27 @@ bool AssignmentTrackingLowering::emitPromotedVarLocs( bool InsertedAnyIntrinsics = false; // Go through every block, translating debug intrinsics for fully promoted // variables into FnVarLocs location defs. No analysis required for these. + auto TranslateDbgRecord = [&](auto *Record) { + // Skip variables that haven't been promoted - we've dealt with those + // already. + if (VarsWithStackSlot->contains(getAggregate(Record))) + return; + Instruction *InsertBefore = getNextNode(Record); + assert(InsertBefore && "Unexpected: debug intrinsics after a terminator"); + FnVarLocs->addVarLoc(InsertBefore, DebugVariable(Record), + Record->getExpression(), Record->getDebugLoc(), + RawLocationWrapper(Record->getRawLocation())); + InsertedAnyIntrinsics = true; + }; for (auto &BB : Fn) { for (auto &I : BB) { // Skip instructions other than dbg.values and dbg.assigns. + for (DPValue &DPV : I.getDbgValueRange()) + if (DPV.isDbgValue() || DPV.isDbgAssign()) + TranslateDbgRecord(&DPV); auto *DVI = dyn_cast(&I); - if (!DVI) - continue; - // Skip variables that haven't been promoted - we've dealt with those - // already. - if (VarsWithStackSlot->contains(getAggregate(DVI))) - continue; - Instruction *InsertBefore = I.getNextNode(); - assert(InsertBefore && "Unexpected: debug intrinsics after a terminator"); - FnVarLocs->addVarLoc(InsertBefore, DebugVariable(DVI), - DVI->getExpression(), DVI->getDebugLoc(), - DVI->getWrappedLocation()); - InsertedAnyIntrinsics = true; + if (DVI) + TranslateDbgRecord(DVI); } } return InsertedAnyIntrinsics; @@ -2508,6 +2648,9 @@ static DenseSet findVarsWithStackSlot(Function &Fn) { for (DbgAssignIntrinsic *DAI : at::getAssignmentMarkers(&I)) { Result.insert({DAI->getVariable(), DAI->getDebugLoc().getInlinedAt()}); } + for (DPValue *DPV : at::getDPAssignmentMarkers(&I)) { + Result.insert({DPV->getVariable(), DPV->getDebugLoc().getInlinedAt()}); + } } } return Result; diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index bef2a8f86a716..79bc23f18064a 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -4514,7 +4514,14 @@ void AssemblyWriter::printDPMarker(const DPMarker &Marker) { void AssemblyWriter::printDPValue(const DPValue &Value) { // There's no formal representation of a DPValue -- print purely as a // debugging aid. - Out << " DPValue { "; + Out << " DPValue "; + + switch(Value.getType()) { + case DPValue::LocationType::Value: Out << "value"; break; + case DPValue::LocationType::Declare: Out << "declare"; break; + case DPValue::LocationType::Assign: Out << "assign"; break; + } + Out << " { "; auto WriterCtx = getContext(); WriteAsOperandInternal(Out, Value.getRawLocation(), WriterCtx, true); Out << ", "; @@ -4522,6 +4529,14 @@ void AssemblyWriter::printDPValue(const DPValue &Value) { Out << ", "; WriteAsOperandInternal(Out, Value.getExpression(), WriterCtx, true); Out << ", "; + if (Value.isDbgAssign()) { + WriteAsOperandInternal(Out, Value.getAssignID(), WriterCtx, true); + Out << ", "; + WriteAsOperandInternal(Out, Value.getRawAddress(), WriterCtx, true); + Out << ", "; + WriteAsOperandInternal(Out, Value.getAddressExpression(), WriterCtx, true); + Out << ", "; + } WriteAsOperandInternal(Out, Value.getDebugLoc().get(), WriterCtx, true); Out << " marker @" << Value.getMarker(); Out << " }"; diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index d14e47e36d786..6d0dcfef82af7 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -64,6 +64,23 @@ TinyPtrVector llvm::FindDbgDeclareUses(Value *V) { return Declares; } +TinyPtrVector llvm::FindDPDeclareUses(Value *V) { + // This function is hot. Check whether the value has any metadata to avoid a + // DenseMap lookup. + if (!V->isUsedByMetadata()) + return {}; + auto *L = LocalAsMetadata::getIfExists(V); + if (!L) + return {}; + + TinyPtrVector Declares; + for (DPValue *DPV : L->getAllDPValueUsers()) { + if (DPV->isDbgDeclare()) + Declares.push_back(DPV); + } + + return Declares; +} template static void findDbgIntrinsics(SmallVectorImpl &Result, @@ -146,6 +163,17 @@ DebugLoc llvm::getDebugValueLoc(DbgVariableIntrinsic *DII) { // with the correct scope / inlinedAt fields. return DILocation::get(DII->getContext(), 0, 0, Scope, InlinedAt); } +DebugLoc llvm::getDebugValueLoc(DPValue *DPV) { + // Original dbg.declare must have a location. + const DebugLoc &DeclareLoc = DPV->getDebugLoc(); + MDNode *Scope = DeclareLoc.getScope(); + DILocation *InlinedAt = DeclareLoc.getInlinedAt(); + // Because no machine insts can come from debug intrinsics, only the scope + // and inlinedAt is significant. Zero line numbers are used in case this + // DebugLoc leaks into any adjacent instructions. Produce an unknown location + // with the correct scope / inlinedAt fields. + return DILocation::get(DPV->getContext(), 0, 0, Scope, InlinedAt); +} //===----------------------------------------------------------------------===// // DebugInfoFinder implementations. @@ -1766,20 +1794,19 @@ AssignmentMarkerRange at::getAssignmentMarkers(DIAssignID *ID) { void at::deleteAssignmentMarkers(const Instruction *Inst) { auto Range = getAssignmentMarkers(Inst); - if (Range.empty()) + SmallVector DPAssigns; + getDPAssignmentMarkers(Inst, DPAssigns); + if (Range.empty() && DPAssigns.empty()) return; SmallVector ToDelete(Range.begin(), Range.end()); for (auto *DAI : ToDelete) DAI->eraseFromParent(); + for (auto *DPV : DPAssigns) + DPV->eraseFromParent(); } void at::RAUW(DIAssignID *Old, DIAssignID *New) { - // Replace MetadataAsValue uses. - if (auto *OldIDAsValue = - MetadataAsValue::getIfExists(Old->getContext(), Old)) { - auto *NewIDAsValue = MetadataAsValue::get(Old->getContext(), New); - OldIDAsValue->replaceAllUsesWith(NewIDAsValue); - } + Old->replaceAllUsesWith(New); // Replace attachments. AssignmentInstRange InstRange = getAssignmentInsts(Old); @@ -1807,7 +1834,8 @@ void at::deleteAll(Function *F) { bool at::calculateFragmentIntersect( const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits, - uint64_t SliceSizeInBits, const DbgAssignIntrinsic *DAI, + uint64_t SliceSizeInBits, + PointerUnion Assign, std::optional &Result) { // There are multiple offsets at play in this function, so let's break it // down. Starting with how variables may be stored in allocas: @@ -1836,7 +1864,7 @@ bool at::calculateFragmentIntersect( // dbg.assign, that has been killed, if any. // // calculateFragmentIntersect(..., SliceOffsetInBits=0, - // SliceSizeInBits=32, Dest=%dest, DAI=dbg.assign) + // SliceSizeInBits=32, Dest=%dest, Assign=dbg.assign) // // Drawing the store (s) in memory followed by the shortened version ($), // then the dbg.assign (d), with the fragment information on a seperate scale @@ -1849,7 +1877,7 @@ bool at::calculateFragmentIntersect( // | | // s[######] - Original stores 64 bits to Dest. // $----[##] - DSE says the lower 32 bits are dead, to be removed. - // d [##] - DAI's address-modifying expression adds 4 bytes to dest. + // d [##] - Assign's address-modifying expression adds 4 bytes to dest. // Variable | | // Fragment 128| // Offsets 159 @@ -1864,10 +1892,10 @@ bool at::calculateFragmentIntersect( // // 3. That offset along with the store size (32) represents the bits of the // variable that'd be affected by the store. Call it SliceOfVariable. - // Intersect that with DAI's fragment info: - // SliceOfVariable ∩ DAI_fragment = none + // Intersect that with Assign's fragment info: + // SliceOfVariable ∩ Assign_fragment = none // - // In this case: none of the dead bits of the store affect DAI. + // In this case: none of the dead bits of the store affect Assign. // // # Example 2 // Similar example with the same goal. This time the upper 16 bits @@ -1878,7 +1906,7 @@ bool at::calculateFragmentIntersect( // !DIExpression(DW_OP_plus_uconst, 4)) // // calculateFragmentIntersect(..., SliceOffsetInBits=48, - // SliceSizeInBits=16, Dest=%dest, DAI=dbg.assign) + // SliceSizeInBits=16, Dest=%dest, Assign=dbg.assign) // // Memory // offset @@ -1887,7 +1915,7 @@ bool at::calculateFragmentIntersect( // | | // s[######] - Original stores 64 bits to Dest. // $[####]-- - DSE says the upper 16 bits are dead, to be removed. - // d [##] - DAI's address-modifying expression adds 4 bytes to dest. + // d [##] - Assign's address-modifying expression adds 4 bytes to dest. // Variable | | // Fragment 128| // Offsets 159 @@ -1896,47 +1924,53 @@ bool at::calculateFragmentIntersect( // 1. SliceOffsetInBits:48 + VarFrag.OffsetInBits:128 = 176 // 2. 176 - (expression_offset:32 + (d.address - dest):0) = 144 // 3. SliceOfVariable offset = 144, size = 16: - // SliceOfVariable ∩ DAI_fragment = (offset: 144, size: 16) - // SliceOfVariable tells us the bits of the variable described by DAI that are + // SliceOfVariable ∩ Assign_fragment = (offset: 144, size: 16) + // SliceOfVariable tells us the bits of the variable described by Assign that are // affected by the DSE. - if (DAI->isKillAddress()) - return false; + auto CalculateFragmentIntersectImpl = [&](const auto *AssignRecord) -> bool { + if (AssignRecord->isKillAddress()) + return false; - DIExpression::FragmentInfo VarFrag = DAI->getFragmentOrEntireVariable(); - if (VarFrag.SizeInBits == 0) - return false; // Variable size is unknown. + DIExpression::FragmentInfo VarFrag = AssignRecord->getFragmentOrEntireVariable(); + if (VarFrag.SizeInBits == 0) + return false; // Variable size is unknown. - // Calculate the difference between Dest and the dbg.assign address + - // address-modifying expression. - int64_t PointerOffsetInBits; - { - auto DestOffsetInBytes = DAI->getAddress()->getPointerOffsetFrom(Dest, DL); - if (!DestOffsetInBytes) - return false; // Can't calculate difference in addresses. + // Calculate the difference between Dest and the dbg.assign address + + // address-modifying expression. + int64_t PointerOffsetInBits; + { + auto DestOffsetInBytes = AssignRecord->getAddress()->getPointerOffsetFrom(Dest, DL); + if (!DestOffsetInBytes) + return false; // Can't calculate difference in addresses. - int64_t ExprOffsetInBytes; - if (!DAI->getAddressExpression()->extractIfOffset(ExprOffsetInBytes)) - return false; + int64_t ExprOffsetInBytes; + if (!AssignRecord->getAddressExpression()->extractIfOffset(ExprOffsetInBytes)) + return false; - int64_t PointerOffsetInBytes = *DestOffsetInBytes + ExprOffsetInBytes; - PointerOffsetInBits = PointerOffsetInBytes * 8; - } + int64_t PointerOffsetInBytes = *DestOffsetInBytes + ExprOffsetInBytes; + PointerOffsetInBits = PointerOffsetInBytes * 8; + } - // Adjust the slice offset so that we go from describing the a slice - // of memory to a slice of the variable. - int64_t NewOffsetInBits = - SliceOffsetInBits + VarFrag.OffsetInBits - PointerOffsetInBits; - if (NewOffsetInBits < 0) - return false; // Fragment offsets can only be positive. - DIExpression::FragmentInfo SliceOfVariable(SliceSizeInBits, NewOffsetInBits); - // Intersect the variable slice with DAI's fragment to trim it down to size. - DIExpression::FragmentInfo TrimmedSliceOfVariable = - DIExpression::FragmentInfo::intersect(SliceOfVariable, VarFrag); - if (TrimmedSliceOfVariable == VarFrag) - Result = std::nullopt; - else - Result = TrimmedSliceOfVariable; - return true; + // Adjust the slice offset so that we go from describing the a slice + // of memory to a slice of the variable. + int64_t NewOffsetInBits = + SliceOffsetInBits + VarFrag.OffsetInBits - PointerOffsetInBits; + if (NewOffsetInBits < 0) + return false; // Fragment offsets can only be positive. + DIExpression::FragmentInfo SliceOfVariable(SliceSizeInBits, NewOffsetInBits); + // Intersect the variable slice with AssignRecord's fragment to trim it down to size. + DIExpression::FragmentInfo TrimmedSliceOfVariable = + DIExpression::FragmentInfo::intersect(SliceOfVariable, VarFrag); + if (TrimmedSliceOfVariable == VarFrag) + Result = std::nullopt; + else + Result = TrimmedSliceOfVariable; + return true; + }; + + if (isa(Assign)) + return CalculateFragmentIntersectImpl(cast(Assign)); + return CalculateFragmentIntersectImpl(cast(Assign)); } /// Collect constant properies (base, size, offset) of \p StoreDest. @@ -2182,6 +2216,8 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) { for (auto &P : DbgDeclares) { const AllocaInst *Alloca = P.first; auto Markers = at::getAssignmentMarkers(Alloca); + SmallVector DPMarkers; + at::getDPAssignmentMarkers(Alloca, DPMarkers); (void)Markers; for (DbgDeclareInst *DDI : P.second) { // Assert that the alloca that DDI uses is now linked to a dbg.assign @@ -2192,8 +2228,11 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) { // trackAssignments will create an alloca-sized fragment for the // dbg.assign. assert(llvm::any_of(Markers, [DDI](DbgAssignIntrinsic *DAI) { - return DebugVariableAggregate(DAI) == DebugVariableAggregate(DDI); - })); + return DebugVariableAggregate(DAI) == DebugVariableAggregate(DDI); + }) || llvm::any_of(DPMarkers, [DDI](DPValue *DPV) { + return DebugVariableAggregate(DPV) == DebugVariableAggregate(DDI); + }) + ); // Delete DDI because the variable location is now tracked using // assignment tracking. DDI->eraseFromParent(); diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp index 534bc74449623..2f3bb8050fbdb 100644 --- a/llvm/lib/IR/DebugInfoMetadata.cpp +++ b/llvm/lib/IR/DebugInfoMetadata.cpp @@ -41,6 +41,10 @@ DebugVariable::DebugVariable(const DbgVariableIntrinsic *DII) : Variable(DII->getVariable()), Fragment(DII->getExpression()->getFragmentInfo()), InlinedAt(DII->getDebugLoc().getInlinedAt()) {} +DebugVariable::DebugVariable(const DPValue *DPV) + : Variable(DPV->getVariable()), + Fragment(DPV->getExpression()->getFragmentInfo()), + InlinedAt(DPV->getDebugLoc().getInlinedAt()) {} DebugVariableAggregate::DebugVariableAggregate(const DbgVariableIntrinsic *DVI) : DebugVariable(DVI->getVariable(), std::nullopt, diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index 5468640838c05..8cab270ed25b2 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -13,9 +13,9 @@ namespace llvm { DPValue::DPValue(const DbgVariableIntrinsic *DVI) - : DebugValueUser(ArrayRef(DVI->getRawLocation())), + : DebugValueUser({DVI->getRawLocation(), nullptr}), Variable(DVI->getVariable()), Expression(DVI->getExpression()), - DbgLoc(DVI->getDebugLoc()) { + DbgLoc(DVI->getDebugLoc()), AddressExpression(nullptr), AssignID() { switch (DVI->getIntrinsicID()) { case Intrinsic::dbg_value: Type = LocationType::Value; @@ -23,6 +23,14 @@ DPValue::DPValue(const DbgVariableIntrinsic *DVI) case Intrinsic::dbg_declare: Type = LocationType::Declare; break; + case Intrinsic::dbg_assign: { + Type = LocationType::Assign; + const DbgAssignIntrinsic* Assign = static_cast(DVI); + resetDebugValue(1, Assign->getRawAddress()); + AddressExpression = Assign->getAddressExpression(); + setAssignId(Assign->getAssignID()); + break; + } default: llvm_unreachable( "Trying to create a DPValue with an invalid intrinsic type!"); @@ -30,17 +38,72 @@ DPValue::DPValue(const DbgVariableIntrinsic *DVI) } DPValue::DPValue(const DPValue &DPV) - : DebugValueUser(DPV.getDebugValues()), Type(DPV.getType()), - Variable(DPV.getVariable()), Expression(DPV.getExpression()), - DbgLoc(DPV.getDebugLoc()) {} + : DebugValueUser(DPV.DebugValues), Variable(DPV.getVariable()), + Type(DPV.getType()), Expression(DPV.getExpression()), + DbgLoc(DPV.getDebugLoc()), AddressExpression(DPV.AddressExpression), + AssignID(DPV.AssignID) {} DPValue::DPValue(Metadata *Location, DILocalVariable *DV, DIExpression *Expr, const DILocation *DI) - : DebugValueUser(ArrayRef(Location)), Variable(DV), Expression(Expr), - DbgLoc(DI), Type(LocationType::Value) {} + : DebugValueUser({Location, nullptr}), Variable(DV), + Type(LocationType::Value), Expression(Expr), DbgLoc(DI), + AddressExpression(nullptr), AssignID() {} + +DPValue::DPValue(Metadata *Value, DILocalVariable *Variable, + DIExpression *Expression, DIAssignID *AssignID, + Metadata *Address, DIExpression *AddressExpression, + const DILocation *DI) + : DebugValueUser({Value, Address}), Variable(Variable), + Expression(Expression), DbgLoc(DI), Type(LocationType::Assign), + AssignID(AssignID), AddressExpression(AddressExpression) {} void DPValue::deleteInstr() { delete this; } +DPValue *DPValue::createDPValue( + Metadata *Location, DILocalVariable *DV, DIExpression *Expr, + const DILocation *DI, Instruction *InsertBefore) { + auto *NewDPValue = new DPValue(Location, DV, Expr, DI); + if (InsertBefore) { + InsertBefore->getParent()->insertDPValueBefore(NewDPValue, + InsertBefore->getIterator()); + } + return NewDPValue; +} +DPValue *DPValue::createDPDeclare( + Value *Address, DILocalVariable *DV, DIExpression *Expr, + const DILocation *DI, Instruction *InsertBefore) { + auto *NewDPDeclare = new DPValue(ValueAsMetadata::get(Address), DV, Expr, DI); + NewDPDeclare->Type = LocationType::Declare; + if (InsertBefore) { + InsertBefore->getParent()->insertDPValueBefore(NewDPDeclare, + InsertBefore->getIterator()); + } + return NewDPDeclare; +} +DPValue *DPValue::createDPAssign( + Metadata *Value, DILocalVariable *Variable, DIExpression *Expression, + DIAssignID *AssignID, Metadata *Address, DIExpression *AddressExpression, + const DILocation *DI, Instruction *InsertBefore) { + auto *NewDPAssign = new DPValue(Value, Variable, Expression, AssignID, + Address, AddressExpression, DI); + if (InsertBefore) { + InsertBefore->getParent()->insertDPValueBefore(NewDPAssign, + InsertBefore->getIterator()); + } + return NewDPAssign; +} +DPValue *DPValue::createLinkedDPAssign( + Instruction *LinkedInstr, Metadata *ValueMD, DILocalVariable *Variable, + DIExpression *Expression, Value *Address, + DIExpression *AddressExpression, const DILocation *DI) { + auto *Link = LinkedInstr->getMetadata(LLVMContext::MD_DIAssignID); + assert(Link && "Linked instruction must have DIAssign metadata attached"); + auto *NewDPAssign = new DPValue(ValueMD, Variable, Expression, + cast(Link), ValueAsMetadata::get(Address), AddressExpression, DI); + LinkedInstr->getParent()->insertDPValueAfter(NewDPAssign, LinkedInstr); + return NewDPAssign; +} + iterator_range DPValue::location_ops() const { auto *MD = getRawLocation(); // If a Value has been deleted, the "location" for this DPValue will be @@ -166,9 +229,6 @@ DPValue::createDebugIntrinsic(Module *M, Instruction *InsertBefore) const { "Cannot clone from BasicBlock that is not part of a Module or " "DICompileUnit!"); LLVMContext &Context = getDebugLoc()->getContext(); - Value *Args[] = {MetadataAsValue::get(Context, getRawLocation()), - MetadataAsValue::get(Context, getVariable()), - MetadataAsValue::get(Context, getExpression())}; Function *IntrinsicFn; // Work out what sort of intrinsic we're going to produce. @@ -179,12 +239,31 @@ DPValue::createDebugIntrinsic(Module *M, Instruction *InsertBefore) const { case DPValue::LocationType::Value: IntrinsicFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_value); break; + case DPValue::LocationType::Assign: + IntrinsicFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_assign); + break; } // Create the intrinsic from this DPValue's information, optionally insert // into the target location. - DbgVariableIntrinsic *DVI = cast( - CallInst::Create(IntrinsicFn->getFunctionType(), IntrinsicFn, Args)); + DbgVariableIntrinsic *DVI; + if (isDbgAssign()) { + Value *AssignArgs[] = { + MetadataAsValue::get(Context, getRawLocation()), + MetadataAsValue::get(Context, getVariable()), + MetadataAsValue::get(Context, getExpression()), + MetadataAsValue::get(Context, getAssignID()), + MetadataAsValue::get(Context, getRawAddress()), + MetadataAsValue::get(Context, getAddressExpression())}; + DVI = cast( + CallInst::Create(IntrinsicFn->getFunctionType(), IntrinsicFn, AssignArgs)); + } else { + Value *Args[] = {MetadataAsValue::get(Context, getRawLocation()), + MetadataAsValue::get(Context, getVariable()), + MetadataAsValue::get(Context, getExpression())}; + DVI = cast( + CallInst::Create(IntrinsicFn->getFunctionType(), IntrinsicFn, Args)); + } DVI->setTailCall(); DVI->setDebugLoc(getDebugLoc()); if (InsertBefore) @@ -193,6 +272,22 @@ DPValue::createDebugIntrinsic(Module *M, Instruction *InsertBefore) const { return DVI; } +///////////////////////////////////////////// +/// DbgAssign Methods + + + Value *DPValue::getAddress() const { + auto *MD = getRawAddress(); + if (auto *V = dyn_cast(MD)) + return V->getValue(); + + // When the value goes to null, it gets replaced by an empty MDNode. + assert(!cast(MD)->getNumOperands() && "Expected an empty MDNode"); + return nullptr; + } + +///////////////////////////////////////////// + const BasicBlock *DPValue::getParent() const { return Marker->MarkedInstr->getParent(); } @@ -217,6 +312,23 @@ const LLVMContext &DPValue::getContext() const { return getBlock()->getContext(); } +void DPValue::insertBefore(DPValue *InsertBefore) { + assert(!getMarker() && + "Cannot insert a DPValue that is already has a DPMarker!"); + assert(InsertBefore->getMarker() && + "Cannot insert a DPValue before a DPValue that does not have a " + "DPMarker!"); + InsertBefore->getMarker()->insertDPValue(this, InsertBefore); +} +void DPValue::insertAfter(DPValue *InsertAfter) { + assert(!getMarker() && + "Cannot insert a DPValue that is already has a DPMarker!"); + assert(InsertAfter->getMarker() && + "Cannot insert a DPValue after a DPValue that does not have a " + "DPMarker!"); + InsertAfter->getMarker()->insertDPValueAfter(this, InsertAfter); +} + /////////////////////////////////////////////////////////////////////////////// // An empty, global, DPMarker for the purpose of describing empty ranges of @@ -307,6 +419,18 @@ void DPMarker::insertDPValue(DPValue *New, bool InsertAtHead) { StoredDPValues.insert(It, *New); New->setMarker(this); } +void DPMarker::insertDPValue(DPValue *New, DPValue *InsertBefore) { + assert(InsertBefore->getMarker() == this && + "DPValue 'InsertBefore' must be contained in this DPMarker!"); + StoredDPValues.insert(InsertBefore->getIterator(), *New); + New->setMarker(this); +} +void DPMarker::insertDPValueAfter(DPValue *New, DPValue *InsertAfter) { + assert(InsertAfter->getMarker() == this && + "DPValue 'InsertAfter' must be contained in this DPMarker!"); + StoredDPValues.insert(++(InsertAfter->getIterator()), *New); + New->setMarker(this); +} void DPMarker::absorbDebugValues(DPMarker &Src, bool InsertAtHead) { auto It = InsertAtHead ? StoredDPValues.begin() : StoredDPValues.end(); diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index 9ceeed886a59f..edf030afab90a 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -155,39 +155,43 @@ const DPValue *DebugValueUser::getUser() const { void DebugValueUser::handleChangedValue(void *Old, Metadata *New) { // NOTE: We could inform the "owner" that a value has changed through // getOwner, if needed. - auto Values = getDebugValues(); - Metadata **It = std::find(Values.begin(), Values.end(), Old); - if (It == Values.end()) - return; - ptrdiff_t Idx = std::distance(Values.begin(), It); + auto OldMD = static_cast(Old); + ptrdiff_t Idx = std::distance(DebugValues.begin(), OldMD); resetDebugValue(Idx, New); } -void DebugValueUser::trackDebugValue(int Idx) { - Metadata *&Operand = DebugValues[Idx]; - MetadataTracking::track(Operand, *Operand, *this); +void DebugValueUser::trackDebugValue(size_t Idx) { + assert(Idx < 2 && "Invalid debug value index."); + Metadata *&MD = DebugValues[Idx]; + if (MD) + MetadataTracking::track(&MD, *MD, *this); } void DebugValueUser::trackDebugValues() { - for (Metadata *&MD : getDebugValues()) - MetadataTracking::track(MD, *MD, *this); + for (Metadata *&MD : DebugValues) + if (MD) + MetadataTracking::track(&MD, *MD, *this); } -void DebugValueUser::untrackDebugValue(int Idx) { - Metadata *&Operand = DebugValues[Idx]; - MetadataTracking::untrack(Operand); +void DebugValueUser::untrackDebugValue(size_t Idx) { + assert(Idx < 2 && "Invalid debug value index."); + Metadata *&MD = DebugValues[Idx]; + if (MD) + MetadataTracking::untrack(MD); } void DebugValueUser::untrackDebugValues() { - for (Metadata *&MD : getDebugValues()) - MetadataTracking::untrack(MD); + for (Metadata *&MD : DebugValues) + if (MD) + MetadataTracking::untrack(MD); } void DebugValueUser::retrackDebugValues(DebugValueUser &X) { assert(DebugValueUser::operator==(X) && "Expected values to match"); - for (const auto &[MD, XMD] : zip(getDebugValues(), X.getDebugValues())) - MetadataTracking::retrack(XMD, MD); - X.DebugValues.reset(); + for (const auto &[MD, XMD] : zip(DebugValues, X.DebugValues)) + if (XMD) + MetadataTracking::retrack(XMD, MD); + X.DebugValues.fill(nullptr); } bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) { @@ -426,27 +430,26 @@ void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) { } ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) { - if (auto ArgList = dyn_cast(&MD)) - return ArgList->Context.getOrCreateReplaceableUses(); - if (auto *N = dyn_cast(&MD)) - return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses(); + if (auto *N = dyn_cast(&MD)) { + return !N->isResolved() || N->isAlwaysReplaceable() + ? N->Context.getOrCreateReplaceableUses() + : nullptr; + } return dyn_cast(&MD); } ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) { - if (auto ArgList = dyn_cast(&MD)) { - return ArgList->Context.getOrCreateReplaceableUses(); + if (auto *N = dyn_cast(&MD)) { + return !N->isResolved() || N->isAlwaysReplaceable() + ? N->Context.getReplaceableUses() + : nullptr; } - if (auto *N = dyn_cast(&MD)) - return N->isResolved() ? nullptr : N->Context.getReplaceableUses(); return dyn_cast(&MD); } bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) { - if (isa(&MD)) - return true; if (auto *N = dyn_cast(&MD)) - return !N->isResolved(); + return !N->isResolved() || N->isAlwaysReplaceable(); return isa(&MD); } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 2cf53bb6d69d4..4444502e8372a 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -170,6 +170,11 @@ struct VerifierSupport { *OS << '\n'; } } + + void Write(const DPValue *V) { + if (V) + *OS << "DPValue { " << V->getVariable() << " }\n"; + } void Write(const Metadata *MD) { if (!MD) @@ -4700,6 +4705,13 @@ void Verifier::visitDIAssignIDMetadata(Instruction &I, MDNode *MD) { "dbg.assign not in same function as inst", DAI, &I); } } + for (DPValue *DPV : cast(MD)->getAllDPValueUsers()) { + CheckDI(DPV->isDbgAssign(), + "!DIAssignID should only be used by llvm.dbg.assign intrinsics", + MD, DPV); + CheckDI(DPV->getFunction() == I.getFunction(), + "dbg.assign not in same function as inst", DPV, &I); + } } void Verifier::visitCallStackMetadata(MDNode *MD) { diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 6fba5bcaedaca..3dda6a892d2a0 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -294,6 +294,10 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) { if (llvm::is_contained(DAI->location_ops(), FillC)) DAI->replaceVariableLocationOp(FillC, FillVal); } + for (auto *DPV : at::getDPAssignmentMarkers(S)) { + if (llvm::is_contained(DPV->location_ops(), FillC)) + DPV->replaceVariableLocationOp(FillC, FillVal); + } S->setAlignment(Alignment); if (isa(MI)) diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index d3064dfea2e80..61081feab9083 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -3999,6 +3999,9 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I, DebugVariable(DPV->getVariable(), DPV->getExpression(), DPV->getDebugLoc()->getInlinedAt()); + if (DPV->isDbgDeclare() || DPV->isDbgAssign()) + continue; + if (!SunkVariables.insert(DbgUserVariable).second) continue; diff --git a/llvm/lib/Transforms/Scalar/ADCE.cpp b/llvm/lib/Transforms/Scalar/ADCE.cpp index d3113199ade78..194a70d1b96c7 100644 --- a/llvm/lib/Transforms/Scalar/ADCE.cpp +++ b/llvm/lib/Transforms/Scalar/ADCE.cpp @@ -548,6 +548,9 @@ ADCEChanged AggressiveDeadCodeElimination::removeDeadInstructions() { // If inhaled, check for any dbg.values attached to this instruction, and // drop any for scopes that aren't alive, like the rest of this loop does. for (DPValue &DPV : make_early_inc_range(I.getDbgValueRange())) { + if (DPV.isDbgAssign()) + if (!at::getAssignmentInsts(&DPV).empty()) + continue; if (AliveScopes.count(DPV.getDebugLoc()->getScope())) continue; I.dropOneDbgValue(&DPV); diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp index b6f9cb6cd2d0b..8e14a8f951ede 100644 --- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp +++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp @@ -489,27 +489,27 @@ static void shortenAssignment(Instruction *Inst, Value *OriginalDest, uint64_t DeadSliceSizeInBits = OldSizeInBits - NewSizeInBits; uint64_t DeadSliceOffsetInBits = OldOffsetInBits + (IsOverwriteEnd ? NewSizeInBits : 0); - auto SetDeadFragExpr = [](DbgAssignIntrinsic *DAI, + auto SetDeadFragExpr = [](auto *Assign, DIExpression::FragmentInfo DeadFragment) { // createFragmentExpression expects an offset relative to the existing // fragment offset if there is one. uint64_t RelativeOffset = DeadFragment.OffsetInBits - - DAI->getExpression() + Assign->getExpression() ->getFragmentInfo() .value_or(DIExpression::FragmentInfo(0, 0)) .OffsetInBits; if (auto NewExpr = DIExpression::createFragmentExpression( - DAI->getExpression(), RelativeOffset, DeadFragment.SizeInBits)) { - DAI->setExpression(*NewExpr); + Assign->getExpression(), RelativeOffset, DeadFragment.SizeInBits)) { + Assign->setExpression(*NewExpr); return; } // Failed to create a fragment expression for this so discard the value, // making this a kill location. auto *Expr = *DIExpression::createFragmentExpression( - DIExpression::get(DAI->getContext(), std::nullopt), + DIExpression::get(Assign->getContext(), std::nullopt), DeadFragment.OffsetInBits, DeadFragment.SizeInBits); - DAI->setExpression(Expr); - DAI->setKillLocation(); + Assign->setExpression(Expr); + Assign->setKillLocation(); }; // A DIAssignID to use so that the inserted dbg.assign intrinsics do not @@ -527,32 +527,37 @@ static void shortenAssignment(Instruction *Inst, Value *OriginalDest, // returned by getAssignmentMarkers so save a copy of the markers to iterate // over. auto LinkedRange = at::getAssignmentMarkers(Inst); + SmallVector LinkedDPAssigns = at::getDPAssignmentMarkers(Inst); SmallVector Linked(LinkedRange.begin(), LinkedRange.end()); - for (auto *DAI : Linked) { + auto InsertAssignForOverlap = [&](auto *Assign) { std::optional NewFragment; if (!at::calculateFragmentIntersect(DL, OriginalDest, DeadSliceOffsetInBits, - DeadSliceSizeInBits, DAI, + DeadSliceSizeInBits, Assign, NewFragment) || !NewFragment) { // We couldn't calculate the intersecting fragment for some reason. Be // cautious and unlink the whole assignment from the store. - DAI->setKillAddress(); - DAI->setAssignId(GetDeadLink()); - continue; + Assign->setKillAddress(); + Assign->setAssignId(GetDeadLink()); + return; } // No intersect. if (NewFragment->SizeInBits == 0) - continue; + return; // Fragments overlap: insert a new dbg.assign for this dead part. - auto *NewAssign = cast(DAI->clone()); - NewAssign->insertAfter(DAI); + auto *NewAssign = static_cast(Assign->clone()); + NewAssign->insertAfter(Assign); NewAssign->setAssignId(GetDeadLink()); if (NewFragment) SetDeadFragExpr(NewAssign, *NewFragment); NewAssign->setKillAddress(); - } + }; + for (auto *DAI : Linked) + InsertAssignForOverlap(DAI); + for (auto *DPV : LinkedDPAssigns) + InsertAssignForOverlap(DPV); } static bool tryToShorten(Instruction *DeadI, int64_t &DeadStart, diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index a6a824c00e64f..e4bef3d842b70 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -191,6 +191,10 @@ static DebugVariable getAggregateVariable(DbgVariableIntrinsic *DVI) { return DebugVariable(DVI->getVariable(), std::nullopt, DVI->getDebugLoc().getInlinedAt()); } +static DebugVariable getAggregateVariable(DPValue *DPV) { + return DebugVariable(DPV->getVariable(), std::nullopt, + DPV->getDebugLoc().getInlinedAt()); +} /// Find linked dbg.assign and generate a new one with the correct /// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the @@ -212,8 +216,9 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, Instruction *Inst, Value *Dest, Value *Value, const DataLayout &DL) { auto MarkerRange = at::getAssignmentMarkers(OldInst); + auto DPAssignMarkerRange = at::getDPAssignmentMarkers(Inst); // Nothing to do if OldInst has no linked dbg.assign intrinsics. - if (MarkerRange.empty()) + if (MarkerRange.empty() && DPAssignMarkerRange.empty()) return; LLVM_DEBUG(dbgs() << " migrateDebugInfo\n"); @@ -234,6 +239,9 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, for (auto *DAI : at::getAssignmentMarkers(OldAlloca)) BaseFragments[getAggregateVariable(DAI)] = DAI->getExpression()->getFragmentInfo(); + for (auto *DPV : at::getDPAssignmentMarkers(OldAlloca)) + BaseFragments[getAggregateVariable(DPV)] = + DPV->getExpression()->getFragmentInfo(); // The new inst needs a DIAssignID unique metadata tag (if OldInst has // one). It shouldn't already have one: assert this assumption. @@ -243,7 +251,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false); assert(OldAlloca->isStaticAlloca()); - for (DbgAssignIntrinsic *DbgAssign : MarkerRange) { + auto MigrateDbgAssign = [&](auto DbgAssign, auto CreateNewAssign) { LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign << "\n"); auto *Expr = DbgAssign->getExpression(); @@ -254,7 +262,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, { auto R = BaseFragments.find(getAggregateVariable(DbgAssign)); if (R == BaseFragments.end()) - continue; + return; BaseFragment = R->second; } std::optional CurrentFragment = @@ -265,7 +273,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, BaseFragment, CurrentFragment, NewFragment); if (Result == Skip) - continue; + return; if (Result == UseFrag && !(NewFragment == CurrentFragment)) { if (CurrentFragment) { // Rewrite NewFragment to be relative to the existing one (this is @@ -297,8 +305,8 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, } ::Value *NewValue = Value ? Value : DbgAssign->getValue(); - auto *NewAssign = DIB.insertDbgAssign( - Inst, NewValue, DbgAssign->getVariable(), Expr, Dest, + auto *NewAssign = CreateNewAssign( + Inst, NewValue, DbgAssign, Expr, Dest, DIExpression::get(Ctx, std::nullopt), DbgAssign->getDebugLoc()); // If we've updated the value but the original dbg.assign has an arglist @@ -317,24 +325,51 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, if (SetKillLocation) NewAssign->setKillLocation(); - // We could use more precision here at the cost of some additional (code) - // complexity - if the original dbg.assign was adjacent to its store, we - // could position this new dbg.assign adjacent to its store rather than the - // old dbg.assgn. That would result in interleaved dbg.assigns rather than - // what we get now: - // split store !1 - // split store !2 - // dbg.assign !1 - // dbg.assign !2 - // This (current behaviour) results results in debug assignments being - // noted as slightly offset (in code) from the store. In practice this - // should have little effect on the debugging experience due to the fact - // that all the split stores should get the same line number. - NewAssign->moveBefore(DbgAssign); - - NewAssign->setDebugLoc(DbgAssign->getDebugLoc()); - LLVM_DEBUG(dbgs() << "Created new assign intrinsic: " << *NewAssign + LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n"); + }; + + for (DbgAssignIntrinsic *DbgAssign : MarkerRange) { + MigrateDbgAssign( + DbgAssign, + [&DIB](Instruction *Inst, ::Value *NewValue, + DbgAssignIntrinsic *DbgAssign, DIExpression *Expr, + ::Value *Dest, DIExpression *AddrExpr, + const DILocation *DL) -> DbgAssignIntrinsic* { + DbgAssignIntrinsic *NewAssign = DIB.insertDbgAssign( + Inst, NewValue, DbgAssign->getVariable(), Expr, Dest, + DIExpression::get(Expr->getContext(), std::nullopt), DbgAssign->getDebugLoc()); + // We could use more precision here at the cost of some additional (code) + // complexity - if the original dbg.assign was adjacent to its store, we + // could position this new dbg.assign adjacent to its store rather than the + // old dbg.assgn. That would result in interleaved dbg.assigns rather than + // what we get now: + // split store !1 + // split store !2 + // dbg.assign !1 + // dbg.assign !2 + // This (current behaviour) results results in debug assignments being + // noted as slightly offset (in code) from the store. In practice this + // should have little effect on the debugging experience due to the fact + // that all the split stores should get the same line number. + NewAssign->moveBefore(DbgAssign); + return NewAssign; + }); + } + for (DPValue *DPAssign : DPAssignMarkerRange) { + MigrateDbgAssign(DPAssign, + [](Instruction *Inst, ::Value *NewValue, DPValue *DPAssign, + DIExpression *Expr, ::Value *Dest, DIExpression *AddrExpr, + const DILocation *DL) -> DPValue* { + DIAssignID *AssignID = cast(Inst->getMetadata(LLVMContext::MD_DIAssignID)); + DPValue *NewAssign = new DPValue( + ValueAsMetadata::get(NewValue), DPAssign->getVariable(), Expr, + AssignID, ValueAsMetadata::get(Dest), AddrExpr, DL); + // We could use more precision here at the cost of some additional (code) + // complexity (see comment above). + NewAssign->insertBefore(DPAssign); + return NewAssign; + }); } } @@ -2995,6 +3030,7 @@ class llvm::sroa::AllocaSliceRewriter // emit dbg.assign intrinsics for mem intrinsics storing through non- // constant geps, or storing a variable number of bytes. assert(at::getAssignmentMarkers(&II).empty() && + at::getDPAssignmentMarkers(&II).empty() && "AT: Unexpected link to non-const GEP"); deleteIfTriviallyDead(OldPtr); return false; @@ -3147,6 +3183,12 @@ class llvm::sroa::AllocaSliceRewriter DAI->getAddress() == II.getDest()) DAI->replaceVariableLocationOp(II.getDest(), AdjustedPtr); } + for (auto *DPV : at::getDPAssignmentMarkers(&II)) { + if (any_of(DPV->location_ops(), + [&](Value *V) { return V == II.getDest(); }) || + DPV->getAddress() == II.getDest()) + DPV->replaceVariableLocationOp(II.getDest(), AdjustedPtr); + } II.setDest(AdjustedPtr); II.setDestAlignment(SliceAlign); } else { @@ -3728,6 +3770,7 @@ class AggLoadStoreRewriter : public InstVisitor { DL); } else { assert(at::getAssignmentMarkers(Store).empty() && + at::getDPAssignmentMarkers(Store).empty() && "AT: unexpected debug.assign linked to store through " "unbounded GEP"); } @@ -4825,19 +4868,29 @@ bool SROAPass::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { // Migrate debug information from the old alloca to the new alloca(s) // and the individual partitions. TinyPtrVector DbgVariables; + TinyPtrVector DPVariables; for (auto *DbgDeclare : FindDbgDeclareUses(&AI)) DbgVariables.push_back(DbgDeclare); for (auto *DbgAssign : at::getAssignmentMarkers(&AI)) DbgVariables.push_back(DbgAssign); - for (DbgVariableIntrinsic *DbgVariable : DbgVariables) { - auto *Expr = DbgVariable->getExpression(); - DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false); + for (auto *DPDeclare : FindDPDeclareUses(&AI)) + DPVariables.push_back(DPDeclare); + for (auto *DPAssign : at::getDPAssignmentMarkers(&AI)) + DPVariables.push_back(DPAssign); + + auto SameVariableFragment = [](const auto *LHS, const auto *RHS) { + return LHS->getVariable() == RHS->getVariable() && + LHS->getDebugLoc()->getInlinedAt() == + RHS->getDebugLoc()->getInlinedAt(); + }; + auto UpdateDebugInfoForFragments = [&](auto *DbgInfo, auto GetDeclares, auto InsertNewDbgInfo) { + auto *Expr = DbgInfo->getExpression(); uint64_t AllocaSize = DL.getTypeSizeInBits(AI.getAllocatedType()).getFixedValue(); for (auto Fragment : Fragments) { // Create a fragment expression describing the new partition or reuse AI's // expression if there is only one partition. - auto *FragmentExpr = Expr; + DIExpression *FragmentExpr = Expr; if (Fragment.Size < AllocaSize || Expr->isFragment()) { // If this alloca is already a scalar replacement of a larger aggregate, // Fragment.Offset describes the offset inside the scalar. @@ -4855,14 +4908,14 @@ bool SROAPass::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { Size = std::min(Size, AbsEnd - Start); } // The new, smaller fragment is stenciled out from the old fragment. - if (auto OrigFragment = FragmentExpr->getFragmentInfo()) { + if (auto OrigFragment = Expr->getFragmentInfo()) { assert(Start >= OrigFragment->OffsetInBits && - "new fragment is outside of original fragment"); + "new fragment is outside of original fragment"); Start -= OrigFragment->OffsetInBits; } // The alloca may be larger than the variable. - auto VarSize = DbgVariable->getVariable()->getSizeInBits(); + auto VarSize = DbgInfo->getVariable()->getSizeInBits(); if (VarSize) { if (Size > *VarSize) Size = *VarSize; @@ -4871,45 +4924,64 @@ bool SROAPass::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { } // Avoid creating a fragment expression that covers the entire variable. - if (!VarSize || *VarSize != Size) { - if (auto E = - DIExpression::createFragmentExpression(Expr, Start, Size)) + if (!VarSize || *VarSize != Size) + if (auto E = DIExpression::createFragmentExpression(Expr, Start, Size); E) FragmentExpr = *E; else continue; - } } // Remove any existing intrinsics on the new alloca describing // the variable fragment. - for (DbgDeclareInst *OldDII : FindDbgDeclareUses(Fragment.Alloca)) { - auto SameVariableFragment = [](const DbgVariableIntrinsic *LHS, - const DbgVariableIntrinsic *RHS) { - return LHS->getVariable() == RHS->getVariable() && - LHS->getDebugLoc()->getInlinedAt() == - RHS->getDebugLoc()->getInlinedAt(); - }; - if (SameVariableFragment(OldDII, DbgVariable)) - OldDII->eraseFromParent(); + for (auto *OldDeclare : GetDeclares(Fragment.Alloca)) { + if (SameVariableFragment(OldDeclare, DbgInfo)) + OldDeclare->eraseFromParent(); } - if (auto *DbgAssign = dyn_cast(DbgVariable)) { + InsertNewDbgInfo(DbgInfo, Fragment, FragmentExpr); + } + }; + for (DPValue *DPV : DPVariables) { + UpdateDebugInfoForFragments(DPV, FindDPDeclareUses, [&AI]( + DPValue *DPV, Fragment Fragment, DIExpression *FragmentExpr) { + if (DPV->isDbgAssign()) { if (!Fragment.Alloca->hasMetadata(LLVMContext::MD_DIAssignID)) { Fragment.Alloca->setMetadata( LLVMContext::MD_DIAssignID, DIAssignID::getDistinct(AI.getContext())); } - auto *NewAssign = DIB.insertDbgAssign( - Fragment.Alloca, DbgAssign->getValue(), DbgAssign->getVariable(), - FragmentExpr, Fragment.Alloca, DbgAssign->getAddressExpression(), - DbgAssign->getDebugLoc()); - NewAssign->setDebugLoc(DbgAssign->getDebugLoc()); - LLVM_DEBUG(dbgs() << "Created new assign intrinsic: " << *NewAssign + auto *NewAssign = DPValue::createLinkedDPAssign( + Fragment.Alloca, DPV->getRawLocation(), DPV->getVariable(), FragmentExpr, + Fragment.Alloca, DPV->getAddressExpression(), DPV->getDebugLoc()); + LLVM_DEBUG(dbgs() << "Created new debug assign: " << *NewAssign << "\n"); } else { - DIB.insertDeclare(Fragment.Alloca, DbgVariable->getVariable(), - FragmentExpr, DbgVariable->getDebugLoc(), &AI); + DPValue::createDPDeclare(Fragment.Alloca, DPV->getVariable(), + FragmentExpr, DPV->getDebugLoc(), &AI); } + }); + for (DbgVariableIntrinsic *DbgVariable : DbgVariables) { + DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false); + UpdateDebugInfoForFragments(DbgVariable, FindDbgDeclareUses, [&AI, &DIB]( + DbgVariableIntrinsic *DVI, Fragment Fragment, DIExpression *FragmentExpr) { + if (auto *DbgAssign = dyn_cast(DVI)) { + if (!Fragment.Alloca->hasMetadata(LLVMContext::MD_DIAssignID)) { + Fragment.Alloca->setMetadata( + LLVMContext::MD_DIAssignID, + DIAssignID::getDistinct(AI.getContext())); + } + auto *NewAssign = DIB.insertDbgAssign( + Fragment.Alloca, DbgAssign->getValue(), DbgAssign->getVariable(), + FragmentExpr, Fragment.Alloca, DbgAssign->getAddressExpression(), + DbgAssign->getDebugLoc()); + NewAssign->setDebugLoc(DbgAssign->getDebugLoc()); + LLVM_DEBUG(dbgs() << "Created new assign intrinsic: " << *NewAssign + << "\n"); + } else { + DIB.insertDeclare(Fragment.Alloca, DVI->getVariable(), + FragmentExpr, DVI->getDebugLoc(), &AI); + } + }); } } return Changed; diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index d3fa8f9603f9f..ccc1184eb5552 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -394,8 +394,17 @@ static bool DDDremoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) { // If the same variable fragment is described more than once it is enough // to keep the last one (i.e. the first found since we for reverse // iteration). - if (!R.second) - ToBeRemoved.push_back(&DPV); + if (R.second) + continue; + + if (DPV.isDbgAssign()) { + // Don't delete dbg.assign intrinsics that are linked to instructions. + if (!at::getAssignmentInsts(&DPV).empty()) + continue; + // Unlinked dbg.assign intrinsics can be treated like dbg.values. + } + + ToBeRemoved.push_back(&DPV); continue; } // Sequence with consecutive dbg.value instrs ended. Clear the map to @@ -481,6 +490,10 @@ static bool DDDremoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) { std::nullopt, DPV.getDebugLoc()->getInlinedAt()); auto VMI = VariableMap.find(Key); + // A dbg.assign with no linked instructions can be treated like a + // dbg.value (i.e. can be deleted). + bool IsDbgValueKind = (!DPV.isDbgAssign() || at::getAssignmentInsts(&DPV).empty()); + // Update the map if we found a new value/expression describing the // variable, or if the variable wasn't mapped already. SmallVector Values(DPV.location_ops()); @@ -500,9 +513,45 @@ static bool DDDremoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) { return !ToBeRemoved.empty(); } + +static bool DDDremomveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) { + assert(BB->isEntryBlock() && "expected entry block"); + SmallVector ToBeRemoved; + DenseSet SeenDefForAggregate; + // Returns the DebugVariable for DVI with no fragment info. + auto GetAggregateVariable = [](const DPValue &DPV) { + return DebugVariable(DPV.getVariable(), std::nullopt, + DPV.getDebugLoc().getInlinedAt()); + }; + + // Remove undef dbg.assign intrinsics that are encountered before + // any non-undef intrinsics from the entry block. + for (auto &I : *BB) { + for (DPValue &DPV : I.getDbgValueRange()) { + if (!DPV.isDbgValue() && !DPV.isDbgAssign()) + continue; + bool IsDbgValueKind = (DPV.isDbgValue() || at::getAssignmentInsts(&DPV).empty()); + DebugVariable Aggregate = GetAggregateVariable(DPV); + if (!SeenDefForAggregate.contains(Aggregate)) { + bool IsKill = DPV.isKillLocation() && IsDbgValueKind; + if (!IsKill) { + SeenDefForAggregate.insert(Aggregate); + } else if (DPV.isDbgAssign()) { + ToBeRemoved.push_back(&DPV); + } + } + } + } + + for (DPValue *DPV : ToBeRemoved) + DPV->eraseFromParent(); + + return !ToBeRemoved.empty(); +} + static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) { if (BB->IsNewDbgInfoFormat) - return DDDremoveRedundantDbgInstrsUsingForwardScan(BB); + return DDDremomveUndefDbgAssignsFromEntryBlock(BB); SmallVector ToBeRemoved; DenseMap, DIExpression *>> @@ -565,6 +614,9 @@ static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) { /// Possible improvements: /// - Keep track of non-overlapping fragments. static bool remomveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) { + if (BB->IsNewDbgInfoFormat) + return DDDremoveRedundantDbgInstrsUsingForwardScan(BB); + assert(BB->isEntryBlock() && "expected entry block"); SmallVector ToBeRemoved; DenseSet SeenDefForAggregate; diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 047fef949dbd2..2414283334f13 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -1724,6 +1724,13 @@ static at::StorageToVarsMap collectEscapedLocals(const DataLayout &DL, LLVM_DEBUG(errs() << " > DEF : " << *DAI << "\n"); EscapedLocals[Base].insert(at::VarRecord(DAI)); } + for (auto *DPV : at::getDPAssignmentMarkers(Base)) { + // Skip variables from inlined functions - they are not local variables. + if (DPV->getDebugLoc().getInlinedAt()) + continue; + LLVM_DEBUG(errs() << " > DEF : " << *DPV << "\n"); + EscapedLocals[Base].insert(at::VarRecord(DPV)); + } } return EscapedLocals; } @@ -1759,6 +1766,10 @@ static void fixupAssignments(Function::iterator Start, Function::iterator End) { I.setMetadata(LLVMContext::MD_DIAssignID, GetNewID(ID)); else if (auto *DAI = dyn_cast(&I)) DAI->setAssignId(GetNewID(DAI->getAssignID())); + for (DPValue &DPV : I.getDbgValueRange()) { + if (DPV.isDbgAssign()) + DPV.setAssignId(GetNewID(DPV.getAssignID())); + } } } } diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index 7ed6d4baa8eb8..eb74c5eb9914a 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -1570,6 +1570,8 @@ static void insertDbgValueOrDPValueAfter(DIBuilder &Builder, Value *DV, DILocalV } } + + /// Inserts a llvm.dbg.value intrinsic before a store to an alloca'd value /// that has an associated llvm.dbg.declare intrinsic. void llvm::ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII, @@ -2066,14 +2068,14 @@ void llvm::salvageDebugInfo(Instruction &I) { salvageDebugInfoForDbgValues(I, DbgUsers, DPUsers); } -/// Salvage the address component of \p DAI. -static void salvageDbgAssignAddress(DbgAssignIntrinsic *DAI) { - Instruction *I = dyn_cast(DAI->getAddress()); +template +static void salvageDbgAssignAddress(T *Assign) { + Instruction *I = dyn_cast(Assign->getAddress()); // Only instructions can be salvaged at the moment. if (!I) return; - assert(!DAI->getAddressExpression()->getFragmentInfo().has_value() && + assert(!Assign->getAddressExpression()->getFragmentInfo().has_value() && "address-expression shouldn't have fragment info"); // The address component of a dbg.assign cannot be variadic. @@ -2087,16 +2089,16 @@ static void salvageDbgAssignAddress(DbgAssignIntrinsic *DAI) { return; DIExpression *SalvagedExpr = DIExpression::appendOpsToArg( - DAI->getAddressExpression(), Ops, 0, /*StackValue=*/false); + Assign->getAddressExpression(), Ops, 0, /*StackValue=*/false); assert(!SalvagedExpr->getFragmentInfo().has_value() && "address-expression shouldn't have fragment info"); // Salvage succeeds if no additional values are required. if (AdditionalValues.empty()) { - DAI->setAddress(NewV); - DAI->setAddressExpression(SalvagedExpr); + Assign->setAddress(NewV); + Assign->setAddressExpression(SalvagedExpr); } else { - DAI->setKillAddress(); + Assign->setKillAddress(); } } @@ -2169,6 +2171,15 @@ void llvm::salvageDebugInfoForDbgValues( } // Duplicate of above block for DPValues. for (auto *DPV : DPUsers) { + if (DPV->isDbgAssign()) { + if (DPV->getAddress() == &I) { + salvageDbgAssignAddress(DPV); + Salvaged = true; + } + if (DPV->getValue() != &I) + continue; + } + // Do not add DW_OP_stack_value for DbgDeclare and DbgAddr, because they // are implicitly pointing out the value as a DWARF memory location // description. diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp index 79fcf13958e3f..b9e8aabff0509 100644 --- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -107,6 +107,7 @@ class AssignmentTrackingInfo { /// fragment. (i.e. not be a comprehensive set if there are multiple /// dbg.assigns for one variable fragment). SmallVector DbgAssigns; + SmallVector DPAssigns; public: void init(AllocaInst *AI) { @@ -115,16 +116,21 @@ class AssignmentTrackingInfo { if (Vars.insert(DebugVariable(DAI)).second) DbgAssigns.push_back(DAI); } + for (DPValue *DPV : at::getDPAssignmentMarkers(AI)) { + if (Vars.insert(DebugVariable(DPV)).second) + DPAssigns.push_back(DPV); + } } /// Update assignment tracking debug info given for the to-be-deleted store /// \p ToDelete that stores to this alloca. void updateForDeletedStore( StoreInst *ToDelete, DIBuilder &DIB, - SmallSet *DbgAssignsToDelete) const { + SmallSet *DbgAssignsToDelete, + SmallSet *DPAssignsToDelete) const { // There's nothing to do if the alloca doesn't have any variables using // assignment tracking. - if (DbgAssigns.empty()) + if (DbgAssigns.empty() && DPAssigns.empty()) return; // Insert a dbg.value where the linked dbg.assign is and remember to delete @@ -141,6 +147,13 @@ class AssignmentTrackingInfo { DAI->getExpression(), DAI->getDebugLoc(), DAI); } + for (DPValue *DPV : at::getDPAssignmentMarkers(ToDelete)) { + VarHasDbgAssignForStore.insert(DebugVariableAggregate(DPV)); + DPAssignsToDelete->insert(DPV); + auto *NewDPValue = new DPValue(DPV->getRawLocation(), DPV->getVariable(), + DPV->getExpression(), DPV->getDebugLoc()); + DPV->getMarker()->insertDPValue(NewDPValue, DPV); + } // It's possible for variables using assignment tracking to have no // dbg.assign linked to this store. These are variables in DbgAssigns that @@ -155,6 +168,11 @@ class AssignmentTrackingInfo { continue; ConvertDebugDeclareToDebugValue(DAI, ToDelete, DIB); } + for (auto *DPV : DPAssigns) { + if (VarHasDbgAssignForStore.contains(DebugVariableAggregate(DPV))) + continue; + ConvertDebugDeclareToDebugValue(DPV, ToDelete, DIB); + } } /// Update assignment tracking debug info given for the newly inserted PHI \p @@ -165,10 +183,12 @@ class AssignmentTrackingInfo { // debug-phi. for (auto *DAI : DbgAssigns) ConvertDebugDeclareToDebugValue(DAI, NewPhi, DIB); + for (auto *DPV : DPAssigns) + ConvertDebugDeclareToDebugValue(DPV, NewPhi, DIB); } - void clear() { DbgAssigns.clear(); } - bool empty() { return DbgAssigns.empty(); } + void clear() { DbgAssigns.clear(); DPAssigns.clear(); } + bool empty() { return DbgAssigns.empty() && DPAssigns.empty(); } }; struct AllocaInfo { @@ -229,11 +249,16 @@ struct AllocaInfo { } } DbgUserVec AllDbgUsers; - findDbgUsers(AllDbgUsers, AI, &DPUsers); + SmallVector AllDPUsers; + findDbgUsers(AllDbgUsers, AI, &AllDPUsers); std::copy_if(AllDbgUsers.begin(), AllDbgUsers.end(), std::back_inserter(DbgUsers), [](DbgVariableIntrinsic *DII) { return !isa(DII); }); + std::copy_if(AllDPUsers.begin(), AllDPUsers.end(), + std::back_inserter(DPUsers), [](DPValue *DPV) { + return !DPV->isDbgAssign(); + }); AssignmentTracking.init(AI); } }; @@ -341,6 +366,7 @@ struct PromoteMem2Reg { /// A set of dbg.assigns to delete because they've been demoted to /// dbg.values. Call cleanUpDbgAssigns to delete them. SmallSet DbgAssignsToDelete; + SmallSet DPAssignsToDelete; /// The set of basic blocks the renamer has already visited. SmallPtrSet Visited; @@ -390,6 +416,9 @@ struct PromoteMem2Reg { for (auto *DAI : DbgAssignsToDelete) DAI->eraseFromParent(); DbgAssignsToDelete.clear(); + for (auto *DPV : DPAssignsToDelete) + DPV->eraseFromParent(); + DPAssignsToDelete.clear(); } }; @@ -465,7 +494,8 @@ static void removeIntrinsicUsers(AllocaInst *AI) { static bool rewriteSingleStoreAlloca( AllocaInst *AI, AllocaInfo &Info, LargeBlockInfo &LBI, const DataLayout &DL, DominatorTree &DT, AssumptionCache *AC, - SmallSet *DbgAssignsToDelete) { + SmallSet *DbgAssignsToDelete, + SmallSet *DPAssignsToDelete) { StoreInst *OnlyStore = Info.OnlyStore; bool StoringGlobalVal = !isa(OnlyStore->getOperand(0)); BasicBlock *StoreBB = OnlyStore->getParent(); @@ -526,7 +556,8 @@ static bool rewriteSingleStoreAlloca( DIBuilder DIB(*AI->getModule(), /*AllowUnresolved*/ false); // Update assignment tracking info for the store we're going to delete. Info.AssignmentTracking.updateForDeletedStore(Info.OnlyStore, DIB, - DbgAssignsToDelete); + DbgAssignsToDelete, + DPAssignsToDelete); // Record debuginfo for the store and remove the declaration's // debuginfo. @@ -580,7 +611,8 @@ static bool rewriteSingleStoreAlloca( static bool promoteSingleBlockAlloca( AllocaInst *AI, const AllocaInfo &Info, LargeBlockInfo &LBI, const DataLayout &DL, DominatorTree &DT, AssumptionCache *AC, - SmallSet *DbgAssignsToDelete) { + SmallSet *DbgAssignsToDelete, + SmallSet *DPAssignsToDelete) { // The trickiest case to handle is when we have large blocks. Because of this, // this code is optimized assuming that large blocks happen. This does not // significantly pessimize the small block case. This uses LargeBlockInfo to @@ -644,7 +676,8 @@ static bool promoteSingleBlockAlloca( while (!AI->use_empty()) { StoreInst *SI = cast(AI->user_back()); // Update assignment tracking info for the store we're going to delete. - Info.AssignmentTracking.updateForDeletedStore(SI, DIB, DbgAssignsToDelete); + Info.AssignmentTracking.updateForDeletedStore(SI, DIB, DbgAssignsToDelete, + DPAssignsToDelete); // Record debuginfo for the store before removing it. for (DbgVariableIntrinsic *DII : Info.DbgUsers) { if (DII->isAddressOfVariable()) { @@ -715,7 +748,7 @@ void PromoteMem2Reg::run() { // it that are directly dominated by the definition with the value stored. if (Info.DefiningBlocks.size() == 1) { if (rewriteSingleStoreAlloca(AI, Info, LBI, SQ.DL, DT, AC, - &DbgAssignsToDelete)) { + &DbgAssignsToDelete, &DPAssignsToDelete)) { // The alloca has been processed, move on. RemoveFromAllocasList(AllocaNum); ++NumSingleStore; @@ -727,7 +760,7 @@ void PromoteMem2Reg::run() { // linear sweep over the block to eliminate it. if (Info.OnlyUsedInOneBlock && promoteSingleBlockAlloca(AI, Info, LBI, SQ.DL, DT, AC, - &DbgAssignsToDelete)) { + &DbgAssignsToDelete, &DPAssignsToDelete)) { // The alloca has been processed, move on. RemoveFromAllocasList(AllocaNum); continue; @@ -1133,7 +1166,8 @@ void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred, // Record debuginfo for the store before removing it. IncomingLocs[AllocaNo] = SI->getDebugLoc(); AllocaATInfo[AllocaNo].updateForDeletedStore(SI, DIB, - &DbgAssignsToDelete); + &DbgAssignsToDelete, + &DPAssignsToDelete); for (DbgVariableIntrinsic *DII : AllocaDbgUsers[ai->second]) if (DII->isAddressOfVariable()) ConvertDebugDeclareToDebugValue(DII, SI, DIB); diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 9257887b1b5d6..a1fabcb440c20 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -3039,6 +3039,10 @@ bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, if (llvm::is_contained(DAI->location_ops(), OrigV)) DAI->replaceVariableLocationOp(OrigV, S); } + for (auto *DPV : at::getDPAssignmentMarkers(SpeculatedStore)) { + if (llvm::is_contained(DPV->location_ops(), OrigV)) + DPV->replaceVariableLocationOp(OrigV, S); + } } // Metadata can be dependent on the condition we are hoisting above. @@ -3065,7 +3069,9 @@ bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, // Hoist the instructions. // jmorse: drop DPValues, in dbg.value mode this is done at end of this block. for (auto &It : make_range(ThenBB->begin(), ThenBB->end())) - It.dropDbgValues(); + for (DPValue &DPV : It.getDbgValueRange()) + if (!DPV.isDbgAssign()) + It.dropOneDbgValue(&DPV); BB->splice(BI->getIterator(), ThenBB, ThenBB->begin(), std::prev(ThenBB->end()));