diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs index 6dac6e2b350..d400c5d2010 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/bridge_subwalk.rs @@ -426,9 +426,6 @@ pub(crate) fn compute_bridge_root_parent_frame( // `root_pc` (`resume_data.frames[0].pc`) is already the post-call resume // point — the slot the inner frame's result lands in — so its Python // coordinate is a direct backtranslation (no `semantic_fallthrough_pc`). - // This local remains needed by the outer-active-box collector; the paused - // parent frame itself carries `ParentResumeCoord::Backxlat(root_pc)`. - let root_py_pc = crate::state::backxlat_py_pc(jitcode_index as i32, root_pc as i32) as u32; // Null the not-yet-produced call-result slot before collecting the active // boxes (the reconstructed callee supplies it on `SubReturn`), mirroring // `compute_inline_caller_frame`. Operate on a clone so `root_sym` stays a @@ -470,8 +467,7 @@ pub(crate) fn compute_bridge_root_parent_frame( ®s_r, root_sym.registers_f(), jitcode_index, - root_py_pc, - None, + false, // Key the query off the same carried root-frame word the snapshot and // decode side read from `frames[0].jitcode_pc`, so both resolve the // identical liveness window. diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs index fce8c7e83a1..ee84a11f9d0 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/diag.rs @@ -20,6 +20,25 @@ pub(crate) fn pcmap_recipe_resultcolor_audit_enabled() -> bool { *ENABLED.get_or_init(|| std::env::var_os("PYRE_PCMAP_RECIPE_RESULTCOLOR_AUDIT").is_some()) } +/// `PYRE_PCMAP_CONTAINING_AUDIT`: assert the Slice-B floor-only depth twin +/// (`depth_containing_for_jitcode_pc`) equals the raw +/// `depth_at_py_pc[vstack_containing_py_pc(jit_pc)]` read at both consumer +/// seams. Off in production; the gated branch is the only added code. +pub(crate) fn pcmap_containing_audit_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var_os("PYRE_PCMAP_CONTAINING_AUDIT").is_some()) +} + +/// `PYRE_PCMAP_AFTERRESIDUAL_AUDIT`: assert the Slice-C after-residual depth +/// twin (`depth_after_residual_for_jitcode_pc`) equals the raw +/// `depth_at_py_pc[semantic_fallthrough_pc(python_pc_for_jitcode_pc(jit_pc))]` +/// read at each consumer seam. Off in production; the gated branch is the only +/// added code. +pub(crate) fn pcmap_afterresidual_audit_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var_os("PYRE_PCMAP_AFTERRESIDUAL_AUDIT").is_some()) +} + pub(crate) fn pcmap_recipe_resultcolor_audit_probe(site: &'static str, verdict: &'static str) { if let Some(path) = std::env::var_os("PYRE_PCMAP_RECIPE_RESULTCOLOR_AUDIT_PROBE") { use std::io::Write; diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs index 4400bfd68bf..1d846721d59 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/inline_call.rs @@ -927,11 +927,30 @@ pub(crate) fn try_walker_call_assembler_self_recursive( let caller_code = unsafe { &*caller_jitcode.payload.code_ptr }; let call_py_pc = python_pc_for_jitcode_pc(&caller_jitcode.payload.metadata, op.pc) as usize; let resume_py_pc = crate::pyjitpl::semantic_fallthrough_pc(caller_code, call_py_pc) as u32; - let resume_depth = crate::liveness::liveness_for(caller_jitcode.payload.code_ptr) - .depth_at_py_pc() - .get(resume_py_pc as usize) - .copied() - .unwrap_or(0) as usize; + let raw_depth = || { + crate::liveness::liveness_for(caller_jitcode.payload.code_ptr) + .depth_at_py_pc() + .get(resume_py_pc as usize) + .copied() + .unwrap_or(0) as usize + }; + let resume_depth = if caller_jitcode.payload.depth_after_residual_populated() { + let depth = caller_jitcode + .payload + .depth_after_residual_for_jitcode_pc(op.pc) + .unwrap_or(0) as usize; + if pcmap_afterresidual_audit_enabled() { + assert_eq!( + depth, + raw_depth(), + "PYRE_PCMAP_AFTERRESIDUAL_AUDIT: self-recursive CA vstack depth twin diverged at jit_pc {} (py {resume_py_pc})", + op.pc + ); + } + depth + } else { + raw_depth() + }; ctx.vstack_boxes.truncate(resume_depth); ctx.vstack_boxes.resize(resume_depth, OpRef::NONE); if resume_depth > 0 { @@ -1964,8 +1983,6 @@ pub(crate) fn try_walker_inline_resolved_user_call( let jc_index = jc.index as u32; (jc_index, jc.payload.resume_marker_for_jitcode_pc(op.pc)) }; - let call_site_py_pc = - crate::state::backxlat_py_pc(call_site_jc_index as i32, op.pc as i32) as u32; let call_site_word = match call_site_marker { Some(m) => m as i32, None => majit_ir::resumedata::NO_JITCODE_PC, @@ -1977,8 +1994,7 @@ pub(crate) fn try_walker_inline_resolved_user_call( ctx.registers_r, ctx.registers_f, call_site_jc_index, - call_site_py_pc, - None, + false, call_site_word, // Keep the marker for the liveness-bank query, but key // entry metadata to the raw CALL offset that produced the diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs index 4329eb36e8a..e1e261a09fb 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/mod.rs @@ -3435,8 +3435,7 @@ fn collect_outer_active_boxes( regs_r: &[OpRef], regs_f: &[OpRef], outer_jitcode_index: u32, - entry_py_pc: u32, - guard_py_pc: Option, + guard_present: bool, // The resume-carried coordinate remains the bank-liveness source. Entry // metadata can be derived from an earlier raw operation coordinate, so it // must travel independently rather than inheriting that marker word. @@ -3523,7 +3522,7 @@ fn collect_outer_active_boxes( // #348: per-snapshot-coordinate color→slot entries — the color→slot source // the inversions below consult for every drained (non-portal) jitcode, the // per-program-point color space the `-live-` markers carry. Branch-guard - // resumes (`guard_py_pc.is_some()`) are fully covered here; a per-opcode-entry + // resumes (`guard_present`) are fully covered here; a per-opcode-entry // resume whose live Ref colors are all constants/leaked carries no entry (the // resume snapshot records Variables only), so `semantic_ref_slot_for_reg_color` // returns `None` and the live color falls to the `regs_r[color]` walk-bank @@ -3556,7 +3555,7 @@ fn collect_outer_active_boxes( (!pcdep_entries.is_empty()).then(|| pcdep_entries.as_slice()); let stack_livereg_gate = fbw_stack_livereg_enabled(); let (guard_pcdep_entries, guard_stack_only) = if stack_livereg_gate { - if let Some(gpc) = guard_py_pc { + if guard_present { if sym.jitcode().is_null() { (Vec::new(), 0usize) } else { @@ -3619,7 +3618,7 @@ fn collect_outer_active_boxes( format!( "collect_outer_active_boxes: liveness-active {bank} \ register {reg_idx} holds OpRef::NONE \ - (outer_jitcode_index={outer_jitcode_index}, entry_py_pc={entry_py_pc}, \ + (outer_jitcode_index={outer_jitcode_index}, entry_jitcode_pc={entry_jitcode_pc}, \ nlocals={nlocals}, owns_vable={owns_vable}, \ vable_len={vable_len}, \ num_regs_i={ni}, num_regs_r={nr}, num_regs_f={nf}, \ @@ -3651,11 +3650,11 @@ fn collect_outer_active_boxes( // the stale `registers_r[merge_color]` / edge-recovery color heuristics // below, which read a color the regalloc reused between the guard pc and // the resume pc (the #424 merge-color-staleness corruption). Scoped to - // the branch-guard reconstruction (`guard_py_pc`); the merge-color + // the branch-guard reconstruction (`guard_present`); the merge-color // heuristics below remain only as the fallback for slots the mirror does // not cover (mirror invalid, or an Int-bank temp the Ref-only mirror does // not hold). - let vstack_mirror: Option<&[OpRef]> = vstack.filter(|_| guard_py_pc.is_some()); + let vstack_mirror: Option<&[OpRef]> = vstack.filter(|_| guard_present); for &idx in &banks.ref_ { let color = idx as usize; if let Some(mirror) = vstack_mirror { @@ -3791,7 +3790,7 @@ fn collect_outer_active_boxes( let walk_real = walk_box.filter(|&v| v != OpRef::NONE && !opref_is_null_const_ptr(v)); let guard_pc_proves_slot = stack_livereg_gate - && guard_py_pc.is_some() + && guard_present && crate::state::semantic_ref_slot_for_reg_color( nlocals, guard_stack_only, @@ -3817,7 +3816,7 @@ fn collect_outer_active_boxes( let walk_is_real = walk_box .is_some_and(|b| b != OpRef::NONE && !opref_is_null_const_ptr(b)); let shadow_is_real = vbox.is_some_and(|b| !opref_is_null_const_ptr(b)); - if guard_py_pc.is_some() && walk_is_real { + if guard_present && walk_is_real { walk_box.unwrap_or_else(fallback) } else if shadow_is_real { vbox.unwrap_or_else(fallback) @@ -4806,11 +4805,28 @@ impl ActiveResumeFrame { return None; } let py_pc = vstack_containing_py_pc(&pjc.metadata, jit_pc); - let depth = crate::liveness::liveness_for(pjc.code_ptr) - .depth_at_py_pc() - .get(py_pc as usize) - .copied() - .unwrap_or(0) as usize; + // Raw py_pc-keyed static-liveness read: the unpopulated-twin fallback + // (skeleton / fixture) and the audit oracle. + let raw_depth = || { + crate::liveness::liveness_for(pjc.code_ptr) + .depth_at_py_pc() + .get(py_pc as usize) + .copied() + .unwrap_or(0) as usize + }; + let depth = if pjc.depth_containing_populated() { + let depth = pjc.depth_containing_for_jitcode_pc(jit_pc).unwrap_or(0) as usize; + if pcmap_containing_audit_enabled() { + assert_eq!( + depth, + raw_depth(), + "PYRE_PCMAP_CONTAINING_AUDIT: resume-frame containing-depth twin diverged at jit_pc {jit_pc} (py {py_pc})" + ); + } + depth + } else { + raw_depth() + }; Some((py_pc, pjc.code_ptr, depth)) } diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs index 388faaa3cfe..d6d9cae664d 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/resume_snapshot.rs @@ -520,13 +520,11 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // dereferences a null operand. Overlay the live operands, build // the snapshot, then restore the shadow so this transient write // never leaks into a later op or merge-point sync. - // Guard's Python pc (derived from the branch handler's scoped - // jitcode pc) — needed by the kept-stack source recovery in the - // overlay below and the resume coordinate further down. - let guard_py_pc = scope.branch_guard_jitcode_pc.map(|guard_jc_pc| unsafe { - let jc = &*sym.jitcode(); - python_pc_for_jitcode_pc(&jc.payload.metadata, guard_jc_pc) - }); + // Whether this is a kept-stack branch guard. Its resume/liveness + // window is keyed by the guard's own jitcode pc + // (`scope.branch_guard_jitcode_pc`, resolved to `entry_jitcode_pc` + // below), so only its presence is needed here. + let has_branch_guard = scope.branch_guard_jitcode_pc.is_some(); let stack_sync: Vec<(usize, OpRef)> = if sym.owns_virtualizable_shadow() { let depth = unsafe { let jc = &*sym.jitcode(); @@ -588,9 +586,9 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // (`resolved_recovered`) is empty for this shape, so the mirror is // the only kept-stack source and it has a hole. // Fill the hole from the guard-PC register file: the per-PC - // `pcdep_color_slots` map at `guard_py_pc` names the Ref-bank color - // that holds operand-stack slot `nlocals + s` at THIS guard PC (the - // same authoritative inversion `collect_outer_active_boxes` reads), + // `pcdep_color_slots` map at the guard's own jitcode pc names the + // Ref-bank color that holds operand-stack slot `nlocals + s` at THIS + // guard PC (the same authoritative twin `collect_outer_active_boxes` reads), // and `ctx.registers_r[color]` holds the live guard-state box. This // is the guard-PC color read (as `resolved_recovered` does for // `registers_r[src]`), NOT the retired stale merge-color read. @@ -600,14 +598,12 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // set. // Capture-only: writes the transient snapshot overlay, never the live // shadow (the trace_opcode.rs bridge-NULL constraint holds). - let mut stack_sync: Vec<(usize, OpRef)> = if guard_py_pc.is_some() + let mut stack_sync: Vec<(usize, OpRef)> = if has_branch_guard && sym.owns_virtualizable_shadow() && !sym.jitcode().is_null() { - let gpc = guard_py_pc.unwrap() as usize; - // `guard_py_pc` above is the plain JitCode-PC inversion (no - // Python-trivia skip), so the pcdep twin here must be the plain - // predecessor-keyed flavor too. + // The pcdep twin here must be the plain predecessor-keyed flavor + // (no Python-trivia skip), keyed by the guard's own jitcode pc. let gjc = scope.branch_guard_jitcode_pc.unwrap(); let nlocals = sym.nlocals(); let nvs = crate::virtualizable_gen::NUM_VABLE_SCALARS; @@ -671,10 +667,9 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( } else { stack_sync }; - // `scope.branch_guard_jitcode_pc` / `guard_py_pc` derived above - // (before the stack overlay, which needs `guard_py_pc` for the #124 - // kept-stack source recovery): map the guard's own jitcode pc to its - // Python opcode so the encoder reads the guard-pc liveness — the + // A branch guard (`scope.branch_guard_jitcode_pc`, consumed by the + // stack overlay above for the #124 kept-stack source recovery) keys + // the encoder liveness window on the guard's own jitcode pc — the // resume `py_pc` is a not-taken merge point whose live colors the // walk has not written. // #124 Approach B (M2): carry the guard's raw JitCode byte offset @@ -690,7 +685,7 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // Carrying `op_pc` for those broke encoder ↔ decoder // symmetry: `collect_outer_active_boxes` resolves the reg banks at // the carried coordinate but `live_locals` / `stack_color_map` at - // `entry_py_pc`, and for a non-branch guard + // the entry coordinate, and for a non-branch guard // `op_pc != marker` // — the two windows diverge and the decoded box layout mismatches. let guard_jitcode_pc: i32 = if let Some(guard_jc_pc) = scope.branch_guard_jitcode_pc { @@ -810,8 +805,8 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( // the ordinary post-call marker. // `py_pc == liveness_py_pc` here: the residual-call path // never supplies `GuardCaptureScope::branch_guard_jitcode_pc` - // (only kept-stack branch guards do), so `guard_py_pc` is - // `None`. + // (only kept-stack branch guards do), so this is not a + // branch guard. // Both offsets are the SAME physical post-call `-live-` insn by // codewriter construction (the plain fallthrough re-key and the // catch-predecessor anchor resolve to one marker), so the @@ -852,17 +847,15 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( None => majit_ir::resumedata::NO_JITCODE_PC, } }; - // #124 Approach B: when the carrier holds the guard's own pc (a - // branch guard whose not-taken arm is reached by RE-EXECUTING - // `goto_if_not`), resume at the guard's Python pc too. Keying the - // snapshot's resume pc on the guard coordinate makes the encoder - // liveness window (`collect_outer_active_boxes`), the blackhole - // `setposition`, and the cranelift bridge re-trace entry all agree - // — the kept operand stack is naturally live at the guard pc and is - // recovered from the walk-level box mirror in - // `collect_outer_active_boxes`. A non-branch guard carries no guard - // pc, so it keeps the merge `py_pc` and its exact resume-translation. - let liveness_py_pc = guard_py_pc.unwrap_or(py_pc); + // The liveness-window py coordinate. A branch guard keys its + // liveness window on the guard's own jitcode pc (via + // `entry_jitcode_pc` below); every other `liveness_py_pc` consumer + // sits on a non-branch-guard path — the `after_residual_call` + // bridge-semantic-maps call below and the fallthrough + // `first_floor_boundary_for_py` entry lookup, both reached only + // when `has_branch_guard` is false — so this is always the merge + // `py_pc`. + let liveness_py_pc = py_pc; let payload = unsafe { &(&*sym.jitcode()).payload }; let resolved = payload .resolve_resume_pc_with_jitcode_pc(guard_jitcode_pc, crate::state::op_live()); @@ -927,7 +920,7 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( ctx.trace_ctx.set_virtualizable_box_at(idx, old); } } - let (entry_jitcode_pc, entry_twin, entry_caller) = if guard_py_pc.is_some() { + let (entry_jitcode_pc, entry_twin, entry_caller) = if has_branch_guard { let entry_jitcode_pc = unsafe { let metadata = &(&*sym.jitcode()).payload.metadata; (guard_jitcode_pc >= 0) @@ -970,8 +963,7 @@ pub(crate) fn walker_capture_snapshot_for_last_guard_impl( ctx.registers_r, ctx.registers_f, jitcode_index, - liveness_py_pc, - guard_py_pc, + has_branch_guard, guard_jitcode_pc, entry_jitcode_pc, entry_twin, @@ -1222,13 +1214,31 @@ pub(crate) fn compute_inline_caller_frame( // A missing marker has no fallthrough-native key. Preserve the // existing Python-keyed path rather than declining a formerly valid // caller frame. - None => unsafe { - crate::liveness::liveness_for(code_ptr) - .depth_at_py_pc() - .get(fallthrough_py_pc as usize) - .copied() - .unwrap_or(0) as usize - }, + None => { + let payload = unsafe { &(*caller_sym.jitcode()).payload }; + let raw = || unsafe { + crate::liveness::liveness_for(code_ptr) + .depth_at_py_pc() + .get(fallthrough_py_pc as usize) + .copied() + .unwrap_or(0) as usize + }; + if payload.depth_after_residual_populated() { + let depth = payload + .depth_after_residual_for_jitcode_pc(call_jit_pc) + .unwrap_or(0) as usize; + if pcmap_afterresidual_audit_enabled() { + assert_eq!( + depth, + raw(), + "PYRE_PCMAP_AFTERRESIDUAL_AUDIT: inline-caller after-residual depth twin diverged at jit_pc {call_jit_pc} (py {fallthrough_py_pc})" + ); + } + depth + } else { + raw() + } + } }; if depth == 0 { return Err(InlineCallerFrameDecline::Unavailable); @@ -1272,8 +1282,7 @@ pub(crate) fn compute_inline_caller_frame( ctx.registers_r, ctx.registers_f, jitcode_index, - fallthrough_py_pc, - None, + false, caller_liveness_word, caller_liveness_word, OuterActiveBoxesEntryTwin::Trivia, @@ -1327,13 +1336,30 @@ pub(crate) fn compute_nested_inline_caller_frame( let depth = match resume_marker_jit_pc { Some(marker) => pjc.depth_trivia_for_jitcode_pc(marker).unwrap_or(0) as usize, None => { - let fallthrough_py_pc = legacy_fallthrough_py_pc(); - unsafe { - crate::liveness::liveness_for(pjc.code_ptr) - .depth_at_py_pc() - .get(fallthrough_py_pc as usize) - .copied() - .unwrap_or(0) as usize + let raw = || { + let fallthrough_py_pc = legacy_fallthrough_py_pc(); + unsafe { + crate::liveness::liveness_for(pjc.code_ptr) + .depth_at_py_pc() + .get(fallthrough_py_pc as usize) + .copied() + .unwrap_or(0) as usize + } + }; + if pjc.depth_after_residual_populated() { + let depth = pjc + .depth_after_residual_for_jitcode_pc(call_jit_pc) + .unwrap_or(0) as usize; + if pcmap_afterresidual_audit_enabled() { + assert_eq!( + depth, + raw(), + "PYRE_PCMAP_AFTERRESIDUAL_AUDIT: nested inline-caller after-residual depth twin diverged at jit_pc {call_jit_pc}" + ); + } + depth + } else { + raw() } } }; @@ -1676,8 +1702,49 @@ pub(crate) fn walker_capture_multi_frame_inline_snapshot( if jc.payload.code_ptr.is_null() { caller_sym.valuestackdepth() as i64 } else { - let lv = crate::liveness::liveness_for(jc.payload.code_ptr); - match lv.depth_at_py_pc().get(resume_py_pc as usize).copied() { + // Raw py_pc-keyed static-liveness read: the fallback where the + // twin does not apply (cross-jitcode parent, or empty twin), + // and the audit oracle. + let raw = || { + crate::liveness::liveness_for(jc.payload.code_ptr) + .depth_at_py_pc() + .get(resume_py_pc as usize) + .copied() + }; + // The raw read resolves `resume_py_pc` against + // `outer.jitcode_index`'s tables but indexes THIS jitcode's + // liveness; the twins key one jitcode, so cut only when both + // are the same jitcode. `Some(inner)` = twin applies (inner is + // its Option depth); `None` = not applicable -> raw. + let twin: Option> = if outer.jitcode_index != jc.index as u32 { + None + } else { + match outer.resume_coord { + ParentResumeCoord::Backxlat(jitcode_pc) => jc + .payload + .depth_trivia_populated() + .then(|| jc.payload.depth_trivia_for_jitcode_pc(jitcode_pc)), + ParentResumeCoord::CallFallthrough(call_jit_pc) => { + jc.payload.depth_after_residual_populated().then(|| { + jc.payload.depth_after_residual_for_jitcode_pc(call_jit_pc) + }) + } + } + }; + let depth = match twin { + Some(d) => { + if pcmap_afterresidual_audit_enabled() { + assert_eq!( + d, + raw(), + "PYRE_PCMAP_AFTERRESIDUAL_AUDIT: multiframe vsd depth twin diverged (py {resume_py_pc})" + ); + } + d + } + None => raw(), + }; + match depth { Some(d) => (caller_sym.nlocals() + d as usize) as i64, None => caller_sym.valuestackdepth() as i64, } diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs index e1277faede0..b0020010a07 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/specialize.rs @@ -3549,8 +3549,7 @@ pub(crate) fn orthodox_list_append_commit( ctx.registers_r, ctx.registers_f, outer_jitcode_index, - call_site_py_pc, - None, + false, call_site_word, // As above, entry metadata is keyed by the append op itself; its // liveness-bank query remains keyed by the resume marker. diff --git a/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs b/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs index c9bc33ae65f..bfd46afaeff 100644 --- a/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs +++ b/pyre/pyre-jit-trace/src/jitcode_dispatch/vstack_mirror.rs @@ -645,11 +645,35 @@ pub(crate) fn step_vstack_mirror(ctx: &mut WalkContext<'_, '_, Sym return; } let py_pc = vstack_step_py_pc(&jc.payload.metadata, jit_pc, ctx.vstack_cur_pypc); - let depth = crate::liveness::liveness_for(jc.payload.code_ptr) - .depth_at_py_pc() - .get(py_pc as usize) - .copied() - .unwrap_or(0) as usize; + // The depth is consumed only when the walk crosses a Python-opcode + // boundary (`py_pc != ctx.vstack_cur_pypc`; see the early-return + // below). On the block-head-marker branch `vstack_step_py_pc` + // returns `current_py_pc` and the value is dead; whenever it is + // live, `py_pc` is the floor segment py, so the floor twin + // reproduces the raw read. + let raw_depth = || { + crate::liveness::liveness_for(jc.payload.code_ptr) + .depth_at_py_pc() + .get(py_pc as usize) + .copied() + .unwrap_or(0) as usize + }; + let depth = if jc.payload.depth_containing_populated() { + let twin = jc + .payload + .depth_containing_for_jitcode_pc(jit_pc) + .unwrap_or(0) as usize; + if pcmap_containing_audit_enabled() && py_pc != ctx.vstack_cur_pypc { + assert_eq!( + twin, + raw_depth(), + "PYRE_PCMAP_CONTAINING_AUDIT: vstack-step depth twin diverged (jit_pc {jit_pc}, py {py_pc})" + ); + } + twin + } else { + raw_depth() + }; (py_pc, jc.payload.code_ptr, depth) } }; @@ -729,16 +753,44 @@ pub(crate) fn seed_vstack_mirror( pyre_interpreter::bytecode::Instruction::ForIter { .. } ) }); - let first_pypc = vstack_initial_py_pc( - &jc.payload.metadata, - start_pc, - predecessor_permuted_stack && target_is_for_iter, - ); - let d = crate::liveness::liveness_for(jc.payload.code_ptr) - .depth_at_py_pc() - .get(first_pypc as usize) - .copied() - .unwrap_or(0) as usize; + let permuted = predecessor_permuted_stack && target_is_for_iter; + let first_pypc = vstack_initial_py_pc(&jc.payload.metadata, start_pc, permuted); + let raw_depth = || { + crate::liveness::liveness_for(jc.payload.code_ptr) + .depth_at_py_pc() + .get(first_pypc as usize) + .copied() + .unwrap_or(0) as usize + }; + let d = if jc.payload.depth_containing_populated() { + // Mirror `vstack_initial_py_pc`'s branch: the exact block-head + // marker depth for a permuted FOR_ITER entry (falling back to the + // floor when `start_pc` is not a marker), else the floor segment + // depth for a normal entry. + let twin = if permuted { + jc.payload + .depth_block_head_for_jitcode_pc(start_pc) + .unwrap_or_else(|| { + jc.payload + .depth_containing_for_jitcode_pc(start_pc) + .unwrap_or(0) + }) + } else { + jc.payload + .depth_containing_for_jitcode_pc(start_pc) + .unwrap_or(0) + } as usize; + if pcmap_containing_audit_enabled() { + assert_eq!( + twin, + raw_depth(), + "PYRE_PCMAP_CONTAINING_AUDIT: vstack-initial depth twin diverged (start_pc {start_pc}, py {first_pypc})" + ); + } + twin + } else { + raw_depth() + }; (first_pypc, d, sym.nlocals()) }; let nvs = crate::virtualizable_gen::NUM_VABLE_SCALARS; @@ -806,7 +858,7 @@ pub(crate) fn vstack_enter_exception_handler( ctx.vstack_valid = false; return; } - let (handler_py, code_ptr) = unsafe { + let (handler_py, code_ptr, twin_depth, twin_populated) = unsafe { let jc = &*sym.jitcode(); if jc.payload.code_ptr.is_null() { ctx.vstack_valid = false; @@ -815,13 +867,32 @@ pub(crate) fn vstack_enter_exception_handler( ( vstack_containing_py_pc(&jc.payload.metadata, handler_jit_pc), jc.payload.code_ptr, + jc.payload.depth_containing_for_jitcode_pc(handler_jit_pc), + jc.payload.depth_containing_populated(), ) }; - let handler_depth = crate::liveness::liveness_for(code_ptr) - .depth_at_py_pc() - .get(handler_py as usize) - .copied() - .unwrap_or(0) as usize; + // Raw py_pc-keyed static-liveness read: the unpopulated-twin fallback + // (skeleton / fixture) and the audit oracle. + let raw_depth = || { + crate::liveness::liveness_for(code_ptr) + .depth_at_py_pc() + .get(handler_py as usize) + .copied() + .unwrap_or(0) as usize + }; + let handler_depth = if twin_populated { + let depth = twin_depth.unwrap_or(0) as usize; + if pcmap_containing_audit_enabled() { + assert_eq!( + depth, + raw_depth(), + "PYRE_PCMAP_CONTAINING_AUDIT: enter-handler containing-depth twin diverged at jit_pc {handler_jit_pc} (py {handler_py})" + ); + } + depth + } else { + raw_depth() + }; ctx.vstack_boxes.clear(); ctx.vstack_boxes.resize(handler_depth, OpRef::NONE); // The unwinder pushes the caught exception onto the new TOS. diff --git a/pyre/pyre-jit-trace/src/pyjitcode.rs b/pyre/pyre-jit-trace/src/pyjitcode.rs index 80dfcbe117b..a6918a652a9 100644 --- a/pyre/pyre-jit-trace/src/pyjitcode.rs +++ b/pyre/pyre-jit-trace/src/pyjitcode.rs @@ -122,6 +122,25 @@ pub struct PyJitCodeMetadata { /// skeleton / fixture. pub depth_trivia_marker_by_jit_pc: Vec<(usize, Option)>, pub depth_trivia_pred_by_jit_pc: Vec<(usize, Option)>, + /// Slice B (#73): floor-only STATIC-liveness depth twin of the + /// containing-opcode resolution. Keys mirror `py_floor_by_jit_pc` exactly + /// (same offsets, same order); each value is + /// `liveness_for(code).depth_at_py_pc().get(py).copied().unwrap_or(0)` for + /// that segment's py. NO block-head marker precedence and NO trivia skip — + /// a floor lookup reproduces `depth_at_py_pc[vstack_containing_py_pc(jit_pc)]` + /// for every jit_pc. Empty for skeleton / fixture metadata. + pub depth_containing_by_jit_pc: Vec<(u32, u16)>, + /// Slice D (#73): exact-match STATIC-liveness depth twin of the block-head + /// marker resolution (`metadata_block_head_py_pc`). Keys mirror + /// `block_head_py_by_jit_pc` exactly (same marker offsets, same order); each + /// value is + /// `liveness_for(code).depth_at_py_pc().get(block_head_py).copied().unwrap_or(0)` + /// for that marker's py. EXACT binary search only (no predecessor, no + /// floor) — a miss means `start_pc` is not a block-head marker and the + /// consumer falls back to the floor twin, mirroring + /// `vstack_initial_py_pc`'s `metadata_block_head_py_pc(...).unwrap_or(floor)`. + /// Empty for skeleton / fixture metadata. + pub depth_block_head_by_jit_pc: Vec<(usize, u16)>, /// Reproduces `pcdep_color_slots[skip_python_trivia_forward( /// python_pc_for_jitcode_pc(jit_pc))]`, resolved with the same exact-marker /// and predecessor-op-start tiers as the trivia-aware depth twin. @@ -167,6 +186,16 @@ pub struct PyJitCodeMetadata { /// fallback. Empty for skeleton / fixture. pub result_color_after_residual_marker_by_jit_pc: Vec<(usize, Option)>, pub result_color_after_residual_pred_by_jit_pc: Vec<(usize, Option)>, + /// Slice C (#73): STATIC-liveness operand-stack depth at the after-residual + /// fallthrough PC, keyed by JitCode byte offset with the same exact-marker / + /// predecessor-op-start split as `after_residual_marker_*`. Value = + /// `liveness_for(code).depth_at_py_pc().get(semantic_fallthrough_pc(py)).copied()` + /// for the RAW resolving py (no trivia skip — the runtime readers invert then + /// take the fallthrough directly, matching the `result_color_after_residual_*` + /// construction). `None` where the fallthrough overshoots the table end. + /// Empty for skeleton / fixture. + pub depth_after_residual_marker_by_jit_pc: Vec<(usize, Option)>, + pub depth_after_residual_pred_by_jit_pc: Vec<(usize, Option)>, /// Whether codewriter register allocation assigned non-identity frame /// colors. Skeleton and portal metadata leave this false. pub has_color_map: bool, @@ -539,6 +568,40 @@ impl PyJitCode { Self::predecessor_index(search).map(|i| table[i].1) } + /// Floor-only containing-opcode STATIC-liveness depth keyed by a JitCode + /// byte offset via the `depth_containing_by_jit_pc` twin. Same + /// `partition_point` floor pivot as `floor_segment_for_jitcode_pc` over the + /// same keys as `py_floor_by_jit_pc`, so it equals + /// `liveness_for(code).depth_at_py_pc().get(vstack_containing_py_pc(jit_pc) as usize).copied().unwrap_or(0)` + /// by construction — no block-head marker precedence, no trivia skip. `None` + /// when the twin is empty (skeleton / fixture) — see + /// [`Self::depth_containing_populated`]. + pub fn depth_containing_for_jitcode_pc(&self, jit_pc: usize) -> Option { + let table = &self.metadata.depth_containing_by_jit_pc; + let end = table.partition_point(|&(off, _)| (off as usize) <= jit_pc); + end.checked_sub(1).map(|idx| table[idx].1) + } + + /// Whether the floor-only containing-depth twin carries entries. `false` for + /// skeleton / fixture installs; the #8/#17 consumers fall back to the raw + /// py_pc-keyed static-liveness read there. + pub fn depth_containing_populated(&self) -> bool { + !self.metadata.depth_containing_by_jit_pc.is_empty() + } + + /// Exact-match block-head marker STATIC-liveness depth keyed by a JitCode + /// byte offset via the `depth_block_head_by_jit_pc` twin. + /// Equals `liveness_for(code).depth_at_py_pc().get(metadata_block_head_py_pc(jit_pc) as usize).copied().unwrap_or(0)` + /// when `jit_pc` is a block-head marker offset; `None` when it is not (the + /// consumer then falls back to the floor twin) or the table is empty. + pub fn depth_block_head_for_jitcode_pc(&self, jit_pc: usize) -> Option { + let table = &self.metadata.depth_block_head_by_jit_pc; + table + .binary_search_by_key(&jit_pc, |&(off, _)| off) + .ok() + .map(|i| table[i].1) + } + /// The const operand-stack slots keyed by a JitCode byte offset via /// the `const_ref_slots_by_jit_pc` predecessor twin. Returns the slots for /// a carried resume coordinate; `None` when the twin is empty (skeleton / @@ -686,6 +749,38 @@ impl PyJitCode { Self::predecessor_index(search).and_then(|i| pred[i].1) } + /// Slice C (#73): STATIC-liveness depth at the after-residual fallthrough + /// PC, keyed by a JitCode byte offset with the SAME exact-marker / + /// predecessor-op-start tiers as `after_residual_marker_for_jitcode_pc`. + /// Equals the raw + /// `liveness_for(code).depth_at_py_pc().get(semantic_fallthrough_pc(python_pc_for_jitcode_pc(jit_pc))).copied()` + /// by construction for a call-op coordinate, including a fallthrough overshoot + /// as an in-table `None`. `None` also when the twin is empty (skeleton / + /// fixture) — distinguish via [`Self::depth_after_residual_populated`]. + pub fn depth_after_residual_for_jitcode_pc(&self, jit_pc: usize) -> Option { + let marker = &self.metadata.depth_after_residual_marker_by_jit_pc; + let pred = &self.metadata.depth_after_residual_pred_by_jit_pc; + if marker.is_empty() && pred.is_empty() { + return None; + } + if let Ok(i) = marker.binary_search_by_key(&jit_pc, |&(off, _)| off) { + return marker[i].1; + } + let search = pred.binary_search_by_key(&jit_pc, |&(off, _)| off); + Self::predecessor_index(search).and_then(|i| pred[i].1) + } + + /// Whether the after-residual depth twin carries entries. `false` for + /// skeleton / fixture installs; consumers fall back to the raw py-keyed + /// static-liveness read there. + pub fn depth_after_residual_populated(&self) -> bool { + !self + .metadata + .depth_after_residual_marker_by_jit_pc + .is_empty() + || !self.metadata.depth_after_residual_pred_by_jit_pc.is_empty() + } + /// Post-`residual_call` catch resume marker keyed by a JitCode byte /// offset, resolved with the SAME exact-marker / predecessor-op-start /// tiers as `python_pc_for_jitcode_pc`. @@ -806,6 +901,8 @@ impl PyJitCode { depth_pred_by_jit_pc: Vec::new(), depth_trivia_marker_by_jit_pc: Vec::new(), depth_trivia_pred_by_jit_pc: Vec::new(), + depth_containing_by_jit_pc: Vec::new(), + depth_block_head_by_jit_pc: Vec::new(), pcdep_trivia_marker_by_jit_pc: Vec::new(), pcdep_trivia_pred_by_jit_pc: Vec::new(), const_ref_trivia_marker_by_jit_pc: Vec::new(), @@ -818,6 +915,8 @@ impl PyJitCode { after_residual_marker_pred_by_jit_pc: Vec::new(), result_color_after_residual_marker_by_jit_pc: Vec::new(), result_color_after_residual_pred_by_jit_pc: Vec::new(), + depth_after_residual_marker_by_jit_pc: Vec::new(), + depth_after_residual_pred_by_jit_pc: Vec::new(), // Encoder/decoder readers in // `get_list_of_active_boxes`, `regalloc::external/input_indices`, // and `setup_bridge_sym::portal_red_regs_at` sentinel-skip both diff --git a/pyre/pyre-jit-trace/src/state.rs b/pyre/pyre-jit-trace/src/state.rs index 6be88a403a7..96c2d921bb8 100644 --- a/pyre/pyre-jit-trace/src/state.rs +++ b/pyre/pyre-jit-trace/src/state.rs @@ -1907,12 +1907,6 @@ pub use crate::liveness::{LiveVars, liveness_for}; pub struct PyreJitState { #[vable(frame)] pub frame: usize, - /// blackhole.py:337 parity: liveness PC from rd_numb (setposition PC). - /// When set, `restore_guard_failure_values` uses this instead of - /// next_instr for liveness lookup — matching RPython's pattern where - /// `blackholeinterp.setposition(jitcode, pc)` is called before - /// `consume_one_section`. - pub resume_pc: Option, } /// Meta information for a trace — describes the shape of the code being traced. @@ -9693,9 +9687,6 @@ impl JitState for PyreJitState { exception: &majit_metainterp::blackhole::ExceptionState, ) -> bool { // resume.py:1077 consume_boxes parity: write values to the frame. - // blackhole.py:337: setposition(jitcode, pc) before consume_one_section — - // frame_pc from rd_numb is the liveness PC (orgpc). - self.resume_pc = Some(_frame_pc as usize); self.restore_guard_failure_values(meta, values, exception) } @@ -10530,10 +10521,7 @@ mod tests { } fn empty_state() -> PyreJitState { - PyreJitState { - frame: 0, - resume_pc: None, - } + PyreJitState { frame: 0 } } fn compile_function_body(src: &str) -> CodeObject { @@ -11002,10 +10990,7 @@ mod tests { frame.fix_array_ptrs(); let frame_ptr = (&mut *frame) as *mut PyFrame as usize; - let mut state = PyreJitState { - frame: frame_ptr, - resume_pc: None, - }; + let mut state = PyreJitState { frame: frame_ptr }; state.set_next_instr(0); state.set_valuestackdepth(4); let meta = PyreMeta { @@ -11645,6 +11630,8 @@ mod tests { depth_pred_by_jit_pc: vec![(0, 2)], depth_trivia_marker_by_jit_pc: vec![(0, Some(2))], depth_trivia_pred_by_jit_pc: vec![(0, Some(2))], + depth_containing_by_jit_pc: Vec::new(), + depth_block_head_by_jit_pc: Vec::new(), pcdep_trivia_marker_by_jit_pc: Vec::new(), pcdep_trivia_pred_by_jit_pc: Vec::new(), const_ref_trivia_marker_by_jit_pc: Vec::new(), @@ -11657,6 +11644,8 @@ mod tests { after_residual_marker_pred_by_jit_pc: Vec::new(), result_color_after_residual_marker_by_jit_pc: Vec::new(), result_color_after_residual_pred_by_jit_pc: Vec::new(), + depth_after_residual_marker_by_jit_pc: Vec::new(), + depth_after_residual_pred_by_jit_pc: Vec::new(), has_color_map: false, portal_frame_reg: 0, portal_ec_reg: 0, diff --git a/pyre/pyre-jit-trace/src/trace.rs b/pyre/pyre-jit-trace/src/trace.rs index 39b9ae95b50..a46e92ad954 100644 --- a/pyre/pyre-jit-trace/src/trace.rs +++ b/pyre/pyre-jit-trace/src/trace.rs @@ -2720,7 +2720,6 @@ fn loop_body_abort_permanent_pc(w_code: *const (), start_pc: usize) -> Option( crate::jitcode_dispatch::census_record("FullBodyWalk::CalleeAbortPermanent"); if crate::jitcode_dispatch::fbw_debug_abort_enabled() { eprintln!( - "[fbw-abort] start_pc={start_pc} callee={} abort_permanent_jit_pc={} \ - marker_py={}; declining callee-abort walk", - hit.callee_name, hit.marker_jit_pc, hit.marker_py_pc + "[fbw-abort] start_pc={start_pc} callee={} abort_permanent_jit_pc={}; \ + declining callee-abort walk", + hit.callee_name, hit.marker_jit_pc ); } fbw_decline(crate::driver::make_green_key(w_code, start_pc)); diff --git a/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs b/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs index f2658f38e9d..9117937af91 100644 --- a/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs +++ b/pyre/pyre-jit-trace/tests/multi_frame_restore_supported.rs @@ -12,10 +12,7 @@ use pyre_jit_trace::state::PyreJitState; #[test] fn pyre_jit_state_supports_multi_frame_restore() { - let state = PyreJitState { - frame: 0, - resume_pc: None, - }; + let state = PyreJitState { frame: 0 }; assert!( state.supports_multi_frame_restore(), "PyreJitState must override JitState::supports_multi_frame_restore \ diff --git a/pyre/pyre-jit/src/eval.rs b/pyre/pyre-jit/src/eval.rs index f57e9b3d034..c23556a7de4 100644 --- a/pyre/pyre-jit/src/eval.rs +++ b/pyre/pyre-jit/src/eval.rs @@ -8792,12 +8792,8 @@ pub(crate) fn decode_and_restore_guard_failure( if rd_numb.is_empty() { (dead_frame_typed.clone(), HashMap::new()) } else { - let (t, rd_numb_pc, virtuals_cache) = + let (t, virtuals_cache) = rebuild_typed_from_rd_numb(raw_values, rd_numb, rd_consts, exit_layout); - // blackhole.py:337 parity: setposition(jitcode, pc) before - // consume_one_section. rd_numb_pc = orgpc used by - // get_list_of_active_boxes during encoding. - jit_state.resume_pc = rd_numb_pc; (t, virtuals_cache) } }; @@ -8930,17 +8926,12 @@ pub(crate) fn decode_and_restore_guard_failure( /// (virtual to materialize). Consumes only the outermost frame's values, /// but splits frames by per-jitcode liveness so the box-section boundary is /// correct for multi-frame (inlined-callee) guards. -/// -/// Returns `(typed_values, rd_numb_frame_pc)`. The frame PC from rd_numb -/// is the liveness PC used by get_list_of_active_boxes during encoding. -/// The recovery side MUST use this same PC for expand — NOT next_instr -/// (which may differ by 1+ due to cache slots). fn rebuild_typed_from_rd_numb( raw_values: &[i64], rd_numb: &[u8], rd_consts: &[majit_ir::Const], exit_layout: &CompiledExitLayout, -) -> (Vec, Option, HashMap) { +) -> (Vec, HashMap) { use majit_ir::resumedata::rebuild_from_numbering; // resume.py:1049-1055 parity: bound each frame's box section by jitcode @@ -9087,15 +9078,7 @@ fn rebuild_typed_from_rd_numb( ); } - // The outer frame's decoded Python position is retained for resume-state - // hygiene. The live resume selection uses the rebuilt frame chain. - // pc=-1 = no-snapshot sentinel; screen it out (as build_resumed_frames does) - // so the negative word never reaches the `as usize` cast. - let rd_numb_pc = frames - .first() - .filter(|f| f.pc >= 0) - .map(|f| pyre_jit_trace::state::backxlat_py_pc(f.jitcode_index, f.pc) as usize); - (typed, rd_numb_pc, virtuals_cache) + (typed, virtuals_cache) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -9731,7 +9714,6 @@ pub(crate) fn build_jit_state( ) -> PyreJitState { let mut jit_state = PyreJitState { frame: frame as *const PyFrame as usize, - resume_pc: None, }; assert!( jit_state.sync_from_virtualizable(virtualizable_info), @@ -10858,10 +10840,7 @@ mod tests { "frame-value count must come from the same compiled jitcode liveness block" ); - let mut state = PyreJitState { - frame: frame_ptr, - resume_pc: Some(resume_pc), - }; + let mut state = PyreJitState { frame: frame_ptr }; state.set_next_instr(0); state.set_valuestackdepth(4); let meta = PyreMeta { diff --git a/pyre/pyre-jit/src/jit/codewriter.rs b/pyre/pyre-jit/src/jit/codewriter.rs index 687bc6e41d0..f82c829f0f6 100644 --- a/pyre/pyre-jit/src/jit/codewriter.rs +++ b/pyre/pyre-jit/src/jit/codewriter.rs @@ -13499,6 +13499,36 @@ impl CodeWriter { py_floor_by_jit_pc.insert(0, (0, 0)); } + // Floor-only depth twin of the containing-opcode resolution + // (`vstack_containing_py_pc`). Shares `py_floor_by_jit_pc`'s keys + // EXACTLY — no block-head marker precedence, no trivia skip — so a + // `partition_point(off <= jit_pc)` floor lookup reproduces + // `liveness_for(code).depth_at_py_pc().get(containing_py).copied().unwrap_or(0)` + // for every jit_pc, including the out-of-range → 0 collapse baked here. + let depth_containing_by_jit_pc: Vec<(u32, u16)> = { + let static_depth = + pyre_jit_trace::state::liveness_for(code as *const _).depth_at_py_pc(); + py_floor_by_jit_pc + .iter() + .map(|&(off, py)| (off, static_depth.get(py as usize).copied().unwrap_or(0))) + .collect() + }; + + // Exact-match depth twin of the block-head marker resolution + // (`metadata_block_head_py_pc`). Same keys as + // `block_head_py_by_jit_pc`, values from the SAME static liveness the + // floor twin reads, so an exact lookup reproduces + // `depth_at_py_pc[metadata_block_head_py_pc(jit_pc)]` at seam #7's + // permuted FOR_ITER-entry arm. Empty when the code has no block heads. + let depth_block_head_by_jit_pc: Vec<(usize, u16)> = { + let static_depth = + pyre_jit_trace::state::liveness_for(code as *const _).depth_at_py_pc(); + block_head_py_by_jit_pc + .iter() + .map(|&(off, py)| (off, static_depth.get(py as usize).copied().unwrap_or(0))) + .collect() + }; + // Sparse carry-forward sidecar: capture ONLY the py_pcs whose // dense marker the on-demand `derive_resume_marker` derivation cannot // reproduce from `first_jit_pc_by_py_pc` + `block_head_py_by_jit_pc`. @@ -13620,6 +13650,8 @@ impl CodeWriter { let mut result_color_after_residual_marker_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut result_color_after_residual_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); + let mut depth_after_residual_marker_by_jit_pc: Vec<(usize, Option)> = Vec::new(); + let mut depth_after_residual_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut after_residual_call_resume_marker_by_jit_pc: Vec<(usize, Option)> = Vec::new(); let mut after_residual_call_resume_pred_by_jit_pc: Vec<(usize, Option)> = Vec::new(); @@ -13747,9 +13779,11 @@ impl CodeWriter { let ft_rc = pyre_jit_trace::pyjitpl::semantic_fallthrough_pc(code, py as usize); result_color_after_residual_marker_by_jit_pc .push((off, result_color_at_pc.get(ft_rc).copied())); + depth_after_residual_marker_by_jit_pc.push((off, static_depth.get(ft_rc).copied())); } after_residual_marker_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); result_color_after_residual_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); + depth_after_residual_marker_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); // Op-start tier: predecessor scan, markers EXCLUDED. for (py, &pos) in first_jit_pc_by_py_pc.iter().enumerate() { if pos != usize::MAX { @@ -13762,10 +13796,13 @@ impl CodeWriter { let ft_rc = pyre_jit_trace::pyjitpl::semantic_fallthrough_pc(code, py); result_color_after_residual_pred_by_jit_pc .push((pos, result_color_at_pc.get(ft_rc).copied())); + depth_after_residual_pred_by_jit_pc + .push((pos, static_depth.get(ft_rc).copied())); } } after_residual_marker_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); result_color_after_residual_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); + depth_after_residual_pred_by_jit_pc.sort_unstable_by_key(|&(off, _)| off); // Post-residual-call catch marker twin: source values from the // same sparse construction inputs, while resolving its key // with the exact block-head / predecessor-op-start split of the @@ -13822,6 +13859,8 @@ impl CodeWriter { depth_pred_by_jit_pc, depth_trivia_marker_by_jit_pc, depth_trivia_pred_by_jit_pc, + depth_containing_by_jit_pc, + depth_block_head_by_jit_pc, pcdep_trivia_marker_by_jit_pc, pcdep_trivia_pred_by_jit_pc, const_ref_trivia_marker_by_jit_pc, @@ -13834,6 +13873,8 @@ impl CodeWriter { after_residual_marker_pred_by_jit_pc, result_color_after_residual_marker_by_jit_pc, result_color_after_residual_pred_by_jit_pc, + depth_after_residual_marker_by_jit_pc, + depth_after_residual_pred_by_jit_pc, has_color_map: !pcdep_color_slots.is_empty(), portal_frame_reg, portal_ec_reg,