-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Optimize coroutine exception handling and fix gen_throw traceback #7166
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,11 @@ | ||
| use crate::{ | ||
| AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, | ||
| builtins::{PyBaseExceptionRef, PyStrRef}, | ||
| builtins::PyStrRef, | ||
| common::lock::PyMutex, | ||
| exceptions::types::PyBaseException, | ||
| frame::{ExecutionResult, FrameOwner, FrameRef}, | ||
| frame::{ExecutionResult, Frame, FrameOwner, FrameRef}, | ||
| function::OptionalArg, | ||
| object::{Traverse, TraverseFn}, | ||
| object::{PyAtomicRef, Traverse, TraverseFn}, | ||
| protocol::PyIterReturn, | ||
| }; | ||
| use crossbeam_utils::atomic::AtomicCell; | ||
|
|
@@ -36,15 +36,17 @@ pub struct Coro { | |
| // _weakreflist | ||
| name: PyMutex<PyStrRef>, | ||
| qualname: PyMutex<PyStrRef>, | ||
| exception: PyMutex<Option<PyBaseExceptionRef>>, // exc_state | ||
| exception: PyAtomicRef<Option<PyBaseException>>, // exc_state | ||
| } | ||
|
|
||
| unsafe impl Traverse for Coro { | ||
| fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { | ||
| self.frame.traverse(tracer_fn); | ||
| self.name.traverse(tracer_fn); | ||
| self.qualname.traverse(tracer_fn); | ||
| self.exception.traverse(tracer_fn); | ||
| if let Some(exc) = self.exception.deref() { | ||
| exc.traverse(tracer_fn); | ||
| } | ||
| } | ||
|
Comment on lines
+39
to
50
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🏁 Script executed: find . -name "*.rs" -type f | xargs grep -l "struct PyAtomicRef" | head -5Repository: RustPython/RustPython Length of output: 94 🏁 Script executed: rg -n "struct PyAtomicRef|type PyAtomicRef" --type rsRepository: RustPython/RustPython Length of output: 91 🏁 Script executed: # Get context around lines 39-50 and 64-74 in coroutine.rs
cat -n crates/vm/src/coroutine.rs | head -80Repository: RustPython/RustPython Length of output: 3152 🏁 Script executed: # Search for PyAtomicRef and Traverse definitions
rg -n "trait Traverse|fn traverse" crates/vm/src --type rs -A 3Repository: RustPython/RustPython Length of output: 91 🏁 Script executed: rg "struct PyAtomicRef" crates/vm/src/object/ -A 10Repository: RustPython/RustPython Length of output: 725 🏁 Script executed: rg "impl.*PyAtomicRef" crates/vm/src/object/ -A 5Repository: RustPython/RustPython Length of output: 5675 🏁 Script executed: rg "fn deref" crates/vm/src/object/ -B 2 -A 5Repository: RustPython/RustPython Length of output: 4162 🏁 Script executed: rg "trait Traverse" crates/vm/src/object/ -A 5Repository: RustPython/RustPython Length of output: 678 🏁 Script executed: rg -n "run_with_context\|exception.*swap\|exception.*deref" crates/vm/src/coroutine.rs -B 2 -A 2Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: cat -n crates/vm/src/coroutine.rs | sed -n '85,130p'Repository: RustPython/RustPython Length of output: 1825 🏁 Script executed: rg -n "\.exception\." crates/vm/src/coroutine.rsRepository: RustPython/RustPython Length of output: 183 🏁 Script executed: # Check how Traverse is called (where traversal happens)
rg -n "\.traverse\(" crates/vm/src/ -B 2 -A 2 | head -50Repository: RustPython/RustPython Length of output: 3083 🏁 Script executed: rg -n "fn.*gc|GC" crates/vm/src/ --type rs | head -20Repository: RustPython/RustPython Length of output: 91 🏁 Script executed: rg -n "\.traverse(" crates/vm/src/object/ -B 3 -A 1 | head -80Repository: RustPython/RustPython Length of output: 137 🏁 Script executed: # Look for GC collection cycle and when it calls traverse
rg -n "impl.*Traverse\|traverse\(" crates/vm/src/gc.rs 2>/dev/null | head -30Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: find crates -name "gc.rs" -type fRepository: RustPython/RustPython Length of output: 91 🏁 Script executed: rg -n "traverse" crates/vm/src/stdlib/gc.rs -B 2 -A 2Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Look at coroutine integration with GC
cat crates/vm/src/stdlib/gc.rs | head -100Repository: RustPython/RustPython Length of output: 3099 🏁 Script executed: # Search for when GC collect is called during execution
rg -n "collect\|gc_collect" crates/vm/src/ --type-listRepository: RustPython/RustPython Length of output: 454 🏁 Script executed: # Better search: look for Traverse implementations and understand execution model
rg -n "unsafe impl Traverse" crates/vm/src/ | head -30Repository: RustPython/RustPython Length of output: 2313 🏁 Script executed: # Let's look at the full context of run_with_context and exception handling
cat -n crates/vm/src/coroutine.rs | sed -n '90,120p'Repository: RustPython/RustPython Length of output: 1298 🏁 Script executed: # Check how resume_gen_frame works
rg -n "fn resume_gen_frame" crates/vm/src/ -A 15Repository: RustPython/RustPython Length of output: 1122 🏁 Script executed: # Look at how exception is used in the frame/vm context
rg -n "current_exception\|set_exception" crates/vm/src/ -B 1 -A 2 | head -80Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Check if there are any comments or discussions about atomic ref safety
rg -n "PyAtomicRef" crates/vm/src/ -B 2 -A 2 | head -60Repository: RustPython/RustPython Length of output: 3902 🏁 Script executed: # Check if there's any GC triggering during frame execution
rg -n "gc_collect\|collect_force" crates/vm/src/ -B 3 -A 3 | head -80Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Look at how the running flag's exclusive access guarantee applies to Traverse
cat -n crates/vm/src/coroutine.rs | sed -n '42,52p'Repository: RustPython/RustPython Length of output: 460 🏁 Script executed: # Check the PyAtomicRef safety documentation and contracts
cat -n crates/vm/src/object/ext.rs | sed -n '244,320p'Repository: RustPython/RustPython Length of output: 3032 🏁 Script executed: # Get more context on PyAtomicRef swap and deref for Option<T>
cat -n crates/vm/src/object/ext.rs | sed -n '340,380p'Repository: RustPython/RustPython Length of output: 1714 🏁 Script executed: # Check the full Traverse trait documentation
cat -n crates/vm/src/object/traverse.rs | head -80Repository: RustPython/RustPython Length of output: 3580 🏁 Script executed: # Verify: Can GC run while running flag is set?
rg -n "running.store\|running.compare_exchange\|running.load" crates/vm/src/coroutine.rsRepository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Look at how GC traversal interacts with running coroutines
cat -n crates/vm/src/gc_state.rs 2>/dev/null | head -50Repository: RustPython/RustPython Length of output: 2071 🏁 Script executed: # Search for where PyAtomicRef deref safety is discussed or tested
rg -n "deref\|swap" crates/vm/src/object/ext.rs | grep -A2 -B2 "unsafe"Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Verify: is there any GC trigger during frame execution that might call traverse?
rg -n "gc_collect\|trigger.*gc\|AUTO_GC" crates/vm/src/ | head -20Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Check if there are existing tests or comments about concurrent traverse/swap safety
rg -n "test.*traverse\|test.*atomic" crates/vm/src/ -B 2 -A 5 | head -40Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Check if exception.deref could be called while exception is being swapped
# Look at the exact sequence in run_with_context more carefully
cat -n crates/vm/src/coroutine.rs | sed -n '99,116p'Repository: RustPython/RustPython Length of output: 926 🏁 Script executed: # Verify the AcqRel vs Relaxed ordering implications
rg -n "Ordering::" crates/vm/src/object/ext.rs | grep -A2 -B2 "swap\|deref"Repository: RustPython/RustPython Length of output: 846 🏁 Script executed: # Check if there's a panic-safety issue with the running flag cleanup
cat -n crates/vm/src/coroutine.rs | sed -n '90,116p'Repository: RustPython/RustPython Length of output: 1213 🏁 Script executed: # Check for panic-safety guard patterns in similar code
rg -n "scopeguard\|guard\|Drop" crates/vm/src/coroutine.rsRepository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Look for how other code handles the running flag or similar state cleanup
rg -n "compare_exchange.*running\|running.*store" crates/vm/src/ -B 3 -A 3 | head -50Repository: RustPython/RustPython Length of output: 47 🏁 Script executed: # Check if resume_gen_frame could panic and what the consequences would be
cat -n crates/vm/src/vm/mod.rs | sed -n '1091,1130p'Repository: RustPython/RustPython Length of output: 1973 Wrap If 🤖 Prompt for AI Agents |
||
| } | ||
|
|
||
|
|
@@ -65,7 +67,7 @@ impl Coro { | |
| frame, | ||
| closed: AtomicCell::new(false), | ||
| running: AtomicCell::new(false), | ||
| exception: PyMutex::default(), | ||
| exception: PyAtomicRef::from(None), | ||
| name: PyMutex::new(name), | ||
| qualname: PyMutex::new(qualname), | ||
| } | ||
|
|
@@ -92,33 +94,20 @@ impl Coro { | |
| func: F, | ||
| ) -> PyResult<ExecutionResult> | ||
| where | ||
| F: FnOnce(FrameRef) -> PyResult<ExecutionResult>, | ||
| F: FnOnce(&Py<Frame>) -> PyResult<ExecutionResult>, | ||
| { | ||
| if self.running.compare_exchange(false, true).is_err() { | ||
| return Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))); | ||
| } | ||
|
|
||
| // swap exception state | ||
| // Get generator's saved exception state from last yield | ||
| let gen_exc = self.exception.lock().take(); | ||
|
|
||
| // Use a slot to capture generator's exception state before with_frame pops | ||
| let exception_slot = &self.exception; | ||
| // SAFETY: running.compare_exchange guarantees exclusive access | ||
| let gen_exc = unsafe { self.exception.swap(None) }; | ||
| let exception_ptr = &self.exception as *const PyAtomicRef<Option<PyBaseException>>; | ||
|
|
||
| // Run the generator frame | ||
| // with_frame does push_exception(None) which creates a new exception context | ||
| // The caller's exception remains in the chain via prev, so topmost_exception() | ||
| // will find it if generator's exception is None | ||
| let result = vm.with_frame(self.frame.clone(), |f| { | ||
| // with_frame pushed None, creating: { exc: None, prev: caller's exc_info } | ||
| // Pop None and push generator's exception instead | ||
| // This maintains the chain: { exc: gen_exc, prev: caller's exc_info } | ||
| vm.pop_exception(); | ||
| vm.push_exception(gen_exc); | ||
| let result = vm.resume_gen_frame(&self.frame, gen_exc, |f| { | ||
| let result = func(f); | ||
| // Save generator's exception state BEFORE with_frame pops | ||
| // This is the generator's current exception context | ||
| *exception_slot.lock() = vm.current_exception(); | ||
| // SAFETY: exclusive access guaranteed by running flag | ||
| let _old = unsafe { (*exception_ptr).swap(vm.current_exception()) }; | ||
| result | ||
| }); | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -463,7 +463,7 @@ impl ExecutingFrame<'_> { | |
| // Execute until return or exception: | ||
| let instructions = &self.code.instructions; | ||
| let mut arg_state = bytecode::OpArgState::default(); | ||
| let mut prev_line: usize = 0; | ||
| let mut prev_line: u32 = 0; | ||
| loop { | ||
| let idx = self.lasti() as usize; | ||
| // Fire 'line' trace event when line number changes. | ||
|
|
@@ -472,9 +472,9 @@ impl ExecutingFrame<'_> { | |
| if vm.use_tracing.get() | ||
| && !vm.is_none(&self.object.trace.lock()) | ||
| && let Some((loc, _)) = self.code.locations.get(idx) | ||
| && loc.line.get() != prev_line | ||
| && loc.line.get() as u32 != prev_line | ||
| { | ||
| prev_line = loc.line.get(); | ||
| prev_line = loc.line.get() as u32; | ||
| vm.trace_event(crate::protocol::TraceEvent::Line, None)?; | ||
| } | ||
| self.update_lasti(|i| *i += 1); | ||
|
|
@@ -543,13 +543,16 @@ impl ExecutingFrame<'_> { | |
| // Check if this is a RERAISE instruction | ||
| // Both AnyInstruction::Raise { kind: Reraise/ReraiseFromStack } and | ||
| // AnyInstruction::Reraise are reraise operations that should not add | ||
| // new traceback entries | ||
| // new traceback entries. | ||
| // EndAsyncFor and CleanupThrow also re-raise non-matching exceptions. | ||
| let is_reraise = match op { | ||
| Instruction::RaiseVarargs { kind } => matches!( | ||
| kind.get(arg), | ||
| bytecode::RaiseKind::BareRaise | bytecode::RaiseKind::ReraiseFromStack | ||
| ), | ||
| Instruction::Reraise { .. } => true, | ||
| Instruction::Reraise { .. } | ||
| | Instruction::EndAsyncFor | ||
| | Instruction::CleanupThrow => true, | ||
| _ => false, | ||
| }; | ||
|
Comment on lines
543
to
557
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🏁 Script executed: # First, let's understand the code changes and find where these instructions are used
rg -n "EndAsyncFor|CleanupThrow" crates/vm/src/ -A 3 -B 3Repository: RustPython/RustPython Length of output: 2834 🏁 Script executed: # Find test coverage for these instructions
rg -n "EndAsyncFor|CleanupThrow|end_async_for|cleanup_throw" --type py tests/ crates/Repository: RustPython/RustPython Length of output: 111 🏁 Script executed: # Look at the full context of frame.rs around the changed lines
sed -n '530,570p' crates/vm/src/frame.rsRepository: RustPython/RustPython Length of output: 2330 🏁 Script executed: # Check if there are any comments or documentation about reraise behavior
rg -n "reraise|RERAISE" crates/vm/src/frame.rs -B 2 -A 2Repository: RustPython/RustPython Length of output: 3124 🏁 Script executed: # Search for where RaiseVarargs and Reraise instructions are handled
rg -n "Instruction::Reraise|Instruction::RaiseVarargs" crates/vm/src/frame.rs -B 2 -A 5Repository: RustPython/RustPython Length of output: 2386 🏁 Script executed: # Find tests related to async for loops and exception handling
find . -name "*.py" -type f | xargs grep -l "async for\|EndAsyncFor" 2>/dev/null | head -20Repository: RustPython/RustPython Length of output: 703 🏁 Script executed: # Search for existing exception handling tests
find . -name "*.py" -type f | xargs grep -l "StopAsyncIteration\|cleanup" 2>/dev/null | head -20Repository: RustPython/RustPython Length of output: 636 🏁 Script executed: # Look for generator/async tests
fd -e py | xargs grep -l "async def\|async for" | head -10Repository: RustPython/RustPython Length of output: 337 🏁 Script executed: # Check bytecode instruction definitions
rg -n "EndAsyncFor|CleanupThrow|CLEANUP_THROW|END_ASYNC_FOR" crates/ --type rust -B 2 -A 2Repository: RustPython/RustPython Length of output: 9658 🏁 Script executed: # Look at EndAsyncFor implementation details
sed -n '1139,1165p' crates/vm/src/frame.rsRepository: RustPython/RustPython Length of output: 1186 🏁 Script executed: # Look at CleanupThrow implementation details
sed -n '2257,2280p' crates/vm/src/frame.rsRepository: RustPython/RustPython Length of output: 1428 Verify targeted test coverage for reraise paths in The classification of these instructions as reraise-like is correct: both conditionally re-raise when their expected exception (StopAsyncIteration/StopIteration) is not matched. When they do re-raise, they should not add new traceback entries, matching CPython semantics. However, ensure targeted coverage exists for the re-raise paths:
Perform a clean build before testing: 🤖 Prompt for AI Agents |
||
|
|
||
|
|
@@ -653,6 +656,19 @@ impl ExecutingFrame<'_> { | |
| Ok(()) | ||
| }; | ||
| if let Err(err) = close_result { | ||
| let idx = self.lasti().saturating_sub(1) as usize; | ||
| if idx < self.code.locations.len() { | ||
| let (loc, _end_loc) = self.code.locations[idx]; | ||
| let next = err.__traceback__(); | ||
| let new_traceback = PyTraceback::new( | ||
| next, | ||
| self.object.to_owned(), | ||
| idx as u32 * 2, | ||
| loc.line, | ||
| ); | ||
| err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); | ||
| } | ||
|
|
||
| self.push_value(vm.ctx.none()); | ||
| vm.contextualize_exception(&err); | ||
| return match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { | ||
|
|
@@ -678,6 +694,23 @@ impl ExecutingFrame<'_> { | |
| Either::B(meth) => meth.call((exc_type, exc_val, exc_tb), vm), | ||
| }; | ||
| return ret.map(ExecutionResult::Yield).or_else(|err| { | ||
| // Add traceback entry for the yield-from/await point. | ||
| // gen_send_ex2 resumes the frame with a pending exception, | ||
| // which goes through error: → PyTraceBack_Here. We add the | ||
| // entry here before calling unwind_blocks. | ||
| let idx = self.lasti().saturating_sub(1) as usize; | ||
| if idx < self.code.locations.len() { | ||
| let (loc, _end_loc) = self.code.locations[idx]; | ||
| let next = err.__traceback__(); | ||
| let new_traceback = PyTraceback::new( | ||
| next, | ||
| self.object.to_owned(), | ||
| idx as u32 * 2, | ||
| loc.line, | ||
| ); | ||
| err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); | ||
| } | ||
|
|
||
| self.push_value(vm.ctx.none()); | ||
| vm.contextualize_exception(&err); | ||
| match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.