X Tutup
Skip to content

Bytecode parity #7507

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
youknowone wants to merge 1 commit into
base: main
Choose a base branch
from
+138 −57
Draft

Bytecode parity #7507

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Lib/test/test_peepholer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,7 +645,6 @@ def containtest():
self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0)
self.check_lnotab(containtest)

@unittest.expectedFailure # TODO: RUSTPYTHON; no BUILD_LIST to BUILD_TUPLE optimization
def test_iterate_literal_list(self):
def forloop():
for x in [a, b]:
Expand Down
92 changes: 64 additions & 28 deletions crates/codegen/src/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1074,12 +1074,12 @@ impl Compiler {
.filter(|(_, s)| {
s.scope == SymbolScope::Cell || s.flags.contains(SymbolFlags::COMP_CELL)
})
.map(|(name, _)| name.clone())
.map(|(name, sym)| (name.clone(), sym.flags))
.collect();
let mut param_cells = Vec::new();
let mut nonparam_cells = Vec::new();
for name in cell_symbols {
if varname_cache.contains(&name) {
for (name, flags) in cell_symbols {
if flags.contains(SymbolFlags::PARAMETER) {
param_cells.push(name);
} else {
nonparam_cells.push(name);
Expand Down Expand Up @@ -1110,8 +1110,9 @@ impl Compiler {
}

// Handle implicit __conditional_annotations__ cell if needed
// Only for class scope - module scope uses NAME operations, not DEREF
if ste.has_conditional_annotations && scope_type == CompilerScope::Class {
if ste.has_conditional_annotations
&& matches!(scope_type, CompilerScope::Class | CompilerScope::Module)
{
cellvar_cache.insert("__conditional_annotations__".to_string());
}

Expand Down Expand Up @@ -1794,8 +1795,27 @@ impl Compiler {
let size_before = self.code_stack.len();
// Set future_annotations from symbol table (detected during symbol table scan)
self.future_annotations = symbol_table.future_annotations;

// Module-level __conditional_annotations__ cell
let has_module_cond_ann = symbol_table.has_conditional_annotations;
if has_module_cond_ann {
self.current_code_info()
.metadata
.cellvars
.insert("__conditional_annotations__".to_string());
}

self.symbol_table_stack.push(symbol_table);

// Emit MAKE_CELL for module-level cells (before RESUME)
if has_module_cond_ann {
let ncells = self.code_stack.last().unwrap().metadata.cellvars.len();
for i in 0..ncells {
let i_varnum: oparg::VarNum = u32::try_from(i).expect("too many cellvars").into();
emit!(self, Instruction::MakeCell { i: i_varnum });
}
}

self.emit_resume_for_scope(CompilerScope::Module, 1);

let (doc, statements) = split_doc(&body.body, &self.opts);
Expand Down Expand Up @@ -5437,7 +5457,24 @@ impl Compiler {
let mut end_async_for_target = BlockIdx::NULL;

// The thing iterated:
self.compile_expression(iter)?;
// Optimize: `for x in [a, b, c]` → use tuple instead of list
// (list creation is wasteful for iteration)
// Skip optimization if any element is starred (e.g., `[a, *b, c]`)
if let ast::Expr::List(ast::ExprList { elts, .. }) = iter
&& !elts.iter().any(|e| matches!(e, ast::Expr::Starred(_)))
{
for elt in elts {
self.compile_expression(elt)?;
}
emit!(
self,
Instruction::BuildTuple {
count: u32::try_from(elts.len()).expect("too many elements"),
}
);
} else {
self.compile_expression(iter)?;
}

if is_async {
if self.ctx.func != FunctionContext::AsyncFunction {
Expand Down Expand Up @@ -7033,6 +7070,7 @@ impl Compiler {
/// For `And`, emits `PopJumpIfFalse`; for `Or`, emits `PopJumpIfTrue`.
fn emit_short_circuit_test(&mut self, op: &ast::BoolOp, target: BlockIdx) {
emit!(self, Instruction::Copy { i: 1 });
emit!(self, Instruction::ToBool);
match op {
ast::BoolOp::And => {
emit!(self, Instruction::PopJumpIfFalse { delta: target });
Expand Down Expand Up @@ -7622,8 +7660,23 @@ impl Compiler {
self.compile_expr_tstring(tstring)?;
}
ast::Expr::StringLiteral(string) => {
let value = self.compile_string_value(string);
self.emit_load_const(ConstantData::Str { value });
let value = string.value.to_str();
if value.contains(char::REPLACEMENT_CHARACTER) {
let value = string
.value
.iter()
.map(|lit| {
let source = self.source_file.slice(lit.range);
crate::string_parser::parse_string_literal(source, lit.flags.into())
})
.collect();
// might have a surrogate literal; should reparse to be sure
self.emit_load_const(ConstantData::Str { value });
} else {
self.emit_load_const(ConstantData::Str {
value: value.into(),
});
}
}
ast::Expr::BytesLiteral(bytes) => {
let iter = bytes.value.iter().flat_map(|x| x.iter().copied());
Expand Down Expand Up @@ -8554,24 +8607,6 @@ impl Compiler {

// fn block_done()

/// Convert a string literal AST node to Wtf8Buf, handling surrogates correctly.
fn compile_string_value(&self, string: &ast::ExprStringLiteral) -> Wtf8Buf {
let value = string.value.to_str();
if value.contains(char::REPLACEMENT_CHARACTER) {
// Might have a surrogate literal; reparse from source to preserve them
string
.value
.iter()
.map(|lit| {
let source = self.source_file.slice(lit.range);
crate::string_parser::parse_string_literal(source, lit.flags.into())
})
.collect()
} else {
value.into()
}
}

fn arg_constant(&mut self, constant: ConstantData) -> oparg::ConstIdx {
let info = self.current_code_info();
info.metadata.consts.insert_full(constant).0.to_u32().into()
Expand Down Expand Up @@ -8601,8 +8636,9 @@ impl Compiler {
}
},
ast::Expr::StringLiteral(s) => {
let value = self.compile_string_value(s);
constants.push(ConstantData::Str { value });
constants.push(ConstantData::Str {
value: s.value.to_string().into(),
});
}
ast::Expr::BytesLiteral(b) => {
constants.push(ConstantData::Bytes {
Expand Down
18 changes: 13 additions & 5 deletions crates/codegen/src/ir.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2172,11 +2172,19 @@ pub(crate) fn label_exception_targets(blocks: &mut [Block]) {
blocks[bi].instructions[i].except_handler = handler_info;

// Track YIELD_VALUE except stack depth
if matches!(
blocks[bi].instructions[i].instr.real(),
Some(Instruction::YieldValue { .. })
) {
last_yield_except_depth = stack.len() as i32;
// Only count for direct yield (arg=0), not yield-from/await (arg=1)
// The yield-from's internal SETUP_FINALLY is not an external except depth
if let Some(Instruction::YieldValue { .. }) =
blocks[bi].instructions[i].instr.real()
{
let yield_arg = u32::from(blocks[bi].instructions[i].arg);
if yield_arg == 0 {
// Direct yield: count actual except depth
last_yield_except_depth = stack.len() as i32;
} else {
// yield-from/await: subtract 1 for the internal SETUP_FINALLY
last_yield_except_depth = (stack.len() as i32) - 1;
}
}

// Set RESUME DEPTH1 flag based on last yield's except depth
Expand Down
37 changes: 23 additions & 14 deletions crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

47 changes: 38 additions & 9 deletions crates/codegen/src/symboltable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,8 @@ fn drop_class_free(symbol_table: &mut SymbolTable, newfree: &mut IndexSet<String
}

// Classes with function definitions need __classdict__ for PEP 649
if !symbol_table.needs_classdict {
// (but not when `from __future__ import annotations` is active)
if !symbol_table.needs_classdict && !symbol_table.future_annotations {
let has_functions = symbol_table.sub_tables.iter().any(|t| {
matches!(
t.typ,
Expand Down Expand Up @@ -571,6 +572,9 @@ impl SymbolTableAnalyzer {
newfree.extend(child_free);
}
if let Some(ann_free) = annotation_free {
// Propagate annotation-scope free names to this scope so
// implicit class-scope cells (__classdict__/__conditional_annotations__)
// can be materialized by drop_class_free when needed.
newfree.extend(ann_free);
}

Expand Down Expand Up @@ -758,6 +762,12 @@ impl SymbolTableAnalyzer {
if let Some(decl_depth) = decl_depth {
// decl_depth is the number of tables between the current one and
// the one that declared the cell var
// For implicit class scope variables (__classdict__, __conditional_annotations__),
// only propagate free to annotation/type-param scopes, not regular functions.
// Regular method functions don't need these in their freevars.
let is_class_implicit =
name == "__classdict__" || name == "__conditional_annotations__";

for (table, typ) in self.tables.iter_mut().rev().take(decl_depth) {
if let CompilerScope::Class = typ {
if let Some(free_class) = table.get_mut(name) {
Expand All @@ -768,10 +778,19 @@ impl SymbolTableAnalyzer {
symbol.scope = SymbolScope::Free;
table.insert(name.to_owned(), symbol);
}
} else if is_class_implicit
&& matches!(
typ,
CompilerScope::Function
| CompilerScope::AsyncFunction
| CompilerScope::Lambda
)
{
// Skip: don't add __classdict__/__conditional_annotations__
// as free vars in regular functions — only annotation/type scopes need them
} else if !table.contains_key(name) {
let mut symbol = Symbol::new(name);
symbol.scope = SymbolScope::Free;
// symbol.is_referenced = true;
table.insert(name.to_owned(), symbol);
}
}
Expand Down Expand Up @@ -1226,20 +1245,30 @@ impl SymbolTableBuilder {
}

fn scan_annotation(&mut self, annotation: &ast::Expr) -> SymbolTableResult {
self.scan_annotation_inner(annotation, false)
}

/// Scan an annotation from an AnnAssign statement (can be conditional)
fn scan_ann_assign_annotation(&mut self, annotation: &ast::Expr) -> SymbolTableResult {
self.scan_annotation_inner(annotation, true)
}

fn scan_annotation_inner(
&mut self,
annotation: &ast::Expr,
is_ann_assign: bool,
) -> SymbolTableResult {
let current_scope = self.tables.last().map(|t| t.typ);

// PEP 649: Check if this is a conditional annotation
// Module-level: always conditional (module may be partially executed)
// Class-level: conditional only when inside if/for/while/etc.
if !self.future_annotations {
// PEP 649: Only AnnAssign annotations can be conditional.
// Function parameter/return annotations are never conditional.
if is_ann_assign && !self.future_annotations {
let is_conditional = matches!(current_scope, Some(CompilerScope::Module))
|| (matches!(current_scope, Some(CompilerScope::Class))
&& self.in_conditional_block);

if is_conditional && !self.tables.last().unwrap().has_conditional_annotations {
self.tables.last_mut().unwrap().has_conditional_annotations = true;
// Register __conditional_annotations__ as both Assigned and Used so that
// it becomes a Cell variable in class scope (children reference it as Free)
self.register_name(
"__conditional_annotations__",
SymbolUsage::Assigned,
Expand Down Expand Up @@ -1571,7 +1600,7 @@ impl SymbolTableBuilder {
// sub_tables that cause mismatch in the annotation scope's sub_table index.
let is_simple_name = *simple && matches!(&**target, Expr::Name(_));
if is_simple_name {
self.scan_annotation(annotation)?;
self.scan_ann_assign_annotation(annotation)?;
} else {
// Still validate annotation for forbidden expressions
// (yield, await, named) even for non-simple targets.
Expand Down
Loading
X Tutup