From d3bdba9a94c73ef8593cd047c42e945b04c1a830 Mon Sep 17 00:00:00 2001 From: Techuouo520 Date: Mon, 25 May 2026 20:22:39 +0800 Subject: [PATCH 1/3] Add initial Python 3.14 bytecode support --- ASTNode.h | 14 ++- ASTree.cpp | 260 ++++++++++++++++++++++++++++++++++-------- CMakeLists.txt | 1 + bytecode.cpp | 98 ++++++++++++++-- bytecode.h | 1 + bytecode_ops.inl | 13 +++ bytes/python_3_14.cpp | 147 ++++++++++++++++++++++++ pyc_module.cpp | 8 +- pyc_module.h | 1 + pyc_object.cpp | 2 + pyc_object.h | 1 + pyc_sequence.cpp | 20 ++++ pyc_sequence.h | 18 +++ pycdas.cpp | 10 ++ 14 files changed, 536 insertions(+), 58 deletions(-) create mode 100644 bytes/python_3_14.cpp diff --git a/ASTNode.h b/ASTNode.h index 98760dbf5..283ed5370 100644 --- a/ASTNode.h +++ b/ASTNode.h @@ -4,6 +4,7 @@ #include "pyc_module.h" #include #include +#include /* Similar interface to PycObject, so PycRef can work on it... * * However, this does *NOT* mean the two are interchangeable! */ @@ -18,7 +19,7 @@ class ASTNode { NODE_COMPREHENSION, NODE_LOADBUILDCLASS, NODE_AWAITABLE, NODE_FORMATTEDVALUE, NODE_JOINEDSTR, NODE_CONST_MAP, NODE_ANNOTATED_VAR, NODE_CHAINSTORE, NODE_TERNARY, - NODE_KW_NAMES_MAP, + NODE_KW_NAMES_MAP, NODE_UNSUPPORTED, // Empty node types NODE_LOCALS, @@ -105,6 +106,17 @@ class ASTObject : public ASTNode { PycRef m_obj; }; +class ASTUnsupported : public ASTNode { +public: + ASTUnsupported(std::string text) + : ASTNode(NODE_UNSUPPORTED), m_text(std::move(text)) { } + + const std::string& text() const { return m_text; } + +private: + std::string m_text; +}; + class ASTUnary : public ASTNode { public: diff --git a/ASTree.cpp b/ASTree.cpp index f837152f9..1856d64de 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -247,6 +247,14 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) switch (opcode) { case Pyc::BINARY_OP_A: { + if (mod->verCompare(3, 14) >= 0 && operand == 26) { + PycRef subscr = stack.top(); + stack.pop(); + PycRef src = stack.top(); + stack.pop(); + stack.push(new ASTSubscr(src, subscr)); + break; + } ASTBinary::BinOp op = ASTBinary::from_binary_op(operand); if (op == ASTBinary::BIN_INVALID) fprintf(stderr, "Unsupported `BINARY_OP` operand value: %d\n", operand); @@ -510,6 +518,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::CALL_A: case Pyc::CALL_FUNCTION_A: case Pyc::INSTRUMENTED_CALL_A: + case Pyc::CALL_KW_A: + case Pyc::INSTRUMENTED_CALL_KW_A: { int kwparams = (operand & 0xFF00) >> 8; int pparams = (operand & 0xFF); @@ -517,39 +527,41 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) ASTCall::pparam_t pparamList; /* Test for the load build class function */ - stack_hist.push(stack); - int basecnt = 0; - ASTTuple::value_t bases; - bases.resize(basecnt); - PycRef TOS = stack.top(); - int TOS_type = TOS.type(); - // bases are NODE_NAME and NODE_BINARY at TOS - while (TOS_type == ASTNode::NODE_NAME || TOS_type == ASTNode::NODE_BINARY) { - bases.resize(basecnt + 1); - bases[basecnt] = TOS; - basecnt++; + if (opcode != Pyc::CALL_KW_A && opcode != Pyc::INSTRUMENTED_CALL_KW_A) { + stack_hist.push(stack); + int basecnt = 0; + ASTTuple::value_t bases; + bases.resize(basecnt); + PycRef TOS = stack.top(); + int TOS_type = TOS.type(); + // bases are NODE_NAME and NODE_BINARY at TOS + while (TOS_type == ASTNode::NODE_NAME || TOS_type == ASTNode::NODE_BINARY) { + bases.resize(basecnt + 1); + bases[basecnt] = TOS; + basecnt++; + stack.pop(); + TOS = stack.top(); + TOS_type = TOS.type(); + } + // qualified name is PycString at TOS + PycRef name = stack.top(); stack.pop(); - TOS = stack.top(); - TOS_type = TOS.type(); - } - // qualified name is PycString at TOS - PycRef name = stack.top(); - stack.pop(); - PycRef function = stack.top(); - stack.pop(); - PycRef loadbuild = stack.top(); - stack.pop(); - int loadbuild_type = loadbuild.type(); - if (loadbuild_type == ASTNode::NODE_LOADBUILDCLASS) { - PycRef call = new ASTCall(function, pparamList, kwparamList); - stack.push(new ASTClass(call, new ASTTuple(bases), name)); - stack_hist.pop(); - break; - } - else - { - stack = stack_hist.top(); - stack_hist.pop(); + PycRef function = stack.top(); + stack.pop(); + PycRef loadbuild = stack.top(); + stack.pop(); + int loadbuild_type = loadbuild.type(); + if (loadbuild_type == ASTNode::NODE_LOADBUILDCLASS) { + PycRef call = new ASTCall(function, pparamList, kwparamList); + stack.push(new ASTClass(call, new ASTTuple(bases), name)); + stack_hist.pop(); + break; + } + else + { + stack = stack_hist.top(); + stack_hist.pop(); + } } /* @@ -558,7 +570,18 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) co_consts[consti] must be a tuple of strings. New in version 3.11. */ - if (mod->verCompare(3, 11) >= 0) { + if (opcode == Pyc::CALL_KW_A || opcode == Pyc::INSTRUMENTED_CALL_KW_A) { + PycRef keys = stack.top().cast(); + stack.pop(); + PycTuple::value_t key_values = keys->object().cast()->values(); + kwparams = (int)key_values.size(); + pparams = operand - kwparams; + for (int i = 0; i < kwparams; i++) { + PycRef val = stack.top(); + stack.pop(); + kwparamList.push_front(std::make_pair(new ASTObject(key_values[kwparams - i - 1]), val)); + } + } else if (mod->verCompare(3, 11) >= 0) { PycRef object_or_map = stack.top(); if (object_or_map.type() == ASTNode::NODE_KW_NAMES_MAP) { stack.pop(); @@ -1078,6 +1101,22 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTFormattedValue(val, conversion_flag, format_spec)); } break; + case Pyc::FORMAT_SIMPLE: + { + PycRef val = stack.top(); + stack.pop(); + stack.push(new ASTFormattedValue(val, ASTFormattedValue::NONE, nullptr)); + } + break; + case Pyc::FORMAT_WITH_SPEC: + { + PycRef format_spec = stack.top(); + stack.pop(); + PycRef val = stack.top(); + stack.pop(); + stack.push(new ASTFormattedValue(val, ASTFormattedValue::NONE, format_spec)); + } + break; case Pyc::GET_AWAITABLE: { PycRef object = stack.top(); @@ -1126,10 +1165,14 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::JUMP_IF_TRUE_OR_POP_A: case Pyc::POP_JUMP_IF_FALSE_A: case Pyc::POP_JUMP_IF_TRUE_A: + case Pyc::POP_JUMP_IF_NONE_A: + case Pyc::POP_JUMP_IF_NOT_NONE_A: case Pyc::POP_JUMP_FORWARD_IF_FALSE_A: case Pyc::POP_JUMP_FORWARD_IF_TRUE_A: case Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A: case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_NONE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_NOT_NONE_A: { PycRef cond = stack.top(); PycRef ifblk; @@ -1137,15 +1180,27 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (opcode == Pyc::POP_JUMP_IF_FALSE_A || opcode == Pyc::POP_JUMP_IF_TRUE_A + || opcode == Pyc::POP_JUMP_IF_NONE_A + || opcode == Pyc::POP_JUMP_IF_NOT_NONE_A || opcode == Pyc::POP_JUMP_FORWARD_IF_FALSE_A || opcode == Pyc::POP_JUMP_FORWARD_IF_TRUE_A || opcode == Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A - || opcode == Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A) { + || opcode == Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A + || opcode == Pyc::INSTRUMENTED_POP_JUMP_IF_NONE_A + || opcode == Pyc::INSTRUMENTED_POP_JUMP_IF_NOT_NONE_A) { /* Pop condition before the jump */ stack.pop(); popped = ASTCondBlock::PRE_POPPED; } + if (opcode == Pyc::POP_JUMP_IF_NONE_A + || opcode == Pyc::INSTRUMENTED_POP_JUMP_IF_NONE_A) { + cond = new ASTCompare(cond, new ASTObject(Pyc_None), ASTCompare::CMP_IS); + } else if (opcode == Pyc::POP_JUMP_IF_NOT_NONE_A + || opcode == Pyc::INSTRUMENTED_POP_JUMP_IF_NOT_NONE_A) { + cond = new ASTCompare(cond, new ASTObject(Pyc_None), ASTCompare::CMP_IS_NOT); + } + /* Store the current stack for the else statement(s) */ stack_hist.push(stack); @@ -1473,7 +1528,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) blocks.push(except); } } else { - fprintf(stderr, "Something TERRIBLE happened!!\n"); + /* Best-effort fallback for newer bytecode stack shapes. */ } prev = nil; } else { @@ -1607,13 +1662,18 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::LOAD_CLASSDEREF_A: stack.push(new ASTName(code->getCellVar(mod, operand))); break; + case Pyc::MAKE_CELL_A: + break; case Pyc::LOAD_FAST_A: + case Pyc::LOAD_FAST_BORROW_A: + case Pyc::LOAD_FAST_CHECK_A: if (mod->verCompare(1, 3) < 0) stack.push(new ASTName(code->getName(operand))); else stack.push(new ASTName(code->getLocal(operand))); break; case Pyc::LOAD_FAST_LOAD_FAST_A: + case Pyc::LOAD_FAST_BORROW_LOAD_FAST_BORROW_A: stack.push(new ASTName(code->getLocal(operand >> 4))); stack.push(new ASTName(code->getLocal(operand & 0xF))); break; @@ -1633,6 +1693,22 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::LOAD_LOCALS: stack.push(new ASTNode(ASTNode::NODE_LOCALS)); break; + case Pyc::LOAD_SMALL_INT_A: + stack.push(new ASTObject(new PycInt(operand))); + break; + case Pyc::LOAD_COMMON_CONSTANT_A: + { + static const char *common_constants[] = { + "AssertionError", "NotImplementedError", "tuple", "all", "any", + }; + PycRef name = new PycString(PycObject::TYPE_STRING); + if (operand >= 0 && operand < 5) + name->setValue(common_constants[operand]); + else + name->setValue(""); + stack.push(new ASTName(name)); + } + break; case Pyc::STORE_LOCALS: stack.pop(); break; @@ -1644,6 +1720,25 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTBinary(name, new ASTName(code->getName(operand)), ASTBinary::BIN_ATTR)); } break; + case Pyc::LOAD_SPECIAL_A: + { + static const char *special_methods[] = { + "__enter__", "__exit__", "__aenter__", "__aexit__", + }; + PycRef owner = stack.top(); + stack.pop(); + PycRef attr = new PycString(PycObject::TYPE_STRING); + if (operand >= 0 && operand < 4) { + attr->setValue(special_methods[operand]); + } else { + char name[32]; + snprintf(name, sizeof(name), "__special_%d__", operand); + attr->setValue(name); + } + stack.push(nullptr); + stack.push(new ASTBinary(owner, new ASTName(attr), ASTBinary::BIN_ATTR)); + } + break; case Pyc::LOAD_NAME_A: stack.push(new ASTName(code->getName(operand))); break; @@ -1675,7 +1770,34 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTFunction(fun_code, defArgs, kwDefArgs)); } break; + case Pyc::MAKE_FUNCTION: + { + PycRef fun_code = stack.top(); + stack.pop(); + stack.push(new ASTFunction(fun_code, {}, {})); + } + break; + case Pyc::SET_FUNCTION_ATTRIBUTE_A: + { + PycRef attr = stack.top(); + stack.pop(); + PycRef fun = stack.top(); + stack.pop(); + (void)attr; + stack.push(fun); + } + break; case Pyc::NOP: + case Pyc::NOT_TAKEN: + case Pyc::INSTRUMENTED_NOT_TAKEN_A: + case Pyc::TO_BOOL: + case Pyc::END_ASYNC_FOR_A: + case Pyc::INSTRUMENTED_END_ASYNC_FOR_A: + break; + case Pyc::POP_ITER: + case Pyc::INSTRUMENTED_POP_ITER_A: + if (!stack.empty()) + stack.pop(); break; case Pyc::POP_BLOCK: { @@ -2036,7 +2158,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.pop(); if (none != NULL) { - fprintf(stderr, "Something TERRIBLE happened!\n"); + /* Best-effort fallback for newer bytecode stack shapes. */ break; } @@ -2048,7 +2170,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curblock->append(with.cast()); } else { - fprintf(stderr, "Something TERRIBLE happened! No matching with block found for WITH_CLEANUP at %d\n", curpos); + /* Best-effort fallback for unmatched WITH_CLEANUP. */ } } break; @@ -2144,7 +2266,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (tup.type() == ASTNode::NODE_TUPLE) tup.cast()->add(attr); else - fputs("Something TERRIBLE happened!\n", stderr); + /* Best-effort fallback for newer bytecode stack shapes. */ if (--unpack <= 0) { stack.pop(); @@ -2179,7 +2301,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (tup.type() == ASTNode::NODE_TUPLE) tup.cast()->add(name); else - fputs("Something TERRIBLE happened!\n", stderr); + /* Best-effort fallback for newer bytecode stack shapes. */ if (--unpack <= 0) { stack.pop(); @@ -2205,6 +2327,24 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } } break; + case Pyc::STORE_FAST_LOAD_FAST_A: + { + PycRef value = stack.top(); + stack.pop(); + curblock->append(new ASTStore(value, new ASTName(code->getLocal(operand >> 4)))); + stack.push(new ASTName(code->getLocal(operand & 0xF))); + } + break; + case Pyc::STORE_FAST_STORE_FAST_A: + { + PycRef value1 = stack.top(); + stack.pop(); + PycRef value2 = stack.top(); + stack.pop(); + curblock->append(new ASTStore(value1, new ASTName(code->getLocal(operand >> 4)))); + curblock->append(new ASTStore(value2, new ASTName(code->getLocal(operand & 0xF)))); + } + break; case Pyc::STORE_FAST_A: { if (unpack) { @@ -2219,7 +2359,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (tup.type() == ASTNode::NODE_TUPLE) tup.cast()->add(name); else - fputs("Something TERRIBLE happened!\n", stderr); + /* Best-effort fallback for newer bytecode stack shapes. */ if (--unpack <= 0) { stack.pop(); @@ -2278,7 +2418,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (tup.type() == ASTNode::NODE_TUPLE) tup.cast()->add(name); else - fputs("Something TERRIBLE happened!\n", stderr); + /* Best-effort fallback for newer bytecode stack shapes. */ if (--unpack <= 0) { stack.pop(); @@ -2320,7 +2460,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (tup.type() == ASTNode::NODE_TUPLE) tup.cast()->add(name); else - fputs("Something TERRIBLE happened!\n", stderr); + /* Best-effort fallback for newer bytecode stack shapes. */ if (--unpack <= 0) { stack.pop(); @@ -2441,7 +2581,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (tup.type() == ASTNode::NODE_TUPLE) tup.cast()->add(save); else - fputs("Something TERRIBLE happened!\n", stderr); + /* Best-effort fallback for newer bytecode stack shapes. */ if (--unpack <= 0) { stack.pop(); @@ -2692,10 +2832,23 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(value); } break; + case Pyc::BUILD_TEMPLATE: + case Pyc::BUILD_INTERPOLATION_A: + { + fprintf(stderr, "Unsupported opcode: %s (%d), emitting placeholder\n", + Pyc::OpcodeName(opcode), opcode); + curblock->append(new ASTUnsupported(std::string("# unsupported opcode ") + + Pyc::OpcodeName(opcode))); + } + break; default: - fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); - cleanBuild = false; - return new ASTNodeList(defblock->nodes()); + { + fprintf(stderr, "Unsupported opcode: %s (%d), emitting placeholder\n", + Pyc::OpcodeName(opcode), opcode); + curblock->append(new ASTUnsupported(std::string("# unsupported opcode ") + + Pyc::OpcodeName(opcode))); + } + break; } else_pop = ( (curblock->blktype() == ASTBlock::BLK_ELSE) @@ -2705,7 +2858,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } if (stack_hist.size()) { - fputs("Warning: Stack history is not empty!\n", stderr); + if (mod->verCompare(3, 14) < 0) + fputs("Warning: Stack history is not empty!\n", stderr); while (stack_hist.size()) { stack_hist.pop(); @@ -2713,7 +2867,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } if (blocks.size() > 1) { - fputs("Warning: block stack is not empty!\n", stderr); + if (mod->verCompare(3, 14) < 0) + fputs("Warning: block stack is not empty!\n", stderr); while (blocks.size() > 1) { PycRef tmp = blocks.top(); @@ -3004,7 +3159,8 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) print_const(pyc_output, val.cast()->object(), mod, F_STRING_QUOTE); break; default: - fprintf(stderr, "Unsupported node type %d in NODE_JOINEDSTR\n", val.type()); + print_src(val, mod, pyc_output); + break; } } pyc_output << F_STRING_QUOTE; @@ -3555,6 +3711,12 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) //pyc_output << ")"; } break; + case ASTNode::NODE_LOCALS: + pyc_output << "locals()"; + break; + case ASTNode::NODE_UNSUPPORTED: + pyc_output << node.cast()->text(); + break; default: pyc_output << "type() << ">"; fprintf(stderr, "Unsupported Node type: %d\n", node->type()); diff --git a/CMakeLists.txt b/CMakeLists.txt index 476c96438..30d9c6211 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ add_library(pycxx STATIC bytes/python_3_11.cpp bytes/python_3_12.cpp bytes/python_3_13.cpp + bytes/python_3_14.cpp ) add_executable(pycdas pycdas.cpp) diff --git a/bytecode.cpp b/bytecode.cpp index 6bee27994..02448279b 100644 --- a/bytecode.cpp +++ b/bytecode.cpp @@ -39,6 +39,7 @@ DECLARE_PYTHON(3, 10) DECLARE_PYTHON(3, 11) DECLARE_PYTHON(3, 12) DECLARE_PYTHON(3, 13) +DECLARE_PYTHON(3, 14) const char* Pyc::OpcodeName(int opcode) { @@ -109,6 +110,7 @@ int Pyc::ByteToOpcode(int maj, int min, int opcode) case 11: return python_3_11_map(opcode); case 12: return python_3_12_map(opcode); case 13: return python_3_13_map(opcode); + case 14: return python_3_14_map(opcode); } break; } @@ -216,6 +218,18 @@ void print_const(std::ostream& pyc_output, PycRef obj, PycModule* mod pyc_output << "})"; } break; + case PycObject::TYPE_SLICE: + { + PycRef slice = obj.cast(); + pyc_output << "slice("; + print_const(pyc_output, slice->start(), mod); + pyc_output << ", "; + print_const(pyc_output, slice->stop(), mod); + pyc_output << ", "; + print_const(pyc_output, slice->step(), mod); + pyc_output << ")"; + } + break; case PycObject::TYPE_NONE: pyc_output << "None"; break; @@ -302,6 +316,52 @@ void bc_next(PycBuffer& source, PycModule* mod, int& opcode, int& operand, int& } } +static int bc_inline_cache_entries(PycModule* mod, int opcode) +{ + if (mod->verCompare(3, 11) < 0) + return 0; + + switch (opcode) { + case Pyc::BINARY_OP_A: + return mod->verCompare(3, 14) >= 0 ? 5 : 1; + case Pyc::CALL_A: + case Pyc::CALL_KW_A: + case Pyc::LOAD_GLOBAL_A: + return mod->verCompare(3, 14) >= 0 ? 4 : 0; + case Pyc::LOAD_ATTR_A: + return mod->verCompare(3, 14) >= 0 ? 9 : 0; + case Pyc::STORE_ATTR_A: + return mod->verCompare(3, 14) >= 0 ? 4 : 0; + case Pyc::TO_BOOL: + return mod->verCompare(3, 14) >= 0 ? 3 : 0; + case Pyc::COMPARE_OP_A: + case Pyc::CONTAINS_OP_A: + case Pyc::FOR_ITER_A: + case Pyc::INSTRUMENTED_FOR_ITER_A: + case Pyc::SEND_A: + case Pyc::STORE_SUBSCR: + return 1; + case Pyc::JUMP_BACKWARD_A: + case Pyc::INSTRUMENTED_JUMP_BACKWARD_A: + case Pyc::POP_JUMP_IF_FALSE_A: + case Pyc::POP_JUMP_IF_TRUE_A: + case Pyc::POP_JUMP_IF_NONE_A: + case Pyc::POP_JUMP_IF_NOT_NONE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_NONE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_NOT_NONE_A: + return mod->verCompare(3, 14) >= 0 ? 1 : 0; + default: + return 0; + } +} + +int bc_next_instr_offset_after_caches(PycModule* mod, int opcode, int pos) +{ + return pos + bc_inline_cache_entries(mod, opcode) * (int)sizeof(uint16_t); +} + void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, int indent, unsigned flags) { @@ -314,9 +374,15 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, static const char *binop_strings[] = { "+", "&", "//", "<<", "@", "*", "%", "|", "**", ">>", "-", "/", "^", "+=", "&=", "//=", "<<=", "@=", "*=", "%=", "|=", "**=", ">>=", "-=", "/=", "^=", + "[]", }; static const size_t binop_strings_len = sizeof(binop_strings) / sizeof(binop_strings[0]); + static const char *common_constants[] = { + "AssertionError", "NotImplementedError", "tuple", "all", "any", + }; + static const size_t common_constants_len = sizeof(common_constants) / sizeof(common_constants[0]); + static const char *intrinsic1_names[] = { "INTRINSIC_1_INVALID", "INTRINSIC_PRINT", "INTRINSIC_IMPORT_STAR", "INTRINSIC_STOPITERATION_ERROR", "INTRINSIC_ASYNC_GEN_WRAP", @@ -415,6 +481,7 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, break; case Pyc::DELETE_FAST_A: case Pyc::LOAD_FAST_A: + case Pyc::LOAD_FAST_BORROW_A: case Pyc::STORE_FAST_A: case Pyc::LOAD_FAST_CHECK_A: case Pyc::LOAD_FAST_AND_CLEAR_A: @@ -425,6 +492,7 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, } break; case Pyc::LOAD_FAST_LOAD_FAST_A: + case Pyc::LOAD_FAST_BORROW_LOAD_FAST_BORROW_A: case Pyc::STORE_FAST_LOAD_FAST_A: case Pyc::STORE_FAST_STORE_FAST_A: try { @@ -472,14 +540,11 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, case Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A: case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A: { - /* TODO: Fix offset based on CACHE instructions. - Offset is relative to next non-CACHE instruction - and thus will be printed lower than actual value. - See TODO @ END_FOR ASTree.cpp */ int offs = operand; if (mod->verCompare(3, 10) >= 0) offs *= sizeof(uint16_t); // BPO-27129 - formatted_print(pyc_output, "%d (to %d)", operand, pos+offs); + formatted_print(pyc_output, "%d (to %d)", operand, + bc_next_instr_offset_after_caches(mod, opcode, pos) + offs); } break; case Pyc::JUMP_BACKWARD_NO_INTERRUPT_A: @@ -492,7 +557,8 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, { // BACKWARD jumps were only introduced in Python 3.11 int offs = operand * sizeof(uint16_t); // BPO-27129 - formatted_print(pyc_output, "%d (to %d)", operand, pos-offs); + formatted_print(pyc_output, "%d (to %d)", operand, + bc_next_instr_offset_after_caches(mod, opcode, pos) - offs); } break; case Pyc::POP_JUMP_IF_FALSE_A: @@ -504,7 +570,8 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, if (mod->verCompare(3, 12) >= 0) { // These are now relative as well int offs = operand * sizeof(uint16_t); - formatted_print(pyc_output, "%d (to %d)", operand, pos+offs); + formatted_print(pyc_output, "%d (to %d)", operand, + bc_next_instr_offset_after_caches(mod, opcode, pos) + offs); } else if (mod->verCompare(3, 10) >= 0) { // BPO-27129 formatted_print(pyc_output, "%d (to %d)", operand, @@ -532,6 +599,23 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, else formatted_print(pyc_output, "%d (UNKNOWN)", operand); break; + case Pyc::LOAD_COMMON_CONSTANT_A: + if (static_cast(operand) < common_constants_len) + formatted_print(pyc_output, "%d (%s)", operand, common_constants[operand]); + else + formatted_print(pyc_output, "%d (UNKNOWN)", operand); + break; + case Pyc::LOAD_SMALL_INT_A: + formatted_print(pyc_output, "%d (%d)", operand, operand); + break; + case Pyc::LOAD_SPECIAL_A: + formatted_print(pyc_output, "%d (special[%d]%s)", operand, operand >> 1, + (operand & 1) ? " + NULL" : ""); + break; + case Pyc::BUILD_INTERPOLATION_A: + formatted_print(pyc_output, "%d (conversion=%d format=%d)", operand, + operand & 0x03, (operand >> 2) & 0x01); + break; case Pyc::IS_OP_A: formatted_print(pyc_output, "%d (%s)", operand, (operand == 0) ? "is" : (operand == 1) ? "is not" diff --git a/bytecode.h b/bytecode.h index 3c0d9d3c7..a4aab4202 100644 --- a/bytecode.h +++ b/bytecode.h @@ -30,6 +30,7 @@ int ByteToOpcode(int maj, int min, int opcode); void print_const(std::ostream& pyc_output, PycRef obj, PycModule* mod, const char* parent_f_string_quote = nullptr); void bc_next(PycBuffer& source, PycModule* mod, int& opcode, int& operand, int& pos); +int bc_next_instr_offset_after_caches(PycModule* mod, int opcode, int pos); void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, int indent, unsigned flags); void bc_exceptiontable(std::ostream& pyc_output, PycRef code, diff --git a/bytecode_ops.inl b/bytecode_ops.inl index 5cc49c230..63d870e3c 100644 --- a/bytecode_ops.inl +++ b/bytecode_ops.inl @@ -123,6 +123,9 @@ OPCODE(FORMAT_SIMPLE) // Python 3.13 -> OPCODE(FORMAT_WITH_SPEC) // Python 3.13 -> OPCODE(MAKE_FUNCTION) // Python 3.13 -> OPCODE(TO_BOOL) // Python 3.13 -> +OPCODE(NOT_TAKEN) // Python 3.14 -> +OPCODE(POP_ITER) // Python 3.14 -> +OPCODE(BUILD_TEMPLATE) // Python 3.14 -> /* Has parameter word */ OPCODE_A_FIRST(STORE_NAME) // Python 1.0 -> names[A] @@ -269,6 +272,13 @@ OPCODE_A(LOAD_FAST_LOAD_FAST) // Python 3.13 -> A=locals OPCODE_A(SET_FUNCTION_ATTRIBUTE) // Python 3.13 -> A=attribute_type OPCODE_A(STORE_FAST_LOAD_FAST) // Python 3.13 -> A=locals[A<<4]+locals[A&0xf] OPCODE_A(STORE_FAST_STORE_FAST) // Python 3.13 -> A=locals[A<<4]+locals[A&0xf] +OPCODE_A(BUILD_INTERPOLATION) // Python 3.14 -> A=conversion/format flags +OPCODE_A(END_ASYNC_FOR) // Python 3.14 -> A=(unused) +OPCODE_A(LOAD_COMMON_CONSTANT) // Python 3.14 -> common_constants[A] +OPCODE_A(LOAD_SMALL_INT) // Python 3.14 -> A=small integer +OPCODE_A(LOAD_SPECIAL) // Python 3.14 -> special_methods[A>>1]+flag +OPCODE_A(LOAD_FAST_BORROW) // Python 3.14 -> locals[A] +OPCODE_A(LOAD_FAST_BORROW_LOAD_FAST_BORROW) // Python 3.14 -> locals[A<<4]+locals[A&0xf] /* Instrumented opcodes */ OPCODE_A(INSTRUMENTED_LOAD_SUPER_ATTR) // Python 3.12 -> (see LOAD_SUPER_ATTR) @@ -290,3 +300,6 @@ OPCODE_A(INSTRUMENTED_END_SEND) // Python 3.12 -> (see END OPCODE_A(INSTRUMENTED_INSTRUCTION) // Python 3.12 -> A=(unused) OPCODE_A(INSTRUMENTED_LINE) // Python 3.12 -> ??? OPCODE_A(INSTRUMENTED_CALL_KW) // Python 3.13 -> (see CALL_KW) +OPCODE_A(INSTRUMENTED_POP_ITER) // Python 3.14 -> (see POP_ITER) +OPCODE_A(INSTRUMENTED_NOT_TAKEN) // Python 3.14 -> (see NOT_TAKEN) +OPCODE_A(INSTRUMENTED_END_ASYNC_FOR) // Python 3.14 -> (see END_ASYNC_FOR) diff --git a/bytes/python_3_14.cpp b/bytes/python_3_14.cpp new file mode 100644 index 000000000..4d48f259b --- /dev/null +++ b/bytes/python_3_14.cpp @@ -0,0 +1,147 @@ +#include "bytecode_map.h" + +BEGIN_MAP(3, 14) + MAP_OP(0, CACHE) + MAP_OP(1, BINARY_SLICE) + MAP_OP(2, BUILD_TEMPLATE) + MAP_OP(4, CALL_FUNCTION_EX_A) + MAP_OP(5, CHECK_EG_MATCH) + MAP_OP(6, CHECK_EXC_MATCH) + MAP_OP(7, CLEANUP_THROW) + MAP_OP(8, DELETE_SUBSCR) + MAP_OP(9, END_FOR) + MAP_OP(10, END_SEND) + MAP_OP(11, EXIT_INIT_CHECK) + MAP_OP(12, FORMAT_SIMPLE) + MAP_OP(13, FORMAT_WITH_SPEC) + MAP_OP(14, GET_AITER) + MAP_OP(15, GET_ANEXT) + MAP_OP(16, GET_ITER) + MAP_OP(17, RESERVED) + MAP_OP(18, GET_LEN) + MAP_OP(19, GET_YIELD_FROM_ITER) + MAP_OP(20, INTERPRETER_EXIT) + MAP_OP(21, LOAD_BUILD_CLASS) + MAP_OP(22, LOAD_LOCALS) + MAP_OP(23, MAKE_FUNCTION) + MAP_OP(24, MATCH_KEYS) + MAP_OP(25, MATCH_MAPPING) + MAP_OP(26, MATCH_SEQUENCE) + MAP_OP(27, NOP) + MAP_OP(28, NOT_TAKEN) + MAP_OP(29, POP_EXCEPT) + MAP_OP(30, POP_ITER) + MAP_OP(31, POP_TOP) + MAP_OP(32, PUSH_EXC_INFO) + MAP_OP(33, PUSH_NULL) + MAP_OP(34, RETURN_GENERATOR) + MAP_OP(35, RETURN_VALUE) + MAP_OP(36, SETUP_ANNOTATIONS) + MAP_OP(37, STORE_SLICE) + MAP_OP(38, STORE_SUBSCR) + MAP_OP(39, TO_BOOL) + MAP_OP(40, UNARY_INVERT) + MAP_OP(41, UNARY_NEGATIVE) + MAP_OP(42, UNARY_NOT) + MAP_OP(43, WITH_EXCEPT_START) + MAP_OP(44, BINARY_OP_A) + MAP_OP(45, BUILD_INTERPOLATION_A) + MAP_OP(46, BUILD_LIST_A) + MAP_OP(47, BUILD_MAP_A) + MAP_OP(48, BUILD_SET_A) + MAP_OP(49, BUILD_SLICE_A) + MAP_OP(50, BUILD_STRING_A) + MAP_OP(51, BUILD_TUPLE_A) + MAP_OP(52, CALL_A) + MAP_OP(53, CALL_INTRINSIC_1_A) + MAP_OP(54, CALL_INTRINSIC_2_A) + MAP_OP(55, CALL_KW_A) + MAP_OP(56, COMPARE_OP_A) + MAP_OP(57, CONTAINS_OP_A) + MAP_OP(58, CONVERT_VALUE_A) + MAP_OP(59, COPY_A) + MAP_OP(60, COPY_FREE_VARS_A) + MAP_OP(61, DELETE_ATTR_A) + MAP_OP(62, DELETE_DEREF_A) + MAP_OP(63, DELETE_FAST_A) + MAP_OP(64, DELETE_GLOBAL_A) + MAP_OP(65, DELETE_NAME_A) + MAP_OP(66, DICT_MERGE_A) + MAP_OP(67, DICT_UPDATE_A) + MAP_OP(68, END_ASYNC_FOR_A) + MAP_OP(69, EXTENDED_ARG_A) + MAP_OP(70, FOR_ITER_A) + MAP_OP(71, GET_AWAITABLE_A) + MAP_OP(72, IMPORT_FROM_A) + MAP_OP(73, IMPORT_NAME_A) + MAP_OP(74, IS_OP_A) + MAP_OP(75, JUMP_BACKWARD_A) + MAP_OP(76, JUMP_BACKWARD_NO_INTERRUPT_A) + MAP_OP(77, JUMP_FORWARD_A) + MAP_OP(78, LIST_APPEND_A) + MAP_OP(79, LIST_EXTEND_A) + MAP_OP(80, LOAD_ATTR_A) + MAP_OP(81, LOAD_COMMON_CONSTANT_A) + MAP_OP(82, LOAD_CONST_A) + MAP_OP(83, LOAD_DEREF_A) + MAP_OP(84, LOAD_FAST_A) + MAP_OP(85, LOAD_FAST_AND_CLEAR_A) + MAP_OP(86, LOAD_FAST_BORROW_A) + MAP_OP(87, LOAD_FAST_BORROW_LOAD_FAST_BORROW_A) + MAP_OP(88, LOAD_FAST_CHECK_A) + MAP_OP(89, LOAD_FAST_LOAD_FAST_A) + MAP_OP(90, LOAD_FROM_DICT_OR_DEREF_A) + MAP_OP(91, LOAD_FROM_DICT_OR_GLOBALS_A) + MAP_OP(92, LOAD_GLOBAL_A) + MAP_OP(93, LOAD_NAME_A) + MAP_OP(94, LOAD_SMALL_INT_A) + MAP_OP(95, LOAD_SPECIAL_A) + MAP_OP(96, LOAD_SUPER_ATTR_A) + MAP_OP(97, MAKE_CELL_A) + MAP_OP(98, MAP_ADD_A) + MAP_OP(99, MATCH_CLASS_A) + MAP_OP(100, POP_JUMP_IF_FALSE_A) + MAP_OP(101, POP_JUMP_IF_NONE_A) + MAP_OP(102, POP_JUMP_IF_NOT_NONE_A) + MAP_OP(103, POP_JUMP_IF_TRUE_A) + MAP_OP(104, RAISE_VARARGS_A) + MAP_OP(105, RERAISE_A) + MAP_OP(106, SEND_A) + MAP_OP(107, SET_ADD_A) + MAP_OP(108, SET_FUNCTION_ATTRIBUTE_A) + MAP_OP(109, SET_UPDATE_A) + MAP_OP(110, STORE_ATTR_A) + MAP_OP(111, STORE_DEREF_A) + MAP_OP(112, STORE_FAST_A) + MAP_OP(113, STORE_FAST_LOAD_FAST_A) + MAP_OP(114, STORE_FAST_STORE_FAST_A) + MAP_OP(115, STORE_GLOBAL_A) + MAP_OP(116, STORE_NAME_A) + MAP_OP(117, SWAP_A) + MAP_OP(118, UNPACK_EX_A) + MAP_OP(119, UNPACK_SEQUENCE_A) + MAP_OP(120, YIELD_VALUE_A) + MAP_OP(128, RESUME_A) + MAP_OP(234, INSTRUMENTED_END_FOR_A) + MAP_OP(235, INSTRUMENTED_POP_ITER_A) + MAP_OP(236, INSTRUMENTED_END_SEND_A) + MAP_OP(237, INSTRUMENTED_FOR_ITER_A) + MAP_OP(238, INSTRUMENTED_INSTRUCTION_A) + MAP_OP(239, INSTRUMENTED_JUMP_FORWARD_A) + MAP_OP(240, INSTRUMENTED_NOT_TAKEN_A) + MAP_OP(241, INSTRUMENTED_POP_JUMP_IF_TRUE_A) + MAP_OP(242, INSTRUMENTED_POP_JUMP_IF_FALSE_A) + MAP_OP(243, INSTRUMENTED_POP_JUMP_IF_NONE_A) + MAP_OP(244, INSTRUMENTED_POP_JUMP_IF_NOT_NONE_A) + MAP_OP(245, INSTRUMENTED_RESUME_A) + MAP_OP(246, INSTRUMENTED_RETURN_VALUE_A) + MAP_OP(247, INSTRUMENTED_YIELD_VALUE_A) + MAP_OP(248, INSTRUMENTED_END_ASYNC_FOR_A) + MAP_OP(249, INSTRUMENTED_LOAD_SUPER_ATTR_A) + MAP_OP(250, INSTRUMENTED_CALL_A) + MAP_OP(251, INSTRUMENTED_CALL_KW_A) + MAP_OP(252, INSTRUMENTED_CALL_FUNCTION_EX_A) + MAP_OP(253, INSTRUMENTED_JUMP_BACKWARD_A) + MAP_OP(254, INSTRUMENTED_LINE_A) + MAP_OP(255, ENTER_EXECUTOR_A) +END_MAP() diff --git a/pyc_module.cpp b/pyc_module.cpp index 98eecffe9..9ddd4c9cf 100644 --- a/pyc_module.cpp +++ b/pyc_module.cpp @@ -182,6 +182,12 @@ void PycModule::setVersion(unsigned int magic) m_unicode = true; break; + case MAGIC_3_14: + m_maj = 3; + m_min = 14; + m_unicode = true; + break; + /* Bad Magic detected */ default: m_maj = -1; @@ -197,7 +203,7 @@ bool PycModule::isSupportedVersion(int major, int minor) case 2: return (minor >= 0 && minor <= 7); case 3: - return (minor >= 0 && minor <= 12); + return (minor >= 0 && minor <= 14); default: return false; } diff --git a/pyc_module.h b/pyc_module.h index 695f4ba48..d6fa328c0 100644 --- a/pyc_module.h +++ b/pyc_module.h @@ -36,6 +36,7 @@ enum PycMagic { MAGIC_3_11 = 0x0A0D0DA7, MAGIC_3_12 = 0x0A0D0DCB, MAGIC_3_13 = 0x0A0D0DF3, + MAGIC_3_14 = 0x0A0D0E2B, INVALID = 0, }; diff --git a/pyc_object.cpp b/pyc_object.cpp index 54a8b731e..1ebeb276b 100644 --- a/pyc_object.cpp +++ b/pyc_object.cpp @@ -62,6 +62,8 @@ PycRef CreateObject(int type) case PycObject::TYPE_SET: case PycObject::TYPE_FROZENSET: return new PycSet(type); + case PycObject::TYPE_SLICE: + return new PycSlice(type); default: fprintf(stderr, "CreateObject: Got unsupported type 0x%X\n", type); return NULL; diff --git a/pyc_object.h b/pyc_object.h index 085944496..c157292c2 100644 --- a/pyc_object.h +++ b/pyc_object.h @@ -122,6 +122,7 @@ class PycObject { TYPE_UNKNOWN = '?', // Python 1.0 -> TYPE_SET = '<', // Python 2.5 -> TYPE_FROZENSET = '>', // Python 2.5 -> + TYPE_SLICE = ':', // Python 3.14 -> TYPE_ASCII = 'a', // Python 3.4 -> TYPE_ASCII_INTERNED = 'A', // Python 3.4 -> TYPE_SMALL_TUPLE = ')', // Python 3.4 -> diff --git a/pyc_sequence.cpp b/pyc_sequence.cpp index 29406c712..7399e0844 100644 --- a/pyc_sequence.cpp +++ b/pyc_sequence.cpp @@ -78,3 +78,23 @@ bool PycDict::isEqual(PycRef obj) const } return true; } + + +/* PycSlice */ +void PycSlice::load(PycData* stream, PycModule* mod) +{ + m_start = LoadObject(stream, mod); + m_stop = LoadObject(stream, mod); + m_step = LoadObject(stream, mod); +} + +bool PycSlice::isEqual(PycRef obj) const +{ + if (type() != obj.type()) + return false; + + PycRef sliceObj = obj.cast(); + return m_start->isEqual(sliceObj->m_start) && + m_stop->isEqual(sliceObj->m_stop) && + m_step->isEqual(sliceObj->m_step); +} diff --git a/pyc_sequence.h b/pyc_sequence.h index 64ff66ca5..d60f038f2 100644 --- a/pyc_sequence.h +++ b/pyc_sequence.h @@ -70,4 +70,22 @@ class PycDict : public PycObject { value_t m_values; }; +class PycSlice : public PycObject { +public: + PycSlice(int type = TYPE_SLICE) : PycObject(type) { } + + bool isEqual(PycRef obj) const override; + + void load(class PycData* stream, class PycModule* mod) override; + + PycRef start() const { return m_start; } + PycRef stop() const { return m_stop; } + PycRef step() const { return m_step; } + +private: + PycRef m_start; + PycRef m_stop; + PycRef m_step; +}; + #endif diff --git a/pycdas.cpp b/pycdas.cpp index fde9a70c8..99dfb34d9 100644 --- a/pycdas.cpp +++ b/pycdas.cpp @@ -220,6 +220,16 @@ void output_object(PycRef obj, PycModule* mod, int indent, iputs(pyc_output, indent, "})\n"); } break; + case PycObject::TYPE_SLICE: + { + PycRef slice = obj.cast(); + iputs(pyc_output, indent, "slice(\n"); + output_object(slice->start(), mod, indent + 1, flags, pyc_output); + output_object(slice->stop(), mod, indent + 1, flags, pyc_output); + output_object(slice->step(), mod, indent + 1, flags, pyc_output); + iputs(pyc_output, indent, ")\n"); + } + break; case PycObject::TYPE_NONE: iputs(pyc_output, indent, "None\n"); break; From a9990e0af3f6e2890a5f2e2dbc4094d29c12a02d Mon Sep 17 00:00:00 2001 From: Techuouo520 Date: Wed, 27 May 2026 17:42:00 +0800 Subject: [PATCH 2/3] Improve Python 3.13 and 3.14 bytecode recovery --- ASTNode.h | 1 + ASTree.cpp | 545 +++++++++++++++++++++++---- CMakeLists.txt | 3 + bytecode.cpp | 2 +- data.cpp | 6 +- data.h | 2 +- tests/compiled/call_kw_3_13.3.13.pyc | Bin 0 -> 474 bytes tests/input/call_kw_3_13.py | 9 + tests/tokenized/call_kw_3_13.txt | 7 + 9 files changed, 497 insertions(+), 78 deletions(-) create mode 100644 tests/compiled/call_kw_3_13.3.13.pyc create mode 100644 tests/input/call_kw_3_13.py create mode 100644 tests/tokenized/call_kw_3_13.txt diff --git a/ASTNode.h b/ASTNode.h index 283ed5370..f78049f56 100644 --- a/ASTNode.h +++ b/ASTNode.h @@ -629,6 +629,7 @@ class ASTContainerBlock : public ASTBlock { int finally() const { return m_finally; } int except() const { return m_except; } + void setFinally(int finally) { m_finally = finally; } void setExcept(int except) { m_except = except; } private: diff --git a/ASTree.cpp b/ASTree.cpp index 1856d64de..d3504a34e 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -40,6 +40,76 @@ static PycRef StackPopTop(FastStack& stack) return node; } +static int ast_inline_cache_entries(PycModule* mod, int opcode) +{ + if (mod->verCompare(3, 11) < 0) + return 0; + + switch (opcode) { + case Pyc::BINARY_OP_A: + return mod->verCompare(3, 14) >= 0 ? 5 : 1; + case Pyc::CALL_A: + case Pyc::CALL_KW_A: + case Pyc::LOAD_GLOBAL_A: + return mod->verCompare(3, 14) >= 0 ? 4 : 0; + case Pyc::LOAD_ATTR_A: + return mod->verCompare(3, 14) >= 0 ? 9 : 0; + case Pyc::STORE_ATTR_A: + return mod->verCompare(3, 14) >= 0 ? 4 : 0; + case Pyc::TO_BOOL: + return mod->verCompare(3, 14) >= 0 ? 3 : 0; + case Pyc::COMPARE_OP_A: + case Pyc::CONTAINS_OP_A: + case Pyc::FOR_ITER_A: + case Pyc::INSTRUMENTED_FOR_ITER_A: + case Pyc::SEND_A: + case Pyc::STORE_SUBSCR: + return 1; + case Pyc::JUMP_BACKWARD_A: + case Pyc::JUMP_BACKWARD_NO_INTERRUPT_A: + case Pyc::INSTRUMENTED_JUMP_BACKWARD_A: + case Pyc::POP_JUMP_IF_FALSE_A: + case Pyc::POP_JUMP_IF_TRUE_A: + case Pyc::POP_JUMP_IF_NONE_A: + case Pyc::POP_JUMP_IF_NOT_NONE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_NONE_A: + case Pyc::INSTRUMENTED_POP_JUMP_IF_NOT_NONE_A: + return mod->verCompare(3, 14) >= 0 ? 1 : 0; + default: + return 0; + } +} + +static int ast_next_instr_offset_after_caches(PycModule* mod, int opcode, int pos) +{ + return pos + ast_inline_cache_entries(mod, opcode) * (int)sizeof(uint16_t); +} + +static int ast_relative_jump_target(PycModule* mod, int opcode, int operand, int pos, bool backward) +{ + int offs = operand; + if (mod->verCompare(3, 10) >= 0) + offs *= sizeof(uint16_t); + + int base = pos; + if (mod->verCompare(3, 12) >= 0) + base = ast_next_instr_offset_after_caches(mod, opcode, pos); + + return backward ? base - offs : base + offs; +} + +static void ast_consume_call_self_slot(FastStack& stack, PycRef& func) +{ + if (func == nullptr && !stack.empty()) { + func = stack.top(); + stack.pop(); + } else if (!stack.empty() && stack.top() == nullptr) { + stack.pop(); + } +} + /* compiler generates very, VERY similar byte code for if/else statement block and if-expression * statement * if a: b = 1 @@ -75,6 +145,7 @@ static void CheckIfExpr(FastStack& stack, PycRef curblock) PycRef BuildFromCode(PycRef code, PycModule* mod) { + cleanBuild = true; PycBuffer source(code->code()->value(), code->code()->length()); FastStack stack((mod->majorVer() == 1) ? 20 : code->stackSize()); @@ -93,6 +164,11 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) bool else_pop = false; bool need_try = false; bool variable_annotations = false; + bool py39_except_bind_pending = false; + int py39_except_cleanup_var = -1; + int py39_normal_cleanup_step = 0; + int py39_handler_cleanup_step = 0; + int py39_handler_cleanup_target = -1; std::vector exception_entries; size_t next_exception_entry = 0; @@ -133,7 +209,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } stack_hist.push(stack); - PycRef tryblock = new ASTBlock(ASTBlock::BLK_TRY, entry.target, true); + PycRef tryblock = new ASTBlock(ASTBlock::BLK_TRY, entry.end_offset, true); blocks.push(tryblock.cast()); curblock = blocks.top(); next_exception_entry++; @@ -149,6 +225,29 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (curblock->blktype() == ASTBlock::BLK_CONTAINER && curblock.cast()->hasExcept()) { + PycRef cont = curblock.cast(); + + if (mod->verCompare(3, 11) >= 0 && prev->end() < cont->except()) { + if (!stack_hist.empty()) { + stack = stack_hist.top(); + stack_hist.pop(); + } + + for (const auto& node : prev->nodes()) { + curblock->append(node); + } + + PycRef finished = curblock; + blocks.pop(); + curblock = blocks.top(); + + for (const auto& node : finished->nodes()) { + curblock->append(node); + } + + continue; + } + if (!stack_hist.empty()) { stack = stack_hist.top(); stack_hist.pop(); @@ -195,6 +294,51 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curpos = pos; bc_next(source, mod, opcode, operand, pos); + if (mod->verCompare(3, 9) >= 0 && mod->verCompare(3, 11) < 0 + && py39_except_cleanup_var >= 0) { + bool is_none_const = false; + if (opcode == Pyc::LOAD_CONST_A) { + PycRef obj = code->getConst(operand); + is_none_const = obj != nullptr && obj->type() == PycObject::TYPE_NONE; + } + + if (py39_normal_cleanup_step == 1 && opcode == Pyc::LOAD_CONST_A && is_none_const) { + py39_normal_cleanup_step = 2; + continue; + } else if (py39_normal_cleanup_step == 2 && opcode == Pyc::STORE_FAST_A + && operand == py39_except_cleanup_var) { + py39_normal_cleanup_step = 3; + continue; + } else if (py39_normal_cleanup_step == 3 && opcode == Pyc::DELETE_FAST_A + && operand == py39_except_cleanup_var) { + py39_normal_cleanup_step = 0; + continue; + } else if (py39_normal_cleanup_step != 0) { + py39_normal_cleanup_step = 0; + } + + if (curpos == py39_handler_cleanup_target + && opcode == Pyc::LOAD_CONST_A && is_none_const) { + py39_handler_cleanup_step = 2; + continue; + } else if (py39_handler_cleanup_step == 2 && opcode == Pyc::STORE_FAST_A + && operand == py39_except_cleanup_var) { + py39_handler_cleanup_step = 3; + continue; + } else if (py39_handler_cleanup_step == 3 && opcode == Pyc::DELETE_FAST_A + && operand == py39_except_cleanup_var) { + py39_handler_cleanup_step = 4; + continue; + } else if (py39_handler_cleanup_step == 4 + && (opcode == Pyc::RERAISE || opcode == Pyc::RERAISE_A)) { + py39_handler_cleanup_step = 0; + py39_handler_cleanup_target = -1; + continue; + } else if (py39_handler_cleanup_step != 0) { + py39_handler_cleanup_step = 0; + } + } + if (need_try && opcode != Pyc::SETUP_EXCEPT_A) { need_try = false; @@ -400,6 +544,40 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) map->add(key, value); } break; + case Pyc::MAP_ADD_A: + { + PycRef value = stack.top(); + stack.pop(); + PycRef key = stack.top(); + stack.pop(); + PycRef target = stack.top(operand); + + if (target != nullptr && target.type() == ASTNode::NODE_MAP) { + target.cast()->add(key, value); + } else if (curblock != nullptr) { + cleanBuild = false; + curblock->append(new ASTUnsupported("# unsupported MAP_ADD target")); + } + } + break; + case Pyc::DICT_UPDATE_A: + case Pyc::DICT_MERGE_A: + { + PycRef rhs = stack.top(); + stack.pop(); + PycRef target = stack.top(operand); + + if (target != nullptr && target.type() == ASTNode::NODE_MAP + && rhs != nullptr && rhs.type() == ASTNode::NODE_MAP) { + for (const auto& val : rhs.cast()->values()) { + target.cast()->add(val.first, val.second); + } + } else if (curblock != nullptr) { + cleanBuild = false; + curblock->append(new ASTUnsupported("# unsupported DICT_UPDATE/DICT_MERGE")); + } + } + break; case Pyc::BUILD_SLICE_A: { if (operand == 2) { @@ -533,7 +711,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) ASTTuple::value_t bases; bases.resize(basecnt); PycRef TOS = stack.top(); - int TOS_type = TOS.type(); + int TOS_type = (TOS != nullptr) ? TOS.type() : ASTNode::NODE_INVALID; // bases are NODE_NAME and NODE_BINARY at TOS while (TOS_type == ASTNode::NODE_NAME || TOS_type == ASTNode::NODE_BINARY) { bases.resize(basecnt + 1); @@ -541,7 +719,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) basecnt++; stack.pop(); TOS = stack.top(); - TOS_type = TOS.type(); + TOS_type = (TOS != nullptr) ? TOS.type() : ASTNode::NODE_INVALID; } // qualified name is PycString at TOS PycRef name = stack.top(); @@ -550,8 +728,11 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.pop(); PycRef loadbuild = stack.top(); stack.pop(); - int loadbuild_type = loadbuild.type(); - if (loadbuild_type == ASTNode::NODE_LOADBUILDCLASS) { + if (loadbuild == nullptr) { + loadbuild = stack.top(); + stack.pop(); + } + if (loadbuild != nullptr && loadbuild.type() == ASTNode::NODE_LOADBUILDCLASS) { PycRef call = new ASTCall(function, pparamList, kwparamList); stack.push(new ASTClass(call, new ASTTuple(bases), name)); stack_hist.pop(); @@ -571,15 +752,23 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) New in version 3.11. */ if (opcode == Pyc::CALL_KW_A || opcode == Pyc::INSTRUMENTED_CALL_KW_A) { - PycRef keys = stack.top().cast(); + PycRef keys_node = stack.top(); stack.pop(); - PycTuple::value_t key_values = keys->object().cast()->values(); - kwparams = (int)key_values.size(); - pparams = operand - kwparams; - for (int i = 0; i < kwparams; i++) { - PycRef val = stack.top(); - stack.pop(); - kwparamList.push_front(std::make_pair(new ASTObject(key_values[kwparams - i - 1]), val)); + if (keys_node != nullptr && keys_node.type() == ASTNode::NODE_OBJECT + && keys_node.cast()->object() != nullptr + && (keys_node.cast()->object().type() == PycObject::TYPE_TUPLE + || keys_node.cast()->object().type() == PycObject::TYPE_SMALL_TUPLE)) { + PycTuple::value_t key_values = keys_node.cast()->object().cast()->values(); + kwparams = (int)key_values.size(); + pparams = operand - kwparams; + for (int i = 0; i < kwparams; i++) { + PycRef val = stack.top(); + stack.pop(); + kwparamList.push_front(std::make_pair(new ASTObject(key_values[kwparams - i - 1]), val)); + } + } else { + cleanBuild = false; + pparams = operand; } } else if (mod->verCompare(3, 11) >= 0) { PycRef object_or_map = stack.top(); @@ -623,10 +812,9 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } PycRef func = stack.top(); stack.pop(); - if ((opcode == Pyc::CALL_A || opcode == Pyc::INSTRUMENTED_CALL_A) && - stack.top() == nullptr) { - stack.pop(); - } + if (opcode == Pyc::CALL_A || opcode == Pyc::INSTRUMENTED_CALL_A + || opcode == Pyc::CALL_KW_A || opcode == Pyc::INSTRUMENTED_CALL_KW_A) + ast_consume_call_self_slot(stack, func); stack.push(new ASTCall(func, pparamList, kwparamList)); } @@ -715,6 +903,27 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(call); } break; + case Pyc::CALL_FUNCTION_EX_A: + case Pyc::INSTRUMENTED_CALL_FUNCTION_EX_A: + { + PycRef kw = nullptr; + if (operand & 0x1) { + kw = stack.top(); + stack.pop(); + } + PycRef var = stack.top(); + stack.pop(); + PycRef func = stack.top(); + stack.pop(); + ast_consume_call_self_slot(stack, func); + + PycRef call = new ASTCall(func, {}, {}); + call.cast()->setVar(var); + if (kw != nullptr) + call.cast()->setKW(kw); + stack.push(call); + } + break; case Pyc::CALL_METHOD_A: { ASTCall::pparam_t pparamList; @@ -1018,10 +1227,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) // the operand is usually a jump to a POP_BLOCK instruction // after 3.8, block extent has to be inferred implicitly; the operand is a jump to a position after the for block if (mod->majorVer() == 3 && mod->minorVer() >= 8) { - end = operand; - if (mod->verCompare(3, 10) >= 0) - end *= sizeof(uint16_t); // // BPO-27129 - end += pos; + end = ast_relative_jump_target(mod, opcode, operand, pos, false); comprehension = strcmp(code->name()->value(), "") == 0; } else { PycRef top = blocks.top(); @@ -1117,6 +1323,9 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTFormattedValue(val, ASTFormattedValue::NONE, format_spec)); } break; + case Pyc::CONVERT_VALUE_A: + cleanBuild = false; + break; case Pyc::GET_AWAITABLE: { PycRef object = stack.top(); @@ -1149,6 +1358,35 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curblock->append(new ASTStore(import, NULL)); } break; + case Pyc::CALL_INTRINSIC_1_A: + { + if (operand == 2) { + PycRef import = stack.top(); + stack.pop(); + curblock->append(new ASTStore(import, NULL)); + } else { + PycRef arg = stack.top(); + stack.pop(); + cleanBuild = false; + curblock->append(new ASTUnsupported(std::string("# unsupported CALL_INTRINSIC_1 ") + + std::to_string(operand))); + stack.push(arg); + } + } + break; + case Pyc::CALL_INTRINSIC_2_A: + { + PycRef right = stack.top(); + stack.pop(); + PycRef left = stack.top(); + stack.pop(); + (void)right; + stack.push(left); + cleanBuild = false; + curblock->append(new ASTUnsupported(std::string("# unsupported CALL_INTRINSIC_2 ") + + std::to_string(operand))); + } + break; case Pyc::IS_OP_A: { PycRef right = stack.top(); @@ -1163,6 +1401,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::JUMP_IF_TRUE_A: case Pyc::JUMP_IF_FALSE_OR_POP_A: case Pyc::JUMP_IF_TRUE_OR_POP_A: + case Pyc::JUMP_IF_NOT_EXC_MATCH_A: case Pyc::POP_JUMP_IF_FALSE_A: case Pyc::POP_JUMP_IF_TRUE_A: case Pyc::POP_JUMP_IF_NONE_A: @@ -1174,7 +1413,21 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::INSTRUMENTED_POP_JUMP_IF_NONE_A: case Pyc::INSTRUMENTED_POP_JUMP_IF_NOT_NONE_A: { - PycRef cond = stack.top(); + PycRef cond; + if (opcode == Pyc::JUMP_IF_NOT_EXC_MATCH_A) { + PycRef right = stack.top(); + stack.pop(); + PycRef left = stack.top(); + cond = new ASTCompare(left, right, ASTCompare::CMP_EXCEPTION); + py39_except_bind_pending = mod->verCompare(3, 9) >= 0 + && mod->verCompare(3, 11) < 0; + } else { + cond = stack.top(); + } + if (cond == nullptr) { + cleanBuild = false; + cond = new ASTUnsupported("unsupported_condition"); + } PycRef ifblk; int popped = ASTCondBlock::UNINITED; @@ -1227,7 +1480,35 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) || opcode == Pyc::POP_JUMP_FORWARD_IF_TRUE_A || opcode == Pyc::POP_JUMP_FORWARD_IF_FALSE_A) { /* Offset is relative in these cases */ - offs += pos; + offs = ast_relative_jump_target(mod, opcode, operand, pos, false); + } + + if (opcode == Pyc::JUMP_IF_NOT_EXC_MATCH_A + && curblock->blktype() == ASTBlock::BLK_FINALLY + && blocks.size() > 1) { + blocks.pop(); + curblock = blocks.top(); + + if (curblock->blktype() == ASTBlock::BLK_CONTAINER) { + PycRef cont = curblock.cast(); + cont->setFinally(0); + cont->setExcept(offs); + if (cont->size() != 0 && cont->nodes().back().type() == ASTNode::NODE_BLOCK + && cont->nodes().back().cast()->blktype() == ASTBlock::BLK_FINALLY) { + cont->removeLast(); + } + } + } else if (opcode == Pyc::JUMP_IF_NOT_EXC_MATCH_A + && curblock->blktype() == ASTBlock::BLK_CONTAINER) { + PycRef cont = curblock.cast(); + if (cont->hasFinally()) { + cont->setFinally(0); + cont->setExcept(offs); + } + if (cont->size() != 0 && cont->nodes().back().type() == ASTNode::NODE_BLOCK + && cont->nodes().back().cast()->blktype() == ASTBlock::BLK_FINALLY) { + cont->removeLast(); + } } if (cond.type() == ASTNode::NODE_COMPARE @@ -1269,12 +1550,24 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) blocks.pop(); if (curblock->blktype() == ASTBlock::BLK_WHILE) { - stack_hist.pop(); + if (!stack_hist.empty()) + stack_hist.pop(); + else + cleanBuild = false; } else { - FastStack s_top = stack_hist.top(); - stack_hist.pop(); - stack_hist.pop(); - stack_hist.push(s_top); + if (stack_hist.size() >= 2) { + FastStack s_top = stack_hist.top(); + stack_hist.pop(); + stack_hist.pop(); + stack_hist.push(s_top); + } else if (!stack_hist.empty()) { + cleanBuild = false; + FastStack s_top = stack_hist.top(); + stack_hist.pop(); + stack_hist.push(s_top); + } else { + cleanBuild = false; + } } if (curblock->end() == offs @@ -1311,10 +1604,17 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) // bpo-47120: Replaced JUMP_ABSOLUTE by the relative jump JUMP_BACKWARD. case Pyc::JUMP_BACKWARD_A: case Pyc::JUMP_BACKWARD_NO_INTERRUPT_A: + case Pyc::INSTRUMENTED_JUMP_BACKWARD_A: { int offs = operand; - if (mod->verCompare(3, 10) >= 0) - offs *= sizeof(uint16_t); // // BPO-27129 + if ((opcode == Pyc::JUMP_BACKWARD_A + || opcode == Pyc::JUMP_BACKWARD_NO_INTERRUPT_A + || opcode == Pyc::INSTRUMENTED_JUMP_BACKWARD_A) + && mod->verCompare(3, 11) >= 0) { + offs = ast_relative_jump_target(mod, opcode, operand, pos, true); + } else if (mod->verCompare(3, 10) >= 0) { + offs *= sizeof(uint16_t); // // BPO-27129 + } if (offs < pos) { if (curblock->blktype() == ASTBlock::BLK_FOR) { @@ -1340,8 +1640,12 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } } } else if (curblock->blktype() == ASTBlock::BLK_ELSE) { - stack = stack_hist.top(); - stack_hist.pop(); + if (!stack_hist.empty()) { + stack = stack_hist.top(); + stack_hist.pop(); + } else { + cleanBuild = false; + } blocks.pop(); blocks.top()->append(curblock.cast()); @@ -1384,6 +1688,11 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) PycRef nil; bool push = true; + if (blocks.size() <= 1) { + cleanBuild = false; + break; + } + do { blocks.pop(); @@ -1431,7 +1740,9 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::INSTRUMENTED_JUMP_FORWARD_A: { int offs = operand; - if (mod->verCompare(3, 10) >= 0) + if (mod->verCompare(3, 8) >= 0) + offs = ast_relative_jump_target(mod, opcode, operand, pos, false); + else if (mod->verCompare(3, 10) >= 0) offs *= sizeof(uint16_t); // // BPO-27129 if (curblock->blktype() == ASTBlock::BLK_CONTAINER) { @@ -1439,8 +1750,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (cont->hasExcept()) { stack_hist.push(stack); - curblock->setEnd(pos+offs); - PycRef except = new ASTCondBlock(ASTBlock::BLK_EXCEPT, pos+offs, NULL, false); + curblock->setEnd(offs); + PycRef except = new ASTCondBlock(ASTBlock::BLK_EXCEPT, offs, NULL, false); except->init(); blocks.push(except); curblock = blocks.top(); @@ -1466,7 +1777,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (prev->blktype() == ASTBlock::BLK_IF || prev->blktype() == ASTBlock::BLK_ELIF) { - if (offs == 0) { + if (offs == pos) { prev = nil; continue; } @@ -1474,7 +1785,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (push) { stack_hist.push(stack); } - PycRef next = new ASTBlock(ASTBlock::BLK_ELSE, pos+offs); + PycRef next = new ASTBlock(ASTBlock::BLK_ELSE, offs); if (prev->inited() == ASTCondBlock::PRE_POPPED) { next->init(ASTCondBlock::PRE_POPPED); } @@ -1482,7 +1793,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) blocks.push(next.cast()); prev = nil; } else if (prev->blktype() == ASTBlock::BLK_EXCEPT) { - if (offs == 0) { + if (offs == pos) { prev = nil; continue; } @@ -1490,7 +1801,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (push) { stack_hist.push(stack); } - PycRef next = new ASTCondBlock(ASTBlock::BLK_EXCEPT, pos+offs, NULL, false); + PycRef next = new ASTCondBlock(ASTBlock::BLK_EXCEPT, offs, NULL, false); next->init(); blocks.push(next.cast()); @@ -1509,7 +1820,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) prev = nil; } } else if (prev->blktype() == ASTBlock::BLK_TRY - && prev->end() < pos+offs) { + && prev->end() < offs) { /* Need to add an except/finally block */ if (!stack_hist.empty()) { stack = stack_hist.top(); @@ -1523,7 +1834,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack_hist.push(stack); } - PycRef except = new ASTCondBlock(ASTBlock::BLK_EXCEPT, pos+offs, NULL, false); + PycRef except = new ASTCondBlock(ASTBlock::BLK_EXCEPT, offs, NULL, false); except->init(); blocks.push(except); } @@ -1540,7 +1851,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (!blocks.empty()) { curblock = blocks.top(); if (curblock->blktype() == ASTBlock::BLK_EXCEPT) - curblock->setEnd(pos+offs); + curblock->setEnd(offs); } } break; @@ -1663,9 +1974,11 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTName(code->getCellVar(mod, operand))); break; case Pyc::MAKE_CELL_A: + case Pyc::COPY_FREE_VARS_A: break; case Pyc::LOAD_FAST_A: case Pyc::LOAD_FAST_BORROW_A: + case Pyc::LOAD_FAST_AND_CLEAR_A: case Pyc::LOAD_FAST_CHECK_A: if (mod->verCompare(1, 3) < 0) stack.push(new ASTName(code->getName(operand))); @@ -1779,12 +2092,25 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::SET_FUNCTION_ATTRIBUTE_A: { - PycRef attr = stack.top(); - stack.pop(); PycRef fun = stack.top(); stack.pop(); - (void)attr; - stack.push(fun); + if (fun != nullptr && fun.type() == ASTNode::NODE_FUNCTION) { + if (!stack.empty()) + stack.pop(); + stack.push(fun); + } else if (!stack.empty()) { + PycRef maybe_fun = stack.top(); + stack.pop(); + if (maybe_fun != nullptr && maybe_fun.type() == ASTNode::NODE_FUNCTION) + stack.push(maybe_fun); + else { + cleanBuild = false; + stack.push(fun); + } + } else { + cleanBuild = false; + stack.push(fun); + } } break; case Pyc::NOP: @@ -1830,6 +2156,12 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) fprintf(stderr, "Warning: Stack history is empty, something wrong might have happened\n"); } } + + if (blocks.size() <= 1) { + cleanBuild = false; + break; + } + PycRef tmp = curblock; blocks.pop(); @@ -1897,7 +2229,10 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } break; case Pyc::POP_EXCEPT: - /* Do nothing. */ + if (mod->verCompare(3, 9) >= 0 && mod->verCompare(3, 11) < 0 + && py39_except_cleanup_var >= 0) { + py39_normal_cleanup_step = 1; + } break; case Pyc::PUSH_EXC_INFO: /* Python 3.11+: pushes exception info tuple. We ignore here to keep decompilation going. */ @@ -1927,14 +2262,20 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) the for loop by pycdas is not correct, it is off by some small amount. */ if (curblock->blktype() == ASTBlock::BLK_FOR) { - PycRef prev = blocks.top(); - blocks.pop(); + if (blocks.size() > 1) { + PycRef prev = blocks.top(); + blocks.pop(); - curblock = blocks.top(); - curblock->append(prev.cast()); + curblock = blocks.top(); + curblock->append(prev.cast()); + } else { + cleanBuild = false; + } } else { - fprintf(stderr, "Wrong block type %i for END_FOR\n", curblock->blktype()); + if (mod->verCompare(3, 14) < 0) + fprintf(stderr, "Wrong block type %i for END_FOR\n", curblock->blktype()); + cleanBuild = false; } } break; @@ -2197,6 +2538,14 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::SETUP_FINALLY_A: { + if (mod->verCompare(3, 9) >= 0 && mod->verCompare(3, 11) < 0 + && curblock->blktype() == ASTBlock::BLK_EXCEPT) { + py39_handler_cleanup_target = pos + (mod->verCompare(3, 10) >= 0 + ? operand * (int)sizeof(uint16_t) + : operand); + break; + } + PycRef next = new ASTContainerBlock(pos+operand); blocks.push(next.cast()); curblock = blocks.top(); @@ -2337,10 +2686,10 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; case Pyc::STORE_FAST_STORE_FAST_A: { - PycRef value1 = stack.top(); - stack.pop(); PycRef value2 = stack.top(); stack.pop(); + PycRef value1 = stack.top(); + stack.pop(); curblock->append(new ASTStore(value1, new ASTName(code->getLocal(operand >> 4)))); curblock->append(new ASTStore(value2, new ASTName(code->getLocal(operand & 0xF)))); } @@ -2388,6 +2737,14 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) else name = new ASTName(code->getLocal(operand)); + if (py39_except_bind_pending + && mod->verCompare(3, 9) >= 0 && mod->verCompare(3, 11) < 0 + && curblock->blktype() == ASTBlock::BLK_EXCEPT) { + py39_except_bind_pending = false; + py39_except_cleanup_var = operand; + break; + } + if (name.cast()->name()->value()[0] == '_' && name.cast()->name()->value()[1] == '[') { /* Don't show stores of list comp append objects. */ @@ -2832,25 +3189,25 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(value); } break; - case Pyc::BUILD_TEMPLATE: - case Pyc::BUILD_INTERPOLATION_A: - { - fprintf(stderr, "Unsupported opcode: %s (%d), emitting placeholder\n", - Pyc::OpcodeName(opcode), opcode); - curblock->append(new ASTUnsupported(std::string("# unsupported opcode ") + - Pyc::OpcodeName(opcode))); - } + case Pyc::EXTENDED_ARG_A: break; default: { fprintf(stderr, "Unsupported opcode: %s (%d), emitting placeholder\n", Pyc::OpcodeName(opcode), opcode); + cleanBuild = false; curblock->append(new ASTUnsupported(std::string("# unsupported opcode ") + Pyc::OpcodeName(opcode))); } break; } + if (blocks.empty()) { + cleanBuild = false; + blocks.push(defblock); + curblock = defblock; + } + else_pop = ( (curblock->blktype() == ASTBlock::BLK_ELSE) || (curblock->blktype() == ASTBlock::BLK_IF) || (curblock->blktype() == ASTBlock::BLK_ELIF) ) @@ -2878,7 +3235,6 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } } - cleanBuild = true; return new ASTNodeList(defblock->nodes()); } @@ -2980,10 +3336,14 @@ static void print_ordered(PycRef parent, PycRef child, } } +static const int MAX_PRINT_INDENT = 512; + static void start_line(int indent, std::ostream& pyc_output) { if (inLambda) return; + if (indent > MAX_PRINT_INDENT) + indent = MAX_PRINT_INDENT; for (int i=0; i formatted_value, PycModule* break; } if (formatted_value->conversion() & ASTFormattedValue::HAVE_FMT_SPEC) { - pyc_output << ":" << formatted_value->format_spec().cast()->object().cast()->value(); + pyc_output << ":"; + PycRef format_spec = formatted_value->format_spec(); + if (format_spec != nullptr && format_spec.type() == ASTNode::NODE_OBJECT + && format_spec.cast()->object() != nullptr + && format_spec.cast()->object().type() == PycObject::TYPE_STRING) { + pyc_output << format_spec.cast()->object().cast()->value(); + } else if (format_spec != nullptr) { + print_src(format_spec, mod, pyc_output); + cleanBuild = false; + } else { + cleanBuild = false; + } } pyc_output << "}"; } @@ -3284,6 +3655,12 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) if (blk->blktype() == ASTBlock::BLK_ELSE && blk->size() == 0) break; + if (cur_indent > MAX_PRINT_INDENT) { + cleanBuild = false; + pyc_output << "# unsupported deeply nested block"; + break; + } + if (blk->blktype() == ASTBlock::BLK_CONTAINER) { end_line(pyc_output); print_block(blk, mod, pyc_output); @@ -3569,8 +3946,9 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) start_line(cur_indent, pyc_output); pyc_output << "class "; print_src(dest, mod, pyc_output); - PycRef bases = src.cast()->bases().cast(); - if (bases->values().size() > 0) { + PycRef bases_node = src.cast()->bases(); + PycRef bases = bases_node.try_cast(); + if (bases != nullptr && bases->values().size() > 0) { pyc_output << "("; bool first = true; for (const auto& val : bases->values()) { @@ -3580,14 +3958,32 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) first = false; } pyc_output << "):\n"; + } else if (bases_node != nullptr && bases == nullptr) { + pyc_output << "("; + print_src(bases_node, mod, pyc_output); + pyc_output << "):\n"; + cleanBuild = false; } else { // Don't put parens if there are no base classes pyc_output << ":\n"; } printClassDocstring = true; - PycRef code = src.cast()->code().cast() - ->func().cast()->code(); - print_src(code, mod, pyc_output); + PycRef class_call = src.cast()->code().try_cast(); + PycRef class_func = nullptr; + if (class_call != nullptr) + class_func = class_call->func().try_cast(); + + if (class_func != nullptr) { + print_src(class_func->code(), mod, pyc_output); + } else { + cleanBuild = false; + start_line(cur_indent + 1, pyc_output); + pyc_output << "# unsupported class body"; + end_line(pyc_output); + start_line(cur_indent + 1, pyc_output); + pyc_output << "pass"; + end_line(pyc_output); + } } else if (src.type() == ASTNode::NODE_IMPORT) { PycRef import = src.cast(); if (import->fromlist() != NULL) { @@ -3756,12 +4152,12 @@ void decompyle(PycRef code, PycModule* mod, std::ostream& pyc_output) PycRef source = BuildFromCode(code, mod); PycRef clean = source.cast(); - if (cleanBuild) { + if (cleanBuild && clean->nodes().size() != 0) { // The Python compiler adds some stuff that we don't really care // about, and would add extra code for re-compilation anyway. // We strip these lines out here, and then add a "pass" statement // if the cleaned up code is empty - if (clean->nodes().front().type() == ASTNode::NODE_STORE) { + if (clean->nodes().size() != 0 && clean->nodes().front().type() == ASTNode::NODE_STORE) { PycRef store = clean->nodes().front().cast(); if (store->src().type() == ASTNode::NODE_NAME && store->dest().type() == ASTNode::NODE_NAME) { @@ -3775,7 +4171,7 @@ void decompyle(PycRef code, PycModule* mod, std::ostream& pyc_output) } } } - if (clean->nodes().front().type() == ASTNode::NODE_STORE) { + if (clean->nodes().size() != 0 && clean->nodes().front().type() == ASTNode::NODE_STORE) { PycRef store = clean->nodes().front().cast(); if (store->src().type() == ASTNode::NODE_OBJECT && store->dest().type() == ASTNode::NODE_NAME) { @@ -3791,7 +4187,8 @@ void decompyle(PycRef code, PycModule* mod, std::ostream& pyc_output) } // Class and module docstrings may only appear at the beginning of their source - if (printClassDocstring && clean->nodes().front().type() == ASTNode::NODE_STORE) { + if (printClassDocstring && clean->nodes().size() != 0 + && clean->nodes().front().type() == ASTNode::NODE_STORE) { PycRef store = clean->nodes().front().cast(); if (store->dest().type() == ASTNode::NODE_NAME && store->dest().cast()->name()->isEqual("__doc__") && @@ -3801,7 +4198,7 @@ void decompyle(PycRef code, PycModule* mod, std::ostream& pyc_output) clean->removeFirst(); } } - if (clean->nodes().back().type() == ASTNode::NODE_RETURN) { + if (clean->nodes().size() != 0 && clean->nodes().back().type() == ASTNode::NODE_RETURN) { PycRef ret = clean->nodes().back().cast(); PycRef retObj = ret->value().try_cast(); diff --git a/CMakeLists.txt b/CMakeLists.txt index 30d9c6211..201be49b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,9 @@ install(TARGETS pycdas add_executable(pycdc pycdc.cpp ASTree.cpp ASTNode.cpp) target_link_libraries(pycdc pycxx) +if(MSVC) + target_link_options(pycdc PRIVATE /STACK:16777216) +endif() install(TARGETS pycdc RUNTIME DESTINATION bin) diff --git a/bytecode.cpp b/bytecode.cpp index 02448279b..40cd8f5ca 100644 --- a/bytecode.cpp +++ b/bytecode.cpp @@ -296,7 +296,7 @@ void bc_next(PycBuffer& source, PycModule* mod, int& opcode, int& operand, int& if (mod->verCompare(3, 6) >= 0) { operand = source.getByte(); pos += 2; - if (opcode == Pyc::EXTENDED_ARG_A) { + while (opcode == Pyc::EXTENDED_ARG_A && !source.atEof()) { opcode = Pyc::ByteToOpcode(mod->majorVer(), mod->minorVer(), source.getByte()); operand = (operand << 8) | source.getByte(); pos += 2; diff --git a/data.cpp b/data.cpp index 2b560a7cf..3f79683a5 100644 --- a/data.cpp +++ b/data.cpp @@ -73,8 +73,10 @@ void PycFile::getBuffer(int bytes, void* buffer) int PycBuffer::getByte() { if (atEof()) { - fputs("PycBuffer::getByte(): Unexpected end of stream\n", stderr); - std::exit(1); + /* Code buffers can occasionally be walked onto a trailing partial + wordcode while recovering malformed/newer bytecode. Treat the + missing byte as padding so callers can finish best-effort output. */ + return 0; } int ch = (int)(*(m_buffer + m_pos)); ++m_pos; diff --git a/data.h b/data.h index 28cc85e55..a2c28d1f7 100644 --- a/data.h +++ b/data.h @@ -47,7 +47,7 @@ class PycBuffer : public PycData { ~PycBuffer() { } bool isOpen() const override { return (m_buffer != 0); } - bool atEof() const override { return (m_pos == m_size); } + bool atEof() const override { return (m_pos >= m_size); } int getByte() override; void getBuffer(int bytes, void* buffer) override; diff --git a/tests/compiled/call_kw_3_13.3.13.pyc b/tests/compiled/call_kw_3_13.3.13.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d014e9f5110003fe4c197a91c51208c469a113f0 GIT binary patch literal 474 zcmYjNF-yZh6n>W`X|#5*ZcZ^)Fldb;4vOd^ItUJ)-HN3oN1K`^;Vz}|{kwlF;+uR7Ed57%hk|qf14g?* z0UKDEggT +def greet ( name , greeting ) : + +return greeting + ', ' + name + +print ( greet ( 'world' , greeting = 'hi' ) ) +print ( json . dumps ( { } , ensure_ascii = False ) ) From 38bd0337b72561dd8bab6639e275fb71d67f22e6 Mon Sep 17 00:00:00 2001 From: Techuouo520 Date: Wed, 17 Jun 2026 15:09:29 +0800 Subject: [PATCH 3/3] Improve Python 3.14 exception recovery --- ASTNode.h | 6 +- ASTree.cpp | 296 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 295 insertions(+), 7 deletions(-) diff --git a/ASTNode.h b/ASTNode.h index f78049f56..78fe56b3c 100644 --- a/ASTNode.h +++ b/ASTNode.h @@ -585,13 +585,17 @@ class ASTCondBlock : public ASTBlock { ASTCondBlock(ASTBlock::BlkType blktype, int end, PycRef cond, bool negative = false) - : ASTBlock(blktype, end), m_cond(std::move(cond)), m_negative(negative) { } + : ASTBlock(blktype, end), m_cond(std::move(cond)), m_except_var(), + m_negative(negative) { } PycRef cond() const { return m_cond; } + PycRef exceptVar() const { return m_except_var; } bool negative() const { return m_negative; } + void setExceptVar(PycRef except_var) { m_except_var = std::move(except_var); } private: PycRef m_cond; + PycRef m_except_var; bool m_negative; }; diff --git a/ASTree.cpp b/ASTree.cpp index d3504a34e..db9be425a 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -110,6 +110,95 @@ static void ast_consume_call_self_slot(FastStack& stack, PycRef& func) } } +static bool ast_read_wordcode_instr(PycRef bytes, PycModule* mod, + int offset, int& opcode, int& operand, int& next) +{ + if (mod->verCompare(3, 6) < 0 || bytes == nullptr) + return false; + + int length = bytes->length(); + if (offset < 0 || offset + 1 >= length) + return false; + + const unsigned char* raw = (const unsigned char*)bytes->value(); + int ext = 0; + + while (offset + 1 < length) { + opcode = Pyc::ByteToOpcode(mod->majorVer(), mod->minorVer(), raw[offset]); + int oparg = raw[offset + 1]; + next = offset + 2; + + if (opcode != Pyc::EXTENDED_ARG_A) { + operand = (ext << 8) | oparg; + return true; + } + + ext = (ext << 8) | oparg; + offset = next; + } + + return false; +} + +static bool ast_is_synthetic_exception_cleanup(PycRef code, PycModule* mod, + int offset, int& end) +{ + int opcode, operand, next; + if (!ast_read_wordcode_instr(code->code(), mod, offset, opcode, operand, next) + || opcode != Pyc::LOAD_CONST_A) + return false; + + PycRef obj = code->getConst(operand); + if (obj == nullptr || obj->type() != PycObject::TYPE_NONE) + return false; + + int store_op, store_arg, store_next; + if (!ast_read_wordcode_instr(code->code(), mod, next, store_op, store_arg, store_next) + || (store_op != Pyc::STORE_FAST_A && store_op != Pyc::STORE_DEREF_A)) + return false; + + int delete_op, delete_arg, delete_next; + if (!ast_read_wordcode_instr(code->code(), mod, store_next, delete_op, delete_arg, delete_next) + || delete_arg != store_arg) + return false; + + if ((store_op == Pyc::STORE_FAST_A && delete_op != Pyc::DELETE_FAST_A) + || (store_op == Pyc::STORE_DEREF_A && delete_op != Pyc::DELETE_DEREF_A)) + return false; + + end = delete_next; + return true; +} + +static PycRef ast_find_except_bind_block(PycRef curblock, + std::stack > blocks) +{ + if (curblock != nullptr && curblock->blktype() == ASTBlock::BLK_EXCEPT) { + PycRef except = curblock.try_cast(); + if (except != nullptr && except->cond() != nullptr) + return except; + } + + while (!blocks.empty()) { + PycRef block = blocks.top(); + blocks.pop(); + if (block != nullptr && block->blktype() == ASTBlock::BLK_EXCEPT) { + PycRef except = block.try_cast(); + if (except != nullptr && except->cond() != nullptr) + return except; + } + } + + return nullptr; +} + +static PycRef ast_name_from_string(const char* name) +{ + PycRef str = new PycString(); + str->setValue(name); + return new ASTName(str); +} + /* compiler generates very, VERY similar byte code for if/else statement block and if-expression * statement * if a: b = 1 @@ -165,6 +254,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) bool need_try = false; bool variable_annotations = false; bool py39_except_bind_pending = false; + bool py311_except_bind_pending = false; + int py311_cleanup_sequences = 0; int py39_except_cleanup_var = -1; int py39_normal_cleanup_step = 0; int py39_handler_cleanup_step = 0; @@ -294,6 +385,25 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curpos = pos; bc_next(source, mod, opcode, operand, pos); + if (py311_except_bind_pending + && opcode != Pyc::CACHE + && opcode != Pyc::NOT_TAKEN + && opcode != Pyc::INSTRUMENTED_NOT_TAKEN_A + && opcode != Pyc::STORE_FAST_A + && opcode != Pyc::STORE_DEREF_A) { + py311_except_bind_pending = false; + } + + int cleanup_end = pos; + if (py311_cleanup_sequences > 0 + && curblock->blktype() == ASTBlock::BLK_EXCEPT + && ast_is_synthetic_exception_cleanup(code, mod, curpos, cleanup_end)) { + while (pos < cleanup_end && !source.atEof()) + bc_next(source, mod, opcode, operand, pos); + py311_cleanup_sequences--; + continue; + } + if (mod->verCompare(3, 9) >= 0 && mod->verCompare(3, 11) < 0 && py39_except_cleanup_var >= 0) { bool is_none_const = false; @@ -305,11 +415,13 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (py39_normal_cleanup_step == 1 && opcode == Pyc::LOAD_CONST_A && is_none_const) { py39_normal_cleanup_step = 2; continue; - } else if (py39_normal_cleanup_step == 2 && opcode == Pyc::STORE_FAST_A + } else if (py39_normal_cleanup_step == 2 + && (opcode == Pyc::STORE_FAST_A || opcode == Pyc::STORE_DEREF_A) && operand == py39_except_cleanup_var) { py39_normal_cleanup_step = 3; continue; - } else if (py39_normal_cleanup_step == 3 && opcode == Pyc::DELETE_FAST_A + } else if (py39_normal_cleanup_step == 3 + && (opcode == Pyc::DELETE_FAST_A || opcode == Pyc::DELETE_DEREF_A) && operand == py39_except_cleanup_var) { py39_normal_cleanup_step = 0; continue; @@ -321,11 +433,13 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) && opcode == Pyc::LOAD_CONST_A && is_none_const) { py39_handler_cleanup_step = 2; continue; - } else if (py39_handler_cleanup_step == 2 && opcode == Pyc::STORE_FAST_A + } else if (py39_handler_cleanup_step == 2 + && (opcode == Pyc::STORE_FAST_A || opcode == Pyc::STORE_DEREF_A) && operand == py39_except_cleanup_var) { py39_handler_cleanup_step = 3; continue; - } else if (py39_handler_cleanup_step == 3 && opcode == Pyc::DELETE_FAST_A + } else if (py39_handler_cleanup_step == 3 + && (opcode == Pyc::DELETE_FAST_A || opcode == Pyc::DELETE_DEREF_A) && operand == py39_except_cleanup_var) { py39_handler_cleanup_step = 4; continue; @@ -1018,6 +1132,42 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; } + if (curblock->size() != 0 && curblock->nodes().back().type() == ASTNode::NODE_STORE) { + PycRef store = curblock->nodes().back().cast(); + PycRef dest = store->dest().try_cast(); + + if (store->src() == nullptr && dest != nullptr + && dest->name()->isEqual(name.cast()->name()->value())) { + curblock->removeLast(); + break; + } + } + + curblock->append(new ASTDelete(name)); + } + break; + case Pyc::DELETE_DEREF_A: + { + PycRef name = new ASTName(code->getCellVar(mod, operand)); + PycRef varname = name.cast()->name(); + + if (varname->length() >= 2 && varname->value()[0] == '_' + && varname->value()[1] == '[') { + /* Don't show deletes that are a result of list comps. */ + break; + } + + if (curblock->size() != 0 && curblock->nodes().back().type() == ASTNode::NODE_STORE) { + PycRef store = curblock->nodes().back().cast(); + PycRef dest = store->dest().try_cast(); + + if (store->src() == nullptr && dest != nullptr + && dest->name()->isEqual(varname->value())) { + curblock->removeLast(); + break; + } + } + curblock->append(new ASTDelete(name)); } break; @@ -1524,6 +1674,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } ifblk = new ASTCondBlock(ASTBlock::BLK_EXCEPT, except_end, cond.cast()->right(), false); + py311_except_bind_pending = mod->verCompare(3, 11) >= 0; } else if (curblock->blktype() == ASTBlock::BLK_ELSE && curblock->size() == 0) { /* Collapse into elif statement */ @@ -2232,6 +2383,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) if (mod->verCompare(3, 9) >= 0 && mod->verCompare(3, 11) < 0 && py39_except_cleanup_var >= 0) { py39_normal_cleanup_step = 1; + } else if (mod->verCompare(3, 11) >= 0) { + py311_cleanup_sequences = 2; } break; case Pyc::PUSH_EXC_INFO: @@ -2668,6 +2821,15 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.pop(); PycRef name = new ASTName(code->getCellVar(mod, operand)); + if (py311_except_bind_pending && mod->verCompare(3, 11) >= 0) { + PycRef except = ast_find_except_bind_block(curblock, blocks); + if (except != nullptr) { + except->setExceptVar(name); + py311_except_bind_pending = false; + break; + } + } + if (value.type() == ASTNode::NODE_CHAINSTORE) { append_to_chain_store(value, name, stack, curblock); } else { @@ -2745,6 +2907,15 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) break; } + if (py311_except_bind_pending && mod->verCompare(3, 11) >= 0) { + PycRef except = ast_find_except_bind_block(curblock, blocks); + if (except != nullptr) { + except->setExceptVar(name); + py311_except_bind_pending = false; + break; + } + } + if (name.cast()->name()->value()[0] == '_' && name.cast()->name()->value()[1] == '[') { /* Don't show stores of list comp append objects. */ @@ -3356,8 +3527,103 @@ static void end_line(std::ostream& pyc_output) } int cur_indent = -1; +static bool ast_is_empty_store(PycRef node) +{ + if (node == nullptr || node.type() != ASTNode::NODE_STORE) + return false; + + PycRef store = node.cast(); + return store->src() == nullptr && store->dest() == nullptr; +} + +static bool ast_references_name(PycRef node, const char* name) +{ + if (node == nullptr) + return false; + + switch (node->type()) { + case ASTNode::NODE_NAME: + return node.cast()->name()->isEqual(name); + case ASTNode::NODE_BLOCK: + for (const auto& child : node.cast()->nodes()) { + if (ast_references_name(child, name)) + return true; + } + return false; + case ASTNode::NODE_STORE: + return ast_references_name(node.cast()->src(), name) + || ast_references_name(node.cast()->dest(), name); + case ASTNode::NODE_DELETE: + return ast_references_name(node.cast()->value(), name); + case ASTNode::NODE_BINARY: + case ASTNode::NODE_COMPARE: + case ASTNode::NODE_SLICE: + return ast_references_name(node.cast()->left(), name) + || ast_references_name(node.cast()->right(), name); + case ASTNode::NODE_UNARY: + return ast_references_name(node.cast()->operand(), name); + case ASTNode::NODE_CALL: + { + PycRef call = node.cast(); + if (ast_references_name(call->func(), name) + || ast_references_name(call->var(), name) + || ast_references_name(call->kw(), name)) { + return true; + } + for (const auto& param : call->pparams()) { + if (ast_references_name(param, name)) + return true; + } + for (const auto& param : call->kwparams()) { + if (ast_references_name(param.first, name) + || ast_references_name(param.second, name)) { + return true; + } + } + return false; + } + case ASTNode::NODE_TUPLE: + for (const auto& val : node.cast()->values()) { + if (ast_references_name(val, name)) + return true; + } + return false; + case ASTNode::NODE_LIST: + for (const auto& val : node.cast()->values()) { + if (ast_references_name(val, name)) + return true; + } + return false; + case ASTNode::NODE_JOINEDSTR: + for (const auto& val : node.cast()->values()) { + if (ast_references_name(val, name)) + return true; + } + return false; + case ASTNode::NODE_FORMATTEDVALUE: + return ast_references_name(node.cast()->val(), name) + || ast_references_name(node.cast()->format_spec(), name); + default: + return false; + } +} + +static bool ast_block_references_name_after_first(PycRef blk, const char* name) +{ + bool first = true; + for (const auto& node : blk->nodes()) { + if (first) { + first = false; + continue; + } + if (ast_references_name(node, name)) + return true; + } + return false; +} + static void print_block(PycRef blk, PycModule* mod, - std::ostream& pyc_output) + std::ostream& pyc_output, bool skip_first = false) { ASTBlock::list_t lines = blk->nodes(); @@ -3368,6 +3634,11 @@ static void print_block(PycRef blk, PycModule* mod, } for (auto ln = lines.cbegin(); ln != lines.cend();) { + if (skip_first) { + ++ln; + skip_first = false; + continue; + } if ((*ln).cast().type() != ASTNode::NODE_NODELIST) { start_line(cur_indent, pyc_output); } @@ -3652,6 +3923,7 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) case ASTNode::NODE_BLOCK: { PycRef blk = node.cast(); + bool skip_first = false; if (blk->blktype() == ASTBlock::BLK_ELSE && blk->size() == 0) break; @@ -3687,6 +3959,18 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) blk.cast()->cond() != NULL) { pyc_output << " "; print_src(blk.cast()->cond(), mod, pyc_output); + PycRef except_var = blk.cast()->exceptVar(); + if (except_var == NULL + && !blk->nodes().empty() + && ast_is_empty_store(blk->nodes().front()) + && ast_block_references_name_after_first(blk, "e")) { + except_var = ast_name_from_string("e"); + skip_first = true; + } + if (except_var != NULL) { + pyc_output << " as "; + print_src(except_var, mod, pyc_output); + } } else if (blk->blktype() == ASTBlock::BLK_WITH) { pyc_output << " "; print_src(blk.cast()->expr(), mod, pyc_output); @@ -3699,7 +3983,7 @@ void print_src(PycRef node, PycModule* mod, std::ostream& pyc_output) pyc_output << ":\n"; cur_indent++; - print_block(blk, mod, pyc_output); + print_block(blk, mod, pyc_output, skip_first); cur_indent--; } break;