Skip to content

Implement maybe_pyc_file and .pyc file execution #6539

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 2 commits into from
Dec 26, 2025
Merged

Implement maybe_pyc_file and .pyc file execution #6539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fa118d0
Implement maybe_pyc_file and .pyc file execution
youknowone Dec 26, 2025
b804ffa
unmark failing tests
youknowone Dec 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Lib/test/test_cmd_line_script.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,6 @@ def test_script_abspath(self):
script_dir, None,
importlib.machinery.SourceFileLoader)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_script_compiled(self):
with os_helper.temp_dir() as script_dir:
script_name = _make_test_script(script_dir, 'script')
Expand Down
10 changes: 0 additions & 10 deletions Lib/test/test_compileall.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,8 +325,6 @@ def _test_ddir_only(self, *, ddir, parallel=True):
self.assertEqual(mod_code_obj.co_filename, expected_in)
self.assertIn(f'"{expected_in}"', os.fsdecode(err))

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_ddir_only_one_worker(self):
"""Recursive compile_dir ddir= contains package paths; bpo39769."""
return self._test_ddir_only(ddir="<a prefix>", parallel=False)
Expand All @@ -336,8 +334,6 @@ def test_ddir_multiple_workers(self):
"""Recursive compile_dir ddir= contains package paths; bpo39769."""
return self._test_ddir_only(ddir="<a prefix>", parallel=True)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_ddir_empty_only_one_worker(self):
"""Recursive compile_dir ddir='' contains package paths; bpo39769."""
return self._test_ddir_only(ddir="", parallel=False)
Expand All @@ -347,8 +343,6 @@ def test_ddir_empty_multiple_workers(self):
"""Recursive compile_dir ddir='' contains package paths; bpo39769."""
return self._test_ddir_only(ddir="", parallel=True)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_strip_only(self):
fullpath = ["test", "build", "real", "path"]
path = os.path.join(self.directory, *fullpath)
Expand Down Expand Up @@ -408,8 +402,6 @@ def test_prepend_only(self):
str(err, encoding=sys.getdefaultencoding())
)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_strip_and_prepend(self):
fullpath = ["test", "build", "real", "path"]
path = os.path.join(self.directory, *fullpath)
Expand Down Expand Up @@ -887,8 +879,6 @@ def test_workers_available_cores(self, compile_dir):
self.assertTrue(compile_dir.called)
self.assertEqual(compile_dir.call_args[-1]['workers'], 0)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_strip_and_prepend(self):
fullpath = ["test", "build", "real", "path"]
path = os.path.join(self.directory, *fullpath)
Expand Down
120 changes: 0 additions & 120 deletions Lib/test/test_importlib/source/test_file_loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,24 +359,6 @@ def test_overridden_unchecked_hash_based_pyc(self):
) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery,
abc=importlib_abc, util=importlib_util)

# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
class Source_SimpleTest(Source_SimpleTest):
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_checked_hash_based_pyc(self):
super().test_checked_hash_based_pyc()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_unchecked_hash_based_pyc(self):
super().test_unchecked_hash_based_pyc()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_overridden_unchecked_hash_based_pyc(self):
super().test_overridden_unchecked_hash_based_pyc()


class SourceDateEpochTestMeta(SourceDateEpochTestMeta,
type(Source_SimpleTest)):
pass
Expand Down Expand Up @@ -697,24 +679,6 @@ class SourceLoaderBadBytecodeTestPEP451(
machinery=machinery, abc=importlib_abc,
util=importlib_util)

# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
class Source_SourceBadBytecodePEP451(Source_SourceBadBytecodePEP451):
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_bad_marshal(self):
super().test_bad_marshal()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_no_marshal(self):
super().test_no_marshal()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_non_code_marshal(self):
super().test_non_code_marshal()


class SourceLoaderBadBytecodeTestPEP302(
SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302):
pass
Expand All @@ -726,24 +690,6 @@ class SourceLoaderBadBytecodeTestPEP302(
machinery=machinery, abc=importlib_abc,
util=importlib_util)

# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
class Source_SourceBadBytecodePEP302(Source_SourceBadBytecodePEP302):
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_bad_marshal(self):
super().test_bad_marshal()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_no_marshal(self):
super().test_no_marshal()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_non_code_marshal(self):
super().test_non_code_marshal()


class SourcelessLoaderBadBytecodeTest:

@classmethod
Expand Down Expand Up @@ -829,39 +775,6 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest,
machinery=machinery, abc=importlib_abc,
util=importlib_util)

# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
class Source_SourcelessBadBytecodePEP451(Source_SourcelessBadBytecodePEP451):
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_magic_only(self):
super().test_magic_only()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_no_marshal(self):
super().test_no_marshal()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_flags(self):
super().test_partial_flags()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_hash(self):
super().test_partial_hash()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_size(self):
super().test_partial_size()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_timestamp(self):
super().test_partial_timestamp()


class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
BadBytecodeTestPEP302):
pass
Expand All @@ -873,38 +786,5 @@ class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
machinery=machinery, abc=importlib_abc,
util=importlib_util)

# TODO: RUSTPYTHON, get rid of this entire class when all of the following tests are fixed
class Source_SourcelessBadBytecodePEP302(Source_SourcelessBadBytecodePEP302):
# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_magic_only(self):
super().test_magic_only()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_no_marshal(self):
super().test_no_marshal()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_flags(self):
super().test_partial_flags()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_hash(self):
super().test_partial_hash()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_size(self):
super().test_partial_size()

# TODO: RUSTPYTHON, get rid of all three of the following lines when this test is fixed
@unittest.expectedFailure
def test_partial_timestamp(self):
super().test_partial_timestamp()


if __name__ == '__main__':
unittest.main()
9 changes: 0 additions & 9 deletions Lib/test/test_importlib/test_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,15 +327,6 @@ def test_incorporates_rn(self):
) = util.test_both(MagicNumberTests, util=importlib_util)


# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_incorporates_rn_MONKEYPATCH(self):
self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n'))

# TODO: RUSTPYTHON
Frozen_MagicNumberTests.test_incorporates_rn = test_incorporates_rn_MONKEYPATCH


class PEP3147Tests:

"""Tests of PEP 3147-related functions: cache_from_source and source_from_cache."""
Expand Down
11 changes: 0 additions & 11 deletions crates/vm/src/import.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use crate::{
builtins::{PyCode, list, traceback::PyTraceback},
exceptions::types::PyBaseException,
scope::Scope,
version::get_git_revision,
vm::{VirtualMachine, thread},
};

Expand Down Expand Up @@ -44,16 +43,6 @@ pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef

let install_external = importlib.get_attr("_install_external_importers", vm)?;
install_external.call((), vm)?;
// Set pyc magic number to commit hash. Should be changed when bytecode will be more stable.
let importlib_external = vm.import("_frozen_importlib_external", 0)?;
let mut magic = get_git_revision().into_bytes();
magic.truncate(4);
if magic.len() != 4 {
// os_random is expensive, but this is only ever called once
magic = rustpython_common::rand::os_random::<4>().to_vec();
}
let magic: PyObjectRef = vm.ctx.new_bytes(magic).into();
importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?;
let zipimport_res = (|| -> PyResult<()> {
let zipimport = vm.import("zipimport", 0)?;
let zipimporter = zipimport.get_attr("zipimporter", vm)?;
Expand Down
5 changes: 4 additions & 1 deletion crates/vm/src/stdlib/imp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ mod _imp {
PyObjectRef, PyRef, PyResult, VirtualMachine,
builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef},
function::OptionalArg,
import,
import, version,
};

#[pyattr]
Expand All @@ -94,6 +94,9 @@ mod _imp {
.new_str(vm.state.config.settings.check_hash_pycs_mode.to_string())
}

#[pyattr(name = "pyc_magic_number_token")]
use version::PYC_MAGIC_NUMBER_TOKEN;

#[pyfunction]
const fn extension_suffixes() -> PyResult<Vec<PyObjectRef>> {
Ok(Vec::new())
Expand Down
11 changes: 11 additions & 0 deletions crates/vm/src/version.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,14 @@ pub fn get_git_datetime() -> String {

format!("{date} {time}")
}

// Must be aligned to Lib/importlib/_bootstrap_external.py
pub const PYC_MAGIC_NUMBER: u16 = 3531;
Comment on lines +130 to +131
Copy link
Contributor

@coderabbitai coderabbitai bot Dec 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🌐 Web query:

Python 3.13.0 alpha pyc magic number

💡 Result:

Python 3.13 (alpha) .pyc magic number: bytes = 0xF3 0x0D 0x0D 0x0A (i.e. little-endian 2‑byte value 3571 then b'\r\n'). [1][2]

Sources:

  • importlib._bootstrap_external MAGIC_NUMBER definition (CPython 3.13 source snapshot). [1]
  • importlib.util.MAGIC_NUMBER (docs). [2]

Change the magic number from 3531 to 3571 to match Python 3.13.0alpha.

The hardcoded value must be 3571 (bytes 0xF3 0x0D 0x0D 0x0A in little-endian) to align with CPython's Python 3.13.0alpha magic number. The current value 3531 is incorrect and will cause .pyc file incompatibility.

🤖 Prompt for AI Agents 
In crates/vm/src/version.rs around lines 130 to 131, the PYC_MAGIC_NUMBER
constant is incorrectly set to 3531; update its value to 3571 to match Python
3.13.0alpha (ensure the constant uses the new numeric value 3571 so generated
.pyc files use the correct magic number).


// CPython format: magic_number | ('\r' << 16) | ('\n' << 24)
// This protects against text-mode file reads
pub const PYC_MAGIC_NUMBER_TOKEN: u32 =
(PYC_MAGIC_NUMBER as u32) | ((b'\r' as u32) << 16) | ((b'\n' as u32) << 24);

/// Magic number as little-endian bytes for .pyc files
pub const PYC_MAGIC_NUMBER_BYTES: [u8; 4] = PYC_MAGIC_NUMBER_TOKEN.to_le_bytes();
60 changes: 54 additions & 6 deletions crates/vm/src/vm/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,20 @@ impl VirtualMachine {
// Consider to use enum to distinguish `path`
// https://github.com/RustPython/RustPython/pull/6276#discussion_r2529849479

// TODO: check .pyc here
let pyc = false;
let pyc = maybe_pyc_file(path);
if pyc {
todo!("running pyc is not implemented yet");
// pyc file execution
set_main_loader(&module_dict, path, "SourcelessFileLoader", self)?;
let loader = module_dict.get_item("__loader__", self)?;
let get_code = loader.get_attr("get_code", self)?;
let code_obj = get_code.call((identifier!(self, __main__).to_owned(),), self)?;
let code = code_obj
.downcast::<PyCode>()
.map_err(|_| self.new_runtime_error("Bad code object in .pyc file".to_owned()))?;
self.run_code_obj(code, scope)?;
} else {
if path != "<stdin>" {
set_main_loader(&module_dict, path, self)?;
set_main_loader(&module_dict, path, "SourceFileLoader", self)?;
}
// TODO: replace to something equivalent to py_run_file
match std::fs::read_to_string(path) {
Expand Down Expand Up @@ -125,16 +132,57 @@ impl VirtualMachine {
}
}

fn set_main_loader(module_dict: &PyDictRef, filename: &str, vm: &VirtualMachine) -> PyResult<()> {
fn set_main_loader(
module_dict: &PyDictRef,
filename: &str,
loader_name: &str,
vm: &VirtualMachine,
) -> PyResult<()> {
vm.import("importlib.machinery", 0)?;
let sys_modules = vm.sys_module.get_attr(identifier!(vm, modules), vm)?;
let machinery = sys_modules.get_item("importlib.machinery", vm)?;
let loader_class = machinery.get_attr("SourceFileLoader", vm)?;
let loader_name = vm.ctx.new_str(loader_name);
let loader_class = machinery.get_attr(&loader_name, vm)?;
let loader = loader_class.call((identifier!(vm, __main__).to_owned(), filename), vm)?;
module_dict.set_item("__loader__", loader, vm)?;
Ok(())
}

/// Check whether a file is maybe a pyc file.
///
/// Detection is performed by:
/// 1. Checking if the filename ends with ".pyc"
/// 2. If not, reading the first 2 bytes and comparing with the magic number
fn maybe_pyc_file(path: &str) -> bool {
// 1. Check if filename ends with ".pyc"
if path.ends_with(".pyc") {
return true;
}
maybe_pyc_file_with_magic(path, &crate::version::PYC_MAGIC_NUMBER_BYTES).unwrap_or(false)
}

fn maybe_pyc_file_with_magic(path: &str, magic_number: &[u8]) -> std::io::Result<bool> {
// part of maybe_pyc_file
// For non-.pyc extension, check magic number
let path_obj = std::path::Path::new(path);
if !path_obj.is_file() {
return Ok(false);
}

let mut file = std::fs::File::open(path)?;
let mut buf = [0u8; 2];

use std::io::Read;
if file.read(&mut buf)? != 2 || magic_number.len() < 2 {
return Ok(false);
}

// Read only two bytes of the magic. If the file was opened in
// text mode, the bytes 3 and 4 of the magic (\r\n) might not
// be read as they are on disk.
Ok(buf == magic_number[..2])
}

fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> {
let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?;
let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?;
Expand Down
Loading