[mypyc] Enable incremental self-compilation

mypyc/build.py

@@ -449,6 +449,52 @@ def write_file(path: str, contents: str) -> None:
         os.utime(path, times=(new_mtime, new_mtime))
 
 
+_setuptools_patch_applied = False
+
+
+def _patch_setuptools_copy_extensions_to_source() -> None:
+    """Skip redundant `.so` copies in --inplace builds.
+
+    setuptools' copy_extensions_to_source rewrites every `.so` in the
+    source tree on every build_ext, even when nothing changed. On macOS
+    this invalidates AMFI's signature cache (~100 ms re-verification per
+    `.so` on the next import), eating most of the separate=True
+    incremental speedup. We patch it to skip the copy when src and dst
+    already match. Idempotent; applied from mypycify().
+    """
+    global _setuptools_patch_applied
+    if _setuptools_patch_applied:
+        return
+    _setuptools_patch_applied = True
+
+    from setuptools.command.build_ext import build_ext as _build_ext
+
+    def _files_match(a: str, b: str) -> bool:
+        try:
+            sa = os.stat(a)
+            sb = os.stat(b)
+        except OSError:
+            return False
+        # Compare size + whole-second mtime. distutils' copy_file
+        # propagates the source mtime, but macOS drops sub-second
+        # precision on write so the float values never match verbatim.
+        return sa.st_size == sb.st_size and int(sa.st_mtime) == int(sb.st_mtime)
+
+    def patched(self: Any) -> None:
+        build_py = self.get_finalized_command("build_py")
+        for ext in self.extensions:
+            inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext)
+            if _files_match(regular_file, inplace_file):
+                continue
+            if os.path.exists(regular_file) or not ext.optional:
+                self.copy_file(regular_file, inplace_file, level=self.verbose)
+            if ext._needs_stub:
+                inplace_stub = self._get_equivalent_stub(ext, inplace_file)
+                self._write_stub_file(inplace_stub, ext, compile=True)
+
+    _build_ext.copy_extensions_to_source = patched  # type: ignore[method-assign]
+
+
 def construct_groups(
     sources: list[BuildSource],
     separate: bool | list[tuple[list[str], str | None]],
@@ -508,7 +554,7 @@ def get_header_deps(cfiles: list[tuple[str, str]]) -> list[str]:
     """
     headers: set[str] = set()
     for _, contents in cfiles:
-        headers.update(re.findall(r'#include "(.*)"', contents))
+        headers.update(re.findall(r'#include [<"]([^>"]+)[>"]', contents))
 
     return sorted(headers)
 
@@ -568,12 +614,21 @@ def mypyc_build(
         cfilenames = []
         for cfile, ctext in cfiles:
             cfile = os.path.join(compiler_options.target_dir, cfile)
-            if not options.mypyc_skip_c_generation:
+            # Empty contents marks a file the previous run already wrote
+            # (fully-cached group): skip the rewrite and just reuse it.
+            if ctext and not options.mypyc_skip_c_generation:
                 write_file(cfile, ctext)
             if os.path.splitext(cfile)[1] == ".c":
                 cfilenames.append(cfile)
 
-        deps = [os.path.join(compiler_options.target_dir, dep) for dep in get_header_deps(cfiles)]
+        # The header regex matches both quote styles, so the result can
+        # include system headers like `<Python.h>` that don't live under
+        # target_dir. Joining those produces non-existent paths which
+        # would force a full rebuild on every run via Extension.depends.
+        candidate_deps = (
+            os.path.join(compiler_options.target_dir, dep) for dep in get_header_deps(cfiles)
+        )
+        deps = [d for d in candidate_deps if os.path.exists(d)]
         group_cfilenames.append((cfilenames, deps))
 
     return groups, group_cfilenames, source_deps
@@ -747,6 +802,9 @@ def mypycify(
                                have no backward compatibility guarantees!
     """
 
+    # Skip redundant inplace .so copies on every build_ext invocation.
+    _patch_setuptools_copy_extensions_to_source()
+
     # Figure out our configuration
     compiler_options = CompilerOptions(
         strip_asserts=strip_asserts,

mypyc/codegen/emit.py

@@ -314,6 +314,18 @@ def get_group_prefix(self, obj: ClassIR | FuncDecl) -> str:
         # See docs above
         return self.get_module_group_prefix(obj.module_name)
 
+    def register_group_dep(self, cl: ClassIR) -> None:
+        """Record `cl`'s defining group as a cross-group dep, if any.
+
+        Call this when emitting code that refers to `cl`'s struct
+        layout: the .c file consuming that layout needs the defining
+        group's `__native_*.h` included, and group_deps drives which
+        headers get pulled in.
+        """
+        target_group = self.context.group_map.get(cl.module_name)
+        if target_group and target_group != self.context.group_name:
+            self.context.group_deps.add(target_group)
+
     def static_name(self, id: str, module: str | None, prefix: str = STATIC_PREFIX) -> str:
         """Create name of a C static variable.
 

mypyc/codegen/emitfunc.py

@@ -348,6 +348,11 @@ def get_attr_expr(self, obj: str, op: GetAttr | SetAttr, decl_cl: ClassIR) -> st
         classes, and *(obj + attr_offset) for attributes defined by traits. We also
         insert all necessary C casts here.
         """
+        # The struct cast below needs the defining group's __native.h
+        # included by the consuming .c file. Record both the receiver
+        # and declaring classes as cross-group deps.
+        self.emitter.register_group_dep(op.class_type.class_ir)
+        self.emitter.register_group_dep(decl_cl)
         cast = f"({op.class_type.struct_name(self.emitter.names)} *)"
         if decl_cl.is_trait and op.class_type.class_ir.is_trait:
             # For pure trait access find the offset first, offsets

mypyc/codegen/emitmodule.py

@@ -362,7 +362,12 @@ def compile_ir_to_c(
             if source.module in modules
         }
         if not group_modules:
-            ctext[group_name] = []
+            # Fully-cached group (e.g. pip's second setup.py invoke for
+            # the wheel phase): no fresh IR was produced. Reuse the file
+            # list recorded in any module's IR cache so the linker still
+            # sees the previous run's outputs; empty content is a "do
+            # not rewrite" sentinel for mypyc_build.
+            ctext[group_name] = _load_cached_group_files(group_sources, result)
             continue
         generator = GroupGenerator(
             group_modules, source_paths, group_name, mapper.group_map, names, compiler_options
@@ -372,6 +377,32 @@ def compile_ir_to_c(
     return ctext
 
 
+def _load_cached_group_files(
+    group_sources: list[BuildSource], result: BuildResult
+) -> list[tuple[str, str]]:
+    """Read the .c/.h paths recorded for this group on the previous run.
+
+    All modules in a group share the same src_hashes map, so the first
+    readable IR cache is sufficient. Returns paths paired with empty
+    content so callers can distinguish "reuse on disk" from "newly
+    generated".
+    """
+    for source in group_sources:
+        state = result.graph.get(source.module)
+        if state is None:
+            continue
+        try:
+            ir_json = result.manager.metastore.read(get_state_ir_cache_name(state))
+        except (FileNotFoundError, OSError):
+            continue
+        try:
+            ir_data = json.loads(ir_json)
+        except json.JSONDecodeError:
+            continue
+        return [(path, "") for path in ir_data.get("src_hashes", {})]
+    return []
+
+
 def get_ir_cache_name(id: str, path: str, options: Options) -> str:
     meta_path, _, _ = get_cache_names(id, path, options)
     # Mypyc uses JSON cache even with --fixed-format-cache (for now).
@@ -614,16 +645,19 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
 
         base_emitter = Emitter(self.context)
         # Optionally just include the runtime library c files to
-        # reduce the number of compiler invocations needed
+        # reduce the number of compiler invocations needed.
+        # Use <> form (only -I paths) so a shim file with the same
+        # basename as a runtime file can't shadow it. Triggered by
+        # mypyc/lower/int_ops.py vs lib-rt/int_ops.c on mypy self-compile.
         if self.compiler_options.include_runtime_files:
             for name in RUNTIME_C_FILES:
-                base_emitter.emit_line(f'#include "{name}"')
+                base_emitter.emit_line(f"#include <{name}>")
             # Include conditional source files
             source_deps = collect_source_dependencies(self.modules)
             for source_dep in sorted(source_deps, key=lambda d: d.path):
-                base_emitter.emit_line(f'#include "{source_dep.path}"')
+                base_emitter.emit_line(f"#include <{source_dep.path}>")
             if self.compiler_options.depends_on_librt_internal:
-                base_emitter.emit_line('#include "internal/librt_internal_api.c"')
+                base_emitter.emit_line("#include <internal/librt_internal_api.c>")
         base_emitter.emit_line(f'#include "__native{self.short_group_suffix}.h"')
         base_emitter.emit_line(f'#include "__native_internal{self.short_group_suffix}.h"')
         emitter = base_emitter

mypyc/irbuild/prepare.py

@@ -182,7 +182,12 @@ def load_type_map(mapper: Mapper, modules: list[MypyFile], deser_ctx: DeserMaps)
                     continue
                 mapper.type_to_ir[node.node] = ir
                 mapper.symbol_fullnames.add(node.node.fullname)
-                mapper.func_to_decl[node.node] = ir.ctor
+                # Trait/builtin-base classes have an ir.ctor FuncDecl
+                # but no emitted CPyDef_<ctor>, so a cross-group direct
+                # call would hit an undefined symbol. Mirror the skip
+                # in prepare_init_method.
+                if not ir.is_trait and not ir.builtin_base:
+                    mapper.func_to_decl[node.node] = ir.ctor
 
     for module in modules:
         for func in get_module_func_defs(module):

mypyc/lib-rt/misc_ops.c

@@ -1281,12 +1281,17 @@ static int CPyImport_SetModuleFile(PyObject *modobj, PyObject *module_name,
         Py_DECREF(file);
         return 0;
     }
-    // Derive __file__ from the shared library's __file__ (for its
-    // directory), the module name (dots -> path separators), and the
-    // extension suffix.  E.g. for module "a.b.c", shared lib
-    // "/path/to/group__mypyc.cpython-312-x86_64-linux-gnu.so",
-    // suffix ".cpython-312-x86_64-linux-gnu.so":
-    //   => "/path/to/a/b/c.cpython-312-x86_64-linux-gnu.so"
+    // Derive __file__ from the shared lib's directory, the module
+    // name, and the extension suffix. Two layouts:
+    //
+    //  Monolithic: one shared lib above the package tree holds many
+    //    modules, so append the full dotted module path.
+    //  separate=True: each module has its own "<segment>__mypyc.so"
+    //    next to the module, so dirname(shared_lib) is already inside
+    //    the parent package. Append only the last segment.
+    //
+    // Detect the separate=True case by matching the shared lib's
+    // basename against "<last_segment>__mypyc<ext>".
     PyObject *derived_file = NULL;
     if (shared_lib_file != NULL && shared_lib_file != Py_None &&
             PyUnicode_Check(shared_lib_file)) {
@@ -1314,30 +1319,65 @@ static int CPyImport_SetModuleFile(PyObject *modobj, PyObject *module_name,
         if (module_path == NULL) {
             return -1;
         }
+
+        // Compute the module's last dotted segment for the separate=True check.
+        Py_ssize_t name_len = PyUnicode_GetLength(module_name);
+        Py_ssize_t last_dot = PyUnicode_FindChar(module_name, '.', 0, name_len, -1);
+        PyObject *last_segment;
+        if (last_dot >= 0) {
+            last_segment = PyUnicode_Substring(module_name, last_dot + 1, name_len);
+        } else {
+            last_segment = module_name;
+            Py_INCREF(last_segment);
+        }
+        if (last_segment == NULL) {
+            Py_DECREF(module_path);
+            return -1;
+        }
+        // Compare shared_lib_file basename against "<last_segment>__mypyc<ext>".
+        PyObject *expected_basename = PyUnicode_FromFormat(
+            "%U__mypyc%U", last_segment, ext_suffix);
+        PyObject *actual_basename;
+        if (sep >= 0) {
+            actual_basename = PyUnicode_Substring(shared_lib_file, sep + 1, sf_len);
+        } else {
+            actual_basename = shared_lib_file;
+            Py_INCREF(actual_basename);
+        }
+        int is_per_module_lib = 0;
+        if (expected_basename != NULL && actual_basename != NULL) {
+            is_per_module_lib =
+                (PyUnicode_Compare(expected_basename, actual_basename) == 0);
+        }
+        Py_XDECREF(expected_basename);
+        Py_XDECREF(actual_basename);
+
         // For packages, __file__ should point to __init__<ext>,
         // e.g. "a/b/__init__.cpython-312-x86_64-linux-gnu.so".
+        PyObject *file_path = is_per_module_lib ? last_segment : module_path;
         if (sep >= 0) {
             PyObject *dir = PyUnicode_Substring(shared_lib_file, 0, sep);
             if (dir != NULL) {
                 if (is_package) {
                     derived_file = PyUnicode_FromFormat(
                         "%U%c%U%c__init__%U", dir, (int)sep_char,
-                        module_path, (int)sep_char, ext_suffix);
+                        file_path, (int)sep_char, ext_suffix);
                 } else {
                     derived_file = PyUnicode_FromFormat(
                         "%U%c%U%U", dir, (int)sep_char,
-                        module_path, ext_suffix);
+                        file_path, ext_suffix);
                 }
                 Py_DECREF(dir);
             }
         } else {
             if (is_package) {
                 derived_file = PyUnicode_FromFormat(
-                    "%U%c__init__%U", module_path, (int)SEP[0], ext_suffix);
+                    "%U%c__init__%U", file_path, (int)SEP[0], ext_suffix);
             } else {
-                derived_file = PyUnicode_FromFormat("%U%U", module_path, ext_suffix);
+                derived_file = PyUnicode_FromFormat("%U%U", file_path, ext_suffix);
             }
         }
+        Py_DECREF(last_segment);
         Py_DECREF(module_path);
     }
     if (derived_file == NULL && !PyErr_Occurred()) {

setup.py

@@ -153,6 +153,7 @@ def run(self) -> None:
     debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1")
     force_multifile = os.getenv("MYPYC_MULTI_FILE", "") == "1"
     log_trace = bool(int(os.getenv("MYPYC_LOG_TRACE", "0")))
+    separate = os.getenv("MYPYC_SEPARATE", "") == "1"
     ext_modules = mypycify(
         mypyc_targets + ["--config-file=mypy_bootstrap.ini"],
         opt_level=opt_level,
@@ -161,6 +162,7 @@ def run(self) -> None:
         # our Appveyor builds run out of memory sometimes.
         multi_file=sys.platform == "win32" or force_multifile,
         log_trace=log_trace,
+        separate=separate,
         # Mypy itself is allowed to use native_internal extension.
         depends_on_librt_internal=True,
     )