Address code duplication in mypyc/lib-rt (#20368)

There is a bunch of code duplication in `mypyc/lib-rt`, but it looks
like there is no way to resolve this in a reasonable way. So instead I
factor out duplicated code into a separate module, and add a test to
verify files are synced.

Note I type-check `lib-rt` separately now. Whatever I tried, I am
getting weird error either like
```
Cannot find implementation or library stub for module named "build_setup"
```
or
```
Source file found twice under different module names: "mypyc.lib-rt.build_setup" and "build_setup"
```
etc. This is kind of a known problem, that we need to address long-term.

Author: ilevkivskyi

mypy/test/testmypyc.py

@@ -2,13 +2,25 @@
 
 from __future__ import annotations
 
+import filecmp
 import os
 from unittest import TestCase
 
 import mypy
+import mypyc
 
 
 class MypycTest(TestCase):
     def test_using_mypyc(self) -> None:
         if os.getenv("TEST_MYPYC", None) == "1":
             assert not mypy.__file__.endswith(".py"), "Expected to find a mypyc-compiled version"
+
+    def test_shared_files_consistent(self) -> None:
+        if os.getenv("TEST_MYPYC", None) != "1":
+            mypyc_path = mypyc.__path__[0]
+            for f in ["build_setup.py"]:
+                assert filecmp.cmp(
+                    os.path.join(mypyc_path, f),
+                    os.path.join(mypyc_path, "lib-rt", f),
+                    shallow=False,
+                ), f"Shared files inconsistent, run cp mypyc/{f} mypyc/lib-rt"

mypy_self_check.ini

@@ -8,7 +8,7 @@ pretty = True
 always_false = MYPYC
 plugins = mypy.plugins.proper_plugin
 python_version = 3.10
-exclude = mypy/typeshed/|mypyc/test-data/
+exclude = mypy/typeshed/|mypyc/test-data/|mypyc/lib-rt/
 enable_error_code = ignore-without-code,redundant-expr
 enable_incomplete_feature = PreciseTupleTypes
 show_error_code_links = True

mypyc/build_setup.py

@@ -1,3 +1,7 @@
+# This file must have the same content for mypyc/build_setup.py and lib-rt/build_setup.py,
+# it exists to work around absence of support for per-file compile flags in setuptools.
+# The version in mypyc/ is the source of truth, and should be copied to lib-rt if modified.
+
 import os
 import platform
 import sys

mypyc/lib-rt/build_setup.py

@@ -0,0 +1,76 @@
+# This file must have the same content for mypyc/build_setup.py and lib-rt/build_setup.py,
+# it exists to work around absence of support for per-file compile flags in setuptools.
+# The version in mypyc/ is the source of truth, and should be copied to lib-rt if modified.
+
+import os
+import platform
+import sys
+
+try:
+    # Import setuptools so that it monkey-patch overrides distutils
+    import setuptools  # noqa: F401
+except ImportError:
+    pass
+
+if sys.version_info >= (3, 12):
+    # From setuptools' monkeypatch
+    from distutils import ccompiler  # type: ignore[import-not-found]
+else:
+    from distutils import ccompiler
+
+EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT = {
+    "unix": {
+        "base64/arch/ssse3": ["-mssse3"],
+        "base64/arch/sse41": ["-msse4.1"],
+        "base64/arch/sse42": ["-msse4.2"],
+        "base64/arch/avx2": ["-mavx2"],
+        "base64/arch/avx": ["-mavx"],
+    },
+    "msvc": {
+        "base64/arch/sse42": ["/arch:SSE4.2"],
+        "base64/arch/avx2": ["/arch:AVX2"],
+        "base64/arch/avx": ["/arch:AVX"],
+    },
+}
+
+ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn  # type: ignore[attr-defined]
+X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64")
+PYODIDE = "PYODIDE" in os.environ
+
+
+def spawn(self, cmd, **kwargs) -> None:  # type: ignore[no-untyped-def]
+    if PYODIDE:
+        new_cmd = list(cmd)
+        for argument in reversed(new_cmd):
+            if not str(argument).endswith(".c"):
+                continue
+            if "base64/arch/" in str(argument):
+                new_cmd.extend(["-msimd128"])
+    else:
+        compiler_type: str = self.compiler_type
+        extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type]
+        new_cmd = list(cmd)
+        if X86_64 and extra_options is not None:
+            # filenames are closer to the end of command line
+            for argument in reversed(new_cmd):
+                # Check if the matching argument contains a source filename.
+                if not str(argument).endswith(".c"):
+                    continue
+
+                for path in extra_options.keys():
+                    if path in str(argument):
+                        if compiler_type == "bcpp":
+                            compiler = new_cmd.pop()
+                            # Borland accepts a source file name at the end,
+                            # insert the options before it
+                            new_cmd.extend(extra_options[path])
+                            new_cmd.append(compiler)
+                        else:
+                            new_cmd.extend(extra_options[path])
+
+                        # path component is found, no need to search any further
+                        break
+    self.__spawn(new_cmd, **kwargs)
+
+
+ccompiler.CCompiler.spawn = spawn  # type: ignore[method-assign]

mypyc/lib-rt/setup.py

@@ -6,12 +6,12 @@
 from __future__ import annotations
 
 import os
-import platform
 import subprocess
 import sys
 from distutils import ccompiler, sysconfig
 from typing import Any
 
+import build_setup  # noqa: F401
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
 
@@ -25,63 +25,6 @@
     "pythonsupport.c",
 ]
 
-EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT = {
-    "unix": {
-        "base64/arch/ssse3": ["-mssse3"],
-        "base64/arch/sse41": ["-msse4.1"],
-        "base64/arch/sse42": ["-msse4.2"],
-        "base64/arch/avx2": ["-mavx2"],
-        "base64/arch/avx": ["-mavx"],
-    },
-    "msvc": {
-        "base64/arch/sse42": ["/arch:SSE4.2"],
-        "base64/arch/avx2": ["/arch:AVX2"],
-        "base64/arch/avx": ["/arch:AVX"],
-    },
-}
-
-ccompiler.CCompiler.__spawn = ccompiler.CCompiler.spawn  # type: ignore[attr-defined]
-X86_64 = platform.machine() in ("x86_64", "AMD64", "amd64")
-PYODIDE = "PYODIDE" in os.environ
-
-
-def spawn(self, cmd, **kwargs) -> None:  # type: ignore[no-untyped-def]
-    if PYODIDE:
-        new_cmd = list(cmd)
-        for argument in reversed(new_cmd):
-            if not str(argument).endswith(".c"):
-                continue
-            if "base64/arch/" in str(argument):
-                new_cmd.extend(["-msimd128"])
-    else:
-        compiler_type: str = self.compiler_type
-        extra_options = EXTRA_FLAGS_PER_COMPILER_TYPE_PER_PATH_COMPONENT[compiler_type]
-        new_cmd = list(cmd)
-        if X86_64 and extra_options is not None:
-            # filenames are closer to the end of command line
-            for argument in reversed(new_cmd):
-                # Check if the matching argument contains a source filename.
-                if not str(argument).endswith(".c"):
-                    continue
-
-                for path in extra_options.keys():
-                    if path in str(argument):
-                        if compiler_type == "bcpp":
-                            compiler = new_cmd.pop()
-                            # Borland accepts a source file name at the end,
-                            # insert the options before it
-                            new_cmd.extend(extra_options[path])
-                            new_cmd.append(compiler)
-                        else:
-                            new_cmd.extend(extra_options[path])
-
-                        # path component is found, no need to search any further
-                        break
-    self.__spawn(new_cmd, **kwargs)
-
-
-ccompiler.CCompiler.spawn = spawn  # type: ignore[method-assign]
-
 
 class BuildExtGtest(build_ext):
     def get_library_names(self) -> list[str]:
@@ -130,13 +73,11 @@ def run(self) -> None:
         cmdclass={"build_ext": BuildExtGtest},
     )
 else:
-    # TODO: we need a way to share our preferred C flags and get_extension() logic with
-    # mypyc/build.py without code duplication.
     compiler = ccompiler.new_compiler()
     sysconfig.customize_compiler(compiler)
     cflags: list[str] = []
     if compiler.compiler_type == "unix":  # type: ignore[attr-defined]
-        cflags += ["-O3"]
+        cflags += ["-O3", "-Wno-unused-function"]
     elif compiler.compiler_type == "msvc":  # type: ignore[attr-defined]
         cflags += ["/O2"]
 

setup.py

@@ -98,6 +98,7 @@ def run(self) -> None:
     ) + (
         # Don't want to grab this accidentally
         os.path.join("mypyc", "lib-rt", "setup.py"),
+        os.path.join("mypyc", "lib-rt", "build_setup.py"),
         # Uses __file__ at top level https://github.com/mypyc/mypyc/issues/700
         os.path.join("mypyc", "__main__.py"),
         os.path.join("mypyc", "build_setup.py"),  # for monkeypatching

tox.ini

@@ -62,3 +62,4 @@ commands =
     python runtests.py self
     python -m mypy --config-file mypy_self_check.ini misc --exclude misc/sync-typeshed.py
     python -m mypy --config-file mypy_self_check.ini test-data/unit/plugins
+    python -m mypy mypyc/lib-rt