From e80db934503fc188c948e24c58d7569cb1145715 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 23:25:34 +0200 Subject: [PATCH 1/9] Remove commented-out blocks, unused globals, and py2-only code. NFC --- python/cppjit/_cpython_cppjit.py | 30 ----- src/interop/interop_wrapper.cxx | 206 +------------------------------ 2 files changed, 3 insertions(+), 233 deletions(-) diff --git a/python/cppjit/_cpython_cppjit.py b/python/cppjit/_cpython_cppjit.py index 0f26343..7ab1918 100644 --- a/python/cppjit/_cpython_cppjit.py +++ b/python/cppjit/_cpython_cppjit.py @@ -1,7 +1,6 @@ """CPython-specific touch-ups""" import ctypes -import sys from . import _stdcpp_fix # noqa: F401 @@ -26,35 +25,6 @@ _w = ctypes.CDLL(_backend.__file__, ctypes.RTLD_GLOBAL) -# some beautification for inspect (only on p2) -if sys.hexversion < 0x3000000: - # TODO: this reliese on CPPOverload cooking up a func_code object, which atm - # is simply not implemented for p3 :/ - - # convince inspect that cppjit method proxies are possible drop-ins for python - # methods and classes for pydoc - import inspect - - inspect._old_isfunction = inspect.isfunction - - def isfunction(object): - if isinstance(object, _backend.CPPOverload) and not object.im_class: - return True - return inspect._old_isfunction(object) - - inspect.isfunction = isfunction - - inspect._old_ismethod = inspect.ismethod - - def ismethod(object): - if isinstance(object, _backend.CPPOverload): - return True - return inspect._old_ismethod(object) - - inspect.ismethod = ismethod - del isfunction, ismethod - - ### template support --------------------------------------------------------- class Template(object): # expected/used by ProxyWrappers.cxx in cpyrt stl_sequence_types = ["std::vector", "std::list", "std::set", "std::deque"] diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 5c3ec82..9986b37 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -55,66 +55,12 @@ static std::set g_builtins = {"bool", "long double", "void"}; -// to filter out ROOT names -static std::set gInitialNames; -static std::set gRootSOs; - // configuration static bool gEnableFastPath = true; // global initialization ----------------------------------------------------- namespace { -// const int kMAXSIGNALS = 16; - -// names copied from TUnixSystem -#ifdef WIN32 -const int SIGBUS = 0; // simple placeholders for ones that don't exist -const int SIGSYS = 0; -const int SIGPIPE = 0; -const int SIGQUIT = 0; -const int SIGWINCH = 0; -const int SIGALRM = 0; -const int SIGCHLD = 0; -const int SIGURG = 0; -const int SIGUSR1 = 0; -const int SIGUSR2 = 0; -#endif - -#if 0 -static struct Signalmap_t { - int fCode; - const char *fSigName; -} gSignalMap[kMAXSIGNALS] = { // the order of the signals should be identical - { SIGBUS, "bus error" }, // to the one in TSysEvtHandler.h - { SIGSEGV, "segmentation violation" }, - { SIGSYS, "bad argument to system call" }, - { SIGPIPE, "write on a pipe with no one to read it" }, - { SIGILL, "illegal instruction" }, - { SIGABRT, "abort" }, - { SIGQUIT, "quit" }, - { SIGINT, "interrupt" }, - { SIGWINCH, "window size change" }, - { SIGALRM, "alarm clock" }, - { SIGCHLD, "death of a child" }, - { SIGURG, "urgent data arrived on an I/O channel" }, - { SIGFPE, "floating point exception" }, - { SIGTERM, "termination signal" }, - { SIGUSR1, "user-defined signal 1" }, - { SIGUSR2, "user-defined signal 2" } -}; -#endif - -static inline void push_tokens_from_string(char* s, - std::vector& tokens) { - char* token = strtok(s, " "); - - while (token) { - tokens.push_back(token); - token = strtok(NULL, " "); - } -} - static inline bool is_integral(std::string& s) { if (s == "false") { s = "0"; @@ -143,29 +89,11 @@ class ApplicationStarter { if (auto existingInterp = Cpp::GetInterpreter()) { Interp = existingInterp; } else { -#ifdef __arm64__ -#ifdef __APPLE__ - // If on apple silicon don't use -march=native - std::vector InterpArgs({"-std=c++17"}); -#else - std::vector InterpArgs({"-std=c++17", "-march=native"}); -#endif -#else - std::vector InterpArgs({"-std=c++17", "-march=native"}); -#endif - char* InterpArgString = getenv("CPPINTEROP_EXTRA_INTERPRETER_ARGS"); - - if (InterpArgString) - push_tokens_from_string(InterpArgString, InterpArgs); - -#ifdef __arm64__ -#ifdef __APPLE__ + // CppInterOp itself appends CPPINTEROP_EXTRA_INTERPRETER_ARGS inside + // CreateInterpreter, so nothing needs to be forwarded from here. +#if defined(__arm64__) && defined(__APPLE__) // If on apple silicon don't use -march=native Interp = Cpp::CreateInterpreter({"-std=c++17"}, /*GpuArgs=*/{}); -#else - Interp = Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, - /*GpuArgs=*/{}); -#endif #else Interp = Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, /*GpuArgs=*/{}); @@ -213,7 +141,6 @@ class ApplicationStarter { "#include \n" "#include \n" // for strcpy "#include \n" - // "#include \n" // defines R__EXTERN "#include \n" "#include \n" "#include \n" @@ -257,26 +184,6 @@ class ApplicationStarter { // helper for multiple inheritance Cpp::Declare("namespace __cppjit_internal { struct Sep; }", /*silent=*/false); - - // std::string libInterOp = - // I->getDynamicLibraryManager()->lookupLibrary("libcling"); void *interopDL - // = dlopen(libInterOp.c_str(), RTLD_LAZY); if (!interopDL) { - // std::cerr << "libInterop could not be opened!\n"; - // exit(1); - // } - - // start off with a reasonable size placeholder for wrappers - // gWrapperHolder.reserve(1024); - - // create an exception handler to process signals - // gExceptionHandler = new TExceptionHandlerImp{}; - } - - ~ApplicationStarter() { - // Cpp::DeleteInterpreter(Interp); - // for (auto wrap : gWrapperHolder) - // delete wrap; - // delete gExceptionHandler; gExceptionHandler = nullptr; } } _applicationStarter; @@ -666,36 +573,6 @@ interop::TCppType_t interop::GetComplexType(const std::string& name) { return Cpp::GetComplexType(Cpp::GetType(name)); } -// //---------------------------------------------------------------------------- -// static std::string extract_namespace(const std::string& name) -// { -// // Find the namespace the named class lives in, take care of templates -// // Note: this code also lives in cpyrt (TODO: refactor?) -// if (name.empty()) -// return name; -// -// int tpl_open = 0; -// for (std::string::size_type pos = name.size()-1; 0 < pos; --pos) { -// std::string::value_type c = name[pos]; -// -// // count '<' and '>' to be able to skip template contents -// if (c == '>') -// ++tpl_open; -// else if (c == '<') -// --tpl_open; -// -// // collect name up to "::" -// else if (tpl_open == 0 && c == ':' && name[pos-1] == ':') { -// // found the extend of the scope ... done -// return name.substr(0, pos-1); -// } -// } -// -// // no namespace; assume outer scope -// return ""; -// } -// - std::string interop::ResolveEnum(TCppScope_t handle) { std::lock_guard Lock(InterOpMutex); std::string type = @@ -1100,83 +977,6 @@ bool interop::IsDefaultConstructable(TCppScope_t scope) { bool interop::IsVariable(TCppScope_t scope) { return Cpp::IsVariable(scope); } -// // helpers for stripping scope names -// static -// std::string outer_with_template(const std::string& name) -// { -// // Cut down to the outer-most scope from , taking proper care of -// templates. -// int tpl_open = 0; -// for (std::string::size_type pos = 0; pos < name.size(); ++pos) { -// std::string::value_type c = name[pos]; -// -// // count '<' and '>' to be able to skip template contents -// if (c == '<') -// ++tpl_open; -// else if (c == '>') -// --tpl_open; -// -// // collect name up to "::" -// else if (tpl_open == 0 && \ -// c == ':' && pos+1 < name.size() && name[pos+1] == ':') { -// // found the extend of the scope ... done -// return name.substr(0, pos-1); -// } -// } -// -// // whole name is apparently a single scope -// return name; -// } -// -// static -// std::string outer_no_template(const std::string& name) -// { -// // Cut down to the outer-most scope from , drop templates -// std::string::size_type first_scope = name.find(':'); -// if (first_scope == std::string::npos) -// return name.substr(0, name.find('<')); -// std::string::size_type first_templ = name.find('<'); -// if (first_templ == std::string::npos) -// return name.substr(0, first_scope); -// return name.substr(0, std::min(first_templ, first_scope)); -// } -// -// #define FILL_COLL(type, filter) { \ -// TIter itr{coll}; \ -// type* obj = nullptr; \ -// while ((obj = (type*)itr.Next())) { \ -// const char* nm = obj->GetName(); \ -// if (nm && nm[0] != '_' && !(obj->Property() & (filter))) { \ -// if (gInitialNames.find(nm) == gInitialNames.end()) \ -// cppnames.insert(nm); \ -// }}} -// -// static inline -// void cond_add(interop::TCppScope_t scope, const std::string& ns_scope, -// std::set& cppnames, const char* name, bool nofilter = false) -// { -// if (!name || name[0] == '_' || strstr(name, ".h") != 0 || strncmp(name, -// "operator", 8) == 0) -// return; -// -// if (scope == GLOBAL_HANDLE) { -// std::string to_add = outer_no_template(name); -// if (nofilter || gInitialNames.find(to_add) == gInitialNames.end()) -// cppnames.insert(outer_no_template(name)); -// } else if (scope == STD_HANDLE) { -// if (strncmp(name, "std::", 5) == 0) { -// name += 5; -// #ifdef __APPLE__ -// if (strncmp(name, "__1::", 5) == 0) name += 5; -// #endif -// } -// cppnames.insert(outer_no_template(name)); -// } else { -// if (strncmp(name, ns_scope.c_str(), ns_scope.size()) == 0) -// cppnames.insert(outer_with_template(name + ns_scope.size())); -// } -// } - void interop::GetAllCppNames(TCppScope_t scope, std::set& cppnames) { // Collect all known names of C++ entities under scope. This is useful for From 761ffff4d6d785b10c0fc08b5a7eda9c971c4772 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 23:25:54 +0200 Subject: [PATCH 2/9] [interop] Drop the cppyy-cling build-tree include paths --- src/interop/interop_wrapper.cxx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 9986b37..32cb586 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -123,14 +123,6 @@ class ApplicationStarter { Cpp::Process(s.str().c_str()); } - // This would give us something like: - // /home/vvassilev/workspace/builds/scratch/cling-build/builddir/lib/clang/13.0.0 - const char* ResourceDir = Cpp::GetResourceDir(); - std::string ClingSrc = std::string(ResourceDir) + "/../../../../cling-src"; - std::string ClingBuildDir = std::string(ResourceDir) + "/../../../"; - Cpp::AddIncludePath((ClingSrc + "/tools/cling/include").c_str()); - Cpp::AddIncludePath((ClingSrc + "/include").c_str()); - Cpp::AddIncludePath((ClingBuildDir + "/include").c_str()); Cpp::AddIncludePath(CPPINTEROP_INCLUDE_DIR); Cpp::LoadLibrary("libstdc++", /* lookup= */ true); From 1640982ea1da5a82407504a2596dbe6c6793254d Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 23:26:14 +0200 Subject: [PATCH 3/9] [interop] No longer fake cling::runtime::gCling --- src/interop/interop_wrapper.cxx | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 32cb586..c55bc05 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -163,16 +163,6 @@ class ApplicationStarter { "(bool)(c1 != c2); } }", /*silent=*/false); - // Define gCling when we run with clang-repl. - // FIXME: We should get rid of all the uses of gCling as this seems to - // break encapsulation. - std::stringstream InterpPtrSS; - InterpPtrSS << "#ifndef __CLING__\n" - << "namespace cling { namespace runtime {\n" - << "void* gCling=(void*)" << Interp.data << ";\n }}\n" - << "#endif \n"; - Cpp::Process(InterpPtrSS.str().c_str()); - // helper for multiple inheritance Cpp::Declare("namespace __cppjit_internal { struct Sep; }", /*silent=*/false); From 28942df2713bba12083f4d8c71991892cc204ee5 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 23:27:00 +0200 Subject: [PATCH 4/9] [interop] Resolve libclangCppInterOp relative to libcppjit at runtime --- CMakeLists.txt | 14 ++++++++------ src/interop/interop_wrapper.cxx | 29 +++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc4d087..03eaeff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,10 +95,11 @@ execute_process( OUTPUT_STRIP_TRAILING_WHITESPACE ) if(_python_platlib) - set(CPPINTEROP_INSTALL_DIR "${_python_platlib}/cppjit_backend") + set(CPPINTEROP_INSTALL_PREFIX "${_python_platlib}") else() - set(CPPINTEROP_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/cppjit_backend") + set(CPPINTEROP_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") endif() +set(CPPINTEROP_INSTALL_DIR "${CPPINTEROP_INSTALL_PREFIX}/cppjit_backend") # Include cmake for CppInterOp config and build using ExternalProject. include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddCppInterOp.cmake) @@ -114,11 +115,12 @@ set(INTEROP_SOURCES add_library(cppjit SHARED ${CPYRT_SOURCES} ${INTEROP_SOURCES}) add_dependencies(cppjit CppInterOp) -# The exact library file the wrapper dlopens and the include dir the -# interpreter boot requires. +# The wrapper anchors these relative spellings at its own load location, +# falling back to the install prefix (see cppinterop_paths()). target_compile_definitions(cppjit PRIVATE - CPPINTEROP_LIBRARY="${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" - CPPINTEROP_INCLUDE_DIR="${CPPINTEROP_INSTALL_DIR}/include" + CPPINTEROP_INSTALL_PREFIX="${CPPINTEROP_INSTALL_PREFIX}" + CPPINTEROP_LIBRARY="cppjit_backend/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" + CPPINTEROP_INCLUDE_DIR="cppjit_backend/include" ) target_include_directories(cppjit PRIVATE diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index c55bc05..7d0cd26 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -23,6 +23,7 @@ using namespace cppjit; #include #include // for getenv #include +#include #include #include #include @@ -74,13 +75,37 @@ static inline bool is_integral(std::string& s) { }) == s.end(); } +struct InterOpPaths { + std::string Library; + std::string IncludeDir; +}; + +// One relative layout, two anchors: prefer CppInterOp next to our own load +// location so wheels relocate; fall back to the build-time install prefix. +static InterOpPaths cppinterop_paths() { + std::filesystem::path anchor = CPPINTEROP_INSTALL_PREFIX; +#ifndef _WIN32 + Dl_info info; + if (dladdr((void*)&cppinterop_paths, &info) && info.dli_fname) { + const std::filesystem::path here = + std::filesystem::path(info.dli_fname).parent_path(); + std::error_code ec; + if (std::filesystem::exists(here / CPPINTEROP_LIBRARY, ec)) + anchor = here; + } +#endif + return {(anchor / CPPINTEROP_LIBRARY).string(), + (anchor / CPPINTEROP_INCLUDE_DIR).string()}; +} + class ApplicationStarter { interop::TInterp_t Interp; public: ApplicationStarter() { std::lock_guard Lock(InterOpMutex); - if (!Cpp::LoadDispatchAPI(CPPINTEROP_LIBRARY)) { + const InterOpPaths Paths = cppinterop_paths(); + if (!Cpp::LoadDispatchAPI(Paths.Library.c_str())) { std::cerr << "[cppjit-backend] Failed to load CppInterOp" << std::endl; return; } @@ -123,7 +148,7 @@ class ApplicationStarter { Cpp::Process(s.str().c_str()); } - Cpp::AddIncludePath(CPPINTEROP_INCLUDE_DIR); + Cpp::AddIncludePath(Paths.IncludeDir.c_str()); Cpp::LoadLibrary("libstdc++", /* lookup= */ true); // load frequently used headers From dd99613407048219a8a66a6b432df79dabecf83e Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 23:27:13 +0200 Subject: [PATCH 5/9] Load CppInterOp explicitly instead of from a global constructor --- python/cppjit/_cpython_cppjit.py | 19 +-- src/interop/interop_wrapper.cxx | 193 +++++++++++++++++-------------- 2 files changed, 120 insertions(+), 92 deletions(-) diff --git a/python/cppjit/_cpython_cppjit.py b/python/cppjit/_cpython_cppjit.py index 7ab1918..d762d4d 100644 --- a/python/cppjit/_cpython_cppjit.py +++ b/python/cppjit/_cpython_cppjit.py @@ -1,6 +1,7 @@ """CPython-specific touch-ups""" import ctypes +import importlib.util from . import _stdcpp_fix # noqa: F401 @@ -16,13 +17,17 @@ "_end_capture_stderr", ] -# the merged libcppjit extension is the backend: importing it loads the -# C++ runtime (no separate loader.load_cpp_backend() step, which would -# initialize the interpreter twice) -import libcppjit as _backend - -# explicitly expose APIs from libcppjit -_w = ctypes.CDLL(_backend.__file__, ctypes.RTLD_GLOBAL) +# preload the merged extension with ctypes and run LoadCppInterOp() first, +# so the interpreter is ready before the extension module initializes +_spec = importlib.util.find_spec("libcppjit") +if _spec is None or not _spec.origin: + raise ImportError("cannot locate the libcppjit extension module") +_w = ctypes.CDLL(_spec.origin, ctypes.RTLD_GLOBAL) +if not _w.LoadCppInterOp(): + raise RuntimeError("failed to load CppInterOp (LoadCppInterOp returned 0)") +del _spec + +import libcppjit as _backend # noqa: E402 ### template support --------------------------------------------------------- diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 7d0cd26..bd2b526 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -98,104 +98,127 @@ static InterOpPaths cppinterop_paths() { (anchor / CPPINTEROP_INCLUDE_DIR).string()}; } -class ApplicationStarter { - interop::TInterp_t Interp; +// The one place libclangCppInterOp is dlopen'd. +static bool loadDispatchAPI(const InterOpPaths& Paths) { + if (!Cpp::LoadDispatchAPI(Paths.Library.c_str())) { + std::cerr << "[cppjit-backend] Failed to load CppInterOp" << std::endl; + return false; + } + return true; +} + +// CppInterOp itself appends CPPINTEROP_EXTRA_INTERPRETER_ARGS inside +// CreateInterpreter, so nothing needs to be forwarded from here. +static interop::TInterp_t acquireOrCreateInterpreter() { + if (auto existingInterp = Cpp::GetInterpreter()) + return existingInterp; -public: - ApplicationStarter() { - std::lock_guard Lock(InterOpMutex); - const InterOpPaths Paths = cppinterop_paths(); - if (!Cpp::LoadDispatchAPI(Paths.Library.c_str())) { - std::cerr << "[cppjit-backend] Failed to load CppInterOp" << std::endl; - return; - } - // Check if somebody already loaded CppInterOp and created an - // interpreter for us. - if (auto existingInterp = Cpp::GetInterpreter()) { - Interp = existingInterp; - } else { - // CppInterOp itself appends CPPINTEROP_EXTRA_INTERPRETER_ARGS inside - // CreateInterpreter, so nothing needs to be forwarded from here. #if defined(__arm64__) && defined(__APPLE__) - // If on apple silicon don't use -march=native - Interp = Cpp::CreateInterpreter({"-std=c++17"}, /*GpuArgs=*/{}); + // If on apple silicon don't use -march=native + return Cpp::CreateInterpreter({"-std=c++17"}, /*GpuArgs=*/{}); #else - Interp = Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, - /*GpuArgs=*/{}); + return Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, + /*GpuArgs=*/{}); #endif - } - - // fill out the builtins - std::set bi{g_builtins}; - for (const auto& name : bi) { - for (const char* a : {"*", "&", "*&", "[]", "*[]"}) - g_builtins.insert(name + a); - } +} - // disable fast path if requested - if (getenv("CPPJIT_DISABLE_FASTPATH")) - gEnableFastPath = false; +static void configureInterpreter(const InterOpPaths& Paths) { + std::set bi{g_builtins}; + for (const auto& name : bi) { + for (const char* a : {"*", "&", "*&", "[]", "*[]"}) + g_builtins.insert(name + a); + } - // set opt level (default to 2 if not given; Cling itself defaults to 0) - int optLevel = 2; + if (getenv("CPPJIT_DISABLE_FASTPATH")) + gEnableFastPath = false; - if (getenv("CPPJIT_OPT_LEVEL")) - optLevel = atoi(getenv("CPPJIT_OPT_LEVEL")); + // set opt level (default to 2 if not given; Cling itself defaults to 0) + int optLevel = 2; - if (optLevel != 0) { - std::ostringstream s; - s << "#pragma cling optimize " << optLevel; - Cpp::Process(s.str().c_str()); - } + if (getenv("CPPJIT_OPT_LEVEL")) + optLevel = atoi(getenv("CPPJIT_OPT_LEVEL")); - Cpp::AddIncludePath(Paths.IncludeDir.c_str()); - Cpp::LoadLibrary("libstdc++", /* lookup= */ true); - - // load frequently used headers - const char* code = "#include \n" - "#include \n" - "#include \n" - "#include \n" - "#include \n" // for strcpy - "#include \n" - "#include \n" - "#include \n" - "#include \n" - "#include \n" // for the dispatcher code to - // use std::function - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#if __has_include()\n" - "#include \n" - "#endif\n" - "#include \n"; - Cpp::Process(code); - - // create helpers for comparing thingies - Cpp::Declare("namespace __cppjit_internal { template" - " bool is_equal(const C1& c1, const C2& c2) { return " - "(bool)(c1 == c2); } }", - /*silent=*/false); - Cpp::Declare("namespace __cppjit_internal { template" - " bool is_not_equal(const C1& c1, const C2& c2) { return " - "(bool)(c1 != c2); } }", - /*silent=*/false); - - // helper for multiple inheritance - Cpp::Declare("namespace __cppjit_internal { struct Sep; }", - /*silent=*/false); + if (optLevel != 0) { + std::ostringstream s; + s << "#pragma cling optimize " << optLevel; + Cpp::Process(s.str().c_str()); } -} _applicationStarter; + + Cpp::AddIncludePath(Paths.IncludeDir.c_str()); + Cpp::LoadLibrary("libstdc++", /* lookup= */ true); +} + +static void preloadHeaders() { + const char* code = "#include \n" + "#include \n" + "#include \n" + "#include \n" + "#include \n" // for strcpy + "#include \n" + "#include \n" + "#include \n" + "#include \n" + "#include \n" // for the dispatcher code to + // use std::function + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#if __has_include()\n" + "#include \n" + "#endif\n" + "#include \n"; + Cpp::Process(code); +} + +static void defineRuntimeHelpers() { + Cpp::Declare("namespace __cppjit_internal { template" + " bool is_equal(const C1& c1, const C2& c2) { return " + "(bool)(c1 == c2); } }", + /*silent=*/false); + Cpp::Declare("namespace __cppjit_internal { template" + " bool is_not_equal(const C1& c1, const C2& c2) { return " + "(bool)(c1 != c2); } }", + /*silent=*/false); + + // helper for multiple inheritance + Cpp::Declare("namespace __cppjit_internal { struct Sep; }", + /*silent=*/false); +} } // unnamed namespace +// Load CppInterOp and set up the interpreter. A dlopen during static +// initialization is unsafe, so _cpython_cppjit.py calls this explicitly +// before the first libcppjit use. Idempotent; returns 1 on success. +extern "C" { +RPY_EXPORTED int LoadCppInterOp(); +} + +extern "C" int LoadCppInterOp() { + std::lock_guard Lock(InterOpMutex); + static bool loaded = false; + if (loaded) + return 1; + + const InterOpPaths Paths = cppinterop_paths(); + if (!loadDispatchAPI(Paths)) + return 0; + + acquireOrCreateInterpreter(); + configureInterpreter(Paths); + preloadHeaders(); + defineRuntimeHelpers(); + + loaded = true; + return 1; +} + // local helpers ------------------------------------------------------------- static inline char* cppstring_to_cstring(const std::string& cppstr) { char* cstr = (char*)malloc(cppstr.size() + 1); From 5e302d581569bdf437e060e028a28b3483e5ca97 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 21:53:36 +0200 Subject: [PATCH 6/9] [cmake] Require only Python Development.Module for the extension build --- CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03eaeff..5fb23ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,13 +16,16 @@ set(CPPINTEROP_GIT_TAG "8d624c621a4b95e36ff73ac708c85a768287478f" CACHE STRING " set(CPPINTEROP_SOURCE_DIR "" CACHE PATH "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source") +# Development.Module, not Development: the full component additionally +# requires libpython (Development.Embed), which manylinux images do not +# ship — extension modules only need headers and module-link rules. set(Python_FIND_VIRTUALENV ONLY) -find_package(Python COMPONENTS Interpreter Development) +find_package(Python COMPONENTS Interpreter Development.Module) if(NOT Python_FOUND) set(Python_FIND_VIRTUALENV STANDARD) - find_package(Python COMPONENTS Interpreter Development) + find_package(Python COMPONENTS Interpreter Development.Module) endif() -if(NOT Python_Development_FOUND) +if(NOT Python_Development.Module_FOUND) message(FATAL_ERROR "Python development headers not found") endif() From f940cf42882b812c0b905cea450f3b5521afb6ab Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 23:28:54 +0200 Subject: [PATCH 7/9] [ci] Build manylinux and macOS arm64 wheels with cibuildwheel --- .github/wheel_smoke.py | 23 +++++++++++++ .github/workflows/wheels.yml | 62 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 31 ++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 .github/wheel_smoke.py create mode 100644 .github/workflows/wheels.yml diff --git a/.github/wheel_smoke.py b/.github/wheel_smoke.py new file mode 100644 index 0000000..5a86263 --- /dev/null +++ b/.github/wheel_smoke.py @@ -0,0 +1,23 @@ +"""Wheel smoke test, run from a clean venv by cibuildwheel's test step: +libcppjit.so must locate libclangCppInterOp relative to its own path (the +build tree is gone by test time), and the template instantiation plus the +header check prove the shipped include tree.""" + +import os + +import cppjit +import cppjit_backend + +cppjit.cppdef("int wheel_smoke(int x) { return x + 1; }") +assert cppjit.gbl.wheel_smoke(41) == 42 + +v = cppjit.gbl.std.vector["int"]() +v.push_back(7) +assert v[0] == 7 + +api = os.path.join( + os.path.dirname(cppjit_backend.__file__), "include", "cpyrt", "API.h" +) +assert os.path.exists(api), api + +print("wheel smoke OK") diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..ae67ee1 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,62 @@ +name: Wheels + +# Build the distributable artifacts -- manylinux and macOS arm64 wheels via +# cibuildwheel (config in pyproject.toml) plus the sdist; no index publishing. +# The weekly schedule is a drift canary for the conda-forge/homebrew LLVM. + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/wheels.yml' + - '.github/wheel_smoke.py' + - 'pyproject.toml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'src/interop/**' + - 'python/cppjit/_cpython_cppjit.py' + push: + tags: ['v*'] + schedule: + - cron: '30 4 * * 1' + +permissions: + contents: read + +concurrency: + group: wheels-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + wheels: + name: wheels ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-24.04, label: manylinux-x86_64 } + - { os: macos-26, label: macosx-arm64 } + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + + - uses: pypa/cibuildwheel@v4.1.1 + + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.label }} + path: wheelhouse/*.whl + + sdist: + name: sdist + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz diff --git a/pyproject.toml b/pyproject.toml index d7501d9..659e58f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,37 @@ cmake.build-type = "Release" metadata.version.provider = "scikit_build_core.metadata.regex" metadata.version.input = "python/cppjit/_version.py" +[tool.cibuildwheel] +build = ["cp312-*", "cp313-*", "cp314-*"] +skip = ["*-musllinux*"] +build-verbosity = 1 +test-command = "python {project}/.github/wheel_smoke.py" + +[tool.cibuildwheel.linux] +archs = ["x86_64"] +manylinux-x86_64-image = "manylinux_2_28" +# LLVM/Clang 21 from conda-forge: built against a glibc within the +# manylinux_2_28 policy, unlike apt.llvm.org or release-tarball builds. +# lld is mandatory: BFD ld mis-relaxes R_X86_64_GOTPCRELX relocations in +# the conda static archives, and the result crashes at the first JIT use. +before-all = """ +curl -fsSL --retry 5 -o /tmp/micromamba.tar.bz2 https://micro.mamba.pm/api/micromamba/linux-64/latest +tar -xjf /tmp/micromamba.tar.bz2 -C /usr/local bin/micromamba +export MAMBA_ROOT_PREFIX=/opt/mamba +micromamba create -y -p /opt/llvm -c conda-forge 'llvmdev=21.*' 'clangdev=21.*' lld zstd zlib libxml2 +""" +# Environment variables reach the CppInterOp ExternalProject sub-configure +# (find_package(zstd) needs the conda env) and auditwheel (LD_LIBRARY_PATH +# resolves the conda DT_NEEDEDs it grafts); ld.lld comes from PATH. +environment = { CMAKE_ARGS = "-DLLVM_DIR=/opt/llvm/lib/cmake/llvm -DClang_DIR=/opt/llvm/lib/cmake/clang", CMAKE_PREFIX_PATH = "/opt/llvm", LDFLAGS = "-fuse-ld=lld", PATH = "/opt/llvm/bin:$PATH", LD_LIBRARY_PATH = "/opt/llvm/lib" } + +[tool.cibuildwheel.macos] +archs = ["arm64"] +before-all = "brew install llvm@21" +# The deployment target must match the Homebrew bottles the wheel grafts +# (bottles target the runner's OS); delocate rejects the default 11.0 label. +environment = { CMAKE_ARGS = "-DLLVM_DIR=/opt/homebrew/opt/llvm@21/lib/cmake/llvm -DClang_DIR=/opt/homebrew/opt/llvm@21/lib/cmake/clang", MACOSX_DEPLOYMENT_TARGET = "26.0" } + [tool.pytest.ini_options] testpaths = ["test"] pythonpath = ["test"] From 272faec0a25c76a3dc47f7553aeed5b226c753d4 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Tue, 18 Aug 2026 10:43:02 +0200 Subject: [PATCH 8/9] [interop] Detect the resource dir of the required system LLVM --- CMakeLists.txt | 4 +++- src/interop/interop_wrapper.cxx | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fb23ab..6342c77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -119,11 +119,13 @@ add_library(cppjit SHARED ${CPYRT_SOURCES} ${INTEROP_SOURCES}) add_dependencies(cppjit CppInterOp) # The wrapper anchors these relative spellings at its own load location, -# falling back to the install prefix (see cppinterop_paths()). +# falling back to the install prefix (see cppinterop_paths()); the clang +# major names the versioned compiler probed for the runtime resource dir. target_compile_definitions(cppjit PRIVATE CPPINTEROP_INSTALL_PREFIX="${CPPINTEROP_INSTALL_PREFIX}" CPPINTEROP_LIBRARY="cppjit_backend/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" CPPINTEROP_INCLUDE_DIR="cppjit_backend/include" + CPPJIT_CLANG_MAJOR="${LLVM_VERSION_MAJOR}" ) target_include_directories(cppjit PRIVATE diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index bd2b526..42b6ebb 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -113,13 +113,22 @@ static interop::TInterp_t acquireOrCreateInterpreter() { if (auto existingInterp = Cpp::GetInterpreter()) return existingInterp; -#if defined(__arm64__) && defined(__APPLE__) - // If on apple silicon don't use -march=native - return Cpp::CreateInterpreter({"-std=c++17"}, /*GpuArgs=*/{}); -#else - return Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, - /*GpuArgs=*/{}); + std::vector args = {"-std=c++17"}; +#if !(defined(__arm64__) && defined(__APPLE__)) + // apple silicon clang rejects -march=native + args.push_back("-march=native"); #endif + // CppInterOp resolves the JIT's builtin headers from its build prefix + // or bare `clang`, but distributions spell the required major + // clang-; when only that spelling resolves, pass it explicitly. + std::string resourceDir; + if (Cpp::DetectResourceDir("clang").empty()) + resourceDir = Cpp::DetectResourceDir("clang-" CPPJIT_CLANG_MAJOR); + if (!resourceDir.empty()) { + args.push_back("-resource-dir"); + args.push_back(resourceDir.c_str()); + } + return Cpp::CreateInterpreter(args, /*GpuArgs=*/{}); } static void configureInterpreter(const InterOpPaths& Paths) { From 1b4e02d7085506c91daacb8e8db232e125bbad83 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Mon, 17 Aug 2026 23:28:54 +0200 Subject: [PATCH 9/9] [ci] Test the linux wheel on a plain runner against the full suite --- .github/workflows/wheels.yml | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ae67ee1..6a17811 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -60,3 +60,59 @@ jobs: with: name: sdist path: dist/*.tar.gz + + # Install the linux wheel on a plain runner, outside the manylinux + # container it was built in, and run the full suite against it. + test-wheel: + name: test wheel (full suite) + needs: wheels + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - uses: actions/download-artifact@v4 + with: + name: wheels-manylinux-x86_64 + path: wheelhouse + + - name: Install the LLVM the wheel requires at runtime + # The JIT needs the builtin headers of the wheel's LLVM major; + # the loader detects them from `clang` or `clang-21` on PATH. + run: wget -qO- https://apt.llvm.org/llvm.sh | sudo bash -s -- 21 + + - name: Install the wheel and the test requirements + run: | + python -m venv wheel-venv + wheel-venv/bin/pip install wheelhouse/cppjit-*cp312*.whl + wheel-venv/bin/pip install -r requirements.txt + + - name: Smoke the wheel outside pytest + # pytest captures output at the fd level, so a native abort during + # collection dies silently; this surfaces interpreter-boot errors. + run: | + clang --version + clang-21 -print-resource-dir + wheel-venv/bin/python -X faulthandler .github/wheel_smoke.py + + - name: Run the test suite against the installed wheel + env: + # match the CI cells, which run the interpreter under C++20 + CPPINTEROP_EXTRA_INTERPRETER_ARGS: -std=c++20 + run: | + cd test + make -j4 PYTHON=$GITHUB_WORKSPACE/wheel-venv/bin/python + rc=0 + $GITHUB_WORKSPACE/wheel-venv/bin/python -X faulthandler -m pytest -ra \ + > pytest.log 2>&1 || rc=$? + tail -80 pytest.log + exit $rc + + - name: Fail on dependency-driven skips + # "0 failed" proves nothing about tests that silently stopped + # running; a skip naming a missing module means the wheel venv + # lost dependency coverage. + run: "! grep -iE '^SKIPPED.*(no module named|could not import|module .* not installed)' test/pytest.log"