diff --git a/CMakeLists.txt b/CMakeLists.txt index fc4d087..03eaeff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,10 +95,11 @@ execute_process( OUTPUT_STRIP_TRAILING_WHITESPACE ) if(_python_platlib) - set(CPPINTEROP_INSTALL_DIR "${_python_platlib}/cppjit_backend") + set(CPPINTEROP_INSTALL_PREFIX "${_python_platlib}") else() - set(CPPINTEROP_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/cppjit_backend") + set(CPPINTEROP_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") endif() +set(CPPINTEROP_INSTALL_DIR "${CPPINTEROP_INSTALL_PREFIX}/cppjit_backend") # Include cmake for CppInterOp config and build using ExternalProject. include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddCppInterOp.cmake) @@ -114,11 +115,12 @@ set(INTEROP_SOURCES add_library(cppjit SHARED ${CPYRT_SOURCES} ${INTEROP_SOURCES}) add_dependencies(cppjit CppInterOp) -# The exact library file the wrapper dlopens and the include dir the -# interpreter boot requires. +# The wrapper anchors these relative spellings at its own load location, +# falling back to the install prefix (see cppinterop_paths()). target_compile_definitions(cppjit PRIVATE - CPPINTEROP_LIBRARY="${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" - CPPINTEROP_INCLUDE_DIR="${CPPINTEROP_INSTALL_DIR}/include" + CPPINTEROP_INSTALL_PREFIX="${CPPINTEROP_INSTALL_PREFIX}" + CPPINTEROP_LIBRARY="cppjit_backend/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" + CPPINTEROP_INCLUDE_DIR="cppjit_backend/include" ) target_include_directories(cppjit PRIVATE diff --git a/python/cppjit/_cpython_cppjit.py b/python/cppjit/_cpython_cppjit.py index 0f26343..d762d4d 100644 --- a/python/cppjit/_cpython_cppjit.py +++ b/python/cppjit/_cpython_cppjit.py @@ -1,7 +1,7 @@ """CPython-specific touch-ups""" import ctypes -import sys +import importlib.util from . import _stdcpp_fix # noqa: F401 @@ -17,42 +17,17 @@ "_end_capture_stderr", ] -# the merged libcppjit extension is the backend: importing it loads the -# C++ runtime (no separate loader.load_cpp_backend() step, which would -# initialize the interpreter twice) -import libcppjit as _backend - -# explicitly expose APIs from libcppjit -_w = ctypes.CDLL(_backend.__file__, ctypes.RTLD_GLOBAL) - - -# some beautification for inspect (only on p2) -if sys.hexversion < 0x3000000: - # TODO: this reliese on CPPOverload cooking up a func_code object, which atm - # is simply not implemented for p3 :/ - - # convince inspect that cppjit method proxies are possible drop-ins for python - # methods and classes for pydoc - import inspect - - inspect._old_isfunction = inspect.isfunction - - def isfunction(object): - if isinstance(object, _backend.CPPOverload) and not object.im_class: - return True - return inspect._old_isfunction(object) - - inspect.isfunction = isfunction - - inspect._old_ismethod = inspect.ismethod - - def ismethod(object): - if isinstance(object, _backend.CPPOverload): - return True - return inspect._old_ismethod(object) - - inspect.ismethod = ismethod - del isfunction, ismethod +# preload the merged extension with ctypes and run LoadCppInterOp() first, +# so the interpreter is ready before the extension module initializes +_spec = importlib.util.find_spec("libcppjit") +if _spec is None or not _spec.origin: + raise ImportError("cannot locate the libcppjit extension module") +_w = ctypes.CDLL(_spec.origin, ctypes.RTLD_GLOBAL) +if not _w.LoadCppInterOp(): + raise RuntimeError("failed to load CppInterOp (LoadCppInterOp returned 0)") +del _spec + +import libcppjit as _backend # noqa: E402 ### template support --------------------------------------------------------- diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 5c3ec82..bd2b526 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -23,6 +23,7 @@ using namespace cppjit; #include #include // for getenv #include +#include #include #include #include @@ -55,66 +56,12 @@ static std::set g_builtins = {"bool", "long double", "void"}; -// to filter out ROOT names -static std::set gInitialNames; -static std::set gRootSOs; - // configuration static bool gEnableFastPath = true; // global initialization ----------------------------------------------------- namespace { -// const int kMAXSIGNALS = 16; - -// names copied from TUnixSystem -#ifdef WIN32 -const int SIGBUS = 0; // simple placeholders for ones that don't exist -const int SIGSYS = 0; -const int SIGPIPE = 0; -const int SIGQUIT = 0; -const int SIGWINCH = 0; -const int SIGALRM = 0; -const int SIGCHLD = 0; -const int SIGURG = 0; -const int SIGUSR1 = 0; -const int SIGUSR2 = 0; -#endif - -#if 0 -static struct Signalmap_t { - int fCode; - const char *fSigName; -} gSignalMap[kMAXSIGNALS] = { // the order of the signals should be identical - { SIGBUS, "bus error" }, // to the one in TSysEvtHandler.h - { SIGSEGV, "segmentation violation" }, - { SIGSYS, "bad argument to system call" }, - { SIGPIPE, "write on a pipe with no one to read it" }, - { SIGILL, "illegal instruction" }, - { SIGABRT, "abort" }, - { SIGQUIT, "quit" }, - { SIGINT, "interrupt" }, - { SIGWINCH, "window size change" }, - { SIGALRM, "alarm clock" }, - { SIGCHLD, "death of a child" }, - { SIGURG, "urgent data arrived on an I/O channel" }, - { SIGFPE, "floating point exception" }, - { SIGTERM, "termination signal" }, - { SIGUSR1, "user-defined signal 1" }, - { SIGUSR2, "user-defined signal 2" } -}; -#endif - -static inline void push_tokens_from_string(char* s, - std::vector& tokens) { - char* token = strtok(s, " "); - - while (token) { - tokens.push_back(token); - token = strtok(NULL, " "); - } -} - static inline bool is_integral(std::string& s) { if (s == "false") { s = "0"; @@ -128,160 +75,150 @@ static inline bool is_integral(std::string& s) { }) == s.end(); } -class ApplicationStarter { - interop::TInterp_t Interp; +struct InterOpPaths { + std::string Library; + std::string IncludeDir; +}; -public: - ApplicationStarter() { - std::lock_guard Lock(InterOpMutex); - if (!Cpp::LoadDispatchAPI(CPPINTEROP_LIBRARY)) { - std::cerr << "[cppjit-backend] Failed to load CppInterOp" << std::endl; - return; - } - // Check if somebody already loaded CppInterOp and created an - // interpreter for us. - if (auto existingInterp = Cpp::GetInterpreter()) { - Interp = existingInterp; - } else { -#ifdef __arm64__ -#ifdef __APPLE__ - // If on apple silicon don't use -march=native - std::vector InterpArgs({"-std=c++17"}); -#else - std::vector InterpArgs({"-std=c++17", "-march=native"}); -#endif -#else - std::vector InterpArgs({"-std=c++17", "-march=native"}); +// One relative layout, two anchors: prefer CppInterOp next to our own load +// location so wheels relocate; fall back to the build-time install prefix. +static InterOpPaths cppinterop_paths() { + std::filesystem::path anchor = CPPINTEROP_INSTALL_PREFIX; +#ifndef _WIN32 + Dl_info info; + if (dladdr((void*)&cppinterop_paths, &info) && info.dli_fname) { + const std::filesystem::path here = + std::filesystem::path(info.dli_fname).parent_path(); + std::error_code ec; + if (std::filesystem::exists(here / CPPINTEROP_LIBRARY, ec)) + anchor = here; + } #endif - char* InterpArgString = getenv("CPPINTEROP_EXTRA_INTERPRETER_ARGS"); + return {(anchor / CPPINTEROP_LIBRARY).string(), + (anchor / CPPINTEROP_INCLUDE_DIR).string()}; +} - if (InterpArgString) - push_tokens_from_string(InterpArgString, InterpArgs); +// The one place libclangCppInterOp is dlopen'd. +static bool loadDispatchAPI(const InterOpPaths& Paths) { + if (!Cpp::LoadDispatchAPI(Paths.Library.c_str())) { + std::cerr << "[cppjit-backend] Failed to load CppInterOp" << std::endl; + return false; + } + return true; +} -#ifdef __arm64__ -#ifdef __APPLE__ - // If on apple silicon don't use -march=native - Interp = Cpp::CreateInterpreter({"-std=c++17"}, /*GpuArgs=*/{}); -#else - Interp = Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, - /*GpuArgs=*/{}); -#endif +// CppInterOp itself appends CPPINTEROP_EXTRA_INTERPRETER_ARGS inside +// CreateInterpreter, so nothing needs to be forwarded from here. +static interop::TInterp_t acquireOrCreateInterpreter() { + if (auto existingInterp = Cpp::GetInterpreter()) + return existingInterp; + +#if defined(__arm64__) && defined(__APPLE__) + // If on apple silicon don't use -march=native + return Cpp::CreateInterpreter({"-std=c++17"}, /*GpuArgs=*/{}); #else - Interp = Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, - /*GpuArgs=*/{}); + return Cpp::CreateInterpreter({"-std=c++17", "-march=native"}, + /*GpuArgs=*/{}); #endif - } - - // fill out the builtins - std::set bi{g_builtins}; - for (const auto& name : bi) { - for (const char* a : {"*", "&", "*&", "[]", "*[]"}) - g_builtins.insert(name + a); - } +} - // disable fast path if requested - if (getenv("CPPJIT_DISABLE_FASTPATH")) - gEnableFastPath = false; +static void configureInterpreter(const InterOpPaths& Paths) { + std::set bi{g_builtins}; + for (const auto& name : bi) { + for (const char* a : {"*", "&", "*&", "[]", "*[]"}) + g_builtins.insert(name + a); + } - // set opt level (default to 2 if not given; Cling itself defaults to 0) - int optLevel = 2; + if (getenv("CPPJIT_DISABLE_FASTPATH")) + gEnableFastPath = false; - if (getenv("CPPJIT_OPT_LEVEL")) - optLevel = atoi(getenv("CPPJIT_OPT_LEVEL")); + // set opt level (default to 2 if not given; Cling itself defaults to 0) + int optLevel = 2; - if (optLevel != 0) { - std::ostringstream s; - s << "#pragma cling optimize " << optLevel; - Cpp::Process(s.str().c_str()); - } + if (getenv("CPPJIT_OPT_LEVEL")) + optLevel = atoi(getenv("CPPJIT_OPT_LEVEL")); - // This would give us something like: - // /home/vvassilev/workspace/builds/scratch/cling-build/builddir/lib/clang/13.0.0 - const char* ResourceDir = Cpp::GetResourceDir(); - std::string ClingSrc = std::string(ResourceDir) + "/../../../../cling-src"; - std::string ClingBuildDir = std::string(ResourceDir) + "/../../../"; - Cpp::AddIncludePath((ClingSrc + "/tools/cling/include").c_str()); - Cpp::AddIncludePath((ClingSrc + "/include").c_str()); - Cpp::AddIncludePath((ClingBuildDir + "/include").c_str()); - Cpp::AddIncludePath(CPPINTEROP_INCLUDE_DIR); - Cpp::LoadLibrary("libstdc++", /* lookup= */ true); - - // load frequently used headers - const char* code = "#include \n" - "#include \n" - "#include \n" - "#include \n" - "#include \n" // for strcpy - "#include \n" - // "#include \n" // defines R__EXTERN - "#include \n" - "#include \n" - "#include \n" - "#include \n" // for the dispatcher code to - // use std::function - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#include \n" // FIXME: Replace with modules - "#if __has_include()\n" - "#include \n" - "#endif\n" - "#include \n"; - Cpp::Process(code); - - // create helpers for comparing thingies - Cpp::Declare("namespace __cppjit_internal { template" - " bool is_equal(const C1& c1, const C2& c2) { return " - "(bool)(c1 == c2); } }", - /*silent=*/false); - Cpp::Declare("namespace __cppjit_internal { template" - " bool is_not_equal(const C1& c1, const C2& c2) { return " - "(bool)(c1 != c2); } }", - /*silent=*/false); - - // Define gCling when we run with clang-repl. - // FIXME: We should get rid of all the uses of gCling as this seems to - // break encapsulation. - std::stringstream InterpPtrSS; - InterpPtrSS << "#ifndef __CLING__\n" - << "namespace cling { namespace runtime {\n" - << "void* gCling=(void*)" << Interp.data << ";\n }}\n" - << "#endif \n"; - Cpp::Process(InterpPtrSS.str().c_str()); - - // helper for multiple inheritance - Cpp::Declare("namespace __cppjit_internal { struct Sep; }", - /*silent=*/false); - - // std::string libInterOp = - // I->getDynamicLibraryManager()->lookupLibrary("libcling"); void *interopDL - // = dlopen(libInterOp.c_str(), RTLD_LAZY); if (!interopDL) { - // std::cerr << "libInterop could not be opened!\n"; - // exit(1); - // } - - // start off with a reasonable size placeholder for wrappers - // gWrapperHolder.reserve(1024); - - // create an exception handler to process signals - // gExceptionHandler = new TExceptionHandlerImp{}; + if (optLevel != 0) { + std::ostringstream s; + s << "#pragma cling optimize " << optLevel; + Cpp::Process(s.str().c_str()); } - ~ApplicationStarter() { - // Cpp::DeleteInterpreter(Interp); - // for (auto wrap : gWrapperHolder) - // delete wrap; - // delete gExceptionHandler; gExceptionHandler = nullptr; - } -} _applicationStarter; + Cpp::AddIncludePath(Paths.IncludeDir.c_str()); + Cpp::LoadLibrary("libstdc++", /* lookup= */ true); +} + +static void preloadHeaders() { + const char* code = "#include \n" + "#include \n" + "#include \n" + "#include \n" + "#include \n" // for strcpy + "#include \n" + "#include \n" + "#include \n" + "#include \n" + "#include \n" // for the dispatcher code to + // use std::function + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#include \n" // FIXME: Replace with modules + "#if __has_include()\n" + "#include \n" + "#endif\n" + "#include \n"; + Cpp::Process(code); +} + +static void defineRuntimeHelpers() { + Cpp::Declare("namespace __cppjit_internal { template" + " bool is_equal(const C1& c1, const C2& c2) { return " + "(bool)(c1 == c2); } }", + /*silent=*/false); + Cpp::Declare("namespace __cppjit_internal { template" + " bool is_not_equal(const C1& c1, const C2& c2) { return " + "(bool)(c1 != c2); } }", + /*silent=*/false); + + // helper for multiple inheritance + Cpp::Declare("namespace __cppjit_internal { struct Sep; }", + /*silent=*/false); +} } // unnamed namespace +// Load CppInterOp and set up the interpreter. A dlopen during static +// initialization is unsafe, so _cpython_cppjit.py calls this explicitly +// before the first libcppjit use. Idempotent; returns 1 on success. +extern "C" { +RPY_EXPORTED int LoadCppInterOp(); +} + +extern "C" int LoadCppInterOp() { + std::lock_guard Lock(InterOpMutex); + static bool loaded = false; + if (loaded) + return 1; + + const InterOpPaths Paths = cppinterop_paths(); + if (!loadDispatchAPI(Paths)) + return 0; + + acquireOrCreateInterpreter(); + configureInterpreter(Paths); + preloadHeaders(); + defineRuntimeHelpers(); + + loaded = true; + return 1; +} + // local helpers ------------------------------------------------------------- static inline char* cppstring_to_cstring(const std::string& cppstr) { char* cstr = (char*)malloc(cppstr.size() + 1); @@ -666,36 +603,6 @@ interop::TCppType_t interop::GetComplexType(const std::string& name) { return Cpp::GetComplexType(Cpp::GetType(name)); } -// //---------------------------------------------------------------------------- -// static std::string extract_namespace(const std::string& name) -// { -// // Find the namespace the named class lives in, take care of templates -// // Note: this code also lives in cpyrt (TODO: refactor?) -// if (name.empty()) -// return name; -// -// int tpl_open = 0; -// for (std::string::size_type pos = name.size()-1; 0 < pos; --pos) { -// std::string::value_type c = name[pos]; -// -// // count '<' and '>' to be able to skip template contents -// if (c == '>') -// ++tpl_open; -// else if (c == '<') -// --tpl_open; -// -// // collect name up to "::" -// else if (tpl_open == 0 && c == ':' && name[pos-1] == ':') { -// // found the extend of the scope ... done -// return name.substr(0, pos-1); -// } -// } -// -// // no namespace; assume outer scope -// return ""; -// } -// - std::string interop::ResolveEnum(TCppScope_t handle) { std::lock_guard Lock(InterOpMutex); std::string type = @@ -1100,83 +1007,6 @@ bool interop::IsDefaultConstructable(TCppScope_t scope) { bool interop::IsVariable(TCppScope_t scope) { return Cpp::IsVariable(scope); } -// // helpers for stripping scope names -// static -// std::string outer_with_template(const std::string& name) -// { -// // Cut down to the outer-most scope from , taking proper care of -// templates. -// int tpl_open = 0; -// for (std::string::size_type pos = 0; pos < name.size(); ++pos) { -// std::string::value_type c = name[pos]; -// -// // count '<' and '>' to be able to skip template contents -// if (c == '<') -// ++tpl_open; -// else if (c == '>') -// --tpl_open; -// -// // collect name up to "::" -// else if (tpl_open == 0 && \ -// c == ':' && pos+1 < name.size() && name[pos+1] == ':') { -// // found the extend of the scope ... done -// return name.substr(0, pos-1); -// } -// } -// -// // whole name is apparently a single scope -// return name; -// } -// -// static -// std::string outer_no_template(const std::string& name) -// { -// // Cut down to the outer-most scope from , drop templates -// std::string::size_type first_scope = name.find(':'); -// if (first_scope == std::string::npos) -// return name.substr(0, name.find('<')); -// std::string::size_type first_templ = name.find('<'); -// if (first_templ == std::string::npos) -// return name.substr(0, first_scope); -// return name.substr(0, std::min(first_templ, first_scope)); -// } -// -// #define FILL_COLL(type, filter) { \ -// TIter itr{coll}; \ -// type* obj = nullptr; \ -// while ((obj = (type*)itr.Next())) { \ -// const char* nm = obj->GetName(); \ -// if (nm && nm[0] != '_' && !(obj->Property() & (filter))) { \ -// if (gInitialNames.find(nm) == gInitialNames.end()) \ -// cppnames.insert(nm); \ -// }}} -// -// static inline -// void cond_add(interop::TCppScope_t scope, const std::string& ns_scope, -// std::set& cppnames, const char* name, bool nofilter = false) -// { -// if (!name || name[0] == '_' || strstr(name, ".h") != 0 || strncmp(name, -// "operator", 8) == 0) -// return; -// -// if (scope == GLOBAL_HANDLE) { -// std::string to_add = outer_no_template(name); -// if (nofilter || gInitialNames.find(to_add) == gInitialNames.end()) -// cppnames.insert(outer_no_template(name)); -// } else if (scope == STD_HANDLE) { -// if (strncmp(name, "std::", 5) == 0) { -// name += 5; -// #ifdef __APPLE__ -// if (strncmp(name, "__1::", 5) == 0) name += 5; -// #endif -// } -// cppnames.insert(outer_no_template(name)); -// } else { -// if (strncmp(name, ns_scope.c_str(), ns_scope.size()) == 0) -// cppnames.insert(outer_with_template(name + ns_scope.size())); -// } -// } - void interop::GetAllCppNames(TCppScope_t scope, std::set& cppnames) { // Collect all known names of C++ entities under scope. This is useful for