Support developer builds of CppInterOp/LLVM, add project verification and robust fallbacks

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.20)
 project(CppJIT LANGUAGES C CXX)
 
 set(CMAKE_CXX_STANDARD 20)
@@ -13,6 +13,8 @@ include(GNUInstallDirs)
 option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF)
 set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "")
 set(CPPINTEROP_GIT_TAG "8d624c621a4b95e36ff73ac708c85a768287478f" CACHE STRING "")
+set(CPPINTEROP_SOURCE_DIR "" CACHE PATH
+    "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source")
 
 set(Python_FIND_VIRTUALENV ONLY)
 find_package(Python COMPONENTS Interpreter Development)
@@ -24,50 +26,69 @@ if(NOT Python_Development_FOUND)
     message(FATAL_ERROR "Python development headers not found")
 endif()
 
+# The LLVM range the pinned CppInterOp supports; update together with the tag.
+set(CPPJIT_LLVM_VERSION_MIN 20)
+set(CPPJIT_LLVM_VERSION_MAX 22)
+
 set(_llvm_hints "")
 
 if(DEFINED ENV{CONDA_PREFIX})
     list(APPEND _llvm_hints "$ENV{CONDA_PREFIX}/lib/cmake/llvm")
-    list(APPEND _llvm_hints "$ENV{CONDA_PREFIX}/lib/cmake/clang")
 endif()
 
-if(NOT DEFINED LLVM_DIR)
+if(DEFINED LLVM_DIR)
+    # An explicit LLVM_DIR is authoritative: fail instead of falling back to a
+    # different LLVM than the one requested. A failed find_package resets
+    # LLVM_DIR to -NOTFOUND, so keep the requested value for the message.
+    set(_llvm_dir_arg "${LLVM_DIR}")
+    find_package(LLVM CONFIG PATHS "${LLVM_DIR}" NO_DEFAULT_PATH)
+    if(NOT LLVM_FOUND)
+        message(FATAL_ERROR
+            "No LLVMConfig.cmake under LLVM_DIR (${_llvm_dir_arg}); expected "
+            "<install prefix or build tree>/lib/cmake/llvm")
+    endif()
+else()
     find_package(LLVM CONFIG QUIET HINTS ${_llvm_hints})
     if(NOT LLVM_FOUND)
         message(FATAL_ERROR
-            "llvm-config not found, please install LLVM 21 (llvmdev, clangdev) using your system package manager or conda.\n"
-            "As an alternative, you can pass -DLLVM_DIR=/path/to/llvm/lib/cmake/llvm to pip\n"
+            "No LLVM CMake package found. Install LLVM "
+            "${CPPJIT_LLVM_VERSION_MIN}-${CPPJIT_LLVM_VERSION_MAX} development packages "
+            "(apt: llvm-${CPPJIT_LLVM_VERSION_MAX}-dev libclang-${CPPJIT_LLVM_VERSION_MAX}-dev; "
+            "conda: llvmdev clangdev), or point cppjit at your own LLVM build with "
+            "-DLLVM_DIR=<prefix or build tree>/lib/cmake/llvm "
             "(pip: --config-settings=cmake.define.LLVM_DIR=...)")
     endif()
-else()
-    find_package(LLVM REQUIRED CONFIG HINTS "${LLVM_DIR}")
 endif()
 
-# somewhat derive Clang_DIR
-if(NOT DEFINED Clang_DIR)
-    get_filename_component(_llvm_cmake_dir "${LLVM_DIR}" DIRECTORY)
-    if(EXISTS "${_llvm_cmake_dir}/clang/ClangConfig.cmake")
-        set(Clang_DIR "${_llvm_cmake_dir}/clang")
-    elseif(EXISTS "${LLVM_DIR}/../clang/ClangConfig.cmake")
-        set(Clang_DIR "${LLVM_DIR}/../clang")
-    endif()
-endif()
-if(DEFINED Clang_DIR)
-    find_package(Clang CONFIG HINTS "${Clang_DIR}")
+message(STATUS "Found LLVM ${LLVM_VERSION} at ${LLVM_DIR}")
+if(LLVM_VERSION_MAJOR LESS CPPJIT_LLVM_VERSION_MIN OR
+   LLVM_VERSION_MAJOR GREATER CPPJIT_LLVM_VERSION_MAX)
+    message(FATAL_ERROR
+        "LLVM ${LLVM_VERSION} is unsupported: the currently supported "
+        "CppInterOp version (${CPPINTEROP_GIT_TAG}) only supports LLVM "
+        "${CPPJIT_LLVM_VERSION_MIN}-${CPPJIT_LLVM_VERSION_MAX}")
 endif()
 
-message(STATUS "Found LLVM ${LLVM_VERSION} at ${LLVM_DIR}")
-if(LLVM_VERSION VERSION_LESS "20.0")
-    message(WARNING "LLVM ${LLVM_VERSION} detected. Minimum supported version is 21. Build may fail.")
+if(DEFINED Clang_DIR)
+    # An explicit Clang_DIR is authoritative, like LLVM_DIR above.
+    set(_clang_dir_arg "${Clang_DIR}")
+    find_package(Clang CONFIG PATHS "${Clang_DIR}" NO_DEFAULT_PATH)
+    if(NOT Clang_FOUND)
+        message(FATAL_ERROR
+            "No ClangConfig.cmake under Clang_DIR (${_clang_dir_arg}); expected "
+            "<install prefix or build tree>/lib/cmake/clang")
+    endif()
+else()
+    # Clang's package sits beside LLVM's in every supported layout; search
+    # only there so an unrelated system clang cannot satisfy the lookup.
+    find_package(Clang CONFIG QUIET HINTS "${LLVM_DIR}/../clang" NO_DEFAULT_PATH)
 endif()
 if(Clang_FOUND)
     message(STATUS "Found Clang at ${Clang_DIR}")
 endif()
 
-# build CppInterOp, we should expose the option to use pre-installed CppInterOp for example using conda
-
-# CPPINTEROP_DIR should resolve to the final installed location at runtime.
-# ask for site-packages path; fall back to CMAKE_INSTALL_PREFIX for standalone builds.
+# CppInterOp is installed at the location cppjit ships at runtime: ask for the
+# site-packages path; fall back to CMAKE_INSTALL_PREFIX for standalone builds.
 execute_process(
     COMMAND ${Python_EXECUTABLE} -c "import sysconfig; print(sysconfig.get_path('platlib'))"
     OUTPUT_VARIABLE _python_platlib
@@ -93,10 +114,11 @@ set(INTEROP_SOURCES
 add_library(cppjit SHARED ${CPYRT_SOURCES} ${INTEROP_SOURCES})
 add_dependencies(cppjit CppInterOp)
 
-
+# The exact library file the wrapper dlopens and the include dir the
+# interpreter boot requires.
 target_compile_definitions(cppjit PRIVATE
-    CPPINTEROP_DIR="${CPPINTEROP_INSTALL_DIR}"
-    CMAKE_SHARED_LIBRARY_SUFFIX="${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    CPPINTEROP_LIBRARY="${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    CPPINTEROP_INCLUDE_DIR="${CPPINTEROP_INSTALL_DIR}/include"
 )
 
 target_include_directories(cppjit PRIVATE

README.md

@@ -9,7 +9,7 @@ CppJIT is the consolidated monorepo packaging of the compiler-research forks of
     -  Installed via your favourite package manager (e.g. `conda install -c conda-forge "llvmdev=21" "clangdev=21"`)
     -  To use a source build of LLVM, pass the path to pip like `pip install . --config-settings=cmake.define.LLVM_DIR=/path/to/build/lib/cmake/llvm`
 - Python 3.12+ with development headers (e.g. `apt install python3.14 python3.14-dev`)
-- CMake 3.16+
+- CMake 3.20+
 
 ### Standard installation:
 

cmake/AddCppInterOp.cmake

@@ -1,5 +1,7 @@
-# Configures the CppInterOp ExternalProject. Default backend is clang-repl;
-# CPPJIT_USE_CLING builds CppInterOp against a provided cling build
+# Configures the CppInterOp ExternalProject, built either from the pinned git
+# tag (default) or from a local checkout (CPPINTEROP_SOURCE_DIR). Default
+# backend is clang-repl; CPPJIT_USE_CLING builds CppInterOp against a provided
+# cling build
 include_guard(GLOBAL)
 include(ExternalProject)
 
@@ -59,17 +61,46 @@ function(cppjit_add_cppinterop)
         list(APPEND _args -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
     endif()
 
-    ExternalProject_Add(CppInterOp
+    set(_source_args
         GIT_REPOSITORY ${CPPINTEROP_GIT_REPOSITORY}
         GIT_TAG        ${CPPINTEROP_GIT_TAG}
-        PREFIX         "${CMAKE_BINARY_DIR}/CppInterOp"
-        CMAKE_ARGS     ${_args}
-        BUILD_BYPRODUCTS
-            "${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    )
+    set(_log_args
         LOG_DOWNLOAD   ON
         LOG_CONFIGURE  ON
         LOG_BUILD      ON
         LOG_INSTALL    ON
         LOG_OUTPUT_ON_FAILURE ON
     )
+    if(CPPINTEROP_SOURCE_DIR)
+        message(STATUS "CppInterOp: building from local source at ${CPPINTEROP_SOURCE_DIR} "
+                       "over the currently supported version ${CPPINTEROP_GIT_TAG}")
+        # BUILD_ALWAYS recompiles uncommitted edits and refreshes the installed
+        # libclangCppInterOp on every build; the sub-build keeps this incremental.
+        set(_source_args
+            SOURCE_DIR   "${CPPINTEROP_SOURCE_DIR}"
+            BUILD_ALWAYS ON
+        )
+        # Stream sub-build output in the dev loop instead of hiding it in log files.
+        set(_log_args "")
+    endif()
+
+    ExternalProject_Add(CppInterOp
+        ${_source_args}
+        PREFIX         "${CMAKE_BINARY_DIR}/CppInterOp"
+        CMAKE_ARGS     ${_args}
+        BUILD_BYPRODUCTS
+            "${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}"
+        ${_log_args}
+    )
+
+    # Verify that the user-provided CppInterOp source override is legitimate.
+    ExternalProject_Add_Step(CppInterOp verify_project
+        COMMAND ${CMAKE_COMMAND}
+            -DCPPINTEROP_BINARY_DIR=<BINARY_DIR>
+            -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/VerifyCppInterOp.cmake"
+        DEPENDEES configure
+        DEPENDERS build
+        COMMENT "Verifying the configured source tree is CppInterOp"
+    )
 endfunction()

cmake/VerifyCppInterOp.cmake

@@ -0,0 +1,14 @@
+# Runs between the CppInterOp ExternalProject's configure and build steps:
+# the sub-configure records its project() name in its cache; reject any
+# source tree that does not declare itself CppInterOp.
+load_cache("${CPPINTEROP_BINARY_DIR}" READ_WITH_PREFIX _sub_ CMAKE_PROJECT_NAME)
+if(NOT DEFINED _sub_CMAKE_PROJECT_NAME)
+    message(FATAL_ERROR
+        "No CMAKE_PROJECT_NAME in ${CPPINTEROP_BINARY_DIR}/CMakeCache.txt "
+        "— the CppInterOp sub-configure did not complete")
+endif()
+if(NOT _sub_CMAKE_PROJECT_NAME STREQUAL "CppInterOp")
+    message(FATAL_ERROR
+        "The provided source path override declares project '${_sub_CMAKE_PROJECT_NAME}', "
+        "not CppInterOp")
+endif()

src/interop/interop_wrapper.cxx

@@ -134,9 +134,7 @@ class ApplicationStarter {
 public:
   ApplicationStarter() {
     std::lock_guard<std::recursive_mutex> Lock(InterOpMutex);
-    if (!Cpp::LoadDispatchAPI(
-            CPPINTEROP_DIR
-            "/lib/libclangCppInterOp" CMAKE_SHARED_LIBRARY_SUFFIX)) {
+    if (!Cpp::LoadDispatchAPI(CPPINTEROP_LIBRARY)) {
       std::cerr << "[cppjit-backend] Failed to load CppInterOp" << std::endl;
       return;
     }
@@ -205,7 +203,7 @@ class ApplicationStarter {
     Cpp::AddIncludePath((ClingSrc + "/tools/cling/include").c_str());
     Cpp::AddIncludePath((ClingSrc + "/include").c_str());
     Cpp::AddIncludePath((ClingBuildDir + "/include").c_str());
-    Cpp::AddIncludePath((std::string(CPPINTEROP_DIR) + "/include").c_str());
+    Cpp::AddIncludePath(CPPINTEROP_INCLUDE_DIR);
     Cpp::LoadLibrary("libstdc++", /* lookup= */ true);
 
     // load frequently used headers