From b38355fe5a98982aaf73ac77642e7e50ae113ef7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:39:30 +0900 Subject: [PATCH 1/2] fix(pypi): do not return rctx.metadata for pip_archive There was a small regression in #3948 that may affect experimental repository cache users. Related to #3791 --- news/4042.fixed.md | 2 ++ python/private/pypi/whl_library.bzl | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 news/4042.fixed.md diff --git a/news/4042.fixed.md b/news/4042.fixed.md new file mode 100644 index 0000000000..35feb31837 --- /dev/null +++ b/news/4042.fixed.md @@ -0,0 +1,2 @@ +Previous refactor that shipped with 2.3 introduced regression for the experimental repository cache +users. This restores the previous behavior ([#3791](https://github.com/bazel-contrib/rules_python/pull/3791)). diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 47b05468ef..281dd863e6 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -532,7 +532,9 @@ def _pip_archive_impl(rctx): if not rctx.delete("whl_file.json"): fail("failed to delete the whl_file.json file") - return _whl_extract(rctx, whl_path = whl_path, logger = logger, sdist_filename = sdist_filename) + # NOTE @aignas 2026-08-14: We never return rctx.metadata for pip archives because the result may + # not be reproducible across all machines given the input args to the repository rule. + _whl_extract(rctx, whl_path = whl_path, logger = logger, sdist_filename = sdist_filename) def _remove_files(rctx, *basenames): paths = list(rctx.path(".").readdir()) From 5a62e1b7d2006be5a091033438077e0370b0a492 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:47:05 +0900 Subject: [PATCH 2/2] refactor: split code into multiple files and move compat shim This move the compatibility shim to the macro for the `dep_template` and since we may set it to an empty string in the future, we do not set it to mandatory. This resulted in a few test fixes for the `pip_archive`. This will hopefully make working with the code easier. The attribute reuse is done in this way on purpose - the most basic repository rules are being used as a basis for the definition of the attributes. --- python/private/pypi/BUILD.bazel | 78 ++ python/private/pypi/patch_and_extract_whl.bzl | 164 ++++ python/private/pypi/pip_archive.bzl | 401 +++++++++ python/private/pypi/whl_archive.bzl | 143 ++++ python/private/pypi/whl_deps_repo.bzl | 93 +++ python/private/pypi/whl_library.bzl | 782 +----------------- tests/integration/whl_library/BUILD.bazel | 8 +- tests/integration/whl_library/MODULE.bazel | 18 +- .../integration/whl_library/test_contents.py | 2 +- 9 files changed, 901 insertions(+), 788 deletions(-) create mode 100644 python/private/pypi/patch_and_extract_whl.bzl create mode 100644 python/private/pypi/pip_archive.bzl create mode 100644 python/private/pypi/whl_archive.bzl create mode 100644 python/private/pypi/whl_deps_repo.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 3c07135d40..311b2520f4 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -547,6 +547,84 @@ bzl_library( deps = [":hash"], ) +bzl_library( + name = "pip_archive", + srcs = ["pip_archive.bzl"], + deps = [ + ":attrs", + ":deps", + ":generate_whl_library_build_bazel", + ":patch_whl", + ":pep508_requirement", + ":pypi_repo_utils", + ":urllib", + ":whl_extract", + ":whl_metadata", + "//python/private:auth", + "//python/private:envsubst", + "//python/private:is_standalone_interpreter", + "//python/private:normalize_name", + "//python/private:repo_utils", + ], +) + +bzl_library( + name = "whl_archive", + srcs = ["whl_archive.bzl"], + deps = [ + ":attrs", + ":deps", + ":generate_whl_library_build_bazel", + ":patch_whl", + ":pep508_requirement", + ":pypi_repo_utils", + ":urllib", + ":whl_extract", + ":whl_metadata", + "//python/private:auth", + "//python/private:envsubst", + "//python/private:is_standalone_interpreter", + "//python/private:normalize_name", + "//python/private:repo_utils", + ], +) + +bzl_library( + name = "whl_deps_repo", + srcs = ["whl_deps_repo.bzl"], + deps = [ + ":attrs", + ":deps", + ":generate_whl_library_build_bazel", + ":patch_whl", + ":pep508_requirement", + ":pypi_repo_utils", + ":urllib", + ":whl_extract", + ":whl_metadata", + "//python/private:auth", + "//python/private:envsubst", + "//python/private:is_standalone_interpreter", + "//python/private:normalize_name", + "//python/private:repo_utils", + ], +) + +bzl_library( + name = "patch_and_extract_whl", + srcs = ["patch_and_extract_whl.bzl"], + deps = [ + ":generate_whl_library_build_bazel", + ":patch_whl", + ":pep508_requirement", + ":pypi_repo_utils", + ":whl_extract", + ":whl_metadata", + "//python/private:normalize_name", + "//python/private:repo_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl new file mode 100644 index 0000000000..b4ecb2f5b5 --- /dev/null +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -0,0 +1,164 @@ +"" + +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:repo_utils.bzl", "repo_utils") +load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":patch_whl.bzl", "patch_whl") +load(":pep508_requirement.bzl", "requirement") +load(":pypi_repo_utils.bzl", "pypi_repo_utils") +load(":whl_extract.bzl", "whl_extract") +load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") + +def _get_entry_points(rctx, install_dir_path, metadata): + dist_info_dir = "{}-{}.dist-info".format( + metadata.name.replace("-", "_"), + metadata.version.replace("-", "_"), + ) + entry_points_txt = install_dir_path.get_child(dist_info_dir).get_child("entry_points.txt") + if entry_points_txt.exists: + return parse_entry_points(rctx.read(entry_points_txt)) + return {} + +def _move_scripts_needing_shebang_rewrite(rctx, entry_points): + bin_dir = rctx.path("bin") + if not bin_dir.exists: + return + + ep_names = {name.lower(): True for name in entry_points} + for script in bin_dir.readdir(): + if script.is_dir: + continue + if script.basename.lower() in ep_names: + rctx.delete(script) + continue + if script.basename.endswith(".exe") or script.basename.endswith(".dll"): + continue + content = rctx.read(script) + if content.startswith("#!python"): + rewrite_bin_dir = rctx.path("rewrite-bin") + repo_utils.mkdir(rctx, rewrite_bin_dir) + repo_utils.rename(rctx, script, rctx.path("rewrite-bin/" + script.basename)) + +def _to_purl(*, index, metadata, filename): + """ + Produce a PyPI PURL from the metadata. + + https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md + """ + + # https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md#name-definition + name = normalize_name(metadata.name).replace("_", "-") + + qualifiers = {} + if index: + qualifiers["repository_url"] = index + if filename: + qualifiers["file_name"] = filename + + return "pkg:pypi/{}@{}?{}".format(name, metadata.version, "&".join(["{}={}".format(key, val) for key, val in qualifiers.items()])) + +def _remove_files(rctx, *basenames): + paths = list(rctx.path(".").readdir()) + for _ in range(10000000): + if not paths: + break + path = paths.pop() + + if path.basename in basenames: + rctx.delete(path) + elif path.is_dir: + paths.extend(path.readdir()) + +def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): + """Extract the wheel, apply patches and generate BUILD.bazel files. + + Reused in pip and http wheel download code. + + Args: + rctx: the repository ctx. + whl_path: the whl path to extract. + logger: The logger to use + sdist_filename: The filename to ignore in the BUILD.bazel files as sources. + + Returns: + The repository metadata if the extraction is reproducible + """ + if rctx.attr.whl_patches: + patches = {} + for patch_file, json_args in rctx.attr.whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if whl_path.basename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + + if patches: + whl_path = patch_whl( + rctx, + whl_path = whl_path, + patches = patches, + ) + + whl_extract(rctx, whl_path = whl_path, logger = logger) + + install_dir_path = whl_path.dirname.get_child("site-packages") + metadata = whl_metadata( + install_dir = install_dir_path, + read_fn = rctx.read, + logger = logger, + ) + rctx.file("metadata.json", json.encode_indent({ + "name": metadata.name, + "provides_extra": metadata.provides_extra, + "requires_dist": metadata.requires_dist, + "version": metadata.version, + })) + namespace_package_files = pypi_repo_utils.find_namespace_package_files(rctx, install_dir_path) + + entry_points = _get_entry_points(rctx, install_dir_path, metadata) + _move_scripts_needing_shebang_rewrite(rctx, entry_points) + + build_file_contents = generate_whl_library_build_bazel( + name = whl_path.basename, + dep_template = rctx.attr.dep_template, + sdist_filename = sdist_filename, + config_load = rctx.attr.config_load, + metadata_name = metadata.name, + metadata_version = metadata.version, + requires_dist = metadata.requires_dist, + # TODO @aignas 2025-05-17: maybe have a build flag for this instead + enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs, + # TODO @aignas 2025-04-14: load through the hub: + annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), + data_exclude = rctx.attr.pip_data_exclude, + group_deps = rctx.attr.group_deps, + group_name = rctx.attr.group_name, + namespace_package_files = namespace_package_files, + extras = requirement(rctx.attr.requirement).extras, + entry_points = entry_points, + purl = _to_purl( + index = rctx.attr.index_url, + metadata = metadata, + filename = sdist_filename or whl_path.basename, + ), + ) + + # Delete these in case the wheel had them. They generally don't cause + # a problem, but let's avoid the chance of that happening. + rctx.file("WORKSPACE") + rctx.file("WORKSPACE.bazel") + rctx.file("MODULE.bazel") + rctx.file("REPO.bazel", """\ +repo( + default_package_metadata = [ + "//:package_metadata", + ], +) +""") + + # BUILD files interfere with globbing and Bazel package boundaries. + _remove_files(rctx, "BUILD", "BUILD.bazel") + rctx.file("BUILD.bazel", build_file_contents) + + if hasattr(rctx, "repo_metadata"): + return rctx.repo_metadata(reproducible = True) + + return None diff --git a/python/private/pypi/pip_archive.bzl b/python/private/pypi/pip_archive.bzl new file mode 100644 index 0000000000..9fc50ccbf9 --- /dev/null +++ b/python/private/pypi/pip_archive.bzl @@ -0,0 +1,401 @@ +"" + +load("//python/private:auth.bzl", "get_auth") +load("//python/private:envsubst.bzl", "envsubst") +load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":attrs.bzl", "ATTRS", "use_isolated") +load(":deps.bzl", "all_repo_names", "record_files") +load(":patch_and_extract_whl.bzl", "patch_and_extract_whl") +load(":pypi_repo_utils.bzl", "pypi_repo_utils") +load(":urllib.bzl", "urllib") +load(":whl_archive.bzl", "whl_archive_attrs") + +_CPPFLAGS = "CPPFLAGS" +_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" + +def _get_xcode_location_cflags(rctx, logger = None): + """Query the xcode sdk location to update cflags + + Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. + Pip won't be able to compile c extensions from sdists with the pre built python distributions from astral-sh + otherwise. See https://github.com/astral-sh/python-build-standalone/issues/103 + """ + + # Only run on MacOS hosts + if not rctx.os.name.lower().startswith("mac os"): + return [] + + xcode_sdk_location = repo_utils.execute_unchecked( + rctx, + op = "GetXcodeLocation", + arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"], + logger = logger, + ) + if xcode_sdk_location.return_code != 0: + return [] + + xcode_root = xcode_sdk_location.stdout.strip() + if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): + # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer + # so we need to change the path to to the macos specific tools which are in a different relative + # path than xcode installed command line tools. + xcode_sdks_json = repo_utils.execute_checked( + rctx, + op = "LocateXCodeSDKs", + arguments = [ + repo_utils.which_checked(rctx, "xcrun"), + "xcodebuild", + "-showsdks", + "-json", + ], + environment = { + "DEVELOPER_DIR": xcode_root, + }, + logger = logger, + ).stdout + xcode_sdks = json.decode(xcode_sdks_json) + potential_sdks = [ + sdk + for sdk in xcode_sdks + if "productName" in sdk and + sdk["productName"] == "macOS" and + "darwinos" not in sdk["canonicalName"] + ] + + # Now we'll get two entries here (one for internal and another one for public) + # It shouldn't matter which one we pick. + xcode_sdk_path = potential_sdks[0]["sdkPath"] + else: + xcode_sdk_path = "{}/SDKs/MacOSX.sdk".format(xcode_root) + + return [ + "-isysroot {}".format(xcode_sdk_path), + ] + +def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None): + """Gather cflags from a standalone toolchain for unix systems. + + Pip won't be able to compile c extensions from sdists with the pre built python distributions from astral-sh + otherwise. See https://github.com/astral-sh/python-build-standalone/issues/103 + """ + + # Only run on Unix systems + if not rctx.os.name.lower().startswith(("mac os", "linux")): + return [] + + # Only update the location when using a standalone toolchain. + if not is_standalone_interpreter(rctx, python_interpreter, logger = logger): + return [] + + stdout = pypi_repo_utils.execute_checked_stdout( + rctx, + op = "GetPythonVersionForUnixCflags", + # python_interpreter by default points to a symlink, however when using bazel in vendor mode, + # and the vendored directory moves around, the execution of python fails, as it's getting confused + # where it's running from. More to the fact that we are executing it in isolated mode "-I", which + # results in PYTHONHOME being ignored. The solution is to run python from it's real directory. + python = python_interpreter.realpath, + arguments = [ + # Run the interpreter in isolated mode, this options implies -E, -P and -s. + # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH, + # which may interfere with this invocation. + "-I", + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", + ], + srcs = [], + logger = logger, + ) + _python_version = stdout + include_path = "{}/include/python{}".format( + python_interpreter.dirname, + _python_version, + ) + + return ["-isystem {}".format(include_path)] + +def _parse_optional_attrs(rctx, args, extra_pip_args = None): + """Helper function to parse common attributes of pip_repository and whl_library repository rules. + + This function also serializes the structured arguments as JSON + so they can be passed on the command line to subprocesses. + + Args: + rctx: Handle to the rule repository context. + args: A list of parsed args for the rule. + extra_pip_args: The pip args to pass. + Returns: Augmented args list. + """ + + if use_isolated(rctx, rctx.attr): + args.append("--isolated") + + # Check for None so we use empty default types from our attrs. + # Some args want to be list, and some want to be dict. + if extra_pip_args != None: + args += [ + "--extra_pip_args", + json.encode(struct(arg = [ + envsubst(pip_arg, rctx.attr.envsubst, rctx.getenv) + for pip_arg in extra_pip_args + ])), + ] + + if rctx.attr.download_only: + args.append("--download_only") + + if rctx.attr.pip_data_exclude != None: + args += [ + "--pip_data_exclude", + json.encode(struct(arg = rctx.attr.pip_data_exclude)), + ] + + env = {} + if rctx.attr.environment != None: + for key, value in rctx.attr.environment.items(): + env[key] = value + + # This is super hacky, but working out something nice is tricky. + # This is in particular needed for psycopg2 which attempts to link libpython.a, + # in order to point the linker at the correct python intepreter. + if rctx.attr.add_libdir_to_library_search_path: + if "LDFLAGS" in env: + fail("Can't set both environment LDFLAGS and add_libdir_to_library_search_path") + command = [ + pypi_repo_utils.resolve_python_interpreter(rctx), + "-c", + "import sys ; sys.stdout.write('{}/lib'.format(sys.exec_prefix))", + ] + result = rctx.execute(command) + if result.return_code != 0: + fail("Failed to get LDFLAGS path: command: {}, exit code: {}, stdout: {}, stderr: {}".format(command, result.return_code, result.stdout, result.stderr)) + libdir = result.stdout + env["LDFLAGS"] = "-L{}".format(libdir) + + args += [ + "--environment", + json.encode(struct(arg = env)), + ] + + return args + +def _get_python_home(rctx, python_interpreter, logger = None): + """Get the PYTHONHOME directory from the selected python interpretter + + Args: + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. + logger: Optional logger to use for operations. + Returns: + String of PYTHONHOME directory. + """ + + return pypi_repo_utils.execute_checked_stdout( + rctx, + op = "GetPythonHome", + # python_interpreter by default points to a symlink, however when using bazel in vendor mode, + # and the vendored directory moves around, the execution of python fails, as it's getting confused + # where it's running from. More to the fact that we are executing it in isolated mode "-I", which + # results in PYTHONHOME being ignored. The solution is to run python from it's real directory. + python = python_interpreter.realpath, + arguments = [ + # Run the interpreter in isolated mode, this options implies -E, -P and -s. + # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH, + # which may interfere with this invocation. + "-I", + "-c", + "import sys; print(f'{sys.prefix}', end='')", + ], + srcs = [], + logger = logger, + ) + +def _create_repository_execution_environment(rctx, python_interpreter, logger = None): + """Create a environment dictionary for processes we spawn with rctx.execute. + + Args: + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. + logger: Optional logger to use for operations. + Returns: + Dictionary of environment variable suitable to pass to rctx.execute. + """ + + env = { + "PYTHONHOME": _get_python_home(rctx, python_interpreter, logger), + "PYTHONPATH": pypi_repo_utils.construct_pythonpath( + rctx, + entries = rctx.attr._python_path_entries, + ), + } + + # Gather any available CPPFLAGS values + # + # We may want to build in an environment without a cc toolchain. + # In those cases, we're limited to --download-only, but we should respect that here. + is_wheel = rctx.attr.filename and rctx.attr.filename.endswith(".whl") + if not (rctx.attr.download_only or is_wheel): + cppflags = [] + cppflags.extend(_get_xcode_location_cflags(rctx, logger = logger)) + cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter, logger = logger)) + env[_CPPFLAGS] = " ".join(cppflags) + return env + +def _pip_archive_impl(rctx): + logger = repo_utils.logger(rctx) + + sdist_filename = None + extra_pip_args = [] + extra_pip_args.extend(rctx.attr.extra_pip_args) + if rctx.attr.urls and rctx.attr.filename: + filename = rctx.attr.filename + urls = rctx.attr.urls + urls = [ + urllib.absolute_url( + rctx.attr.index_url, + url, + envsubst = rctx.attr.envsubst, + getenv = rctx.getenv, + ) + for url in urls + ] + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", + auth = get_auth(rctx, urls), + ) + if not rctx.attr.sha256 and not rctx.attr.integrity: + # this is only seen when there is a direct URL reference without a hash + logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( + rctx.attr.requirement, + result.sha256, + )) + + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + + if filename.endswith(".whl"): + fail("Only sdists are supported") + else: + sdist_filename = filename + + # It is an sdist and we need to tell PyPI to use a file in this directory + # and, allow getting build dependencies from PYTHONPATH, which we + # setup in this repository rule, but still download any necessary + # build deps from PyPI (e.g. `flit_core`) if they are missing. + extra_pip_args.extend(["--find-links", "."]) + + # When we already have a wheel, Python isn't used, + # so there's no need to setup env vars to run Python, unless we need to + # build an sdist or resolve a requirement. + python_interpreter = pypi_repo_utils.resolve_python_interpreter( + rctx, + python_interpreter = rctx.attr.python_interpreter, + python_interpreter_target = rctx.attr.python_interpreter_target, + ) + args = [ + "-m", + "python.private.pypi.whl_installer.wheel_installer", + "--requirement", + rctx.attr.requirement, + ] + args = _parse_optional_attrs(rctx, args, extra_pip_args) + + # Manually construct the PYTHONPATH since we cannot use the toolchain here + environment = _create_repository_execution_environment(rctx, python_interpreter, logger = logger) + + if rctx.attr.urls: + op_tmpl = "whl_library.BuildWheelFromSource({name}, {requirement})" + elif rctx.attr.download_only: + op_tmpl = "whl_library.DownloadWheel({name}, {requirement})" + else: + op_tmpl = "whl_library.ResolveRequirement({name}, {requirement})" + + pypi_repo_utils.execute_checked( + rctx, + # truncate the requirement value when logging it / reporting + # progress since it may contain several ' --hash=sha256:... + # --hash=sha256:...' substrings that fill up the console + python = python_interpreter, + op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]), + arguments = args, + environment = environment, + srcs = rctx.attr._python_srcs, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + logger = logger, + ) + + whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) + if not rctx.delete("whl_file.json"): + fail("failed to delete the whl_file.json file") + + # NOTE @aignas 2026-08-14: We never return rctx.metadata for pip archives because the result may + # not be reproducible across all machines given the input args to the repository rule. + patch_and_extract_whl(rctx, whl_path = whl_path, logger = logger, sdist_filename = sdist_filename) + +# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique +_attrs = whl_archive_attrs | { + k: ATTRS[k] + for k in [ + # used for pulling deps with pip + "download_only", + "add_libdir_to_library_search_path", + "environment", + "extra_pip_args", + "isolated", + "python_interpreter", + "python_interpreter_target", + "quiet", + "timeout", + ] +} | { + "_python_path_entries": attr.label_list( + # Get the root directory of these rules and keep them as a default attribute + # in order to avoid unnecessary repository fetching restarts. + # + # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 + default = [ + Label("//:BUILD.bazel"), + ] + [ + # Includes all the external dependencies from repositories.bzl + Label("@" + repo + "//:BUILD.bazel") + for repo in all_repo_names + ], + ), + "_python_srcs": attr.label_list( + # Used as a default value in a rule to ensure we fetch the dependencies. + default = [ + Label("//python/private/pypi/whl_installer:wheel_installer.py"), + Label("//python/private/pypi/whl_installer:arguments.py"), + ] + record_files.values(), + ), +} + +pip_archive = repository_rule( + attrs = _attrs | { + "_rule_name": attr.string(default = "pip_archive"), + }, + doc = """ +Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. +Instantiated from pip_repository and inherits config options from there. + +:::{versionchanged} 1.9.0 +The `whl_library` is marked as reproducible if using starlark to extract and parse the +wheel contents without building an `sdist` first. +::: + +:::{versionchanged} 2.3.0 +The whl-only pure Starlark operations have been refactored into {obj}`whl_archive` and the +previously named {obj}`whl_library` repository became renamed to `pip_archive`. +::: +""", + implementation = _pip_archive_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/whl_archive.bzl b/python/private/pypi/whl_archive.bzl new file mode 100644 index 0000000000..75a2e6b290 --- /dev/null +++ b/python/private/pypi/whl_archive.bzl @@ -0,0 +1,143 @@ +"" + +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":attrs.bzl", "ATTRS") +load(":patch_and_extract_whl.bzl", "patch_and_extract_whl") +load(":urllib.bzl", "urllib") +load(":whl_deps_repo.bzl", "whl_deps_attrs") + +def _whl_archive_impl(rctx): + logger = repo_utils.logger(rctx) + + whl_path = None + if rctx.attr.whl_file: + rctx.watch(rctx.attr.whl_file) + whl_path = rctx.path(rctx.attr.whl_file) + + # Simulate the behaviour where the whl is present in the current directory. + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) + elif rctx.attr.urls and rctx.attr.filename: + filename = rctx.attr.filename + urls = rctx.attr.urls + urls = [ + urllib.absolute_url( + rctx.attr.index_url, + url, + envsubst = rctx.attr.envsubst, + getenv = rctx.getenv, + ) + for url in urls + ] + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", + auth = get_auth(rctx, urls), + ) + if not rctx.attr.sha256 and not rctx.attr.integrity: + # this is only seen when there is a direct URL reference without a hash + logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( + rctx.attr.requirement, + result.sha256, + )) + + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + + if filename.endswith(".whl"): + whl_path = rctx.path(filename) + else: + fail("Only wheels are supported") + else: + fail("Either 'whl_file' or 'urls' and 'filename' needs to be specified") + + return patch_and_extract_whl(rctx, whl_path = whl_path, logger = logger) + +whl_archive_attrs = whl_deps_attrs | { + "annotation": attr.label( + doc = ( + "Optional json encoded file containing annotation to apply to the extracted wheel. " + + "See `package_annotation`" + ), + allow_files = True, + ), + "filename": attr.string( + doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", + ), + "index_url": attr.string( + doc = "The index_url that the package will be downloaded from.", + ), + "integrity": attr.string( + doc = """\ +The expected checksum of the downloaded whl in Subresource Integrity format +(e.g. `sha256-...` or `sha512-...`). Only used when `urls` is passed. If +`sha256` is also set, it takes precedence over this attribute. + +:::{versionadded} 2.3.0 +::: +""", + ), + "sha256": attr.string( + doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.", + ), + "urls": attr.string_list( + doc = """\ +The list of urls of the whl to be downloaded using bazel downloader. Using this +attr makes `extra_pip_args` and `download_only` ignored.""", + ), + "whl_patches": attr.label_keyed_string_dict( + doc = """ +A label-keyed-string dict with patch files as keys and json-strings as values. + +The keys are labels to the patch file to apply. + +The values describe what to apply the patch to and how to apply it. +It is encoded as `json.encode(struct([whls], patch_strip])`, +where `whls` is a `list[str`] of wheel filenames, and `patch_strip` +is a number. + +So it will look something like this: +``` +"//path/to/package:my.patch": json.encode(struct( + whls = ["something-2.7.1-py3-none-any.whl"], + patch_strip = 1, +)), +``` +The patch is applied within the scope of the .whl file. +I.e. you should create the patch from the same place you unziped the wheel. + + +This is to maintain flexibility and correct bzlmod extension interface until we have a better +way to define whl_library and move whl patching to a separate place. INTERNAL USE ONLY.""", + ), +} | { + k: ATTRS[k] + for k in [ + # legacy parameters that are global to the entire hub. + "enable_implicit_namespace_pkgs", + "envsubst", + "pip_data_exclude", + ] +} | AUTH_ATTRS + +whl_archive = repository_rule( + attrs = whl_archive_attrs | { + # attributes only relevant to this rule and not reusable outside + "whl_file": attr.label( + doc = "The whl file that should be used instead of downloading or building the whl.", + ), + "_rule_name": attr.string(default = "whl_archive"), + }, + doc = """ +Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. + +Does not depend on any python. +""", + implementation = _whl_archive_impl, + environ = [ + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/whl_deps_repo.bzl b/python/private/pypi/whl_deps_repo.bzl new file mode 100644 index 0000000000..20891da397 --- /dev/null +++ b/python/private/pypi/whl_deps_repo.bzl @@ -0,0 +1,93 @@ +"" + +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":pep508_requirement.bzl", "requirement") + +# Reusable common attributes for generating BUILD.bazel from Requires-Dist in the wheel METADATA. +whl_deps_attrs = { + "config_load": attr.string( + doc = "The load location for configuration for pipstar.", + ), + "dep_template": attr.string( + doc = """ +The dep template to use for referencing the dependencies. It should have `{name}` +and `{target}` tokens that will be replaced with the normalized distribution name +and the target that we need respectively. + +For example if your whl depends on `numpy` and your Python package repo is named +`pip` so that you would normally do `@pip//numpy`, then this should be: `@pip//{name}`. +""", + ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), + "requirement": attr.string( + mandatory = True, + doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", + ), +} + +def _whl_deps_repo_impl(rctx): + logger = repo_utils.logger(rctx) + + if rctx.attr.metadata_file and rctx.attr.metadata: + logger.fail("Only one of 'metadata_file' and 'metadata' can be specified") + return + if not (rctx.attr.metadata_file or rctx.attr.metadata): + logger.fail("At least one of 'metadata_file' and 'metadata' must be specified") + return + + if rctx.attr.metadata_file: + metadata_contents = rctx.read(rctx.attr.metadata_file) + else: + metadata_contents = rctx.attr.metadata + + metadata = struct(**json.decode(metadata_contents)) + + build_file_contents = generate_whl_library_build_bazel( + dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( + rctx.attr.repo_prefix, + ), + config_load = rctx.attr.config_load, + metadata_name = metadata.name, + metadata_version = metadata.version, + requires_dist = metadata.requires_dist, + group_deps = rctx.attr.group_deps, + group_name = rctx.attr.group_name, + repo = rctx.attr.repo or ( + str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None + ), + extras = requirement(rctx.attr.requirement).extras, + ) + rctx.file("BUILD.bazel", build_file_contents) + +whl_deps_repo = repository_rule( + attrs = whl_deps_attrs | { + "metadata": attr.string( + doc = """ +The subset of the METADATA contents that is needed for generation of the dependencies. +* name: {type}`str` +* version: {type}`str` +* provides_extra: {type}`list[str]` +* requires_dist: {type}`list[str]` +""", + ), + "metadata_file": attr.label(doc = "An alternative way to pass {attr}`metadata` but as a file."), + "repo": attr.label(doc = "A label at the root of the repo to get stuff from."), + } | { + "_rule_name": attr.string(default = "whl_deps_repo"), + }, + doc = """ +A repo rule that reuses the sources from a different place and then creates the necessary targets +so that this can be used in the repo. + +Does not depend on any python. +""", + implementation = _whl_deps_repo_impl, + environ = [REPO_DEBUG_ENV_VAR], +) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 281dd863e6..87ec063e20 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,782 +14,8 @@ "" -load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") -load("//python/private:envsubst.bzl", "envsubst") -load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") -load(":attrs.bzl", "ATTRS", "use_isolated") -load(":deps.bzl", "all_repo_names", "record_files") -load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load(":patch_whl.bzl", "patch_whl") -load(":pep508_requirement.bzl", "requirement") -load(":pypi_repo_utils.bzl", "pypi_repo_utils") -load(":urllib.bzl", "urllib") -load(":whl_extract.bzl", "whl_extract") -load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") - -_CPPFLAGS = "CPPFLAGS" -_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" - -def _get_xcode_location_cflags(rctx, logger = None): - """Query the xcode sdk location to update cflags - - Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. - Pip won't be able to compile c extensions from sdists with the pre built python distributions from astral-sh - otherwise. See https://github.com/astral-sh/python-build-standalone/issues/103 - """ - - # Only run on MacOS hosts - if not rctx.os.name.lower().startswith("mac os"): - return [] - - xcode_sdk_location = repo_utils.execute_unchecked( - rctx, - op = "GetXcodeLocation", - arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"], - logger = logger, - ) - if xcode_sdk_location.return_code != 0: - return [] - - xcode_root = xcode_sdk_location.stdout.strip() - if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): - # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer - # so we need to change the path to to the macos specific tools which are in a different relative - # path than xcode installed command line tools. - xcode_sdks_json = repo_utils.execute_checked( - rctx, - op = "LocateXCodeSDKs", - arguments = [ - repo_utils.which_checked(rctx, "xcrun"), - "xcodebuild", - "-showsdks", - "-json", - ], - environment = { - "DEVELOPER_DIR": xcode_root, - }, - logger = logger, - ).stdout - xcode_sdks = json.decode(xcode_sdks_json) - potential_sdks = [ - sdk - for sdk in xcode_sdks - if "productName" in sdk and - sdk["productName"] == "macOS" and - "darwinos" not in sdk["canonicalName"] - ] - - # Now we'll get two entries here (one for internal and another one for public) - # It shouldn't matter which one we pick. - xcode_sdk_path = potential_sdks[0]["sdkPath"] - else: - xcode_sdk_path = "{}/SDKs/MacOSX.sdk".format(xcode_root) - - return [ - "-isysroot {}".format(xcode_sdk_path), - ] - -def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None): - """Gather cflags from a standalone toolchain for unix systems. - - Pip won't be able to compile c extensions from sdists with the pre built python distributions from astral-sh - otherwise. See https://github.com/astral-sh/python-build-standalone/issues/103 - """ - - # Only run on Unix systems - if not rctx.os.name.lower().startswith(("mac os", "linux")): - return [] - - # Only update the location when using a standalone toolchain. - if not is_standalone_interpreter(rctx, python_interpreter, logger = logger): - return [] - - stdout = pypi_repo_utils.execute_checked_stdout( - rctx, - op = "GetPythonVersionForUnixCflags", - # python_interpreter by default points to a symlink, however when using bazel in vendor mode, - # and the vendored directory moves around, the execution of python fails, as it's getting confused - # where it's running from. More to the fact that we are executing it in isolated mode "-I", which - # results in PYTHONHOME being ignored. The solution is to run python from it's real directory. - python = python_interpreter.realpath, - arguments = [ - # Run the interpreter in isolated mode, this options implies -E, -P and -s. - # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH, - # which may interfere with this invocation. - "-I", - "-c", - "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", - ], - srcs = [], - logger = logger, - ) - _python_version = stdout - include_path = "{}/include/python{}".format( - python_interpreter.dirname, - _python_version, - ) - - return ["-isystem {}".format(include_path)] - -def _parse_optional_attrs(rctx, args, extra_pip_args = None): - """Helper function to parse common attributes of pip_repository and whl_library repository rules. - - This function also serializes the structured arguments as JSON - so they can be passed on the command line to subprocesses. - - Args: - rctx: Handle to the rule repository context. - args: A list of parsed args for the rule. - extra_pip_args: The pip args to pass. - Returns: Augmented args list. - """ - - if use_isolated(rctx, rctx.attr): - args.append("--isolated") - - # Check for None so we use empty default types from our attrs. - # Some args want to be list, and some want to be dict. - if extra_pip_args != None: - args += [ - "--extra_pip_args", - json.encode(struct(arg = [ - envsubst(pip_arg, rctx.attr.envsubst, rctx.getenv) - for pip_arg in extra_pip_args - ])), - ] - - if rctx.attr.download_only: - args.append("--download_only") - - if rctx.attr.pip_data_exclude != None: - args += [ - "--pip_data_exclude", - json.encode(struct(arg = rctx.attr.pip_data_exclude)), - ] - - env = {} - if rctx.attr.environment != None: - for key, value in rctx.attr.environment.items(): - env[key] = value - - # This is super hacky, but working out something nice is tricky. - # This is in particular needed for psycopg2 which attempts to link libpython.a, - # in order to point the linker at the correct python intepreter. - if rctx.attr.add_libdir_to_library_search_path: - if "LDFLAGS" in env: - fail("Can't set both environment LDFLAGS and add_libdir_to_library_search_path") - command = [ - pypi_repo_utils.resolve_python_interpreter(rctx), - "-c", - "import sys ; sys.stdout.write('{}/lib'.format(sys.exec_prefix))", - ] - result = rctx.execute(command) - if result.return_code != 0: - fail("Failed to get LDFLAGS path: command: {}, exit code: {}, stdout: {}, stderr: {}".format(command, result.return_code, result.stdout, result.stderr)) - libdir = result.stdout - env["LDFLAGS"] = "-L{}".format(libdir) - - args += [ - "--environment", - json.encode(struct(arg = env)), - ] - - return args - -def _get_python_home(rctx, python_interpreter, logger = None): - """Get the PYTHONHOME directory from the selected python interpretter - - Args: - rctx (repository_ctx): The repository context. - python_interpreter (path): The resolved python interpreter. - logger: Optional logger to use for operations. - Returns: - String of PYTHONHOME directory. - """ - - return pypi_repo_utils.execute_checked_stdout( - rctx, - op = "GetPythonHome", - # python_interpreter by default points to a symlink, however when using bazel in vendor mode, - # and the vendored directory moves around, the execution of python fails, as it's getting confused - # where it's running from. More to the fact that we are executing it in isolated mode "-I", which - # results in PYTHONHOME being ignored. The solution is to run python from it's real directory. - python = python_interpreter.realpath, - arguments = [ - # Run the interpreter in isolated mode, this options implies -E, -P and -s. - # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH, - # which may interfere with this invocation. - "-I", - "-c", - "import sys; print(f'{sys.prefix}', end='')", - ], - srcs = [], - logger = logger, - ) - -def _create_repository_execution_environment(rctx, python_interpreter, logger = None): - """Create a environment dictionary for processes we spawn with rctx.execute. - - Args: - rctx (repository_ctx): The repository context. - python_interpreter (path): The resolved python interpreter. - logger: Optional logger to use for operations. - Returns: - Dictionary of environment variable suitable to pass to rctx.execute. - """ - - env = { - "PYTHONHOME": _get_python_home(rctx, python_interpreter, logger), - "PYTHONPATH": pypi_repo_utils.construct_pythonpath( - rctx, - entries = rctx.attr._python_path_entries, - ), - } - - # Gather any available CPPFLAGS values - # - # We may want to build in an environment without a cc toolchain. - # In those cases, we're limited to --download-only, but we should respect that here. - is_wheel = rctx.attr.filename and rctx.attr.filename.endswith(".whl") - if not (rctx.attr.download_only or is_wheel): - cppflags = [] - cppflags.extend(_get_xcode_location_cflags(rctx, logger = logger)) - cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter, logger = logger)) - env[_CPPFLAGS] = " ".join(cppflags) - return env - -def _get_entry_points(rctx, install_dir_path, metadata): - dist_info_dir = "{}-{}.dist-info".format( - metadata.name.replace("-", "_"), - metadata.version.replace("-", "_"), - ) - entry_points_txt = install_dir_path.get_child(dist_info_dir).get_child("entry_points.txt") - if entry_points_txt.exists: - return parse_entry_points(rctx.read(entry_points_txt)) - return {} - -def _move_scripts_needing_shebang_rewrite(rctx, entry_points): - bin_dir = rctx.path("bin") - if not bin_dir.exists: - return - - ep_names = {name.lower(): True for name in entry_points} - for script in bin_dir.readdir(): - if script.is_dir: - continue - if script.basename.lower() in ep_names: - rctx.delete(script) - continue - if script.basename.endswith(".exe") or script.basename.endswith(".dll"): - continue - content = rctx.read(script) - if content.startswith("#!python"): - rewrite_bin_dir = rctx.path("rewrite-bin") - repo_utils.mkdir(rctx, rewrite_bin_dir) - repo_utils.rename(rctx, script, rctx.path("rewrite-bin/" + script.basename)) - -def _to_purl(*, index, metadata, filename): - """ - Produce a PyPI PURL from the metadata. - - https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md - """ - - # https://github.com/package-url/purl-spec/blob/main/types-doc/pypi-definition.md#name-definition - name = normalize_name(metadata.name).replace("_", "-") - - qualifiers = {} - if index: - qualifiers["repository_url"] = index - if filename: - qualifiers["file_name"] = filename - - return "pkg:pypi/{}@{}?{}".format(name, metadata.version, "&".join(["{}={}".format(key, val) for key, val in qualifiers.items()])) - -def _whl_extract(rctx, *, whl_path, logger, sdist_filename = None): - """Extract the wheel, apply patches and generate BUILD.bazel files.""" - if rctx.attr.whl_patches: - patches = {} - for patch_file, json_args in rctx.attr.whl_patches.items(): - patch_dst = struct(**json.decode(json_args)) - if whl_path.basename in patch_dst.whls: - patches[patch_file] = patch_dst.patch_strip - - if patches: - whl_path = patch_whl( - rctx, - whl_path = whl_path, - patches = patches, - ) - - whl_extract(rctx, whl_path = whl_path, logger = logger) - - install_dir_path = whl_path.dirname.get_child("site-packages") - metadata = whl_metadata( - install_dir = install_dir_path, - read_fn = rctx.read, - logger = logger, - ) - rctx.file("metadata.json", json.encode_indent({ - "name": metadata.name, - "provides_extra": metadata.provides_extra, - "requires_dist": metadata.requires_dist, - "version": metadata.version, - })) - namespace_package_files = pypi_repo_utils.find_namespace_package_files(rctx, install_dir_path) - - entry_points = _get_entry_points(rctx, install_dir_path, metadata) - _move_scripts_needing_shebang_rewrite(rctx, entry_points) - - build_file_contents = generate_whl_library_build_bazel( - name = whl_path.basename, - dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( - rctx.attr.repo_prefix, - ), - sdist_filename = sdist_filename, - config_load = rctx.attr.config_load, - metadata_name = metadata.name, - metadata_version = metadata.version, - requires_dist = metadata.requires_dist, - # TODO @aignas 2025-05-17: maybe have a build flag for this instead - enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs, - # TODO @aignas 2025-04-14: load through the hub: - annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), - data_exclude = rctx.attr.pip_data_exclude, - group_deps = rctx.attr.group_deps, - group_name = rctx.attr.group_name, - namespace_package_files = namespace_package_files, - extras = requirement(rctx.attr.requirement).extras, - entry_points = entry_points, - purl = _to_purl( - index = rctx.attr.index_url, - metadata = metadata, - filename = sdist_filename or whl_path.basename, - ), - ) - - # Delete these in case the wheel had them. They generally don't cause - # a problem, but let's avoid the chance of that happening. - rctx.file("WORKSPACE") - rctx.file("WORKSPACE.bazel") - rctx.file("MODULE.bazel") - rctx.file("REPO.bazel", """\ -repo( - default_package_metadata = [ - "//:package_metadata", - ], -) -""") - - # BUILD files interfere with globbing and Bazel package boundaries. - _remove_files(rctx, "BUILD", "BUILD.bazel") - rctx.file("BUILD.bazel", build_file_contents) - - if hasattr(rctx, "repo_metadata"): - return rctx.repo_metadata(reproducible = True) - - return None - -def _whl_archive_impl(rctx): - logger = repo_utils.logger(rctx) - - whl_path = None - if rctx.attr.whl_file: - rctx.watch(rctx.attr.whl_file) - whl_path = rctx.path(rctx.attr.whl_file) - - # Simulate the behaviour where the whl is present in the current directory. - rctx.symlink(whl_path, whl_path.basename) - whl_path = rctx.path(whl_path.basename) - elif rctx.attr.urls and rctx.attr.filename: - filename = rctx.attr.filename - urls = rctx.attr.urls - urls = [ - urllib.absolute_url( - rctx.attr.index_url, - url, - envsubst = rctx.attr.envsubst, - getenv = rctx.getenv, - ) - for url in urls - ] - result = rctx.download( - url = urls, - output = filename, - sha256 = rctx.attr.sha256, - integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", - auth = get_auth(rctx, urls), - ) - if not rctx.attr.sha256 and not rctx.attr.integrity: - # this is only seen when there is a direct URL reference without a hash - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, - result.sha256, - )) - - if not result.success: - fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) - - if filename.endswith(".whl"): - whl_path = rctx.path(filename) - else: - fail("Only wheels are supported") - else: - fail("Either 'whl_file' or 'urls' and 'filename' needs to be specified") - - return _whl_extract(rctx, whl_path = whl_path, logger = logger) - -def _pip_archive_impl(rctx): - logger = repo_utils.logger(rctx) - - sdist_filename = None - extra_pip_args = [] - extra_pip_args.extend(rctx.attr.extra_pip_args) - if rctx.attr.urls and rctx.attr.filename: - filename = rctx.attr.filename - urls = rctx.attr.urls - urls = [ - urllib.absolute_url( - rctx.attr.index_url, - url, - envsubst = rctx.attr.envsubst, - getenv = rctx.getenv, - ) - for url in urls - ] - result = rctx.download( - url = urls, - output = filename, - sha256 = rctx.attr.sha256, - integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", - auth = get_auth(rctx, urls), - ) - if not rctx.attr.sha256 and not rctx.attr.integrity: - # this is only seen when there is a direct URL reference without a hash - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, - result.sha256, - )) - - if not result.success: - fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) - - if filename.endswith(".whl"): - fail("Only sdists are supported") - else: - sdist_filename = filename - - # It is an sdist and we need to tell PyPI to use a file in this directory - # and, allow getting build dependencies from PYTHONPATH, which we - # setup in this repository rule, but still download any necessary - # build deps from PyPI (e.g. `flit_core`) if they are missing. - extra_pip_args.extend(["--find-links", "."]) - - # When we already have a wheel, Python isn't used, - # so there's no need to setup env vars to run Python, unless we need to - # build an sdist or resolve a requirement. - python_interpreter = pypi_repo_utils.resolve_python_interpreter( - rctx, - python_interpreter = rctx.attr.python_interpreter, - python_interpreter_target = rctx.attr.python_interpreter_target, - ) - args = [ - "-m", - "python.private.pypi.whl_installer.wheel_installer", - "--requirement", - rctx.attr.requirement, - ] - args = _parse_optional_attrs(rctx, args, extra_pip_args) - - # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = _create_repository_execution_environment(rctx, python_interpreter, logger = logger) - - if rctx.attr.urls: - op_tmpl = "whl_library.BuildWheelFromSource({name}, {requirement})" - elif rctx.attr.download_only: - op_tmpl = "whl_library.DownloadWheel({name}, {requirement})" - else: - op_tmpl = "whl_library.ResolveRequirement({name}, {requirement})" - - pypi_repo_utils.execute_checked( - rctx, - # truncate the requirement value when logging it / reporting - # progress since it may contain several ' --hash=sha256:... - # --hash=sha256:...' substrings that fill up the console - python = python_interpreter, - op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]), - arguments = args, - environment = environment, - srcs = rctx.attr._python_srcs, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - logger = logger, - ) - - whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) - if not rctx.delete("whl_file.json"): - fail("failed to delete the whl_file.json file") - - # NOTE @aignas 2026-08-14: We never return rctx.metadata for pip archives because the result may - # not be reproducible across all machines given the input args to the repository rule. - _whl_extract(rctx, whl_path = whl_path, logger = logger, sdist_filename = sdist_filename) - -def _remove_files(rctx, *basenames): - paths = list(rctx.path(".").readdir()) - for _ in range(10000000): - if not paths: - break - path = paths.pop() - - if path.basename in basenames: - rctx.delete(path) - elif path.is_dir: - paths.extend(path.readdir()) - -# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique -_pip_archive_attrs = dict({ - "annotation": attr.label( - doc = ( - "Optional json encoded file containing annotation to apply to the extracted wheel. " + - "See `package_annotation`" - ), - allow_files = True, - ), - "config_load": attr.string( - doc = "The load location for configuration for pipstar.", - ), - "dep_template": attr.string( - doc = """ -The dep template to use for referencing the dependencies. It should have `{name}` -and `{target}` tokens that will be replaced with the normalized distribution name -and the target that we need respectively. - -For example if your whl depends on `numpy` and your Python package repo is named -`pip` so that you would normally do `@pip//numpy`, then this should be: `@pip//{name}`. -""", - ), - "filename": attr.string( - doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", - ), - "group_deps": attr.string_list( - doc = "List of dependencies to skip in order to break the cycles within a dependency group.", - default = [], - ), - "group_name": attr.string( - doc = "Name of the group, if any.", - ), - "index_url": attr.string( - doc = "The index_url that the package will be downloaded from.", - ), - "integrity": attr.string( - doc = """\ -The expected checksum of the downloaded whl in Subresource Integrity format -(e.g. `sha256-...` or `sha512-...`). Only used when `urls` is passed. If -`sha256` is also set, it takes precedence over this attribute. - -:::{versionadded} 2.3.0 -::: -""", - ), - "repo": attr.string( - doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", - ), - "repo_prefix": attr.string( - doc = """ -Prefix for the generated packages will be of the form `@//...` - -DEPRECATED. Only left for people who vendor requirements.bzl. -""", - ), - "requirement": attr.string( - mandatory = True, - doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", - ), - "sha256": attr.string( - doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.", - ), - "urls": attr.string_list( - doc = """\ -The list of urls of the whl to be downloaded using bazel downloader. Using this -attr makes `extra_pip_args` and `download_only` ignored.""", - ), - "whl_patches": attr.label_keyed_string_dict( - doc = """ -A label-keyed-string dict with patch files as keys and json-strings as values. - -The keys are labels to the patch file to apply. - -The values describe what to apply the patch to and how to apply it. -It is encoded as `json.encode(struct([whls], patch_strip])`, -where `whls` is a `list[str`] of wheel filenames, and `patch_strip` -is a number. - -So it will look something like this: -``` -"//path/to/package:my.patch": json.encode(struct( - whls = ["something-2.7.1-py3-none-any.whl"], - patch_strip = 1, -)), -``` -The patch is applied within the scope of the .whl file. -I.e. you should create the patch from the same place you unziped the wheel. - - -This is to maintain flexibility and correct bzlmod extension interface until we have a better -way to define whl_library and move whl patching to a separate place. INTERNAL USE ONLY.""", - ), - "_python_path_entries": attr.label_list( - # Get the root directory of these rules and keep them as a default attribute - # in order to avoid unnecessary repository fetching restarts. - # - # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 - default = [ - Label("//:BUILD.bazel"), - ] + [ - # Includes all the external dependencies from repositories.bzl - Label("@" + repo + "//:BUILD.bazel") - for repo in all_repo_names - ], - ), - "_python_srcs": attr.label_list( - # Used as a default value in a rule to ensure we fetch the dependencies. - default = [ - Label("//python/private/pypi/whl_installer:wheel_installer.py"), - Label("//python/private/pypi/whl_installer:arguments.py"), - ] + record_files.values(), - ), - "_rule_name": attr.string(default = "whl_library"), -}, **ATTRS) -_pip_archive_attrs.update(AUTH_ATTRS) - -pip_archive = repository_rule( - attrs = _pip_archive_attrs, - doc = """ -Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. -Instantiated from pip_repository and inherits config options from there. - -:::{versionchanged} 1.9.0 -The `whl_library` is marked as reproducible if using starlark to extract and parse the -wheel contents without building an `sdist` first. -::: - -:::{versionchanged} 2.3.0 -The whl-only pure Starlark operations have been refactored into {obj}`whl_archive` and the -previously named {obj}`whl_library` repository became renamed to `pip_archive`. -::: -""", - implementation = _pip_archive_impl, - environ = [ - "RULES_PYTHON_PIP_ISOLATED", - REPO_DEBUG_ENV_VAR, - ], -) - -whl_archive = repository_rule( - attrs = { - k: _pip_archive_attrs[k] - for k in [ - "annotation", - "config_load", - "dep_template", - "filename", - "group_deps", - "group_name", - "index_url", - "integrity", - "repo_prefix", - "requirement", - "sha256", - "urls", - "whl_patches", - "enable_implicit_namespace_pkgs", - "envsubst", - "pip_data_exclude", - ] - } | { - "whl_file": attr.label( - doc = "The whl file that should be used instead of downloading or building the whl.", - ), - } | AUTH_ATTRS, - doc = """ -Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. - -Does not depend on any python. -""", - implementation = _whl_archive_impl, - environ = [ - REPO_DEBUG_ENV_VAR, - ], -) - -def _whl_deps_library_impl(rctx): - logger = repo_utils.logger(rctx) - - if rctx.attr.metadata_file and rctx.attr.metadata: - logger.fail("Only one of 'metadata_file' and 'metadata' can be specified") - return - if not (rctx.attr.metadata_file or rctx.attr.metadata): - logger.fail("At least one of 'metadata_file' and 'metadata' must be specified") - return - - if rctx.attr.metadata_file: - metadata_contents = rctx.read(rctx.attr.metadata_file) - else: - metadata_contents = rctx.attr.metadata - - metadata = struct(**json.decode(metadata_contents)) - - build_file_contents = generate_whl_library_build_bazel( - dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( - rctx.attr.repo_prefix, - ), - config_load = rctx.attr.config_load, - metadata_name = metadata.name, - metadata_version = metadata.version, - requires_dist = metadata.requires_dist, - group_deps = rctx.attr.group_deps, - group_name = rctx.attr.group_name, - repo = rctx.attr.repo or ( - str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None - ), - extras = requirement(rctx.attr.requirement).extras, - ) - rctx.file("BUILD.bazel", build_file_contents) - -whl_deps_library = repository_rule( - attrs = { - k: _pip_archive_attrs[k] - for k in [ - "config_load", - "dep_template", - "group_deps", - "group_name", - "requirement", - ] - } | { - "metadata": attr.string( - doc = """ -The subset of the METADATA contents that is needed for generation of the dependencies. -* name: {type}`str` -* version: {type}`str` -* provides_extra: {type}`list[str]` -* requires_dist: {type}`list[str]` -""", - ), - "metadata_file": attr.label(doc = "An alternative way to pass {attr}`metadata` but as a file."), - "repo": attr.label(doc = "A label at the root of the repo to get stuff from."), - }, - doc = """ -A repo rule that reuses the sources from a different place and then creates the necessary targets -so that this can be used in the repo. - -Does not depend on any python. -""", - implementation = _whl_deps_library_impl, - environ = [REPO_DEBUG_ENV_VAR], -) +load(":pip_archive.bzl", "pip_archive") +load(":whl_archive.bzl", "whl_archive") def whl_library(name, repo = None, **kwargs): """Create a whl_library. @@ -811,6 +37,10 @@ def whl_library(name, repo = None, **kwargs): whl_file = kwargs.get("whl_file") urls = kwargs.get("urls", []) filename = kwargs.get("filename") + + # compatibility shim for cases for repo_prefix is still used by the called + kwargs.setdefault("dep_template", "@{}{{name}}//:{{target}}".format(kwargs.pop("repo_prefix", ""))) + if whl_file or (urls and filename and filename.endswith(".whl")): whl_archive(name = name, **kwargs) else: diff --git a/tests/integration/whl_library/BUILD.bazel b/tests/integration/whl_library/BUILD.bazel index 0c26fee4cf..a53bcc7258 100644 --- a/tests/integration/whl_library/BUILD.bazel +++ b/tests/integration/whl_library/BUILD.bazel @@ -37,8 +37,8 @@ genquery( # Extract all deps genquery( name = "whl_deps_target_deps", - expression = "deps(@whl_deps_library//:pkg)", - scope = ["@whl_deps_library//:pkg"], + expression = "deps(@whl_deps_repo//:pkg)", + scope = ["@whl_deps_repo//:pkg"], ) py_test( @@ -51,13 +51,13 @@ py_test( "@pip_sdist_archive//:srcs", "@whl_archive//:srcs", "@whl_archive//:whl", - "@whl_deps_library//:whl", + "@whl_deps_repo//:whl", ], env = { "SDIST_SRC_FILES": "$(locations @pip_sdist_archive//:srcs)", "SRC_FILES": "$(locations @pip_archive//:srcs)", "WHL_DEPS": "$(location :whl_target_deps)", - "WHL_DEPS_LOCATION": "$(location @whl_deps_library//:whl)", + "WHL_DEPS_LOCATION": "$(location @whl_deps_repo//:whl)", "WHL_FILES": "$(locations @whl_archive//:srcs)", "WHL_LOCATION": "$(location @whl_archive//:whl)", "WHL_TARGET_DEPS": "$(location :whl_deps_target_deps)", diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index 39a579f79e..d3886d7650 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel @@ -13,10 +13,13 @@ python.toolchain( ) use_repo(python, pbs_host = "python_3_14_host") -pip_archive = use_repo_rule("@rules_python//python/private/pypi:whl_library.bzl", "pip_archive") +pip_archive = use_repo_rule("@rules_python//python/private/pypi:pip_archive.bzl", "pip_archive") + +DEP_TEMPLATE = "@integration_test//:{name}_{target}" pip_archive( name = "pip_sdist_archive", + dep_template = DEP_TEMPLATE, filename = "requests-2.34.2.tar.gz", python_interpreter_target = "@pbs_host//:python", requirement = "requests", @@ -29,15 +32,16 @@ pip_archive( pip_archive( name = "pip_archive", + dep_template = DEP_TEMPLATE, python_interpreter_target = "@pbs_host//:python", requirement = "requests", ) -whl_archive = use_repo_rule("@rules_python//python/private/pypi:whl_library.bzl", "whl_archive") +whl_archive = use_repo_rule("@rules_python//python/private/pypi:whl_archive.bzl", "whl_archive") whl_archive( name = "whl_archive", - dep_template = "@integration_test//:{name}_{target}", + dep_template = DEP_TEMPLATE, filename = "requests-2.34.2-py3-none-any.whl", # https://pypi.org/project/requests/#requests-2.34.2-py3-none-any.whl requirement = "requests", @@ -47,11 +51,11 @@ whl_archive( ], ) -whl_deps_library = use_repo_rule("@rules_python//python/private/pypi:whl_library.bzl", "whl_deps_library") +whl_deps_repo = use_repo_rule("@rules_python//python/private/pypi:whl_deps_repo.bzl", "whl_deps_repo") -whl_deps_library( - name = "whl_deps_library", - dep_template = "@integration_test//:{name}_{target}", +whl_deps_repo( + name = "whl_deps_repo", + dep_template = DEP_TEMPLATE, metadata_file = "@whl_archive//:metadata.json", requirement = "requests", ) diff --git a/tests/integration/whl_library/test_contents.py b/tests/integration/whl_library/test_contents.py index ab352a5d98..6db7245d24 100644 --- a/tests/integration/whl_library/test_contents.py +++ b/tests/integration/whl_library/test_contents.py @@ -120,7 +120,7 @@ def _normalize_label(label: str) -> str: def test_whl_deps_ar_the_same(self): for var, main_dep in { "WHL_DEPS": "@whl_archive//:pkg", - "WHL_TARGET_DEPS": "@whl_deps_library//:pkg", + "WHL_TARGET_DEPS": "@whl_deps_repo//:pkg", }.items(): self.assertEqual( {