From 149a84052b66ea0f01de29fe2330760648c5a23b Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 14 Dec 2025 18:38:57 -0800 Subject: [PATCH 01/10] basic impl --- CHANGELOG.md | 4 + docs/BUILD.bazel | 3 + python/BUILD.bazel | 1 + python/private/BUILD.bazel | 20 +- python/private/common.bzl | 119 +++++++++ python/private/py_executable.bzl | 122 ++++++--- python/private/py_executable_info.bzl | 33 +++ python/private/py_interpreter_program.bzl | 22 +- python/private/py_runtime_rule.bzl | 2 +- python/private/stage2_bootstrap_template.py | 2 +- python/private/zipapp/BUILD.bazel | 48 ++++ python/private/zipapp/py_zipapp_rule.bzl | 251 ++++++++++++++++++ .../private/{ => zipapp}/zip_main_template.py | 12 +- python/private/zipapp/zip_shell_template.sh | 89 +++++++ .../zipapp_stage2_bootstrap_template.py | 10 + python/zipapp/BUILD.bazel | 29 ++ python/zipapp/py_zipapp_binary.bzl | 15 ++ python/zipapp/py_zipapp_test.bzl | 15 ++ tests/py_zipapp/BUILD.bazel | 28 ++ tests/py_zipapp/main.py | 12 + tests/py_zipapp/zipapp_test.py | 68 +++++ tests/tools/zipapp/BUILD.bazel | 13 + tests/tools/zipapp/exe_zip_maker_test.py | 63 +++++ tests/tools/zipapp/zipper_test.py | 220 +++++++++++++++ tools/zipapp/BUILD.bazel | 36 +++ tools/zipapp/exe_zip_maker.py | 40 +++ tools/zipapp/zipper.py | 220 +++++++++++++++ 27 files changed, 1432 insertions(+), 65 deletions(-) create mode 100644 python/private/zipapp/BUILD.bazel create mode 100644 python/private/zipapp/py_zipapp_rule.bzl rename python/private/{ => zipapp}/zip_main_template.py (96%) create mode 100644 python/private/zipapp/zip_shell_template.sh create mode 100644 python/private/zipapp/zipapp_stage2_bootstrap_template.py create mode 100644 python/zipapp/BUILD.bazel create mode 100644 python/zipapp/py_zipapp_binary.bzl create mode 100644 python/zipapp/py_zipapp_test.bzl create mode 100644 tests/py_zipapp/BUILD.bazel create mode 100644 tests/py_zipapp/main.py create mode 100644 tests/py_zipapp/zipapp_test.py create mode 100644 tests/tools/zipapp/BUILD.bazel create mode 100644 tests/tools/zipapp/exe_zip_maker_test.py create mode 100644 tests/tools/zipapp/zipper_test.py create mode 100644 tools/zipapp/BUILD.bazel create mode 100644 tools/zipapp/exe_zip_maker.py create mode 100644 tools/zipapp/zipper.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c9f812f3bd..2180418273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,10 @@ END_UNRELEASED_TEMPLATE * (binaries/tests) Build information is now included in binaries and tests. Use the `bazel_binary_info` module to access it. The {flag}`--stamp` flag will add {flag}`--workspace_status` information. +* (zipapp) {obj}`py_zipapp_binary` and {obj}`py_zipapp_test` rules added. These + will replace `--build_python_zip` and the zip output group of + `py_binary/py_test`. The zipapp rules support more functionality, correctness, + and have better build performance. {#v1-8-0} ## [1.8.0] - 2025-12-19 diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 55d8f75a74..632c3dd613 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -125,10 +125,13 @@ sphinx_stardocs( "//python/private/pypi:pkg_aliases_bzl", "//python/private/pypi:whl_config_setting_bzl", "//python/private/pypi:whl_library_bzl", + "//python/private/zipapp:py_zipapp_rule_bzl", "//python/uv:lock_bzl", "//python/uv:uv_bzl", "//python/uv:uv_toolchain_bzl", "//python/uv:uv_toolchain_info_bzl", + "//python/zipapp:py_zipapp_binary_bzl", + "//python/zipapp:py_zipapp_test_bzl", ] + ([ # This depends on @pythons_hub, which is only created under bzlmod, "//python/extensions:pip_bzl", diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 5c4626ee55..6577ff805f 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -47,6 +47,7 @@ filegroup( "//python/runfiles:distribution", "//python/runtime_env_toolchains:distribution", "//python/uv:distribution", + "//python/zipapp:distribution", ], visibility = ["//:__pkg__"], ) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index efc1dd7319..2b3d6bcfc0 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -38,6 +38,7 @@ filegroup( "//python/private/cc:distribution", "//python/private/pypi:distribution", "//python/private/whl_filegroup:distribution", + "//python/private/zipapp:distribution", "//tools/build_defs/python/private:distribution", ], visibility = ["//python:__pkg__"], @@ -366,7 +367,7 @@ bzl_library( name = "py_cc_toolchain_rule_bzl", srcs = ["py_cc_toolchain_rule.bzl"], deps = [ - ":common_labels.bzl", + ":common_labels_bzl", ":py_cc_toolchain_info_bzl", ":rules_cc_srcs_bzl", ":sentinel_bzl", @@ -460,7 +461,10 @@ bzl_library( bzl_library( name = "py_interpreter_program_bzl", srcs = ["py_interpreter_program.bzl"], - deps = ["@bazel_skylib//rules:common_settings"], + deps = [ + ":sentinel_bzl", + "@bazel_skylib//rules:common_settings", + ], ) bzl_library( @@ -740,8 +744,8 @@ bzl_library( srcs = ["venv_runfiles.bzl"], deps = [ ":common_bzl", - ":py_info.bzl", - ":py_internal.bzl", + ":py_info_bzl", + ":py_internal_bzl", "@bazel_skylib//lib:paths", ], ) @@ -786,14 +790,6 @@ filegroup( visibility = ["//visibility:public"], ) -filegroup( - name = "zip_main_template", - srcs = ["zip_main_template.py"], - # Not actually public. Only public because it's an implicit dependency of - # py_runtime. - visibility = ["//visibility:public"], -) - filegroup( name = "site_init_template", srcs = ["site_init_template.py"], diff --git a/python/private/common.bzl b/python/private/common.bzl index 2d4afca3f5..8ddb81db3a 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -16,6 +16,8 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("//python/private:py_interpreter_program.bzl", "PyInterpreterProgramInfo") +load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE") load(":cc_helper.bzl", "cc_helper") load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load(":py_info.bzl", "PyInfo", "PyInfoBuilder") @@ -484,3 +486,120 @@ def collect_deps(ctx, extra_deps = []): deps = list(deps) deps.extend(extra_deps) return deps + +def maybe_create_repo_mapping(ctx, *, runfiles): + """Creates a repo mapping manifest if bzlmod is enabled. + + There isn't a way to reference the repo mapping Bazel implicitly + creates, so we have to manually create it ourselves. + + Args: + ctx: rule ctx. + runfiles: runfiles object to generate mapping for. + + Returns: + File object if the repo mapping manifest was created, None otherwise. + """ + if not py_internal.is_bzlmod_enabled(ctx): + return None + + # We have to add `.custom` because `{name}.repo_mapping` is used by Bazel + # internally. + repo_mapping_manifest = ctx.actions.declare_file(ctx.label.name + ".custom.repo_mapping") + py_internal.create_repo_mapping_manifest( + ctx = ctx, + runfiles = runfiles, + output = repo_mapping_manifest, + ) + return repo_mapping_manifest + +def actions_run( + ctx, + *, + executable, + toolchain = None, + **kwargs): + """Runs a tool as an action, supporting py_interpreter_program targets. + + This is wrapper around `ctx.actions.run()` that sets some useful defaults, + supports handling `py_interpreter_program` targets, and some other features + to let the target being run influence the action invocation. + + Args: + ctx: The rule context. The rule must have the + `//python:exec_tools_toolchain_type` toolchain available. + executable: The executable to run. This can be a target that provides + `PyInterpreterProgramInfo` or a regular executable target. If it + provides `testing.ExecutionInfo`, the requirements will be added to + the execution requirements. + toolchain: The toolchain type to use. Must be None or + `//python:exec_tools_toolchain_type`. + **kwargs: Additional arguments to pass to `ctx.actions.run()`. + `mnemonic` and `progress_message` are required. + """ + mnemonic = kwargs.pop("mnemonic", None) + if not mnemonic: + fail("actions_run: missing required argument 'mnemonic'") + + progress_message = kwargs.pop("progress_message", None) + if not progress_message: + fail("actions_run: missing required argument 'progress_message'") + + tools = kwargs.pop("tools", None) + tools = list(tools) if tools else [] + arguments = kwargs.pop("arguments", []) + + action_arguments = [] + action_env = { + "PYTHONHASHSEED": "0", # Helps avoid non-deterministic behavior + "PYTHONNOUSERSITE": "1", # Helps avoid non-deterministic behavior + "PYTHONSAFEPATH": "1", # Helps avoid incorrect import issues + } + default_info = executable[DefaultInfo] + if PyInterpreterProgramInfo in executable: + if toolchain and toolchain != EXEC_TOOLS_TOOLCHAIN_TYPE: + fail(("Action {}: tool {} provides PyInterpreterProgramInfo, which " + + "requires the `toolchain` arg be " + + "None or {}, got: {}").format( + mnemonic, + executable, + EXEC_TOOLS_TOOLCHAIN_TYPE, + toolchain, + )) + exec_tools = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE].exec_tools + action_exe = exec_tools.exec_interpreter[DefaultInfo].files_to_run + + program_info = executable[PyInterpreterProgramInfo] + + interpreter_args = ctx.actions.args() + interpreter_args.add_all(program_info.interpreter_args) + interpreter_args.add(default_info.files_to_run.executable) + action_arguments.append(interpreter_args) + + action_env.update(program_info.env) + tools.append(default_info.files_to_run) + toolchain = EXEC_TOOLS_TOOLCHAIN_TYPE + else: + action_exe = executable[DefaultInfo].files_to_run + + execution_requirements = {} + if testing.ExecutionInfo in executable: + execution_requirements.update(executable[testing.ExecutionInfo].requirements) + + # Give precedence to caller's execution requirements. + execution_requirements.update(kwargs.pop("execution_requirements", None) or {}) + + # Give precedence to caller's env. + action_env.update(kwargs.pop("env", None) or {}) + action_arguments.extend(arguments) + ctx.actions.run( + executable = action_exe, + arguments = action_arguments, + tools = tools, + env = action_env, + execution_requirements = execution_requirements, + toolchain = toolchain, + mnemonic = mnemonic, + progress_message = progress_message, + **kwargs + ) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 5bac6247ef..50e8ba5449 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -42,13 +42,13 @@ load( "collect_runfiles", "create_binary_semantics_struct", "create_cc_details_struct", - "create_executable_result_struct", "create_instrumented_files_info", "create_output_group_info", "create_py_info", "csv", "filter_to_py_srcs", "is_bool", + "maybe_create_repo_mapping", "relative_path", "runfiles_root_path", "target_platform_has_any_constraint", @@ -455,14 +455,31 @@ def _create_executable( build_zip_enabled = build_zip_enabled, )) + app_runfiles = builders.RunfilesBuilder() + app_runfiles.add(runfiles_details.app_runfiles) + app_runfiles.add(venv.files_without_interpreter) + app_runfiles.add(venv.lib_runfiles) + # The interpreter is added this late in the process so that it isn't # added to the zipped files. if venv and venv.interpreter: extra_runfiles = extra_runfiles.merge(ctx.runfiles([venv.interpreter])) - return create_executable_result_struct( + return struct( + # depset[File] of additional files that should be included as default + # outputs. extra_files_to_build = depset(extra_files_to_build), + # dict[str, depset[File]]; additional output groups that should be + # returned. output_groups = {"python_zip_file": depset([zip_file])}, + # runfiles; additional runfiles to include. extra_runfiles = extra_runfiles, + # File|None; the stage2 bootstrap file, if any + stage2_bootstrap = stage2_bootstrap, + # runfiles; runfiles for the app itself (e.g its deps, but no Python + # runtime files) + app_runfiles = app_runfiles.build(ctx), + # File|None; the venv `bin/python3` file, if any. + venv_python_exe = venv.interpreter, ) def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): @@ -858,16 +875,11 @@ def _create_zip_file(ctx, *, output, zip_main, runfiles): manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) inputs = [zip_main] - if _py_builtins.is_bzlmod_enabled(ctx): - zip_repo_mapping_manifest = ctx.actions.declare_file( - output.basename + ".repo_mapping", - sibling = output, - ) - _py_builtins.create_repo_mapping_manifest( - ctx = ctx, - runfiles = runfiles, - output = zip_repo_mapping_manifest, - ) + zip_repo_mapping_manifest = maybe_create_repo_mapping( + ctx = ctx, + runfiles = runfiles, + ) + if zip_repo_mapping_manifest: manifest.add("{}/_repo_mapping={}".format( _ZIP_RUNFILES_DIRECTORY_NAME, zip_repo_mapping_manifest.path, @@ -1060,8 +1072,8 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = required_pyc_files = required_pyc_files, implicit_pyc_files = implicit_pyc_files, implicit_pyc_source_files = implicit_pyc_source_files, + runtime_runfiles = runtime_details.runfiles, extra_common_runfiles = [ - runtime_details.runfiles, cc_details.extra_runfiles, native_deps_details.runfiles, ], @@ -1092,6 +1104,8 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = ) )) + app_runfiles = exec_result.app_runfiles + return _create_providers( ctx = ctx, executable = executable, @@ -1108,6 +1122,10 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = cc_info = cc_details.cc_info_for_propagating, inherited_environment = inherited_environment, output_groups = exec_result.output_groups, + stage2_bootstrap = exec_result.stage2_bootstrap, + app_runfiles = app_runfiles, + venv_python_exe = exec_result.venv_python_exe, + interpreter_args = ctx.attr.interpreter_args, ) def _get_build_info(ctx, cc_toolchain): @@ -1253,6 +1271,7 @@ def _get_base_runfiles_for_binary( required_pyc_files, implicit_pyc_files, implicit_pyc_source_files, + runtime_runfiles, extra_common_runfiles): """Returns the set of runfiles necessary prior to executable creation. @@ -1273,6 +1292,7 @@ def _get_base_runfiles_for_binary( collection is enabled. implicit_pyc_source_files: `depset[File]` source files for implicit pyc files that are used when the implicit pyc files are not. + runtime_runfiles: runfiles for the python runtime. extra_common_runfiles: List of runfiles; additional runfiles that will be added to the common runfiles. @@ -1282,60 +1302,70 @@ def _get_base_runfiles_for_binary( * data_runfiles: The data runfiles * runfiles_without_exe: The default runfiles, but without the executable or files specific to the original program/executable. - * build_data_file: A file with build stamp information if stamping is enabled, otherwise - None. + * build_data_file: A file with build stamp information if stamping is + enabled, otherwise None. + * app_runfiles: Runfiles for user-space dependencies (doesn't + include the runtime or build data files) """ - common_runfiles = builders.RunfilesBuilder() - common_runfiles.files.add(required_py_files) - common_runfiles.files.add(required_pyc_files) + app_runfiles = builders.RunfilesBuilder() + app_runfiles.files.add(required_py_files) + app_runfiles.files.add(required_pyc_files) pyc_collection_enabled = PycCollectionAttr.is_pyc_collection_enabled(ctx) if pyc_collection_enabled: - common_runfiles.files.add(implicit_pyc_files) + app_runfiles.files.add(implicit_pyc_files) else: - common_runfiles.files.add(implicit_pyc_source_files) + app_runfiles.files.add(implicit_pyc_source_files) for dep in (ctx.attr.deps + extra_deps): if not (PyInfo in dep or (BuiltinPyInfo != None and BuiltinPyInfo in dep)): continue info = dep[PyInfo] if PyInfo in dep else dep[BuiltinPyInfo] - common_runfiles.files.add(info.transitive_sources) + app_runfiles.files.add(info.transitive_sources) # Everything past this won't work with BuiltinPyInfo if not hasattr(info, "transitive_pyc_files"): continue - common_runfiles.files.add(info.transitive_pyc_files) + app_runfiles.files.add(info.transitive_pyc_files) if pyc_collection_enabled: - common_runfiles.files.add(info.transitive_implicit_pyc_files) + app_runfiles.files.add(info.transitive_implicit_pyc_files) else: - common_runfiles.files.add(info.transitive_implicit_pyc_source_files) + app_runfiles.files.add(info.transitive_implicit_pyc_source_files) - common_runfiles.runfiles.append(collect_runfiles(ctx)) + app_runfiles.runfiles.append(collect_runfiles(ctx)) if extra_deps: - common_runfiles.add_targets(extra_deps) - common_runfiles.add(extra_common_runfiles) - - build_data_file = _write_build_data(ctx) - common_runfiles.add(build_data_file) + app_runfiles.add_targets(extra_deps) + app_runfiles.add(extra_common_runfiles) - common_runfiles = common_runfiles.build(ctx) + app_runfiles = app_runfiles.build(ctx) if _should_create_init_files(ctx): - common_runfiles = _py_builtins.merge_runfiles_with_generated_inits_empty_files_supplier( + app_runfiles = _py_builtins.merge_runfiles_with_generated_inits_empty_files_supplier( ctx = ctx, - runfiles = common_runfiles, + runfiles = app_runfiles, ) - runfiles_with_exe = common_runfiles.merge(ctx.runfiles([executable])) + runfiles_without_exe = builders.RunfilesBuilder() + runfiles_without_exe.add(app_runfiles) + runfiles_without_exe.add(runtime_runfiles) + build_data_file = _write_build_data(ctx) + runfiles_without_exe.add(build_data_file) - data_runfiles = runfiles_with_exe - default_runfiles = runfiles_with_exe + runfiles_without_exe = runfiles_without_exe.build(ctx) + runfiles_with_exe = runfiles_without_exe.merge(ctx.runfiles([executable])) + + # There are three types of runfiles: + # 1. app: Deps added by a user. This is akin to the typical files that would + # be in a traditional venv. No Python runtime files or build data files. + # 2. without-exe: (1) + build data + python runtime + # 3. binary (default/data runfiles): (2) + main executable return struct( - runfiles_without_exe = common_runfiles, - default_runfiles = default_runfiles, + app_runfiles = app_runfiles, build_data_file = build_data_file, - data_runfiles = data_runfiles, + data_runfiles = runfiles_with_exe, + default_runfiles = runfiles_with_exe, + runfiles_without_exe = runfiles_without_exe, ) def _write_build_data(ctx): @@ -1645,7 +1675,11 @@ def _create_providers( cc_info, inherited_environment, runtime_details, - output_groups): + output_groups, + stage2_bootstrap, + app_runfiles, + venv_python_exe, + interpreter_args): """Creates the providers an executable should return. Args: @@ -1674,6 +1708,10 @@ def _create_providers( is run within. runtime_details: struct of runtime information; see _get_runtime_details() output_groups: dict[str, depset[File]]; used to create OutputGroupInfo + stage2_bootstrap: File; the stage 2 bootstrap script. + app_runfiles: runfiles; the runfiles for the application (deps, etc). + venv_python_exe: File; the python executable in the venv. + interpreter_args: list of strings; arguments to pass to the interpreter. Returns: A list of modern providers. @@ -1698,6 +1736,10 @@ def _create_providers( runfiles_without_exe = runfiles_details.runfiles_without_exe, build_data_file = runfiles_details.build_data_file, interpreter_path = runtime_details.executable_interpreter_path, + stage2_bootstrap = stage2_bootstrap, + app_runfiles = app_runfiles, + venv_python_exe = venv_python_exe, + interpreter_args = interpreter_args, ), ] diff --git a/python/private/py_executable_info.bzl b/python/private/py_executable_info.bzl index deb119428d..03dd57390d 100644 --- a/python/private/py_executable_info.bzl +++ b/python/private/py_executable_info.bzl @@ -10,10 +10,24 @@ This provider is for executable-specific information (e.g. tests and binaries). ::: """, fields = { + "app_runfiles": """ +:type: runfiles + +The runfiles for the executable's "user" dependencies. These are things in e.g. +`deps` (or similar), but doesn't include "external" or "implicit" pieces, +e.g. the Python runtime itself. It's roughly akin to the files a traditional +venv would have installed into it. +""", "build_data_file": """ :type: None | File A symlink to build_data.txt if stamping is enabled, otherwise None. +""", + "interpreter_args": """ +:type: list[str] + +Args that should be passed to the interpreter before regular args +(e.g. `-X whatever`). """, "interpreter_path": """ :type: None | str @@ -28,6 +42,12 @@ should be within `runtime_files`) The user-level entry point file. Usually a `.py` file, but may also be `.pyc` file if precompiling is enabled. + +:::{seealso} + +The {obj}`stage2_bootstrap` attribute, which bootstraps an executable to run +the user main file. +::: """, "runfiles_without_exe": """ :type: runfiles @@ -35,6 +55,19 @@ file if precompiling is enabled. The runfiles the program needs, but without the original executable, files only added to support the original executable, or files specific to the original program. +""", + "stage2_bootstrap": """ +:type: File | None + +The Bazel-executable-level entry point to the program, which handles Bazel-specific +setup before running the file in {obj}`main`. May be None if a two-stage bootstrap +implementation isn't being used. +""", + "venv_python_exe": """ +:type: File | None + +The `bin/python3` file within the venv this binary uses. May be None if venv +mode is not enabled. """, }, ) diff --git a/python/private/py_interpreter_program.bzl b/python/private/py_interpreter_program.bzl index cd62a7190d..7eb3e28bd9 100644 --- a/python/private/py_interpreter_program.bzl +++ b/python/private/py_interpreter_program.bzl @@ -15,6 +15,7 @@ """Internal only bootstrap level binary-like rule.""" load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python/private:sentinel.bzl", "SentinelInfo") PyInterpreterProgramInfo = provider( doc = "Information about how to run a program with an external interpreter.", @@ -28,14 +29,20 @@ PyInterpreterProgramInfo = provider( def _py_interpreter_program_impl(ctx): # Bazel requires the executable file to be an output created by this target. - executable = ctx.actions.declare_file(ctx.label.name) + # To avoid colliding with the source file (e.g. target=foo, main=foo.py), + # we append an underscore to the name, but keep the extension so that + # the original extension is preserved. + extension = ctx.file.main.extension + executable_name = "{}_.{}".format(ctx.label.name, extension) + executable = ctx.actions.declare_file(executable_name) ctx.actions.symlink(output = executable, target_file = ctx.file.main) execution_requirements = {} - execution_requirements.update([ - value.split("=", 1) - for value in ctx.attr.execution_requirements[BuildSettingInfo].value - if value.strip() - ]) + if BuildSettingInfo in ctx.attr.execution_requirements: + execution_requirements.update([ + value.split("=", 1) + for value in ctx.attr.execution_requirements[BuildSettingInfo].value + if value.strip() + ]) return [ DefaultInfo( @@ -85,8 +92,9 @@ ctx.actions.run( doc = "Environment variables that should set prior to running.", ), "execution_requirements": attr.label( + default = "//python:none", doc = "Execution requirements to set when running it as an action", - providers = [BuildSettingInfo], + providers = [[BuildSettingInfo], [SentinelInfo]], ), "interpreter_args": attr.string_list( doc = "Args that should be passed to the interpreter.", diff --git a/python/private/py_runtime_rule.bzl b/python/private/py_runtime_rule.bzl index 3bcee4cfd7..09e245a58e 100644 --- a/python/private/py_runtime_rule.bzl +++ b/python/private/py_runtime_rule.bzl @@ -361,7 +361,7 @@ See {obj}`PyRuntimeInfo.supports_build_time_venv` for docs. default = True, ), "zip_main_template": attr.label( - default = "//python/private:zip_main_template", + default = "//python/private/zipapp:zip_main_template", allow_single_file = True, doc = """ The template to use for a zip's top-level `__main__.py` file. diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 3595a43110..959e631ad1 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -27,7 +27,7 @@ # ===== Template substitutions start ===== # We just put them in one place so its easy to tell which are used. -# Runfiles-relative path to the main Python source file. +# Runfiles-root-relative path to the main Python source file. # Empty if MAIN_MODULE is used MAIN_PATH = "%main%" diff --git a/python/private/zipapp/BUILD.bazel b/python/private/zipapp/BUILD.bazel new file mode 100644 index 0000000000..2e96178144 --- /dev/null +++ b/python/private/zipapp/BUILD.bazel @@ -0,0 +1,48 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package( + default_visibility = ["//:__subpackages__"], +) + +licenses(["notice"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), +) + +bzl_library( + name = "py_zipapp_rule_bzl", + srcs = ["py_zipapp_rule.bzl"], + deps = [ + "//python/private:builders_bzl", + "//python/private:common_bzl", + "//python/private:py_executable_info_bzl", + "//python/private:py_info_bzl", + "//python/private:py_internal_bzl", + "//python/private:py_interpreter_program_bzl", + "//python/private:py_runtime_info_bzl", + "//python/private:toolchain_types_bzl", + "@bazel_skylib//lib:paths", + ], +) + + + +filegroup( + name = "zip_main_template", + srcs = ["zip_main_template.py"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "zip_shell_template", + srcs = ["zip_shell_template.sh"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "zipapp_stage2_bootstrap_template", + srcs = ["zipapp_stage2_bootstrap_template.py"], + visibility = ["//visibility:public"], +) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl new file mode 100644 index 0000000000..068aeabac6 --- /dev/null +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -0,0 +1,251 @@ +"""Implementation of the zipapp rules.""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("//python/private:builders.bzl", "builders") +load("//python/private:common.bzl", "actions_run", "maybe_create_repo_mapping", "runfiles_root_path") +load("//python/private:py_executable_info.bzl", "PyExecutableInfo") +load("//python/private:py_internal.bzl", "py_internal") +load("//python/private:py_runtime_info.bzl", "PyRuntimeInfo") +load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE") + +def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap): + python_exe = py_executable.venv_python_exe + python_exe_path = runfiles_root_path(ctx, python_exe.short_path) + + if py_runtime.interpreter: + python_binary_actual_path = runfiles_root_path(ctx, py_runtime.interpreter.short_path) + else: + python_binary_actual_path = py_runtime.interpreter_path + + zip_main_py = ctx.actions.declare_file(ctx.label.name + ".zip_main.py") + ctx.actions.expand_template( + template = py_runtime.zip_main_template, + output = zip_main_py, + substitutions = { + "%python_binary%": python_exe_path, + "%python_binary_actual%": python_binary_actual_path, + "%stage2_bootstrap%": runfiles_root_path(ctx, stage2_bootstrap.short_path), + "%workspace_name%": ctx.workspace_name, + }, + ) + return zip_main_py + +def _map_zip_empty_filenames(list_paths_cb): + return ["rf-empty|" + path for path in list_paths_cb().to_list()] + +def _map_zip_runfiles(file): + return "rf-file|" + str(int(file.is_symlink)) + "|" + file.short_path + "|" + file.path + +def _map_zip_symlinks(entry): + return "rf-symlink|" + str(int(entry.target_file.is_symlink)) + "|" + entry.path + "|" + entry.target_file.path + +def _map_zip_root_symlinks(entry): + return "rf-root-symlink|" + str(int(entry.target_file.is_symlink)) + "|" + entry.path + "|" + entry.target_file.path + +def _build_manifest(ctx, manifest, runfiles, zip_main): + manifest.add("regular|0|__main__.py|{}".format(zip_main.path)) + + manifest.add_all( + # NOTE: Accessing runfiles.empty_filenames materializes them. A lambda + # is used to defer that. + [lambda: runfiles.empty_filenames], + map_each = _map_zip_empty_filenames, + allow_closure = True, + ) + + manifest.add_all(runfiles.files, map_each = _map_zip_runfiles) + manifest.add_all(runfiles.symlinks, map_each = _map_zip_symlinks) + manifest.add_all(runfiles.root_symlinks, map_each = _map_zip_root_symlinks) + + inputs = [zip_main] + zip_repo_mapping_manifest = maybe_create_repo_mapping( + ctx = ctx, + runfiles = runfiles, + ) + if zip_repo_mapping_manifest: + # NOTE: rf-root-symlink is used to make it show up under the runfiles + # subdirectory within the zip. + manifest.add( + zip_repo_mapping_manifest.path, + format = "rf-root-symlink|0|_repo_mapping|%s", + ) + inputs.append(zip_repo_mapping_manifest) + return inputs + +def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): + output = ctx.actions.declare_file(ctx.label.name + ".zip") + manifest = ctx.actions.args() + manifest.use_param_file("%s", use_always = True) + manifest.set_param_file_format("multiline") + + runfiles = builders.RunfilesBuilder() + + runfiles.add(py_runtime.files) + runfiles.add(py_executable.venv_python_exe) + runfiles.add(py_executable.app_runfiles) + runfiles.add(stage2_bootstrap) + + runfiles = runfiles.build(ctx) + + zip_main = _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap) + inputs = _build_manifest(ctx, manifest, runfiles, zip_main) + + zipper_args = ctx.actions.args() + zipper_args.add(output) + zipper_args.add(ctx.workspace_name, format = "--workspace-name=%s") + zipper_args.add( + str(int(py_internal.get_legacy_external_runfiles(ctx))), + format = "--legacy-external-runfiles=%s", + ) + if ctx.attr.compression: + zipper_args.add(ctx.attr.compression, "--compression=%s") + zipper_args.add("--runfiles-dir=runfiles") + + actions_run( + ctx, + executable = ctx.attr._zipper, + arguments = [manifest, zipper_args], + inputs = depset(inputs, transitive = [runfiles.files]), + outputs = [output], + mnemonic = "PyZipAppCreateZip", + progress_message = "Reticulating zipapp archive: %{label} into %{output}", + ) + return output + +def _create_shell_bootstrap(ctx, py_runtime, py_executable, stage2_bootstrap): + preamble = ctx.actions.declare_file(ctx.label.name + ".preamble.sh") + + bundled_pyexe_path = "" + external_pyexe_path = "" + if py_runtime.interpreter_path: + external_pyexe_path = py_runtime.interpreter_path + else: + bundled_pyexe_path = runfiles_root_path(ctx, py_runtime.interpreter.short_path) + + substitutions = { + "%BUNDLED_PYEXE_PATH%": bundled_pyexe_path, + "%EXTERNAL_PYEXE_PATH%": external_pyexe_path, + "%EXTRACT_DIR%": paths.join( + (ctx.label.repo_name or "_main"), + ctx.label.package, + ctx.label.name, + ), + "%INTERPRETER_ARGS%": "\n".join([ + '"{}"'.format(v) + for v in py_executable.interpreter_args + ]), + "%STAGE2_BOOTSTRAP%": runfiles_root_path(ctx, stage2_bootstrap.short_path), + } + ctx.actions.expand_template( + template = ctx.file._zip_shell_template, + output = preamble, + substitutions = substitutions, + is_executable = True, + ) + return preamble + +def _create_self_executable_zip(ctx, preamble, zip_file): + pyz = ctx.actions.declare_file(ctx.label.name + ".pyz") + args = ctx.actions.args() + args.add(preamble) + args.add(zip_file) + args.add(pyz) + actions_run( + ctx, + executable = ctx.attr._exe_zip_maker, + arguments = [args], + inputs = [preamble, zip_file], + outputs = [pyz], + mnemonic = "PyZipAppCreateExecutableZip", + progress_message = "Reticulating zipapp executable: %{label} into %{output}", + ) + return pyz + +def _py_zipapp_executable_impl(ctx): + py_executable = ctx.attr.binary[PyExecutableInfo] + py_runtime = ctx.attr.binary[PyRuntimeInfo] + + stage2_bootstrap = py_executable.stage2_bootstrap + + zip_file = _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap) + if ctx.attr.executable: + preamble = _create_shell_bootstrap(ctx, py_runtime, py_executable, stage2_bootstrap) + executable = _create_self_executable_zip(ctx, preamble, zip_file) + default_output = executable + else: + # Bazel requires executable=True rules to have an executable given, so give + # a fake one to satisfy that. + default_output = zip_file + executable = ctx.actions.declare_file(ctx.label.name + "-not-executable") + ctx.actions.write(executable, "echo 'ERROR: Non executable zip file'; exit 1") + + return [ + DefaultInfo( + files = depset([default_output]), + runfiles = ctx.runfiles(files = [default_output]), + executable = executable, + ), + ] + +_ATTRS = { + "binary": attr.label( + doc = """ +A `py_binary` or `py_test` (or equivalent) target to package. +""", + providers = [PyExecutableInfo, PyRuntimeInfo], + mandatory = True, + ), + "compression": attr.string( + doc = """ +The compression level to use. + +Typically 0 to 9, with higher numbers being to compress more. +""", + default = "", + ), + "executable": attr.bool( + doc = """ +Whether the output should be an executable zip file. +""", + default = True, + ), + "_exe_zip_maker": attr.label( + cfg = "exec", + default = "//tools/zipapp:exe_zip_maker", + ), + "_zip_shell_template": attr.label( + default = ":zip_shell_template", + allow_single_file = True, + ), + "_zipper": attr.label( + cfg = "exec", + default = "//tools/zipapp:zipper", + ), +} +_TOOLCHAINS = [EXEC_TOOLS_TOOLCHAIN_TYPE] + +py_zipapp_binary = rule( + doc = """ +Packages a `py_binary` as a Python zipapp. +""", + implementation = _py_zipapp_executable_impl, + attrs = _ATTRS, + # NOTE: While this is marked executable, it is conditionally executable + # based on the `executable` attribute. + executable = True, + toolchains = _TOOLCHAINS, +) + +py_zipapp_test = rule( + doc = """ +Packages a `py_test` as a Python zipapp. + +This target is also a valid test target to run. +""", + implementation = _py_zipapp_executable_impl, + attrs = _ATTRS, + # NOTE: While this is marked as a test, it is conditionally executable + # based on the `executable` attribute. + test = True, + toolchains = _TOOLCHAINS, +) diff --git a/python/private/zip_main_template.py b/python/private/zipapp/zip_main_template.py similarity index 96% rename from python/private/zip_main_template.py rename to python/private/zipapp/zip_main_template.py index d1489b46aa..35db1645bc 100644 --- a/python/private/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -3,11 +3,15 @@ # NOTE: This file is a "stage 1" bootstrap, so it's responsible for locating the # desired runtime and having it run the stage 2 bootstrap. This means it can't # assume much about the current runtime and environment. e.g., the current -# runtime may not be the correct one, the zip may not have been extract, the +# runtime may not be the correct one, the zip may not have been extracted, the # runfiles env vars may not be set, etc. # # NOTE: This program must retain compatibility with a wide variety of Python # versions since it is run by an unknown Python interpreter. +# +# NOTE: For a self-executable zip, this file may not be the entry point +# for the program and may be skipped entirely; the self-executable zip +# preamble may jump directly to the stage2 bootstrap. import sys @@ -23,11 +27,11 @@ import tempfile import zipfile -# runfiles-relative path +# runfiles-root-relative path _STAGE2_BOOTSTRAP = "%stage2_bootstrap%" -# runfiles-relative path to venv's bin/python3. Empty if venv not being used. +# runfiles-root-relative path to venv's bin/python3. Empty if venv not being used. _PYTHON_BINARY = "%python_binary%" -# runfiles-relative path, absolute path, or single word. The actual Python +# runfiles-root-relative path, absolute path, or single word. The actual Python # executable to use. _PYTHON_BINARY_ACTUAL = "%python_binary_actual%" _WORKSPACE_NAME = "%workspace_name%" diff --git a/python/private/zipapp/zip_shell_template.sh b/python/private/zipapp/zip_shell_template.sh new file mode 100644 index 0000000000..464fc0f861 --- /dev/null +++ b/python/private/zipapp/zip_shell_template.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +set -e + +if [[ -n "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then + set -x +fi + +# runfiles-root-relative path +BUNDLED_PYEXE_PATH="%BUNDLED_PYEXE_PATH%" +# Absolute path or single word +EXTERNAL_PYEXE_PATH="%EXTERNAL_PYEXE_PATH%" +# runfiles-root-relative path +STAGE2_BOOTSTRAP="%STAGE2_BOOTSTRAP%" +EXTRACT_DIR="%EXTRACT_DIR%" +ZIP_HASH="%ZIP_HASH%" +declare -a INTERPRETER_ARGS_FROM_TARGET=( +%INTERPRETER_ARGS% +) + +declare -a interpreter_env +declare -a interpreter_args +declare -a additional_interpreter_args + +if [[ -z "${PYTHONSAFEPATH+x}" ]]; then + # ${FOO-WORD} expands to WORD if $FOO is undefined, and $FOO otherwise + interpreter_env+=("PYTHONSAFEPATH=${PYTHONSAFEPATH-1}") +fi + + +if [[ -n "${RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS}" ]]; then + read -a additional_interpreter_args <<< "${RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS}" + interpreter_args+=("${additional_interpreter_args[@]}") + unset RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS +fi + + +if [[ -n "$RULES_PYTHON_EXTRACT_ROOT" ]]; then + zip_dir="$RULES_PYTHON_EXTRACT_ROOT/$EXTRACT_DIR/$ZIP_HASH" + if [[ ! -e "$zip_dir/__main__.py" ]]; then + mkdir -p "$zip_dir" + # Unzip emits a warning and exits 1 with the prelude + ( unzip -q -d "$zip_dir" "$0" 2>/dev/null || true ) + fi +else + # NOTE: Macs have an old version of mktemp, so we must use only the + # minimal functionality of it. + zip_dir=$(mktemp -d) + # Unzip emits a warning and exits 1 with the prelude + ( unzip -q -d "$zip_dir" "$0" 2>/dev/null || true ) + if [[ -n "$zip_dir" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then + trap 'rm -fr "$zip_dir"' EXIT + fi + trap 'rm -rf "$zip_dir"' EXIT +fi + +export RUNFILES_DIR="$zip_dir/runfiles" +if [[ ! -d "$RUNFILES_DIR" ]]; then + echo "Runfiles dir not found: zip extraction likely failed" 1>&2 + echo "Run with RULES_PYTHON_BOOTSTRAP_VERBOSE=1 to aid debugging" 1>&2 + exit 1 +fi + +if [[ -n "$BUNDLED_PYEXE_PATH" ]]; then + python_exe="$RUNFILES_DIR/$BUNDLED_PYEXE_PATH" +else + python_exe="$EXTERNAL_PYEXE_PATH" +fi + +command=( + env + "${interpreter_env[@]}" + "$python_exe" + "-XRULES_PYTHON_ZIP_DIR=$zip_dir" + "${interpreter_args[@]}" + "${INTERPRETER_ARGS_FROM_TARGET[@]}" + "$RUNFILES_DIR/$STAGE2_BOOTSTRAP" + "$@" +) + +# NOTE: because exec isn't used, signals don't propagate to the child +# TODO: Use exec and let the program handle cleanup. Without exec, +# signals don't propagate to the child nicely. +# See https://github.com/bazel-contrib/rules_python/issues/2043#issuecomment-2215469971 +# for more information. +"${command[@]}" +# Explicit exit is needed because the implicit next line the zip file this +# template is prepended to. +exit 0 diff --git a/python/private/zipapp/zipapp_stage2_bootstrap_template.py b/python/private/zipapp/zipapp_stage2_bootstrap_template.py new file mode 100644 index 0000000000..9fd71fefb9 --- /dev/null +++ b/python/private/zipapp/zipapp_stage2_bootstrap_template.py @@ -0,0 +1,10 @@ +import runpy +import shutil +import sys + +try: + sys.argv.pop(0) # Remove zipapp_stage2_bootstrap from args + runpy.run_path(sys.argv[0], run_name="__main__") +finally: + if zip_dir := sys._xoptions.get("RULES_PYTHON_ZIP_DIR"): + shutil.rmtree(zip_dir, True) diff --git a/python/zipapp/BUILD.bazel b/python/zipapp/BUILD.bazel new file mode 100644 index 0000000000..71249ae45c --- /dev/null +++ b/python/zipapp/BUILD.bazel @@ -0,0 +1,29 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python:__pkg__"], +) + +bzl_library( + name = "py_zipapp_binary_bzl", + srcs = ["py_zipapp_binary.bzl"], + deps = [ + "//python/private:util_bzl", + "//python/private/zipapp:py_zipapp_rule_bzl", + ], +) + +bzl_library( + name = "py_zipapp_test_bzl", + srcs = ["py_zipapp_test.bzl"], + deps = [ + "//python/private:util_bzl", + "//python/private/zipapp:py_zipapp_rule_bzl", + ], +) diff --git a/python/zipapp/py_zipapp_binary.bzl b/python/zipapp/py_zipapp_binary.bzl new file mode 100644 index 0000000000..4c40786200 --- /dev/null +++ b/python/zipapp/py_zipapp_binary.bzl @@ -0,0 +1,15 @@ +"""`py_zipapp_binary` macro.""" + +load("//python/private:util.bzl", "add_tag") +load("//python/private/zipapp:py_zipapp_rule.bzl", _py_zipapp_binary_rule = "py_zipapp_binary") + +def py_zipapp_binary(**kwargs): + """Builds a Python zipapp from a py_binary/py_test target. + + Args: + **kwargs: Args passed onto {rule}`py_zipapp_binary`. + """ + add_tag(kwargs, "@rules_python//python:py_zipapp_binary") + _py_zipapp_binary_rule( + **kwargs + ) diff --git a/python/zipapp/py_zipapp_test.bzl b/python/zipapp/py_zipapp_test.bzl new file mode 100644 index 0000000000..bc113ad4ed --- /dev/null +++ b/python/zipapp/py_zipapp_test.bzl @@ -0,0 +1,15 @@ +"""`py_zipapp_test` macro.""" + +load("//python/private:util.bzl", "add_tag") +load("//python/private/zipapp:py_zipapp_rule.bzl", _py_zipapp_test = "py_zipapp_test") + +def py_zipapp_test(**kwargs): + """Builds a Python zipapp from a py_binary/py_test target. + + Args: + **kwargs: Args passed onto {rule}`py_zipapp_test`. + """ + add_tag(kwargs, "@rules_python//python:py_zipapp_test") + _py_zipapp_test( + **kwargs + ) diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel new file mode 100644 index 0000000000..51d8ff7c56 --- /dev/null +++ b/tests/py_zipapp/BUILD.bazel @@ -0,0 +1,28 @@ +load("//python:py_binary.bzl", "py_binary") +load("//python:py_test.bzl", "py_test") +load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") + +py_binary( + name = "bin", + srcs = ["main.py"], + config_settings = { + "//python/config_settings:bootstrap_impl": "script", + "//python/config_settings:venvs_site_packages": "yes", + }, + main = "main.py", + deps = ["@dev_pip//absl_py"], +) + +py_zipapp_binary( + name = "bin_zipapp", + binary = ":bin", +) + +py_test( + name = "zipapp_test", + srcs = ["zipapp_test.py"], + data = [":bin_zipapp"], + env = { + "TEST_ZIPAPP": "$(location :bin_zipapp)", + }, +) diff --git a/tests/py_zipapp/main.py b/tests/py_zipapp/main.py new file mode 100644 index 0000000000..b8fdbe365e --- /dev/null +++ b/tests/py_zipapp/main.py @@ -0,0 +1,12 @@ +"A trivial zipapp that prints a message" + + +def main(): + print("Hello from zipapp") + import absl + + print(f"absl: {absl}") + + +if __name__ == "__main__": + main() diff --git a/tests/py_zipapp/zipapp_test.py b/tests/py_zipapp/zipapp_test.py new file mode 100644 index 0000000000..c2933ae133 --- /dev/null +++ b/tests/py_zipapp/zipapp_test.py @@ -0,0 +1,68 @@ +import os +import subprocess +import unittest +import zipfile + + +class PyZipAppTest(unittest.TestCase): + def test_zipapp_contents(self): + zipapp_path = os.environ["TEST_ZIPAPP"] + + self.assertTrue(os.path.exists(zipapp_path)) + self.assertTrue(os.path.isfile(zipapp_path)) + + # The zipapp itself is a shell script prepended to the zip file. + with open(zipapp_path, "rb") as f: + content = f.read() + self.assertTrue(content.startswith(b"#!/usr/bin/env bash")) + + output = subprocess.check_output([zipapp_path]).decode("utf-8").strip() + self.assertIn("Hello from zipapp", output) + self.assertIn("absl", output) + + def assertHasPathMatchingSuffix(self, namelist, suffix, msg=None): + if not any(name.endswith(suffix) for name in namelist): + self.fail(msg or f"No path in zipapp matching suffix '{suffix}'") + + def assertZipEntryIsSymlink(self, zip_file, path, msg=None): + try: + info = zip_file.getinfo(path) + except KeyError: + self.fail(msg or f"Path '{path}' not found in zipfile") + + # S_IFLNK is 0o120000. + # ZipInfo.external_attr is 32 bits: the high 16 bits are Unix attributes. + is_symlink = (info.external_attr >> 16) & 0o170000 == 0o120000 + if not is_symlink: + self.fail(msg or f"Path '{path}' is not a symlink") + + def test_zipapp_structure(self): + zipapp_path = os.environ["TEST_ZIPAPP"] + + with zipfile.ZipFile(zipapp_path, "r") as zf: + namelist = zf.namelist() + self.assertIn("runfiles/_repo_mapping", namelist) + + self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") + + # The venv directory name depends on the target name, so find it + # by looking for pyvenv.cfg. + venv_config = next( + (name for name in namelist if name.endswith("/pyvenv.cfg")), None + ) + self.assertIsNotNone(venv_config) + + venv_root = os.path.dirname(venv_config) + + # Verify bin/python3 exists and is a symlink + python_bin = f"{venv_root}/bin/python3" + self.assertZipEntryIsSymlink(zf, python_bin) + + # Verify _bazel_site_init.py exists in site-packages + self.assertHasPathMatchingSuffix( + namelist, "/site-packages/_bazel_site_init.py" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/tools/zipapp/BUILD.bazel b/tests/tools/zipapp/BUILD.bazel new file mode 100644 index 0000000000..9383fb7adf --- /dev/null +++ b/tests/tools/zipapp/BUILD.bazel @@ -0,0 +1,13 @@ +load("//python:py_test.bzl", "py_test") + +py_test( + name = "zipper_test", + srcs = ["zipper_test.py"], + deps = ["//tools/zipapp:zipper_lib"], +) + +py_test( + name = "exe_zip_maker_test", + srcs = ["exe_zip_maker_test.py"], + deps = ["//tools/zipapp:exe_zip_maker_lib"], +) diff --git a/tests/tools/zipapp/exe_zip_maker_test.py b/tests/tools/zipapp/exe_zip_maker_test.py new file mode 100644 index 0000000000..b130f2810c --- /dev/null +++ b/tests/tools/zipapp/exe_zip_maker_test.py @@ -0,0 +1,63 @@ +import hashlib +import os +import shutil +import stat +import tempfile +import unittest + +from tools.zipapp import exe_zip_maker + + +class ExeZipMakerTest(unittest.TestCase): + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.preamble_path = os.path.join(self.test_dir, "preamble.txt") + self.zip_path = os.path.join(self.test_dir, "data.zip") + self.output_path = os.path.join(self.test_dir, "output.exe") + + def tearDown(self): + shutil.rmtree(self.test_dir) + + def test_create_exe_zip(self): + # Create dummy zip file + zip_content = b"PK\x03\x04dummyzipcontent" + with open(self.zip_path, "wb") as f: + f.write(zip_content) + + # Calculate expected hash + expected_hash = hashlib.sha256(zip_content).hexdigest() + + # Create preamble with placeholder + preamble_text = "#!/bin/bash\nEXPECTED_HASH='%ZIP_HASH%'\n# ... logic ...\n" + with open(self.preamble_path, "w") as f: + f.write(preamble_text) + + # Call create_exe_zip directly + exe_zip_maker.create_exe_zip( + self.preamble_path, self.zip_path, self.output_path + ) + + # Verify output exists + self.assertTrue(os.path.exists(self.output_path)) + + # Verify executable bit + st = os.stat(self.output_path) + self.assertTrue(st.st_mode & stat.S_IEXEC) + + # Verify content + with open(self.output_path, "rb") as f: + content = f.read() + + # Split content back into preamble and zip + # We know the preamble text length after substitution. + expected_preamble = preamble_text.replace("%ZIP_HASH%", expected_hash).encode( + "utf-8" + ) + + self.assertTrue(content.startswith(expected_preamble)) + self.assertTrue(content.endswith(zip_content)) + self.assertEqual(len(content), len(expected_preamble) + len(zip_content)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/tools/zipapp/zipper_test.py b/tests/tools/zipapp/zipper_test.py new file mode 100644 index 0000000000..993b079b9c --- /dev/null +++ b/tests/tools/zipapp/zipper_test.py @@ -0,0 +1,220 @@ +import os +import pathlib +import shutil +import tempfile +import time +import unittest +import zipfile + +from tools.zipapp import zipper + + +class ZipperTest(unittest.TestCase): + def setUp(self): + self.test_dir = pathlib.Path(tempfile.mkdtemp()) + self.manifest_path = self.test_dir / "manifest.txt" + self.output_zip = self.test_dir / "output.zip" + + def tearDown(self): + shutil.rmtree(self.test_dir) + + def _create_zip(self, **kwargs): + defaults = { + "manifest_path": self.manifest_path, + "output_zip": self.output_zip, + "compress_level": 0, + "workspace_name": "my_ws", + "legacy_external_runfiles": False, + "runfiles_dir": "runfiles", + } + defaults.update(kwargs) + zipper.create_zip(**defaults) + + def assertZipFileContent( + self, zf, path, content=None, is_symlink=False, target=None + ): + info = zf.getinfo(path) + if is_symlink: + self.assertTrue( + self.is_symlink(info), + f"{path} should be a symlink but is not", + ) + self.assertEqual(zf.read(path).decode(), target) + else: + self.assertFalse( + self.is_symlink(info), + f"{path} should NOT be a symlink but is", + ) + self.assertEqual(zf.read(path).decode(), content) + + def test_create_zip_with_files_and_symlinks(self): + file1_path = self.test_dir / "file1.txt" + file1_path.write_text("content1") + + link_target_path = "target.txt" # Relative target + symlink_path = self.test_dir / "symlink_source" + symlink_path.symlink_to(link_target_path) + + manifest_content = [ + f"regular|0|file1.txt|{file1_path}", + f"rf-file|0|foo/bar.txt|{file1_path}", + f"rf-symlink|1|link1|{symlink_path}", # Should read target 'target.txt' + f"rf-root-symlink|0|root_file|{file1_path}", + f"rf-empty|empty_file", + ] + self.manifest_path.write_text("\n".join(manifest_content)) + + self._create_zip() + + self.assertTrue(self.output_zip.exists()) + + with zipfile.ZipFile(self.output_zip, "r") as zf: + self.assertEqual( + set(zf.namelist()), + { + "file1.txt", + "runfiles/my_ws/foo/bar.txt", + "runfiles/my_ws/link1", + "runfiles/root_file", + "runfiles/my_ws/empty_file", + }, + ) + + self.assertZipFileContent(zf, "file1.txt", content="content1") + self.assertZipFileContent( + zf, "runfiles/my_ws/foo/bar.txt", content="content1" + ) + self.assertZipFileContent( + zf, "runfiles/my_ws/link1", is_symlink=True, target="target.txt" + ) + self.assertZipFileContent(zf, "runfiles/root_file", content="content1") + self.assertZipFileContent(zf, "runfiles/my_ws/empty_file", content="") + + def test_timestamps_are_deterministic(self): + # Create a content file with a specific recent timestamp + file1_path = self.test_dir / "file1.txt" + file1_path.write_text("content1") + + # Set mtime to something recent (e.g. now) + os.utime(file1_path, None) + + manifest_content = [ + f"regular|0|file1.txt|{file1_path}", + ] + + self.manifest_path.write_text("\n".join(manifest_content)) + + self._create_zip() + + with zipfile.ZipFile(self.output_zip, "r") as zf: + info = zf.getinfo("file1.txt") + # DOS epoch is 1980-01-01 00:00:00 + expected_date_time = (1980, 1, 1, 0, 0, 0) + self.assertEqual(info.date_time, expected_date_time) + + def test_runfiles_mapping_with_cross_repo_paths(self): + # Create content file + file1_path = self.test_dir / "file1.txt" + file1_path.write_text("content1") + + manifest_content = [ + f"rf-file|0|../other_repo/foo.txt|{file1_path}", + f"rf-empty|../other_repo/empty_file", + ] + + self.manifest_path.write_text("\n".join(manifest_content)) + + self._create_zip(workspace_name="my_ws") + + with zipfile.ZipFile(self.output_zip, "r") as zf: + self.assertEqual( + set(zf.namelist()), + { + "runfiles/other_repo/foo.txt", + "runfiles/other_repo/empty_file", + }, + ) + self.assertZipFileContent( + zf, "runfiles/other_repo/foo.txt", content="content1" + ) + self.assertZipFileContent(zf, "runfiles/other_repo/empty_file", content="") + + def test_runfiles_mapping_with_legacy_external_paths(self): + file1_path = self.test_dir / "file1.txt" + file1_path.write_text("content1") + + manifest_content = [ + f"rf-file|0|external/other_repo/foo.txt|{file1_path}", + f"rf-empty|external/other_repo/empty_file", + ] + + self.manifest_path.write_text("\n".join(manifest_content)) + + self._create_zip(workspace_name="my_ws", legacy_external_runfiles=True) + + with zipfile.ZipFile(self.output_zip, "r") as zf: + self.assertEqual( + set(zf.namelist()), + { + "runfiles/other_repo/foo.txt", + "runfiles/other_repo/empty_file", + }, + ) + self.assertZipFileContent( + zf, "runfiles/other_repo/foo.txt", content="content1" + ) + self.assertZipFileContent(zf, "runfiles/other_repo/empty_file", content="") + + def test_output_deterministic(self): + # Create files + file1 = self.test_dir / "file1" + file1.write_text("1") + file2 = self.test_dir / "file2" + file2.write_text("2") + file3 = self.test_dir / "file3" + file3.write_text("3") + + # Manifest entries mixed up + # We want the final order to be: + # 1. a/regular (regular) + # 2. runfiles/a_root_link (rf-root-symlink) + # 3. runfiles/my_ws/b_rf_file (rf-file) + # 4. runfiles/my_ws/c_rf_link (rf-symlink) + # 5. runfiles/my_ws/d_rf_empty (rf-empty) + # 6. z/regular (regular) + + manifest_content = [ + f"regular|0|z/regular|{file1}", + f"rf-file|0|b_rf_file|{file2}", # -> runfiles/my_ws/b_rf_file + f"rf-root-symlink|0|a_root_link|{file3}", # -> runfiles/a_root_link + f"regular|0|a/regular|{file3}", + f"rf-empty|d_rf_empty", # -> runfiles/my_ws/d_rf_empty + f"rf-symlink|0|c_rf_link|{file3}", # -> runfiles/my_ws/c_rf_link + ] + + self.manifest_path.write_text("\n".join(manifest_content)) + + self._create_zip(workspace_name="my_ws") + + with zipfile.ZipFile(self.output_zip, "r") as zf: + self.assertEqual( + zf.namelist(), + [ + "a/regular", + "runfiles/a_root_link", + "runfiles/my_ws/b_rf_file", + "runfiles/my_ws/c_rf_link", + "runfiles/my_ws/d_rf_empty", + "z/regular", + ], + ) + + def is_symlink(self, zip_info): + # Check upper 4 bits of external_attr for S_IFLNK + # S_IFLNK is 0o120000 = 0xA000 + attr = zip_info.external_attr >> 16 + return (attr & 0xF000) == 0xA000 + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/zipapp/BUILD.bazel b/tools/zipapp/BUILD.bazel new file mode 100644 index 0000000000..7a2002cd72 --- /dev/null +++ b/tools/zipapp/BUILD.bazel @@ -0,0 +1,36 @@ +load("//python:py_library.bzl", "py_library") +load("//python/private:py_interpreter_program.bzl", "py_interpreter_program") # buildifier: disable=bzl-visibility + +package( + default_visibility = ["//:__subpackages__"], +) + +py_interpreter_program( + name = "zipper", + main = "zipper.py", + visibility = [ + # Not actually public. Only public so rules_python-generated toolchains + # are able to reference it. + "//visibility:public", + ], +) + +py_library( + name = "zipper_lib", + srcs = ["zipper.py"], +) + +py_interpreter_program( + name = "exe_zip_maker", + main = "exe_zip_maker.py", + visibility = [ + # Not actually public. Only public so rules_python-generated toolchains + # are able to reference it. + "//visibility:public", + ], +) + +py_library( + name = "exe_zip_maker_lib", + srcs = ["exe_zip_maker.py"], +) diff --git a/tools/zipapp/exe_zip_maker.py b/tools/zipapp/exe_zip_maker.py new file mode 100644 index 0000000000..29391c86da --- /dev/null +++ b/tools/zipapp/exe_zip_maker.py @@ -0,0 +1,40 @@ +import hashlib +import os +import shutil +import stat +import sys + +BLOCK_SIZE = 256 * 1024 + + +def create_exe_zip(preamble_path, zip_path, output_path): + sha256_hash = hashlib.sha256() + with open(zip_path, "rb", buffering=BLOCK_SIZE) as f: + for byte_block in iter(lambda: f.read(BLOCK_SIZE), b""): + sha256_hash.update(byte_block) + zip_hash = sha256_hash.hexdigest() + + with open(preamble_path, "rb") as f: + preamble_content = f.read() + + preamble_content = preamble_content.replace(b"%ZIP_HASH%", zip_hash.encode("utf-8")) + + with open(output_path, "wb") as out_f: + out_f.write(preamble_content) + with open(zip_path, "rb") as zip_f: + shutil.copyfileobj(zip_f, out_f, length=BLOCK_SIZE) + + st = os.stat(output_path) + os.chmod(output_path, st.st_mode | stat.S_IEXEC) + + +def main(): + if len(sys.argv) != 4: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + create_exe_zip(sys.argv[1], sys.argv[2], sys.argv[3]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/zipapp/zipper.py b/tools/zipapp/zipper.py new file mode 100644 index 0000000000..2263f13125 --- /dev/null +++ b/tools/zipapp/zipper.py @@ -0,0 +1,220 @@ +import argparse +import os +import shutil +import stat +import sys +import zipfile + +# Unix permission bit for symlink (S_IFLNK) +# S_IFLNK is usually 0o120000 +S_IFLNK = 0o120000 + + +def _get_zip_runfiles_path( + path, workspace_name, legacy_external_runfiles, runfiles_dir +): + if legacy_external_runfiles and path.startswith("external/"): + path = path[len("external/") :] + elif path.startswith("../"): + path = path[3:] + else: + path = os.path.join(workspace_name, path) + return os.path.join(runfiles_dir, path) + + +def _parse_entry( + line, + line_idx, + workspace_name, + legacy_external_runfiles, + runfiles_dir, +): + line = line.strip() + if not line: + return None + + parts = line.split("|") + type_ = parts[0] + + if type_ == "regular": + _, is_symlink_str, zip_path, content_path = parts + elif type_ == "rf-empty": + _, runfile_path = parts + zip_path = _get_zip_runfiles_path( + runfile_path, workspace_name, legacy_external_runfiles, runfiles_dir + ) + content_path = None # Empty file + is_symlink_str = "0" + elif type_ == "rf-file": + _, is_symlink_str, runfile_path, content_path = parts + zip_path = _get_zip_runfiles_path( + runfile_path, workspace_name, legacy_external_runfiles, runfiles_dir + ) + elif type_ == "rf-symlink": + _, is_symlink_str, runfile_path, content_path = parts + zip_path = os.path.join(runfiles_dir, workspace_name, runfile_path) + elif type_ == "rf-root-symlink": + _, is_symlink_str, runfile_path, content_path = parts + zip_path = os.path.join(runfiles_dir, runfile_path) + else: + raise ValueError( + f"Error: Unknown entry type or invalid format at line {line_idx + 1}: {line}" + ) + + return type_, is_symlink_str, zip_path, content_path + + +def read_manifest( + manifest_path, workspace_name, legacy_external_runfiles, runfiles_dir +): + with open(manifest_path, "r") as f: + entries = [] + for line_idx, line in enumerate(f): + try: + entry = _parse_entry( + line, + line_idx, + workspace_name, + legacy_external_runfiles, + runfiles_dir, + ) + if entry: + entries.append(entry) + except ValueError as e: + e.add_note(f"Error processing line {line_idx + 1}: {line.strip()}") + raise + + # Sort by zip path (3rd element in tuple) + entries.sort(key=lambda x: x[2]) + return entries + + +def _write_entry(zf, entry, compress_type): + type_, is_symlink_str, zip_path, content_path = entry + + if type_ == "rf-empty": + zi = zipfile.ZipInfo(zip_path) + zi.date_time = (1980, 1, 1, 0, 0, 0) + zi.create_system = 3 # Unix + zi.compress_type = compress_type + # Create empty file + zi.external_attr = (0o644 & 0xFFFF) << 16 + zf.writestr(zi, "") + return + + is_symlink = is_symlink_str == "1" + + if is_symlink: + zi = zipfile.ZipInfo(zip_path) + zi.date_time = (1980, 1, 1, 0, 0, 0) + zi.create_system = 3 # Unix + zi.compress_type = compress_type + target = os.readlink(content_path) + # Set permissions to 777 for symlink (standard) + zi.external_attr = (S_IFLNK | 0o777) << 16 + zf.writestr(zi, target) + else: + st = os.stat(content_path) + zi = zipfile.ZipInfo(zip_path) + zi.date_time = (1980, 1, 1, 0, 0, 0) + zi.create_system = 3 # Unix + zi.compress_type = compress_type + # Preserve permissions, otherwise execute is dropped. + zi.external_attr = (st.st_mode & 0xFFFF) << 16 + with open(content_path, "rb") as src, zf.open(zi, "w") as dst: + shutil.copyfileobj(src, dst) + + +def create_zip( + *, + manifest_path, + output_zip, + compress_level, + workspace_name, + legacy_external_runfiles, + runfiles_dir, +): + compress_type = zipfile.ZIP_STORED if compress_level == 0 else zipfile.ZIP_DEFLATED + zf_level = compress_level if compress_level != 0 else None + + entries = read_manifest( + manifest_path, workspace_name, legacy_external_runfiles, runfiles_dir + ) + + with zipfile.ZipFile( + output_zip, "w", compress_type, allowZip64=True, compresslevel=zf_level + ) as zf: + for entry in entries: + _write_entry(zf, entry, compress_type) + + +def main(): + parser = argparse.ArgumentParser(description="Create a zip file from a manifest.") + parser.add_argument( + "manifest", + help=""" +Path to the manifest file. Lines have one of the following formats: + +1. `regular|is_symlink|zip_path|content_path`: This form stores the `zip_path` + in the zip, whose content is taken from `content_path` + +2. `rf-empty|runfile_path`: A `runfiles.empty_filenames` value. The stored + zip path is computed from `runfile_path` + +3. `rf-file|is_symlink|runfile_path|content_path`: Store a file in + the zip. The zip path is computed from `runfile_path`. + +4. `rf-symlink|is_symlink|runfile_symlink_path|content_path`: Store a + main-repo-relative path in the zip. + +5. `rf-root-symlink|is_symlink|runfile_root_path|content_path`: Store a + runfiles-root-relative path in the zip. + +In all cases, `is_symlink` is `1` or `0` if the path should be stored +as a symlink whose value is read (using `readlink()`) from `content_path`. +The `runfiles_path` + +For runfiles entries, they have `--runfiles-dir` prepended to their computed +zip path. + +Compute `zip_path` from `runfile_path`: Computing the final zip path for +runfiles entries is a bit complicated, but boils down to computing what the +runfiles-root-relative path would be, with `--legacy-external-runfiles` taken +into account. +""", + ) + parser.add_argument("output", help="Path to the output zip file.") + parser.add_argument( + "--compression", + type=int, + default=0, + help="Compression level (0 for stored, others for deflated)", + ) + parser.add_argument("--workspace-name", default="", help="Name of the workspace") + parser.add_argument( + "--legacy-external-runfiles", + default="0", + choices=["0", "1"], + help="Whether to use legacy external runfiles behavior", + ) + parser.add_argument( + "--runfiles-dir", default="runfiles", help="Name of the runfiles directory" + ) + args = parser.parse_args() + + try: + create_zip( + manifest_path=args.manifest, + output_zip=args.output, + compress_level=args.compression, + workspace_name=args.workspace_name, + legacy_external_runfiles=args.legacy_external_runfiles == "1", + runfiles_dir=args.runfiles_dir, + ) + except Exception as e: + e.add_note(f"Error creating zip {args.output}") + raise + + +if __name__ == "__main__": + sys.exit(main()) From 1f22bdcea35958985ed83fdc376171ec01cd0b92 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 24 Jan 2026 19:44:42 -0800 Subject: [PATCH 02/10] disable test pre-bazel 8 --- tests/py_zipapp/BUILD.bazel | 2 ++ tests/support/support.bzl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel index 51d8ff7c56..cfcc0e93f8 100644 --- a/tests/py_zipapp/BUILD.bazel +++ b/tests/py_zipapp/BUILD.bazel @@ -1,6 +1,7 @@ load("//python:py_binary.bzl", "py_binary") load("//python:py_test.bzl", "py_test") load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") +load("//tests/support:support.bzl", "SUPPORTS_BAZEL_8") py_binary( name = "bin", @@ -16,6 +17,7 @@ py_binary( py_zipapp_binary( name = "bin_zipapp", binary = ":bin", + target_compatible_with = SUPPORTS_BAZEL_8, ) py_test( diff --git a/tests/support/support.bzl b/tests/support/support.bzl index b767ec2714..2d4f8d64fb 100644 --- a/tests/support/support.bzl +++ b/tests/support/support.bzl @@ -45,6 +45,8 @@ NOT_WINDOWS = select({ "//conditions:default": [], }) +BAZEL_8_OR_LATER = [] if config.bazel_8_or_later else ["@platforms//:incompatible"] + def maybe_builtin_build_python_zip(value): settings = {} if not config.bazel_10_or_later: From adb589f9daef43b213091c1f36fa7dfd761c923d Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 25 Jan 2026 10:05:05 -0800 Subject: [PATCH 03/10] add bazel 7 support --- python/private/zipapp/py_zipapp_rule.bzl | 12 +++++++++--- tests/py_zipapp/BUILD.bazel | 4 ++-- tools/zipapp/zipper.py | 16 +++++++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index 068aeabac6..9a4810667a 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -8,6 +8,12 @@ load("//python/private:py_internal.bzl", "py_internal") load("//python/private:py_runtime_info.bzl", "PyRuntimeInfo") load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE") +def _is_symlink(f): + if hasattr(f, "is_symlink"): + return str(int(f.is_symlink)) + else: + return "-1" + def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap): python_exe = py_executable.venv_python_exe python_exe_path = runfiles_root_path(ctx, python_exe.short_path) @@ -34,13 +40,13 @@ def _map_zip_empty_filenames(list_paths_cb): return ["rf-empty|" + path for path in list_paths_cb().to_list()] def _map_zip_runfiles(file): - return "rf-file|" + str(int(file.is_symlink)) + "|" + file.short_path + "|" + file.path + return "rf-file|" + _is_symlink(file) + "|" + file.short_path + "|" + file.path def _map_zip_symlinks(entry): - return "rf-symlink|" + str(int(entry.target_file.is_symlink)) + "|" + entry.path + "|" + entry.target_file.path + return "rf-symlink|" + _is_symlink(entry.target_file) + "|" + entry.path + "|" + entry.target_file.path def _map_zip_root_symlinks(entry): - return "rf-root-symlink|" + str(int(entry.target_file.is_symlink)) + "|" + entry.path + "|" + entry.target_file.path + return "rf-root-symlink|" + _is_symlink(entry.target_file) + "|" + entry.path + "|" + entry.target_file.path def _build_manifest(ctx, manifest, runfiles, zip_main): manifest.add("regular|0|__main__.py|{}".format(zip_main.path)) diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel index cfcc0e93f8..a148ec92c7 100644 --- a/tests/py_zipapp/BUILD.bazel +++ b/tests/py_zipapp/BUILD.bazel @@ -1,7 +1,7 @@ load("//python:py_binary.bzl", "py_binary") load("//python:py_test.bzl", "py_test") load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") -load("//tests/support:support.bzl", "SUPPORTS_BAZEL_8") +load("//tests/support:support.bzl", "BAZEL_8_OR_LATER") py_binary( name = "bin", @@ -17,7 +17,7 @@ py_binary( py_zipapp_binary( name = "bin_zipapp", binary = ":bin", - target_compatible_with = SUPPORTS_BAZEL_8, + #target_compatible_with = BAZEL_8_OR_LATER, ) py_test( diff --git a/tools/zipapp/zipper.py b/tools/zipapp/zipper.py index 2263f13125..b5afdc4223 100644 --- a/tools/zipapp/zipper.py +++ b/tools/zipapp/zipper.py @@ -102,6 +102,12 @@ def _write_entry(zf, entry, compress_type): zf.writestr(zi, "") return + if is_symlink_str == "-1": + if not os.path.exists(content_path): + is_symlink_str = "1" + else: + is_symlink_str = "0" + is_symlink = is_symlink_str == "1" if is_symlink: @@ -170,9 +176,13 @@ def main(): 5. `rf-root-symlink|is_symlink|runfile_root_path|content_path`: Store a runfiles-root-relative path in the zip. -In all cases, `is_symlink` is `1` or `0` if the path should be stored -as a symlink whose value is read (using `readlink()`) from `content_path`. -The `runfiles_path` +In all cases, `is_symlink` has the following values: +* `1` means it should be stored as a symlink whose value is read + (using `readlink()`) from `content_path`. +* `0` means to store it as a regular file, read from `content_path` +* `-1` occurs with Bazel 7 (because it lacks `File.is_symlink`), which means + to infer whether it's a symlink (files to be stored as symlinks can be + determined by looking for symlinks that point to non-existent files). For runfiles entries, they have `--runfiles-dir` prepended to their computed zip path. From 8ced1dbd8ca82afbbda87c3d663f74b5692e0fbe Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 25 Jan 2026 10:34:42 -0800 Subject: [PATCH 04/10] make test support workspace mode --- tests/py_zipapp/BUILD.bazel | 16 +++++++++------- .../{zipapp_test.py => venv_zipapp_test.py} | 7 ++++++- 2 files changed, 15 insertions(+), 8 deletions(-) rename tests/py_zipapp/{zipapp_test.py => venv_zipapp_test.py} (92%) diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel index a148ec92c7..01c63b7198 100644 --- a/tests/py_zipapp/BUILD.bazel +++ b/tests/py_zipapp/BUILD.bazel @@ -1,10 +1,11 @@ load("//python:py_binary.bzl", "py_binary") load("//python:py_test.bzl", "py_test") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") load("//tests/support:support.bzl", "BAZEL_8_OR_LATER") py_binary( - name = "bin", + name = "venv_bin", srcs = ["main.py"], config_settings = { "//python/config_settings:bootstrap_impl": "script", @@ -15,16 +16,17 @@ py_binary( ) py_zipapp_binary( - name = "bin_zipapp", - binary = ":bin", + name = "venv_zipapp", + binary = ":venv_bin", #target_compatible_with = BAZEL_8_OR_LATER, ) py_test( - name = "zipapp_test", - srcs = ["zipapp_test.py"], - data = [":bin_zipapp"], + name = "venv_zipapp_test", + srcs = ["venv_zipapp_test.py"], + data = [":venv_zipapp"], env = { - "TEST_ZIPAPP": "$(location :bin_zipapp)", + "TEST_ZIPAPP": "$(location :venv_zipapp)", + "BZLMOD_ENABLED": str(int(BZLMOD_ENABLED)), }, ) diff --git a/tests/py_zipapp/zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py similarity index 92% rename from tests/py_zipapp/zipapp_test.py rename to tests/py_zipapp/venv_zipapp_test.py index c2933ae133..b9840fb40a 100644 --- a/tests/py_zipapp/zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -36,12 +36,17 @@ def assertZipEntryIsSymlink(self, zip_file, path, msg=None): if not is_symlink: self.fail(msg or f"Path '{path}' is not a symlink") + def _is_bzlmod_enabled(self): + return os.environ["BZLMOD_ENABLED"] == "1" + def test_zipapp_structure(self): zipapp_path = os.environ["TEST_ZIPAPP"] with zipfile.ZipFile(zipapp_path, "r") as zf: namelist = zf.namelist() - self.assertIn("runfiles/_repo_mapping", namelist) + + if self._is_bzlmod_enabled(): + self.assertIn("runfiles/_repo_mapping", namelist) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") From 5dbfab4cd3e7c4d67da19d691b9691bb6bff7f29 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 25 Jan 2026 10:36:04 -0800 Subject: [PATCH 05/10] cleanup --- tests/py_zipapp/BUILD.bazel | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel index 01c63b7198..91f0d5163f 100644 --- a/tests/py_zipapp/BUILD.bazel +++ b/tests/py_zipapp/BUILD.bazel @@ -2,7 +2,6 @@ load("//python:py_binary.bzl", "py_binary") load("//python:py_test.bzl", "py_test") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") -load("//tests/support:support.bzl", "BAZEL_8_OR_LATER") py_binary( name = "venv_bin", @@ -18,7 +17,6 @@ py_binary( py_zipapp_binary( name = "venv_zipapp", binary = ":venv_bin", - #target_compatible_with = BAZEL_8_OR_LATER, ) py_test( From 89b438ac3a8a732c34f551e71cafc0f72dbb8b31 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 25 Jan 2026 10:46:11 -0800 Subject: [PATCH 06/10] basic impl of config_settings attr --- python/private/zipapp/py_zipapp_rule.bzl | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index 9a4810667a..c42945bbcf 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -1,12 +1,14 @@ """Implementation of the zipapp rules.""" load("@bazel_skylib//lib:paths.bzl", "paths") +load("//python/private:attributes.bzl", "apply_config_settings_attr") load("//python/private:builders.bzl", "builders") load("//python/private:common.bzl", "actions_run", "maybe_create_repo_mapping", "runfiles_root_path") load("//python/private:py_executable_info.bzl", "PyExecutableInfo") load("//python/private:py_internal.bzl", "py_internal") load("//python/private:py_runtime_info.bzl", "PyRuntimeInfo") load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE") +load("//python/private:transition_labels.bzl", "TRANSITION_LABELS") def _is_symlink(f): if hasattr(f, "is_symlink"): @@ -193,6 +195,15 @@ def _py_zipapp_executable_impl(ctx): ), ] +def _transition_zipapp_impl(settings, attr): + return apply_config_settings_attr(dict(settings), attr) + +_zipapp_transition = transition( + implementation = _transition_zipapp_impl, + inputs = TRANSITION_LABELS, + outputs = TRANSITION_LABELS, +) + _ATTRS = { "binary": attr.label( doc = """ @@ -209,12 +220,45 @@ Typically 0 to 9, with higher numbers being to compress more. """, default = "", ), + "config_settings": attr.label_keyed_string_dict( + doc = """ +Config settings to change for this target. + +The keys are labels for settings, and the values are strings for the new value +to use. Pass `Label` objects or canonical label strings for the keys to ensure +they resolve as expected (canonical labels start with `@@` and can be +obtained by calling `str(Label(...))`). + +Most `@rules_python//python/config_setting` settings can be used here, which +allows, for example, making only a certain `py_binary` use +{obj}`--boostrap_impl=script`. + +Additional or custom config settings can be registered using the +{obj}`add_transition_setting` API. This allows, for example, forcing a +particular CPU, or defining a custom setting that `select()` uses elsewhere +to pick between `pip.parse` hubs. See the [How to guide on multiple +versions of a library] for a more concrete example. + +:::{note} +These values are transitioned on, so will affect the analysis graph and the +associated memory overhead. The more unique configurations in your overall +build, the more memory and (often unnecessary) re-analysis and re-building +can occur. See +https://bazel.build/extending/config#memory-performance-considerations for +more information about risks and considerations. +::: +""", + ), "executable": attr.bool( doc = """ Whether the output should be an executable zip file. """, default = True, ), + # Required to opt-in to the transition feature. + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), "_exe_zip_maker": attr.label( cfg = "exec", default = "//tools/zipapp:exe_zip_maker", @@ -240,6 +284,7 @@ Packages a `py_binary` as a Python zipapp. # based on the `executable` attribute. executable = True, toolchains = _TOOLCHAINS, + cfg = _zipapp_transition, ) py_zipapp_test = rule( @@ -254,4 +299,5 @@ This target is also a valid test target to run. # based on the `executable` attribute. test = True, toolchains = _TOOLCHAINS, + cfg = _zipapp_transition, ) From 5ae496ed222fbad9e678842377084793bae39f1a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 25 Jan 2026 18:51:47 -0800 Subject: [PATCH 07/10] add missing bzl deps --- python/private/BUILD.bazel | 2 ++ python/private/zipapp/BUILD.bazel | 2 ++ 2 files changed, 4 insertions(+) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 18aeb0eeeb..a9a4537a0f 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -78,7 +78,9 @@ bzl_library( ":py_info_bzl", ":py_internal_bzl", ":reexports_bzl", + ":rule_builders_bzl", ":rules_cc_srcs_bzl", + "@bazel_skylib//lib:dicts", "@bazel_skylib//rules:common_settings", ], ) diff --git a/python/private/zipapp/BUILD.bazel b/python/private/zipapp/BUILD.bazel index 2e96178144..41c0bfd2fb 100644 --- a/python/private/zipapp/BUILD.bazel +++ b/python/private/zipapp/BUILD.bazel @@ -15,6 +15,7 @@ bzl_library( name = "py_zipapp_rule_bzl", srcs = ["py_zipapp_rule.bzl"], deps = [ + "//python/private:attributes_bzl", "//python/private:builders_bzl", "//python/private:common_bzl", "//python/private:py_executable_info_bzl", @@ -23,6 +24,7 @@ bzl_library( "//python/private:py_interpreter_program_bzl", "//python/private:py_runtime_info_bzl", "//python/private:toolchain_types_bzl", + "//python/private:transition_labels_bzl", "@bazel_skylib//lib:paths", ], ) From 3518a8bdca4277f0b0fe330545a37dafff6fc51f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 25 Jan 2026 19:06:39 -0800 Subject: [PATCH 08/10] format code --- python/private/zipapp/BUILD.bazel | 2 -- tests/py_zipapp/BUILD.bazel | 2 +- tests/py_zipapp/venv_zipapp_test.py | 4 ++-- tools/zipapp/zipper.py | 8 ++++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/python/private/zipapp/BUILD.bazel b/python/private/zipapp/BUILD.bazel index 41c0bfd2fb..9b02ae5bc4 100644 --- a/python/private/zipapp/BUILD.bazel +++ b/python/private/zipapp/BUILD.bazel @@ -29,8 +29,6 @@ bzl_library( ], ) - - filegroup( name = "zip_main_template", srcs = ["zip_main_template.py"], diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel index 91f0d5163f..f77bc91a52 100644 --- a/tests/py_zipapp/BUILD.bazel +++ b/tests/py_zipapp/BUILD.bazel @@ -24,7 +24,7 @@ py_test( srcs = ["venv_zipapp_test.py"], data = [":venv_zipapp"], env = { - "TEST_ZIPAPP": "$(location :venv_zipapp)", "BZLMOD_ENABLED": str(int(BZLMOD_ENABLED)), + "TEST_ZIPAPP": "$(location :venv_zipapp)", }, ) diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index b9840fb40a..40d20fedb4 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -37,7 +37,7 @@ def assertZipEntryIsSymlink(self, zip_file, path, msg=None): self.fail(msg or f"Path '{path}' is not a symlink") def _is_bzlmod_enabled(self): - return os.environ["BZLMOD_ENABLED"] == "1" + return os.environ["BZLMOD_ENABLED"] == "1" def test_zipapp_structure(self): zipapp_path = os.environ["TEST_ZIPAPP"] @@ -46,7 +46,7 @@ def test_zipapp_structure(self): namelist = zf.namelist() if self._is_bzlmod_enabled(): - self.assertIn("runfiles/_repo_mapping", namelist) + self.assertIn("runfiles/_repo_mapping", namelist) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") diff --git a/tools/zipapp/zipper.py b/tools/zipapp/zipper.py index b5afdc4223..6f41c1e663 100644 --- a/tools/zipapp/zipper.py +++ b/tools/zipapp/zipper.py @@ -103,10 +103,10 @@ def _write_entry(zf, entry, compress_type): return if is_symlink_str == "-1": - if not os.path.exists(content_path): - is_symlink_str = "1" - else: - is_symlink_str = "0" + if not os.path.exists(content_path): + is_symlink_str = "1" + else: + is_symlink_str = "0" is_symlink = is_symlink_str == "1" From b23f4529fe9be20b45420cf6d4fa035618d0fb59 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 25 Jan 2026 19:17:30 -0800 Subject: [PATCH 09/10] disable buildifier warning --- .bazelrc.deleted_packages | 4 ---- tests/py_zipapp/BUILD.bazel | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index d11f96d664..c683ec9068 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -8,10 +8,6 @@ common --deleted_packages=examples/bzlmod/libs/my_lib common --deleted_packages=examples/bzlmod/other_module common --deleted_packages=examples/bzlmod/other_module/other_module/pkg common --deleted_packages=examples/bzlmod/patches -common --deleted_packages=examples/bzlmod/py_proto_library -common --deleted_packages=examples/bzlmod/py_proto_library/example.com/another_proto -common --deleted_packages=examples/bzlmod/py_proto_library/example.com/proto -common --deleted_packages=examples/bzlmod/py_proto_library/foo_external common --deleted_packages=examples/bzlmod/runfiles common --deleted_packages=examples/bzlmod/tests common --deleted_packages=examples/bzlmod/tests/other_module diff --git a/tests/py_zipapp/BUILD.bazel b/tests/py_zipapp/BUILD.bazel index f77bc91a52..b1830265dc 100644 --- a/tests/py_zipapp/BUILD.bazel +++ b/tests/py_zipapp/BUILD.bazel @@ -1,6 +1,6 @@ load("//python:py_binary.bzl", "py_binary") load("//python:py_test.bzl", "py_test") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") py_binary( From 6528660d9baa7ffa58533ec3f6473ea2d9991f54 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 26 Jan 2026 17:34:02 -0800 Subject: [PATCH 10/10] fix case when venv is none --- python/private/py_executable.bzl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 1295ded1e3..b16a415b69 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -457,8 +457,9 @@ def _create_executable( app_runfiles = builders.RunfilesBuilder() app_runfiles.add(runfiles_details.app_runfiles) - app_runfiles.add(venv.files_without_interpreter) - app_runfiles.add(venv.lib_runfiles) + if venv: + app_runfiles.add(venv.files_without_interpreter) + app_runfiles.add(venv.lib_runfiles) # The interpreter is added this late in the process so that it isn't # added to the zipped files.