Fix windows-specific issues for pywrap rules

PiperOrigin-RevId: 694978320
This commit is contained in:
Vadym Matsishevskyi 2024-11-09 21:24:31 -08:00 committed by TensorFlower Gardener
parent 41c434855a
commit 641de23853
21 changed files with 1339 additions and 20 deletions

View File

@ -370,6 +370,8 @@ build:windows --host_copt=-D_ENABLE_EXTENDED_ALIGNED_STORAGE
# runfiles symlink tree to decide what to put into the Python wheel.
startup --windows_enable_symlinks
build:windows --enable_runfiles
build:windows --nobuild_python_zip
build:windows --dynamic_mode=off
# Default paths for TF_SYSTEM_LIBS
build:linux --define=PREFIX=/usr
@ -443,6 +445,7 @@ build:avx_win --copt=/arch:AVX
# Use Clang-cl compiler on Windows
build:win_clang --copt=/clang:-Weverything
build:win_clang --host_copt=/clang:-Weverything
build:win_clang --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
build:win_clang --extra_execution_platforms=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
build:win_clang --host_platform=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
@ -454,6 +457,7 @@ test:win_clang --host_linkopt=/FORCE:MULTIPLE
# Same config as above but for XLA, which has different toolchain paths
build:win_clang_xla --copt=/clang:-Weverything
build:win_clang_xla --host_copt=/clang:-Weverything
build:win_clang_xla --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
build:win_clang_xla --extra_execution_platforms=//tools/toolchains/win:x64_windows-clang-cl
build:win_clang_xla --host_platform=//tools/toolchains/win:x64_windows-clang-cl
@ -811,10 +815,12 @@ build:macos_x86_pycpp_test --config=macos_x86_pycpp_test_filters -- //tensorflow
build:cross_compile_macos_x86_pycpp_test --config=macos_x86_pycpp_test
build:cross_compile_macos_x86_pycpp_test -//tensorflow/core/kernels:quantized_conv_ops_test -//tensorflow/core/kernels:quantized_matmul_op_test -//tensorflow/python/ops:quantized_conv_ops_test -//tensorflow/tools/graph_transforms:transforms_test -//tensorflow/python/tools:aot_compiled_test
# WINDOWS X86-64 CPU PYCPP
build:windows_x86_cpu_pycpp_test_build_opts --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions --dynamic_mode=off
build:windows_x86_cpu_pycpp_test_build_opts_debug --config=windows_x86_cpu_pycpp_test_build_opts --linkopt=/demangle:no --host_linkopt=/demangle:no --linkopt=/errorlimit:0 --host_linkopt=/errorlimit:0
test:windows_x86_cpu_pycpp_test_filters --test_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-gpu,-tpu,-benchmark-test
test:windows_x86_cpu_pycpp_test_filters --build_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-benchmark-test
test:windows_x86_cpu_pycpp_test_filters --test_lang_filters=cc,py --test_size_filters=small,medium --test_timeout="300,450,1200,3600"
test:windows_x86_cpu_pycpp_test_opts --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions --dynamic_mode=off --build_tests_only
test:windows_x86_cpu_pycpp_test_opts --config=windows_x86_cpu_pycpp_test_build_opts --build_tests_only
test:windows_x86_cpu_pycpp_test --config=windows_x86_cpu_pycpp_test_opts --config=windows_x86_cpu_pycpp_test_filters -- //tensorflow/... -//tensorflow/java/... -//tensorflow/lite/... -//tensorflow/compiler/...
# END TF TEST SUITE OPTIONS

View File

@ -1,3 +1,4 @@
load("@local_tsl//tsl/platform:build_config_root.bzl", "if_pywrap")
load("//tensorflow:pytype.default.bzl", "pytype_strict_library")
load(
"//tensorflow:tensorflow.default.bzl",
@ -156,11 +157,13 @@ tf_python_pybind_extension(
],
# Each dependency MUST be either header-only or exclusive.
deps = [
":pywrap_quantization_lib_header_only",
"//tensorflow/compiler/mlir/quantization/tensorflow/python:type_casters",
"@pybind11",
"@pybind11_abseil//pybind11_abseil:absl_casters",
"@pybind11_abseil//pybind11_abseil:import_status_module",
"@pybind11_abseil//pybind11_abseil:status_casters",
],
] + if_pywrap(
if_false = [":pywrap_quantization_lib_header_only"],
if_true = [":pywrap_quantization_lib_impl"],
),
)

View File

@ -49,7 +49,7 @@ def gen_op_libraries(
srcs = [],
outs = [name + ".inc.cc"],
cmd =
"PYWRAP_TARGET='//third_party/tensorflow/python:_pywrap_tensorflow' " +
"PYWRAP_TARGET='//tensorflow/python:_pywrap_tensorflow' " +
"$(location %s) --output=$@ --gen_register_op=true" % gen_op_lib_exec,
tools = [":" + gen_op_lib_exec],
tags = tags,
@ -114,7 +114,7 @@ def gen_op_libraries(
srcs = [],
outs = [name + ".mlir"],
cmd =
"PYWRAP_TARGET='//third_party/tensorflow/python:_pywrap_tensorflow' " +
"PYWRAP_TARGET='//tensorflow/python:_pywrap_tensorflow' " +
"$(location %s) --output=$@ --gen_register_op=false" % gen_tfr_lib_exec,
tools = [":" + gen_tfr_lib_exec],
tags = tags,

View File

@ -27,6 +27,10 @@ alias(
tf_proto_library(
name = "op_performance_data",
srcs = ["op_performance_data.proto"],
local_defines = [
"PROTOBUF_USE_DLLS",
"LIBPROTOBUF_EXPORTS",
],
make_default_target_header_only = True,
protodeps = [
"//tensorflow/core/framework:attr_value_proto",

View File

@ -196,6 +196,10 @@ tf_proto_library(
"core_platform_payloads.proto",
"fingerprint.proto",
],
local_defines = [
"PROTOBUF_USE_DLLS",
"LIBPROTOBUF_EXPORTS",
],
make_default_target_header_only = True,
protodeps = [
":error_codes_proto_impl",

View File

@ -567,7 +567,7 @@ tf_python_pybind_extension(
"//tensorflow/python/lib/core:pybind11_lib",
"//third_party/python_runtime:headers",
"@pybind11",
],
] + if_pywrap(["//tensorflow/lite/toco/python:toco_python_api"]),
)
# TODO(edloper): Remove unused dependency on safe_ptr. (Blocker: there are
@ -1461,6 +1461,7 @@ pywrap_library(
"-DEFAULTLIB:advapi32.lib",
"-DEFAULTLIB:crypt32.lib",
"-DEFAULTLIB:Normaliz.lib",
"-DEFAULTLIB:ntdll.lib",
],
"//conditions:default": [],
}),
@ -1501,6 +1502,7 @@ pywrap_library(
],
}),
visibility = ["//visibility:public"],
win_def_file = "_pywrap_tensorflow.def",
# win_def_file = "_pywrap_tensorflow.def",
deps = [
":_pywrap_quantize_training",

File diff suppressed because it is too large Load Diff

View File

@ -502,7 +502,7 @@ genrule(
name = "create_models_for_aot_compile",
outs = EMITTED_AOT_SAVE_MODEL_OBJECTS,
cmd = (
"PYWRAP_TARGET='//third_party/tensorflow/python:_pywrap_tensorflow' " +
"PYWRAP_TARGET='//tensorflow/python:_pywrap_tensorflow' " +
"$(location :make_aot_compile_models) --out_dir $(@D)"
),
tags = ["no_rocm"],

View File

@ -132,7 +132,7 @@ def saved_model_compile_aot(
"{}_makefile.inc".format(name),
],
cmd = (
"PYWRAP_TARGET='//third_party/tensorflow/python:_pywrap_tensorflow' " +
"PYWRAP_TARGET='//tensorflow/python:_pywrap_tensorflow' " +
"$(location {}) aot_compile_cpu ".format(
clean_dep("//tensorflow/python/tools:saved_model_cli"),
) +

View File

@ -234,7 +234,7 @@ tf_python_pybind_extension(
"//tensorflow/core/platform:status",
"//tensorflow/python/lib/core:pybind11_status",
"@pybind11",
],
] + if_pywrap(["//tensorflow/tools/graph_transforms:transform_graph_lib"]),
)
tf_python_pybind_extension(

14
third_party/absl/build_dll.patch vendored Normal file
View File

@ -0,0 +1,14 @@
Index: absl/copts/GENERATED_copts.bzl
<+>UTF-8
===================================================================
diff --git a/absl/copts/GENERATED_copts.bzl b/absl/copts/GENERATED_copts.bzl
--- a/absl/copts/GENERATED_copts.bzl (revision fb3621f4f897824c0dbe0615fa94543df6192f30)
+++ b/absl/copts/GENERATED_copts.bzl (date 1729733688013)
@@ -11,6 +11,7 @@
"/D_CRT_SECURE_NO_WARNINGS",
"/D_SCL_SECURE_NO_WARNINGS",
"/D_ENABLE_EXTENDED_ALIGNED_STORAGE",
+ "/DABSL_BUILD_DLL",
]
ABSL_CLANG_CL_TEST_FLAGS = [

View File

@ -44,5 +44,8 @@ def repo():
system_link_files = SYS_LINKS,
strip_prefix = "abseil-cpp-{commit}".format(commit = ABSL_COMMIT),
urls = tf_mirror_urls("https://github.com/abseil/abseil-cpp/archive/{commit}.tar.gz".format(commit = ABSL_COMMIT)),
patch_file = ["//third_party/absl:nvidia_jetson.patch"],
patch_file = [
"//third_party/absl:nvidia_jetson.patch",
"//third_party/absl:build_dll.patch",
],
)

View File

@ -1,9 +1,10 @@
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 38970d9929b9..2690c97aa3e0 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -34,6 +34,36 @@ exports_files([
"utils/lit/lit.py",
"utils/lldbDataFormatters.py",
])
+config_setting(

View File

@ -23,7 +23,6 @@ def repo(name):
"//third_party/llvm:toolchains.patch",
"//third_party/llvm:zstd.patch",
"//third_party/llvm:rocdl_shuffle_down.patch",
"//third_party/llvm:intrinsic_memcpy.patch",
],
link_files = {"//third_party/llvm:run_lit.sh": "mlir/run_lit.sh"},
)

View File

@ -370,6 +370,8 @@ build:windows --host_copt=-D_ENABLE_EXTENDED_ALIGNED_STORAGE
# runfiles symlink tree to decide what to put into the Python wheel.
startup --windows_enable_symlinks
build:windows --enable_runfiles
build:windows --nobuild_python_zip
build:windows --dynamic_mode=off
# Default paths for TF_SYSTEM_LIBS
build:linux --define=PREFIX=/usr
@ -443,6 +445,7 @@ build:avx_win --copt=/arch:AVX
# Use Clang-cl compiler on Windows
build:win_clang --copt=/clang:-Weverything
build:win_clang --host_copt=/clang:-Weverything
build:win_clang --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
build:win_clang --extra_execution_platforms=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
build:win_clang --host_platform=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
@ -454,6 +457,7 @@ test:win_clang --host_linkopt=/FORCE:MULTIPLE
# Same config as above but for XLA, which has different toolchain paths
build:win_clang_xla --copt=/clang:-Weverything
build:win_clang_xla --host_copt=/clang:-Weverything
build:win_clang_xla --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
build:win_clang_xla --extra_execution_platforms=//tools/toolchains/win:x64_windows-clang-cl
build:win_clang_xla --host_platform=//tools/toolchains/win:x64_windows-clang-cl
@ -811,10 +815,12 @@ build:macos_x86_pycpp_test --config=macos_x86_pycpp_test_filters -- //tensorflow
build:cross_compile_macos_x86_pycpp_test --config=macos_x86_pycpp_test
build:cross_compile_macos_x86_pycpp_test -//tensorflow/core/kernels:quantized_conv_ops_test -//tensorflow/core/kernels:quantized_matmul_op_test -//tensorflow/python/ops:quantized_conv_ops_test -//tensorflow/tools/graph_transforms:transforms_test -//tensorflow/python/tools:aot_compiled_test
# WINDOWS X86-64 CPU PYCPP
build:windows_x86_cpu_pycpp_test_build_opts --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions --dynamic_mode=off
build:windows_x86_cpu_pycpp_test_build_opts_debug --config=windows_x86_cpu_pycpp_test_build_opts --linkopt=/demangle:no --host_linkopt=/demangle:no --linkopt=/errorlimit:0 --host_linkopt=/errorlimit:0
test:windows_x86_cpu_pycpp_test_filters --test_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-gpu,-tpu,-benchmark-test
test:windows_x86_cpu_pycpp_test_filters --build_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-benchmark-test
test:windows_x86_cpu_pycpp_test_filters --test_lang_filters=cc,py --test_size_filters=small,medium --test_timeout="300,450,1200,3600"
test:windows_x86_cpu_pycpp_test_opts --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions --dynamic_mode=off --build_tests_only
test:windows_x86_cpu_pycpp_test_opts --config=windows_x86_cpu_pycpp_test_build_opts --build_tests_only
test:windows_x86_cpu_pycpp_test --config=windows_x86_cpu_pycpp_test_opts --config=windows_x86_cpu_pycpp_test_filters -- //tensorflow/... -//tensorflow/java/... -//tensorflow/lite/... -//tensorflow/compiler/...
# END TF TEST SUITE OPTIONS

View File

@ -370,6 +370,8 @@ build:windows --host_copt=-D_ENABLE_EXTENDED_ALIGNED_STORAGE
# runfiles symlink tree to decide what to put into the Python wheel.
startup --windows_enable_symlinks
build:windows --enable_runfiles
build:windows --nobuild_python_zip
build:windows --dynamic_mode=off
# Default paths for TF_SYSTEM_LIBS
build:linux --define=PREFIX=/usr
@ -443,6 +445,7 @@ build:avx_win --copt=/arch:AVX
# Use Clang-cl compiler on Windows
build:win_clang --copt=/clang:-Weverything
build:win_clang --host_copt=/clang:-Weverything
build:win_clang --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
build:win_clang --extra_execution_platforms=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
build:win_clang --host_platform=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
@ -454,6 +457,7 @@ test:win_clang --host_linkopt=/FORCE:MULTIPLE
# Same config as above but for XLA, which has different toolchain paths
build:win_clang_xla --copt=/clang:-Weverything
build:win_clang_xla --host_copt=/clang:-Weverything
build:win_clang_xla --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
build:win_clang_xla --extra_execution_platforms=//tools/toolchains/win:x64_windows-clang-cl
build:win_clang_xla --host_platform=//tools/toolchains/win:x64_windows-clang-cl
@ -811,10 +815,12 @@ build:macos_x86_pycpp_test --config=macos_x86_pycpp_test_filters -- //tensorflow
build:cross_compile_macos_x86_pycpp_test --config=macos_x86_pycpp_test
build:cross_compile_macos_x86_pycpp_test -//tensorflow/core/kernels:quantized_conv_ops_test -//tensorflow/core/kernels:quantized_matmul_op_test -//tensorflow/python/ops:quantized_conv_ops_test -//tensorflow/tools/graph_transforms:transforms_test -//tensorflow/python/tools:aot_compiled_test
# WINDOWS X86-64 CPU PYCPP
build:windows_x86_cpu_pycpp_test_build_opts --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions --dynamic_mode=off
build:windows_x86_cpu_pycpp_test_build_opts_debug --config=windows_x86_cpu_pycpp_test_build_opts --linkopt=/demangle:no --host_linkopt=/demangle:no --linkopt=/errorlimit:0 --host_linkopt=/errorlimit:0
test:windows_x86_cpu_pycpp_test_filters --test_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-gpu,-tpu,-benchmark-test
test:windows_x86_cpu_pycpp_test_filters --build_tag_filters=-no_windows,-windows_excluded,-no_oss,-oss_excluded,-benchmark-test
test:windows_x86_cpu_pycpp_test_filters --test_lang_filters=cc,py --test_size_filters=small,medium --test_timeout="300,450,1200,3600"
test:windows_x86_cpu_pycpp_test_opts --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions --dynamic_mode=off --build_tests_only
test:windows_x86_cpu_pycpp_test_opts --config=windows_x86_cpu_pycpp_test_build_opts --build_tests_only
test:windows_x86_cpu_pycpp_test --config=windows_x86_cpu_pycpp_test_opts --config=windows_x86_cpu_pycpp_test_filters -- //tensorflow/... -//tensorflow/java/... -//tensorflow/lite/... -//tensorflow/compiler/...
# END TF TEST SUITE OPTIONS

View File

@ -0,0 +1,14 @@
Index: absl/copts/GENERATED_copts.bzl
<+>UTF-8
===================================================================
diff --git a/absl/copts/GENERATED_copts.bzl b/absl/copts/GENERATED_copts.bzl
--- a/absl/copts/GENERATED_copts.bzl (revision fb3621f4f897824c0dbe0615fa94543df6192f30)
+++ b/absl/copts/GENERATED_copts.bzl (date 1729733688013)
@@ -11,6 +11,7 @@
"/D_CRT_SECURE_NO_WARNINGS",
"/D_SCL_SECURE_NO_WARNINGS",
"/D_ENABLE_EXTENDED_ALIGNED_STORAGE",
+ "/DABSL_BUILD_DLL",
]
ABSL_CLANG_CL_TEST_FLAGS = [

View File

@ -44,5 +44,8 @@ def repo():
system_link_files = SYS_LINKS,
strip_prefix = "abseil-cpp-{commit}".format(commit = ABSL_COMMIT),
urls = tf_mirror_urls("https://github.com/abseil/abseil-cpp/archive/{commit}.tar.gz".format(commit = ABSL_COMMIT)),
patch_file = ["//third_party/absl:nvidia_jetson.patch"],
patch_file = [
"//third_party/absl:nvidia_jetson.patch",
"//third_party/absl:build_dll.patch",
],
)

View File

@ -108,7 +108,7 @@ def pywrap_library(
testonly,
compatible_with,
py_cc_win_def_file,
["PROTOBUF_USE_DLLS"],
None,
)
common_deps = extra_deps + [
@ -158,7 +158,6 @@ def pywrap_library(
win_def_file = ":%s" % win_def_name,
testonly = testonly,
compatible_with = compatible_with,
local_defines = ["PROTOBUF_USE_DLLS"],
)
shared_objects.append(":%s" % shared_object_name)
@ -465,7 +464,7 @@ def pybind_extension(
visibility = visibility,
testonly = testonly,
compatible_with = compatible_with,
local_defines = ["PROTOBUF_USE_DLLS"],
local_defines = ["PROTOBUF_USE_DLLS", "ABSL_CONSUME_DLL"],
**kwargs
)

View File

@ -461,7 +461,8 @@ def tf_proto_library_cc(
create_service = False,
create_java_proto = False,
create_kotlin_proto = False,
make_default_target_header_only = False):
make_default_target_header_only = False,
local_defines = None):
js_codegen = js_codegen # unused argument
native.filegroup(
name = name + "_proto_srcs",
@ -548,6 +549,7 @@ def tf_proto_library_cc(
visibility = visibility,
deps = cc_deps,
protolib_deps = protolib_deps,
local_defines = local_defines,
)
def tf_proto_library_py(
@ -612,7 +614,8 @@ def tf_proto_library(
create_grpc_library = False,
make_default_target_header_only = False,
exports = [],
tags = []):
tags = [],
local_defines = None):
"""Make a proto library, possibly depending on other proto libraries."""
# TODO(b/145545130): Add docstring explaining what rules this creates and how
@ -651,6 +654,7 @@ def tf_proto_library(
make_default_target_header_only = make_default_target_header_only,
protodeps = protodeps,
visibility = visibility,
local_defines = local_defines,
)
if create_grpc_library:

View File

@ -15,6 +15,10 @@ package_group(
tf_proto_library(
name = "xplane_proto",
srcs = ["xplane.proto"],
local_defines = [
"PROTOBUF_USE_DLLS",
"LIBPROTOBUF_EXPORTS",
],
make_default_target_header_only = True,
visibility = internal_visibility([":friends"]),
)