tensorflow/third_party/protobuf/protobuf.patch
Vadym Matsishevskyi 5720ab7845 Introduce pywrap bazel rules and migrate Tensorflow to it
The gist of this change (the new rules implementation) is contained within `rules_pywrap` folder. The rules are generic and not tensorflow-specific

1) (internal-specific)

2) (internal-specific)

3) It provides same linking strategy of final artifacts on all 3 supported platforms (no major differences between Linux, Mac and Windows).

4) It makes it possible to abandon usage of header-only targets to prevent ODR violations. Simply speaking you can now depend on generated protobuf message classes normally, without need to worry how that is linked afterwards.

5) The current version is backward-compatible and unless explicitly enabled is a no-op. To enable the new rules pass `--repo_env=USE_PYWRAP_RULES=True` flag to build/test command.

6) The `if_pywrap` construct is temporary and will be removed once full migration is completed. Currently if_pywrap is mainly used to pass normal dependencies (instead of header-only). The header-only stuff is kept for backward compatibility and smoother migration but will be eventually removed.

7) This CL migrates TF and the most problematic among all google ML repositories. Once TF is sabilized the other repositories, such as JAX and XLA will be migrated too (which should be way easier than migrating TF anyways)

PiperOrigin-RevId: 684324990
2024-10-10 00:12:03 -07:00

165 lines
4.9 KiB
Diff

diff --git a/BUILD.bazel b/BUILD.bazel
--- a/BUILD.bazel (revision 90b73ac3f0b10320315c2ca0d03a5a9b095d2f66)
+++ b/BUILD.bazel (date 1714620794503)
@@ -68,6 +68,8 @@
copts = COPTS,
includes = ["src/"],
linkopts = LINK_OPTS,
+ local_defines = ["PROTOBUF_USE_DLLS", "LIBPROTOBUF_EXPORTS"],
+ alwayslink = 1,
visibility = ["//visibility:public"],
)
@@ -135,6 +137,8 @@
copts = COPTS,
includes = ["src/"],
linkopts = LINK_OPTS,
+ local_defines = ["PROTOBUF_USE_DLLS", "LIBPROTOBUF_EXPORTS"],
+ alwayslink = 1,
visibility = ["//visibility:public"],
deps = [":protobuf_lite"] + select({
"//build_defs:config_msvc": [],
@@ -1074,7 +1078,8 @@
"@com_google_protobuf//:type_proto",
"@com_google_protobuf//:wrappers_proto",
],
- command_line = "--cpp_out=$(OUT)",
+ command_line = "--cpp_out=dllexport_decl=PROTOBUF_EXPORT:$(OUT)",
+# command_line = "--cpp_out=$(OUT)",
runtime = ":protobuf",
visibility = ["//visibility:public"],
)
diff --git a/protobuf.bzl b/protobuf.bzl
--- a/protobuf.bzl (revision 90b73ac3f0b10320315c2ca0d03a5a9b095d2f66)
+++ b/protobuf.bzl (date 1714611573270)
@@ -127,7 +127,7 @@
use_grpc_plugin = (ctx.attr.plugin_language == "grpc" and ctx.attr.plugin)
path_tpl = "$(realpath %s)" if in_gen_dir else "%s"
if ctx.attr.gen_cc:
- args += [("--cpp_out=" + path_tpl) % gen_dir]
+ args += [("--cpp_out=dllexport_decl=PROTOBUF_EXPORT:" + path_tpl) % gen_dir]
outs.extend(_CcOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
if ctx.attr.gen_py:
args += [("--python_out=" + path_tpl) % gen_dir]
diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc
index 162531226..e93ec4809 100644
--- a/python/google/protobuf/pyext/descriptor.cc
+++ b/python/google/protobuf/pyext/descriptor.cc
@@ -58,6 +58,37 @@
: 0) \
: PyBytes_AsStringAndSize(ob, (charpp), (sizep)))
+#if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION)
+static PyCodeObject* PyFrame_GetCode(PyFrameObject *frame)
+{
+ Py_INCREF(frame->f_code);
+ return frame->f_code;
+}
+
+static PyFrameObject* PyFrame_GetBack(PyFrameObject *frame)
+{
+ Py_XINCREF(frame->f_back);
+ return frame->f_back;
+}
+#endif
+
+#if PY_VERSION_HEX < 0x030B00A7 && !defined(PYPY_VERSION)
+static PyObject* PyFrame_GetLocals(PyFrameObject *frame)
+{
+ if (PyFrame_FastToLocalsWithError(frame) < 0) {
+ return NULL;
+ }
+ Py_INCREF(frame->f_locals);
+ return frame->f_locals;
+}
+
+static PyObject* PyFrame_GetGlobals(PyFrameObject *frame)
+{
+ Py_INCREF(frame->f_globals);
+ return frame->f_globals;
+}
+#endif
+
namespace google {
namespace protobuf {
namespace python {
@@ -96,48 +127,66 @@ bool _CalledFromGeneratedFile(int stacklevel) {
// This check is not critical and is somewhat difficult to implement correctly
// in PyPy.
PyFrameObject* frame = PyEval_GetFrame();
+ PyCodeObject* frame_code = nullptr;
+ PyObject* frame_globals = nullptr;
+ PyObject* frame_locals = nullptr;
+ bool result = false;
+
if (frame == nullptr) {
- return false;
+ goto exit;
}
+ Py_INCREF(frame);
while (stacklevel-- > 0) {
- frame = frame->f_back;
+ PyFrameObject* next_frame = PyFrame_GetBack(frame);
+ Py_DECREF(frame);
+ frame = next_frame;
if (frame == nullptr) {
- return false;
+ goto exit;
}
}
- if (frame->f_code->co_filename == nullptr) {
- return false;
+ frame_code = PyFrame_GetCode(frame);
+ if (frame_code->co_filename == nullptr) {
+ goto exit;
}
char* filename;
Py_ssize_t filename_size;
- if (PyString_AsStringAndSize(frame->f_code->co_filename,
+ if (PyString_AsStringAndSize(frame_code->co_filename,
&filename, &filename_size) < 0) {
// filename is not a string.
PyErr_Clear();
- return false;
+ goto exit;
}
if ((filename_size < 3) ||
(strcmp(&filename[filename_size - 3], ".py") != 0)) {
// Cython's stack does not have .py file name and is not at global module
// scope.
- return true;
+ result = true;
+ goto exit;
}
if (filename_size < 7) {
// filename is too short.
- return false;
+ goto exit;
}
if (strcmp(&filename[filename_size - 7], "_pb2.py") != 0) {
// Filename is not ending with _pb2.
- return false;
+ goto exit;
}
- if (frame->f_globals != frame->f_locals) {
+ frame_globals = PyFrame_GetGlobals(frame);
+ frame_locals = PyFrame_GetLocals(frame);
+ if (frame_globals != frame_locals) {
// Not at global module scope
- return false;
+ goto exit;
}
#endif
- return true;
+ result = true;
+exit:
+ Py_XDECREF(frame_globals);
+ Py_XDECREF(frame_locals);
+ Py_XDECREF(frame_code);
+ Py_XDECREF(frame);
+ return result;
}
// If the calling code is not a _pb2.py file, raise AttributeError.