Merge pull request #246 from FloopCZ/system-protobuf

Add support for matching Protobuf install

Author: FloopCZ

.dockerignore

@@ -0,0 +1 @@
+tensorflow_cc/build/

Dockerfiles/install-archlinux.sh

@@ -15,6 +15,7 @@ done
 # install requirements
 pacman -Syu --noconfirm --needed \
   base-devel \
+  gcc11 \
   cmake \
   git \
   python \

Dockerfiles/install-common.sh

@@ -19,7 +19,7 @@ chmod go+rX tensorflow_cc/tensorflow_cc/build
 cd tensorflow_cc/tensorflow_cc/build
 
 # build and install
-cmake -DLOCAL_RAM_RESOURCES=2048 -DLOCAL_CPU_RESOURCES=2 ..
+cmake -DLOCAL_RAM_RESOURCES=2048 -DLOCAL_CPU_RESOURCES=2 -DINSTALL_PROTOBUF=ON ..
 make
 rm -rf /home/tensorflow_cc/.cache
 rm -rf /root/.cache

README.md

@@ -67,7 +67,12 @@ sudo pacman -S cuda cudnn nvidia
 ```
 
 **Warning:** Newer versions of TensorFlow sometimes fail to build with the latest version of Bazel. You may wish
-to install an older version of Bazel (e.g., 3.7.2).
+to install an older version of Bazel (e.g., 5.1.1).
+
+**Warning:** If your program uses Protobuf and you encounter linkage or other problems, you can
+try `-DINSTALL_PROTOBUF=ON` option to install a Protobuf version matching the version bundled with TensorFlow.
+Our Dockerfiles are already built with the right version of Protobuf, so you may want to try
+your program in the Dockerfile first.
 
 #### 2) Clone this repository
 ```

tensorflow_cc/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.3 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.7 FATAL_ERROR)
 cmake_policy(SET CMP0048 NEW)  # Enable version parameter in project().
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/PROJECT_VERSION" version)
 project(
@@ -11,6 +11,7 @@ option(REQUIRE_CUDA "Make sure to find and use CUDA (implies ALLOW_CUDA)." OFF)
 set(LOCAL_RAM_RESOURCES 4096 CACHE STRING "The amount of local RAM resources passed to bazel scheduler (e.g., 4096).")
 set(LOCAL_CPU_RESOURCES HOST_CPUS CACHE STRING "The amount of local CPU cores passed to bazel scheduler (e.g., 2).")
 set(TENSORFLOW_TAG "v${version}" CACHE STRING "The tensorflow release tag to be checked out (default v${version}).")
+option(INSTALL_PROTOBUF "Install protobuf compatible with tensorflow version." OFF)
 set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ standard for building and linking the library (e.g., 14).")
 
 # -------------
@@ -31,6 +32,9 @@ configure_file("cmake/build_tensorflow.sh.in" "build_tensorflow.sh" @ONLY)
 # ----------------------------------------------
 
 include(TensorflowBase)
+if(INSTALL_PROTOBUF)
+  include(ProtobufExternal)
+endif()
 
 # ------------------------------
 # Define Tensorflow_CC Interface
@@ -39,6 +43,10 @@ include(TensorflowBase)
 add_library(tensorflow_cc INTERFACE)
 target_compile_features(tensorflow_cc INTERFACE "cxx_std_${CMAKE_CXX_STANDARD}")
 
+if(INSTALL_PROTOBUF)
+    add_dependencies(tensorflow_cc protobuf-external)
+endif()
+
 # The include folders are sometimes contained under bazel-bin/bin/ and sometimes just bazel-bin.
 target_include_directories(
   tensorflow_cc INTERFACE

tensorflow_cc/cmake/ProtobufExternal.cmake

@@ -0,0 +1,31 @@
+# Find the proper protobuf archive url.
+file(DOWNLOAD
+  "https://raw.githubusercontent.com/tensorflow/tensorflow/${TENSORFLOW_TAG}/tensorflow/workspace2.bzl"
+  "${CMAKE_CURRENT_BINARY_DIR}/tmp/workspace2.bzl"
+)
+file(READ
+  "${CMAKE_CURRENT_BINARY_DIR}/tmp/workspace2.bzl"
+  workspace2_str
+)
+string(REGEX MATCH
+  "https://github.com/protocolbuffers/protobuf/archive/v[.0-9]+.zip"
+  PROTOBUF_ARCHIVE
+  "${workspace2_str}"
+)
+message("Will build Protobuf from '${PROTOBUF_ARCHIVE}'.")
+
+ExternalProject_Add(
+  protobuf-external
+  PREFIX protobuf
+  URL "${PROTOBUF_ARCHIVE}"
+  CMAKE_CACHE_ARGS
+    "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}"
+    "-Dprotobuf_BUILD_TESTS:BOOL=OFF"
+    "-Dprotobuf_BUILD_EXAMPLES:BOOL=OFF"
+    "-DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}"
+  SOURCE_SUBDIR cmake
+  INSTALL_COMMAND ""
+)
+
+install(SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf-external-build/cmake_install.cmake")
+

tensorflow_cc/cmake/TensorflowBase.cmake

@@ -11,7 +11,8 @@ ExternalProject_Add(
   BUILD_IN_SOURCE 1
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
-  BUILD_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/build_tensorflow.sh"
+  BUILD_COMMAND echo "*" > "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/.bazelversion"
+        COMMAND "${CMAKE_CURRENT_BINARY_DIR}/build_tensorflow.sh"
   INSTALL_COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/cmake/copy_links.sh" bazel-bin
           # The include folders are sometimes contained under bazel-bin/bin/ and sometimes just bazel-bin.
           # Later, we include both so let's make sure they both exist so that CMake does not complain.

tensorflow_cc/cmake/build_tensorflow.sh.in

@@ -64,25 +64,17 @@ if [ "$cuda_allowed" == true ] && [ "$cuda_available" == true ]; then
     export TF_CUDNN_VERSION="$(find /opt /usr -name 'libcudnn.so.*' -path '*/cuda*' | tail -n1 | sed -r 's/^.*\.so\.//')"
 
     # choose the right version of CUDA compiler
-    if [ -z "$GCC_HOST_COMPILER_PATH" ]; then
-        if hash gcc-11 2>/dev/null && version_gt 11.1 `gcc-11 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-11"}
+    if [ -z "${GCC_HOST_COMPILER_PATH}" ]; then
+        if hash gcc-11 2>/dev/null && version_gt 11.4 `gcc-11 -dumpversion`; then
+            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-11`}
         elif hash gcc-10 2>/dev/null && version_gt 10.3 `gcc-10 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-10"}
+            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-10`}
         elif hash gcc-9 2>/dev/null && version_gt 9.4 `gcc-9 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-9"}
+            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-9`}
         elif hash gcc-8 2>/dev/null && version_gt 8.5 `gcc-8 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-8"}
-        elif hash gcc-7 2>/dev/null && version_gt 7.5 `gcc-7 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-7"}
-        elif hash gcc-6 2>/dev/null && version_gt 6.4 `gcc-6 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-6"}
-        elif hash gcc-5 2>/dev/null && version_gt 5.5 `gcc-5 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-5"}
-        elif hash gcc-4 2>/dev/null && version_gt 4.9 `gcc-4 -dumpversion`; then
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-4"}
+            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-8`}
         else
-            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc"}
+            export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc`}
         fi
     fi