[ML-8669] Build against TensorFlow 1.15.0 (#172)

Build against TensorFlow 1.15.0. It uses pip to install the Python package

Author: Hanyu Cui

Dockerfile

@@ -42,10 +42,9 @@ ENV PYTHONPATH /opt/spark/python/lib/py4j-0.10.7-src.zip:/opt/spark/python/lib/p
 ENV PYSPARK_PYTHON python
 
 # Workaround for https://github.com/tensorflow/tensorflow/issues/30635.
-RUN wget https://repo1.maven.org/maven2/org/tensorflow/libtensorflow_jni/1.14.0/libtensorflow_jni-1.14.0.jar && \
-    jar xf libtensorflow_jni-1.14.0.jar org/tensorflow/native/linux-x86_64/libtensorflow_framework.so.1 && \
-    mv org/tensorflow/native/linux-x86_64/libtensorflow_framework.so.1 /usr/lib && \
-    rm libtensorflow_jni-1.14.0.jar
+RUN wget -q https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-linux-x86_64-1.15.0.tar.gz && \
+    tar xf libtensorflow_jni-cpu-linux-x86_64-1.15.0.tar.gz && \
+    cp libtensorflow_framework.so.1 /usr/lib
 
 # The tensorframes dir will be mounted here.
 VOLUME /mnt/tensorframes

docker-compose.yml

@@ -9,3 +9,4 @@ services:
     volumes:
     - .:/mnt/tensorframes
     - ~/.ivy2:/root/.ivy2
+    - ~/.m2:/root/.m2

environment.yml

@@ -8,6 +8,7 @@ dependencies:
 - pandas=0.24.2
 - nomkl
 - protobuf=3.6.1
-- tensorflow=1.14.0
 # test
 - nose=1.3.7
+- pip:
+    - tensorflow==1.15.0

project/Dependencies.scala

@@ -1,5 +1,5 @@
 object Dependencies {
   // The spark version
   val targetSparkVersion = sys.props.getOrElse("spark.version", "2.4.4")
-  val targetTensorFlowVersion = "1.14.0"
+  val targetTensorFlowVersion = "1.15.0"
 }

python/requirements.txt

@@ -3,4 +3,4 @@ nose>=1.3.3
 pandas>=0.19.1
 # The proto files under src/main/protobuf must be in sync with the TF version here.
 # You can use update-tf-proto.sh under dev/ to update the files.
-tensorflow==1.14.0
+tensorflow==1.15.0

src/main/protobuf/tensorflow/core/framework/node_def.proto

@@ -11,7 +11,7 @@ import "tensorflow/core/framework/attr_value.proto";
 message NodeDef {
   // The name given to this operator. Used for naming inputs,
   // logging, visualization, etc.  Unique within a single GraphDef.
-  // Must match the regexp "[A-Za-z0-9.][A-Za-z0-9_./]*".
+  // Must match the regexp "[A-Za-z0-9.][A-Za-z0-9_>./]*".
   string name = 1;
 
   // The operation name.  There may be custom parameters in attrs.
@@ -70,6 +70,15 @@ message NodeDef {
     // be {A, B}. This information can be used to map errors originating at the
     // current node to some top level source code.
     repeated string original_node_names = 1;
+
+    // This is intended to store the list of names of the functions from the
+    // original graph that this node was derived. For example if this node, say
+    // C, was result of a fusion of node A in function FA and node B in function
+    // FB, then `original_funcs` would be {FA, FB}. If the node is in the top
+    // level graph, the `original_func` is empty. This information, with the
+    // `original_node_names` can be used to map errors originating at the
+    // current ndoe to some top level source code.
+    repeated string original_func_names = 2;
   };
 
   // This stores debug information associated with the node.

src/main/protobuf/tensorflow/core/framework/op_def.proto

@@ -14,7 +14,7 @@ import "tensorflow/core/framework/types.proto";
 // LINT.IfChange
 message OpDef {
   // Op names starting with an underscore are reserved for internal use.
-  // Names should be CamelCase and match the regexp "[A-Z][a-zA-Z0-9_]*".
+  // Names should be CamelCase and match the regexp "[A-Z][a-zA-Z0-9>_]*".
   string name = 1;
 
   // For describing inputs and outputs.

src/main/protobuf/tensorflow/core/framework/resource_handle.proto

@@ -7,6 +7,9 @@ option java_multiple_files = true;
 option java_package = "org.tensorflow.framework";
 option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework";
 
+import "tensorflow/core/framework/tensor_shape.proto";
+import "tensorflow/core/framework/types.proto";
+
 // Protocol buffer representing a handle to a tensorflow resource. Handles are
 // not valid across executions, but can be serialized back and forth from within
 // a single run.
@@ -27,4 +30,13 @@ message ResourceHandleProto {
   // For debug-only, the name of the type pointed to by this handle, if
   // available.
   string maybe_type_name = 5;
+
+  // Protocol buffer representing a pair of (data type, tensor shape).
+  message DtypeAndShape {
+    DataType dtype = 1;
+    TensorShapeProto shape = 2;
+  }
+
+  // Data types and shapes for the underlying resource.
+  repeated DtypeAndShape dtypes_and_shapes = 6;
 };

src/main/protobuf/tensorflow/core/framework/types.proto

@@ -67,7 +67,7 @@ enum DataType {
   DT_UINT64_REF = 123;
 }
 // LINT.ThenChange(
-//    https://www.tensorflow.org/code/tensorflow/c/c_api.h,
+//    https://www.tensorflow.org/code/tensorflow/c/tf_datatype.h,
 //    https://www.tensorflow.org/code/tensorflow/go/tensor.go,
 //    https://www.tensorflow.org/code/tensorflow/core/framework/tensor.cc,
 //    https://www.tensorflow.org/code/tensorflow/core/framework/types.h,

src/main/protobuf/tensorflow/core/framework/variable.proto

@@ -7,7 +7,7 @@ option java_outer_classname = "VariableProtos";
 option java_multiple_files = true;
 option java_package = "org.tensorflow.framework";
 
-// add go_package externally with copybara
+option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework";
 
 // Indicates when a distributed variable will be synced.
 enum VariableSynchronization {