Open
Description
I am trying to use tf.keras.estimator.model_to_estimator to convert tf.keras model to be distributed, however, I found that keras-metrics does not work as desired, Is there any idea or work around for me ? thanks
Traceback:
Traceback (most recent call last):
File "1.py", line 204, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 125, in run
_sys.exit(main(argv))
File "1.py", line 190, in main
config=Rcfg
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/estimator/__init__.py", line 73, in model_to_estimator
config=config)
File "/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py", line 486, in model_to_estimator
config)
File "/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py", line 354, in _save_first_checkpoint
custom_objects)
File "/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py", line 201, in _clone_and_build_model
optimizer_iterations=global_step)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/models.py", line 511, in clone_and_build_model
target_tensors=target_tensors)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/checkpointable/base.py", line 442, in _method_wrapper
method(self, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/engine/training.py", line 499, in compile
sample_weights=self.sample_weights)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/engine/training.py", line 1844, in _handle_metrics
return_stateful_result=return_stateful_result))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/engine/training.py", line 1800, in _handle_per_output_metrics
stateful_metric_result = _call_stateful_fn(stateful_fn)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/engine/training.py", line 1773, in _call_stateful_fn
fn, y_true, y_pred, weights=weights, mask=mask)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/engine/training_utils.py", line 852, in call_metric_function
return metric_fn(y_true, y_pred, sample_weight=weights)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/metrics.py", line 438, in __call__
update_op = self.update_state(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/metrics.py", line 160, in inner
return func.__get__(instance_ref(), cls)(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/metrics.py", line 98, in decorated
update_op = update_state_fn(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/keras/metrics.py", line 649, in update_state
matches = self._fn(y_true, y_pred, **self._fn_kwargs)
File "/usr/local/lib/python2.7/dist-packages/keras_metrics/metrics.py", line 192, in __call__
tp = self.tp(y_true, y_pred)
File "/usr/local/lib/python2.7/dist-packages/keras_metrics/metrics.py", line 50, in __call__
tp_update = K.update_add(self.tp, tp)
File "/usr/local/lib/python2.7/dist-packages/keras/backend/tensorflow_backend.py", line 986, in update_add
return tf.assign_add(x, increment)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/state_ops.py", line 190, in assign_add
ref, value, use_locking=use_locking, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_state_ops.py", line 107, in assign_add
"AssignAdd", ref=ref, value=value, use_locking=use_locking, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 350, in _apply_op_helper
g = ops._get_graph_from_inputs(_Flatten(keywords.values()))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 5713, in _get_graph_from_inputs
_assert_same_graph(original_graph_element, graph_element)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 5649, in _assert_same_graph
original_item))
ValueError: Tensor("metrics/precision/Sum:0", shape=(), dtype=int32) must be from the same graph as Tensor("Variable:0", shape=(), dtype=int32_ref).
Buggy code (a little bit messy...)
If I dont add keras_metrics.sparse_categorical_precision() into Accuracy part, it DOES work but fail when I add sparse_categorical_precision...
Tested in Py2.7/3.7 TF 1.13.1