1# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Unit tests for debug_gradients module.""" 16 17import tempfile 18 19from tensorflow.core.protobuf import config_pb2 20from tensorflow.core.protobuf import rewriter_config_pb2 21from tensorflow.python.client import session 22from tensorflow.python.debug.lib import debug_data 23from tensorflow.python.debug.lib import debug_gradients 24from tensorflow.python.debug.lib import debug_utils 25from tensorflow.python.framework import ops 26from tensorflow.python.framework import test_util 27from tensorflow.python.lib.io import file_io 28from tensorflow.python.ops import gradients_impl 29from tensorflow.python.ops import math_ops 30from tensorflow.python.ops import variables 31from tensorflow.python.platform import googletest 32from tensorflow.python.training import gradient_descent 33 34 35@test_util.run_v1_only("Sessions are not available in TF 2.x") 36class IdentifyGradientTest(test_util.TensorFlowTestCase): 37 38 def setUp(self): 39 rewriter_config = rewriter_config_pb2.RewriterConfig( 40 disable_model_pruning=True, 41 dependency_optimization=rewriter_config_pb2.RewriterConfig.OFF) 42 graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_config) 43 config = config_pb2.ConfigProto(graph_options=graph_options) 44 self.sess = session.Session(config=config) 45 with self.sess.as_default(): 46 self.u = variables.Variable(2.0, name="u") 47 self.v = variables.Variable(3.0, name="v") 48 self.w = math_ops.multiply(self.u.value(), self.v.value(), name="w") 49 50 def tearDown(self): 51 ops.reset_default_graph() 52 debug_gradients.clear_gradient_debuggers() 53 54 def testIdentifyGradientGivesCorrectTensorObjectWithoutContextManager(self): 55 grad_debugger = debug_gradients.GradientsDebugger() 56 id_grad_w = grad_debugger.identify_gradient(self.w) 57 y = math_ops.add(id_grad_w, -1.0, name="y") 58 59 grads = gradients_impl.gradients(y, [self.u, self.v]) 60 self.assertEqual(2, len(grads)) 61 u_grad = grads[0] 62 v_grad = grads[1] 63 64 self.sess.run(variables.global_variables_initializer()) 65 self.assertAllClose(5.0, self.sess.run(y)) 66 self.assertAllClose(3.0, self.sess.run(u_grad)) 67 self.assertAllClose(2.0, self.sess.run(v_grad)) 68 69 # Fetch the gradient tensor with the x-tensor object. 70 w_grad = grad_debugger.gradient_tensor(self.w) 71 self.assertIsInstance(w_grad, ops.Tensor) 72 self.assertAllClose(1.0, self.sess.run(w_grad)) 73 74 # Fetch the gradient tensor with the x-tensor's name. 75 w_grad = grad_debugger.gradient_tensor(self.w.name) 76 self.assertIsInstance(w_grad, ops.Tensor) 77 self.assertAllClose(1.0, self.sess.run(w_grad)) 78 79 # Fetch the gradient tensor with the x-tensor name. 80 w_grad = grad_debugger.gradient_tensor(self.w.name) 81 self.assertIsInstance(w_grad, ops.Tensor) 82 self.assertAllClose(1.0, self.sess.run(w_grad)) 83 84 def testIdentifyGradientGivesCorrectTensorObjectWithTfGradients(self): 85 grad_debugger = debug_gradients.GradientsDebugger() 86 id_grad_w = grad_debugger.identify_gradient(self.w) 87 y = math_ops.add(id_grad_w, -1.0, name="y") 88 89 with grad_debugger: 90 grads = gradients_impl.gradients(y, [self.u, self.v]) 91 self.assertEqual(2, len(grads)) 92 u_grad = grads[0] 93 v_grad = grads[1] 94 95 self.sess.run(variables.global_variables_initializer()) 96 self.assertAllClose(5.0, self.sess.run(y)) 97 self.assertAllClose(3.0, self.sess.run(u_grad)) 98 self.assertAllClose(2.0, self.sess.run(v_grad)) 99 100 # Fetch the gradient tensor with the x-tensor object. 101 w_grad = grad_debugger.gradient_tensor(self.w) 102 self.assertIsInstance(w_grad, ops.Tensor) 103 self.assertAllClose(1.0, self.sess.run(w_grad)) 104 105 # Fetch the gradient tensor with the x-tensor's name. 106 w_grad = grad_debugger.gradient_tensor(self.w.name) 107 self.assertIsInstance(w_grad, ops.Tensor) 108 self.assertAllClose(1.0, self.sess.run(w_grad)) 109 110 # Fetch the gradient tensor with the x-tensor name. 111 w_grad = grad_debugger.gradient_tensor(self.w.name) 112 self.assertIsInstance(w_grad, ops.Tensor) 113 self.assertAllClose(1.0, self.sess.run(w_grad)) 114 115 def testCallingIdentifyGradientTwiceWithTheSameGradientsDebuggerErrors(self): 116 grad_debugger = debug_gradients.GradientsDebugger() 117 grad_debugger.identify_gradient(self.w) 118 with self.assertRaisesRegex(ValueError, 119 "The graph already contains an op named .*"): 120 grad_debugger.identify_gradient(self.w) 121 122 def testIdentifyGradientWorksOnMultipleLosses(self): 123 grad_debugger_1 = debug_gradients.GradientsDebugger() 124 grad_debugger_2 = debug_gradients.GradientsDebugger() 125 126 y = math_ops.add(self.w, -1.0, name="y") 127 debug_y = grad_debugger_1.identify_gradient(y) 128 z1 = math_ops.square(debug_y, name="z1") 129 130 debug_y = grad_debugger_2.identify_gradient(y) 131 z2 = math_ops.sqrt(debug_y, name="z2") 132 133 with grad_debugger_1: 134 gradient_descent.GradientDescentOptimizer(0.1).minimize(z1) 135 with grad_debugger_2: 136 gradient_descent.GradientDescentOptimizer(0.1).minimize(z2) 137 138 dz1_dy = grad_debugger_1.gradient_tensor(y) 139 dz2_dy = grad_debugger_2.gradient_tensor(y) 140 self.assertIsInstance(dz1_dy, ops.Tensor) 141 self.assertIsInstance(dz2_dy, ops.Tensor) 142 self.assertIsNot(dz1_dy, dz2_dy) 143 144 self.sess.run(variables.global_variables_initializer()) 145 self.assertAllClose(5.0**2, self.sess.run(z1)) 146 self.assertAllClose(5.0**0.5, self.sess.run(z2)) 147 self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy)) 148 self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy)) 149 150 def testIdentifyGradientRaisesLookupErrorForUnknownXTensor(self): 151 grad_debugger_1 = debug_gradients.GradientsDebugger() 152 grad_debugger_2 = debug_gradients.GradientsDebugger() 153 id_grad_w = grad_debugger_1.identify_gradient(self.w) 154 y = math_ops.add(id_grad_w, -1.0, name="y") 155 156 # There are >1 gradient debuggers registered, and grad_debugger is not used 157 # as a context manager here, so the gradient w.r.t. self.w will not be 158 # registered. 159 gradients_impl.gradients(y, [self.u, self.v]) 160 161 with self.assertRaisesRegex( 162 LookupError, 163 r"This GradientsDebugger has not received any gradient tensor for "): 164 grad_debugger_1.gradient_tensor(self.w) 165 with self.assertRaisesRegex( 166 LookupError, 167 r"This GradientsDebugger has not received any gradient tensor for "): 168 grad_debugger_2.gradient_tensor(self.w) 169 170 def testIdentifyGradientRaisesTypeErrorForNonTensorOrTensorNameInput(self): 171 grad_debugger = debug_gradients.GradientsDebugger() 172 with self.assertRaisesRegex( 173 TypeError, 174 r"x_tensor must be a str or tf\.Tensor or tf\.Variable, but instead " 175 r"has type .*Operation.*"): 176 grad_debugger.gradient_tensor(variables.global_variables_initializer()) 177 178 def testIdentifyGradientTensorWorksWithGradientDescentOptimizer(self): 179 grad_debugger = debug_gradients.GradientsDebugger() 180 id_grad_w = grad_debugger.identify_gradient(self.w) 181 y = math_ops.add(id_grad_w, -1.0, name="y") 182 183 with grad_debugger: 184 gradient_descent.GradientDescentOptimizer(0.1).minimize(y) 185 186 self.sess.run(variables.global_variables_initializer()) 187 188 # Fetch the gradient tensor with the x-tensor object. 189 w_grad = grad_debugger.gradient_tensor(self.w) 190 self.assertIsInstance(w_grad, ops.Tensor) 191 self.assertAllClose(1.0, self.sess.run(w_grad)) 192 193 def testWatchGradientsByXTensorNamesWorks(self): 194 y = math_ops.add(self.w, -1.0, name="y") 195 196 # The constructrion of the forward graph has completed. 197 # But we can still get the gradient tensors by using 198 # watch_gradients_by_tensor_names(). 199 grad_debugger = debug_gradients.GradientsDebugger() 200 with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "w:0$"): 201 grads = gradients_impl.gradients(y, [self.u, self.v]) 202 self.assertEqual(2, len(grads)) 203 u_grad = grads[0] 204 v_grad = grads[1] 205 206 self.sess.run(variables.global_variables_initializer()) 207 self.assertAllClose(5.0, self.sess.run(y)) 208 self.assertAllClose(3.0, self.sess.run(u_grad)) 209 self.assertAllClose(2.0, self.sess.run(v_grad)) 210 211 w_grad = grad_debugger.gradient_tensor(self.w) 212 self.assertIsInstance(w_grad, ops.Tensor) 213 self.assertAllClose(1.0, self.sess.run(w_grad)) 214 215 w_grad = grad_debugger.gradient_tensor("w:0") 216 self.assertIsInstance(w_grad, ops.Tensor) 217 self.assertAllClose(1.0, self.sess.run(w_grad)) 218 219 def testWatchGradientsByXTensorNamesWorksWithoutContextManager(self): 220 y = math_ops.add(self.w, -1.0, name="y") 221 222 # The constructrion of the forward graph has completed. 223 # But we can still get the gradient tensors by using 224 # watch_gradients_by_tensor_names(). 225 grad_debugger = debug_gradients.GradientsDebugger() 226 grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "w:0$") 227 grads = gradients_impl.gradients(y, [self.u, self.v]) 228 self.assertEqual(2, len(grads)) 229 u_grad = grads[0] 230 v_grad = grads[1] 231 232 self.sess.run(variables.global_variables_initializer()) 233 self.assertAllClose(5.0, self.sess.run(y)) 234 self.assertAllClose(3.0, self.sess.run(u_grad)) 235 self.assertAllClose(2.0, self.sess.run(v_grad)) 236 237 w_grad = grad_debugger.gradient_tensor(self.w) 238 self.assertIsInstance(w_grad, ops.Tensor) 239 self.assertAllClose(1.0, self.sess.run(w_grad)) 240 241 w_grad = grad_debugger.gradient_tensor("w:0") 242 self.assertIsInstance(w_grad, ops.Tensor) 243 self.assertAllClose(1.0, self.sess.run(w_grad)) 244 245 def testWatchGradientsWorksOnRefTensor(self): 246 y = math_ops.add(self.w, -1.0, name="y") 247 248 grad_debugger = debug_gradients.GradientsDebugger() 249 with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, "u:0$"): 250 grads = gradients_impl.gradients(y, [self.u, self.v]) 251 self.assertEqual(2, len(grads)) 252 u_grad = grads[0] 253 v_grad = grads[1] 254 255 self.assertIs(u_grad, grad_debugger.gradient_tensor("u:0")) 256 257 self.sess.run(variables.global_variables_initializer()) 258 self.assertAllClose(3.0, self.sess.run(u_grad)) 259 self.assertAllClose(2.0, self.sess.run(v_grad)) 260 self.assertAllClose(3.0, self.sess.run( 261 grad_debugger.gradient_tensor("u:0"))) 262 263 def testWatchGradientsWorksOnMultipleTensors(self): 264 y = math_ops.add(self.w, -1.0, name="y") 265 266 grad_debugger = debug_gradients.GradientsDebugger() 267 with grad_debugger.watch_gradients_by_tensor_names(self.sess.graph, 268 "(u|w):0$"): 269 grads = gradients_impl.gradients(y, [self.u, self.v]) 270 self.assertEqual(2, len(grads)) 271 u_grad = grads[0] 272 273 self.assertEqual(2, len(grad_debugger.gradient_tensors())) 274 self.assertIs(u_grad, grad_debugger.gradient_tensor("u:0")) 275 self.assertIsInstance(grad_debugger.gradient_tensor("w:0"), ops.Tensor) 276 277 self.sess.run(variables.global_variables_initializer()) 278 self.assertAllClose(1.0, self.sess.run( 279 grad_debugger.gradient_tensor("w:0"))) 280 self.assertAllClose(3.0, self.sess.run( 281 grad_debugger.gradient_tensor("u:0"))) 282 283 def testWatchGradientsByXTensorsWorks(self): 284 y = math_ops.add(self.w, -1.0, name="foo/y") 285 z = math_ops.square(y, name="foo/z") 286 287 # The constructrion of the forward graph has completed. 288 # But we can still get the gradient tensors by using 289 # watch_gradients_by_x_tensors(). 290 grad_debugger = debug_gradients.GradientsDebugger() 291 with grad_debugger.watch_gradients_by_tensors(self.sess.graph, 292 [self.w, self.u, y]): 293 gradient_descent.GradientDescentOptimizer(0.1).minimize(z) 294 295 self.assertEqual(3, len(grad_debugger.gradient_tensors())) 296 u_grad = grad_debugger.gradient_tensor(self.u) 297 w_grad = grad_debugger.gradient_tensor(self.w) 298 y_grad = grad_debugger.gradient_tensor(y) 299 300 self.sess.run(variables.global_variables_initializer()) 301 self.assertAllClose(10.0, self.sess.run(y_grad)) 302 self.assertAllClose(10.0, self.sess.run(w_grad)) 303 self.assertAllClose(30.0, self.sess.run(u_grad)) 304 305 def testWatchGradientsByTensorCanWorkOnMultipleLosses(self): 306 y = math_ops.add(self.w, -1.0, name="y") 307 z1 = math_ops.square(y, name="z1") 308 z2 = math_ops.sqrt(y, name="z2") 309 310 grad_debugger_1 = debug_gradients.GradientsDebugger() 311 with grad_debugger_1.watch_gradients_by_tensors(self.sess.graph, y): 312 gradient_descent.GradientDescentOptimizer(0.1).minimize(z1) 313 314 grad_debugger_2 = debug_gradients.GradientsDebugger() 315 with grad_debugger_2.watch_gradients_by_tensors(self.sess.graph, y): 316 gradient_descent.GradientDescentOptimizer(0.1).minimize(z2) 317 318 dz1_dy = grad_debugger_1.gradient_tensor(y) 319 dz2_dy = grad_debugger_2.gradient_tensor(y) 320 self.assertIsInstance(dz1_dy, ops.Tensor) 321 self.assertIsInstance(dz2_dy, ops.Tensor) 322 self.assertIsNot(dz1_dy, dz2_dy) 323 324 self.sess.run(variables.global_variables_initializer()) 325 self.assertAllClose(5.0**2, self.sess.run(z1)) 326 self.assertAllClose(5.0**0.5, self.sess.run(z2)) 327 self.assertAllClose(2.0 * 5.0, self.sess.run(dz1_dy)) 328 self.assertAllClose(0.5 * (5.0**-0.5), self.sess.run(dz2_dy)) 329 330 def testGradientsValuesFromDumpWorks(self): 331 y = math_ops.add(self.w, -1.0, name="y") 332 z = math_ops.square(y, name="z") 333 334 grad_debugger = debug_gradients.GradientsDebugger() 335 with grad_debugger.watch_gradients_by_tensors(self.sess.graph, 336 [self.w, self.u, y]): 337 train_op = gradient_descent.GradientDescentOptimizer(0.1).minimize(z) 338 339 self.sess.run(variables.global_variables_initializer()) 340 341 run_options = config_pb2.RunOptions(output_partition_graphs=True) 342 dump_dir = tempfile.mkdtemp() 343 debug_url = "file://" + dump_dir 344 debug_utils.watch_graph(run_options, self.sess.graph, debug_urls=debug_url) 345 run_metadata = config_pb2.RunMetadata() 346 self.assertAllClose(2.0, self.sess.run(self.u)) 347 self.sess.run(train_op, options=run_options, run_metadata=run_metadata) 348 self.assertAllClose(-1.0, self.sess.run(self.u)) 349 350 dump = debug_data.DebugDumpDir( 351 dump_dir, partition_graphs=run_metadata.partition_graphs) 352 dump.set_python_graph(self.sess.graph) 353 354 y_grad_values = debug_gradients.gradient_values_from_dump( 355 grad_debugger, y, dump) 356 self.assertEqual(1, len(y_grad_values)) 357 self.assertAllClose(10.0, y_grad_values[0]) 358 359 w_grad_values = debug_gradients.gradient_values_from_dump( 360 grad_debugger, self.w, dump) 361 self.assertEqual(1, len(w_grad_values)) 362 self.assertAllClose(10.0, w_grad_values[0]) 363 364 u_grad_values = debug_gradients.gradient_values_from_dump( 365 grad_debugger, self.u, dump) 366 self.assertEqual(1, len(u_grad_values)) 367 self.assertAllClose(30.0, u_grad_values[0]) 368 369 with self.assertRaisesRegex( 370 LookupError, 371 r"This GradientsDebugger has not received any gradient tensor for " 372 r"x-tensor v:0"): 373 debug_gradients.gradient_values_from_dump(grad_debugger, self.v, dump) 374 375 # Cleanup. 376 file_io.delete_recursively(dump_dir) 377 378 379if __name__ == "__main__": 380 googletest.main() 381