1from __future__ import absolute_import, print_function, division 2import unittest 3 4import numpy as np 5from nose.plugins.skip import SkipTest 6from six.moves import xrange 7 8import theano 9from theano import config 10from theano import tensor as T 11from theano import tensor 12from theano import gof 13from theano.gof.opt import check_stack_trace 14from theano.tests import unittest_tools as utt 15from theano import printing 16from theano.tensor.nnet import (categorical_crossentropy, 17 crossentropy_categorical_1hot, 18 crossentropy_softmax_1hot, 19 crossentropy_softmax_1hot_with_bias, 20 crossentropy_softmax_1hot_with_bias_dx, 21 crossentropy_softmax_argmax_1hot_with_bias, 22 CrossentropySoftmax1HotWithBiasDx, 23 CrossentropySoftmaxArgmax1HotWithBias, 24 CrossentropyCategorical1Hot, 25 CrossentropyCategorical1HotGrad, 26 sigmoid, softplus, Softmax, softmax, 27 softmax_op, softmax_graph, SoftmaxWithBias, 28 softmax_with_bias, logsoftmax_op, 29 softmax_grad, SoftmaxGrad, 30 Prepend_scalar_constant_to_each_row, 31 Prepend_scalar_to_each_row, 32 relu, 33 h_softmax, 34 elu, 35 selu, 36 binary_crossentropy, 37 sigmoid_binary_crossentropy, 38 confusion_matrix) 39from theano.tensor import matrix, vector, lvector, scalar 40from theano.tensor.nnet.nnet import softsign 41from theano.tensor.tests.test_basic import (makeBroadcastTester, check_floatX, 42 _good_broadcast_unary_normal_float_no_complex, 43 upcast_int8_nfunc) 44 45 46class T_sigmoid(unittest.TestCase): 47 48 def setUp(self): 49 utt.seed_rng() 50 51 def test_elemwise(self): 52 utt.verify_grad(sigmoid, [np.random.rand(3, 4)]) 53 54 55class T_softplus(unittest.TestCase): 56 57 def setUp(self): 58 utt.seed_rng() 59 60 def test_elemwise(self): 61 utt.verify_grad(softplus, [np.random.rand(3, 4)]) 62 63 64class T_Softmax(utt.InferShapeTester): 65 66 def test0(self): 67 def f(a): 68 return softmax_op(a)[:, 0] 69 utt.verify_grad(f, [np.random.rand(3, 4)]) 70 71 def test1(self): 72 def f(a): 73 return softmax_op(a)[:, 1] 74 utt.verify_grad(f, [np.random.rand(3, 4)]) 75 76 def test2(self): 77 def f(a): 78 return softmax_op(a)[:, 2] 79 utt.verify_grad(f, [np.random.rand(3, 4)]) 80 81 def test3(self): 82 def f(a): 83 return softmax_op(a)[:, 3] 84 utt.verify_grad(f, [np.random.rand(3, 4)]) 85 86 def test_infer_shape(self): 87 admat = matrix() 88 admat_val = np.random.rand(3, 4).astype(config.floatX) 89 self._compile_and_check([admat], [Softmax()(admat)], 90 [admat_val], Softmax) 91 92 def test_vector(self): 93 x = T.vector() 94 f = theano.function([x], softmax_op(x)) 95 96 xv = np.random.randn(6).astype(config.floatX) 97 assert np.allclose(f(xv), np.exp(xv) / np.exp(xv).sum()) 98 99 def test_vector_grad(self): 100 def f(a): 101 return softmax_op(a) 102 utt.verify_grad(f, [np.random.rand(4)]) 103 104 105class T_SoftmaxWithBias(utt.InferShapeTester): 106 107 def test0(self): 108 def f(a, b): 109 return softmax_with_bias(a, b)[:, 0] 110 utt.verify_grad(f, [np.random.rand(3, 4), 111 np.random.rand(4)]) 112 113 def test1(self): 114 def f(a, b): 115 return softmax_with_bias(a, b)[:, 1] 116 utt.verify_grad(f, [np.random.rand(3, 4), 117 np.random.rand(4)]) 118 119 def test2(self): 120 def f(a, b): 121 return softmax_with_bias(a, b)[:, 2] 122 utt.verify_grad(f, [np.random.rand(3, 4), 123 np.random.rand(4)]) 124 125 def test3(self): 126 def f(a, b): 127 return softmax_with_bias(a, b)[:, 3] 128 utt.verify_grad(f, [np.random.rand(3, 4), 129 np.random.rand(4)]) 130 131 def test_broadcast(self): 132 # test that we don't raise an error during optimization for no good 133 # reason as softmax_with_bias don't support correctly some/all 134 # broadcasted inputs pattern 135 initial_W = np.asarray([[0.1, 0.1, 0.1], 136 [0.1, 0.1, 0.1], 137 [0.1, 0.1, 0.1]], 138 dtype=theano.config.floatX) 139 W = theano.shared(value=initial_W, name='W') 140 vbias = theano.shared(value=0.1, name='vbias') # 0.01 141 hid = T.vector('hid') 142 f = theano.function([hid], 143 T.nnet.softmax_op(T.dot(hid, W.T) + vbias)) 144 ops = [node.op for node in f.maker.fgraph.toposort()] 145 assert softmax_with_bias not in ops 146 assert softmax_op in ops 147 148 f([0, 1, 0]) 149 # print f.maker.fgraph.toposort() 150 151 def test_softmax_with_bias_trace(self): 152 a = theano.shared( 153 np.random.randn(3).astype(config.floatX)) 154 b = theano.shared(np.float32(np.random.randn())) 155 sm = T.nnet.softmax(a + b) 156 f = theano.function([], sm) 157 assert check_stack_trace(f, ops_to_check='last') 158 159 def test_infer_shape(self): 160 admat = matrix() 161 advec = vector() 162 admat_val = np.random.rand(3, 4).astype(config.floatX) 163 advec_val = np.random.rand(4).astype(config.floatX) 164 self._compile_and_check([admat, advec], 165 [SoftmaxWithBias()(admat, advec)], 166 [admat_val, advec_val], SoftmaxWithBias) 167 168 169class T_LogSoftmax(utt.InferShapeTester): 170 171 def test0(self): 172 def f(a): 173 return logsoftmax_op(a)[:, 0] 174 utt.verify_grad(f, [np.random.rand(3, 4)]) 175 176 def test1(self): 177 def f(a): 178 return logsoftmax_op(a)[:, 1] 179 utt.verify_grad(f, [np.random.rand(3, 4)]) 180 181 def test2(self): 182 def f(a): 183 return logsoftmax_op(a)[:, 2] 184 utt.verify_grad(f, [np.random.rand(3, 4)]) 185 186 def test3(self): 187 def f(a): 188 return logsoftmax_op(a)[:, 3] 189 utt.verify_grad(f, [np.random.rand(3, 4)]) 190 191 def test_matrix(self): 192 def f(a): 193 return logsoftmax_op(a) 194 utt.verify_grad(f, [np.random.rand(3, 4)]) 195 196 def test_vector(self): 197 x = T.vector() 198 f = theano.function([x], logsoftmax_op(x)) 199 200 xv = np.random.randn(6).astype(config.floatX) 201 assert np.allclose(f(xv), 202 np.log(np.exp(xv) / np.exp(xv).sum())) 203 204 def test_vector_grad(self): 205 def f(a): 206 return logsoftmax_op(a) 207 utt.verify_grad(f, [np.random.rand(4)]) 208 209 def test_allclose(self): 210 m = theano.config.mode 211 m = theano.compile.get_mode(m) 212 m.check_isfinite = False 213 x, y = tensor.matrices('xy') 214 # regular softmax and crossentropy 215 sm = tensor.nnet.softmax(x) 216 cm = tensor.nnet.categorical_crossentropy(sm, y) 217 218 # numerically stable log-softmax with crossentropy 219 logsm = tensor.nnet.logsoftmax(x) 220 sm2 = tensor.exp(logsm) # just used to show equivalence with sm 221 cm2 = -tensor.sum(y * logsm, axis=1) 222 grad = tensor.grad(cm2.mean(), x) 223 224 # create some inputs into a softmax that are large and labels 225 a = np.exp(10 * np.random.rand(5, 10).astype(theano.config.floatX)) 226 # create some one-hot coded labels 227 b = np.eye(5, 10).astype(theano.config.floatX) 228 229 # show equivalence of softmax and exponentiated numerically stable 230 # log-softmax 231 f1 = theano.function([x], [sm, sm2]) 232 sm_, sm2_ = f1(a) 233 utt.assert_allclose(sm_, sm2_) 234 235 # now show that the two versions result in the same crossentropy cost 236 # this indicates that the forward function does provide some numerical 237 # stability 238 f2 = theano.function([x, y], [cm, cm2], mode=m) 239 cm_, cm2_ = f2(a, b) 240 utt.assert_allclose(cm_, cm2_) 241 242 # now, show that in the standard softmax case the gradients blow up 243 # while in the log-softmax case they don't 244 f3 = theano.function([x, y], [grad]) 245 grad_ = f3(a, b) 246 assert not np.any(np.isnan(grad_)) 247 248 def test_isclose(self): 249 def f(a): 250 return logsoftmax_op(a) 251 252 def test_local_softmax_optimization(self): 253 # Test the Logsoftmax substitution 254 # 255 # Check that Log(Softmax(x)) is substituted with Logsoftmax(x). Note that 256 # only the forward pass is checked (i.e., doesn't check the gradient) 257 258 x, y = tensor.matrices('xy') 259 sm = tensor.nnet.softmax(x) 260 logsm = tensor.log(sm) 261 f = theano.function([x], logsm) 262 assert isinstance(f.maker.fgraph.outputs[0].owner.op, 263 theano.tensor.nnet.nnet.LogSoftmax) 264 assert check_stack_trace( 265 f, ops_to_check=theano.tensor.nnet.nnet.LogSoftmax) 266 267 def test_local_softmax_grad_optimization_and_big_input(self): 268 # Test the Logsoftmax's grad substitution. 269 # 270 # Check that Log(Softmax(x))'s grad is substituted with Logsoftmax(x)'s 271 # grad and that the new operation does not explode for big inputs. 272 # Note that only the grad is checked. 273 274 m = theano.config.mode 275 m = theano.compile.get_mode(m) 276 m.check_isfinite = False 277 # some inputs that are large to make the gradient explode in the non 278 # optimized case 279 a = np.exp( 280 10 * np.random.rand(5, 10).astype(theano.config.floatX)) 281 282 def myfunc(x): 283 sm = tensor.nnet.softmax(x) 284 logsm = tensor.log(sm) 285 return logsm 286 # We set step to 0.1 because for big values we need a big epsilon 287 utt.verify_grad(myfunc, [a], eps=0.1, mode=m) 288 sa = theano.shared(a) 289 f = theano.function([], myfunc(sa)) 290 self.assertTrue(check_stack_trace(f, ops_to_check='all')) 291 292 def test_logsoftmax_grad_true_div_elemwise(self): 293 # Checks that the gradient of an expression similar to a log(softmax) 294 # but with a different elemwise operation than true_div is not 295 # optimized. 296 297 x = T.matrix('x') 298 y = T.log(T.nnet.softmax(x)) 299 g = T.grad(y.sum(), x) 300 301 softmax_grad_node = g.owner 302 assert softmax_grad_node.op == softmax_grad 303 true_div_node = softmax_grad_node.inputs[0].owner 304 assert true_div_node.op == tensor.true_div 305 306 # We replace the elemwise true_div op by an elemwise add. 307 new_g = softmax_grad(tensor.add(*true_div_node.inputs), 308 softmax_grad_node.inputs[1]) 309 310 fgraph = gof.FunctionGraph([x], [new_g]) 311 theano.compile.mode.optdb.query( 312 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 313 314 assert softmax_grad in [n.op for n in fgraph.toposort()] 315 316 317class T_SoftmaxGrad(utt.InferShapeTester): 318 319 def test_infer_shape(self): 320 admat = matrix() 321 bdmat = matrix() 322 admat_val = np.random.rand(3, 4).astype(config.floatX) 323 bdmat_val = np.random.rand(3, 4).astype(config.floatX) 324 self._compile_and_check([admat, bdmat], [SoftmaxGrad()(admat, bdmat)], 325 [admat_val, bdmat_val], SoftmaxGrad) 326 327 328class T_CrossentropySoftmax1Hot(unittest.TestCase): 329 330 def setUp(self): 331 utt.seed_rng() 332 333 def test0(self): 334 y_idx = [0, 1, 3] 335 336 def f(a, b): 337 return crossentropy_softmax_1hot_with_bias(a, b, y_idx)[0] 338 utt.verify_grad(f, [np.random.rand(3, 4), 339 np.random.rand(4)]) 340 341 def test1(self): 342 y_idx = [0, 1, 3] 343 344 def f(a): 345 return crossentropy_softmax_1hot(a, y_idx)[0] 346 utt.verify_grad(f, [np.random.rand(3, 4)]) 347 348 def test_vector(self): 349 y_idx = [3] 350 351 def f(a): 352 return crossentropy_softmax_1hot(T.shape_padleft(a), y_idx)[0] 353 utt.verify_grad(f, [np.random.rand(4)]) 354 355 def test_vectors(self): 356 y_idx = [3] 357 358 def f(a, b): 359 return crossentropy_softmax_1hot(T.shape_padleft(a) + b, y_idx)[0] 360 utt.verify_grad(f, [np.random.rand(4), np.random.rand(4)]) 361 362 363class T_CrossentropySoftmax1HotWithBiasDx(utt.InferShapeTester): 364 365 def test0(self): 366 def ff(class_dtype): 367 def f(sm): 368 # Class indices 369 y = np.random.randint(low=0, high=5, size=10).astype(class_dtype) 370 return theano.tensor.nnet.crossentropy_softmax_1hot_with_bias_dx( 371 np.random.rand(10), # Gradient w.r.t. NLL. 372 sm, # Softmax output. 373 y) 374 return f 375 # Build a random softmax output whose rows sum to 1. 376 softmax_output = np.random.rand(10, 5) 377 softmax_output /= softmax_output.sum(axis=1).reshape(10, 1) 378 for dtype in ['uint8', 'int8', 'uint64', 'int64']: 379 utt.verify_grad(ff(dtype), [softmax_output]) 380 381 def test1(self): 382 rng = np.random.RandomState(utt.fetch_seed()) 383 softmax_output = rng.rand(10, 5) 384 softmax_output /= softmax_output.sum(axis=1).reshape(10, 1) 385 386 def f(dy): 387 return (theano.tensor.nnet.crossentropy_softmax_1hot_with_bias_dx( 388 dy, 389 softmax_output, 390 rng.randint(low=0, high=5, size=10))) 391 utt.verify_grad(f, [rng.rand(10)]) 392 393 def test_infer_shape(self): 394 admat = matrix() 395 advec = vector() 396 alvec = lvector() 397 rng = np.random.RandomState(utt.fetch_seed()) 398 admat_val = rng.rand(10, 5).astype(config.floatX) 399 admat_val /= admat_val.sum(axis=1).reshape(10, 1) 400 advec_val = rng.rand(10).astype(config.floatX) 401 alvec_val = rng.randint(low=0, high=5, size=10) 402 self._compile_and_check( 403 [advec, admat, alvec], 404 [CrossentropySoftmax1HotWithBiasDx()(advec, admat, alvec)], 405 [advec_val, admat_val, alvec_val], 406 CrossentropySoftmax1HotWithBiasDx) 407 408 def test_neg_idx(self): 409 admat = matrix() 410 advec = vector() 411 alvec = lvector() 412 rng = np.random.RandomState(utt.fetch_seed()) 413 admat_val = rng.rand(10, 5).astype(config.floatX) 414 admat_val /= admat_val.sum(axis=1).reshape(10, 1) 415 advec_val = rng.rand(10).astype(config.floatX) 416 alvec_val = rng.randint(low=0, high=5, size=10) 417 alvec_val[1] = -1 418 out = CrossentropySoftmax1HotWithBiasDx()(advec, admat, alvec) 419 f = theano.function([advec, admat, alvec], out) 420 self.assertRaises(ValueError, f, advec_val, admat_val, alvec_val) 421 422 423class T_CrossentropySoftmaxArgmax1HotWithBias(utt.InferShapeTester): 424 425 def setUp(self): 426 super(T_CrossentropySoftmaxArgmax1HotWithBias, self).setUp() 427 self.op = theano.tensor.nnet.crossentropy_softmax_argmax_1hot_with_bias 428 429 def test0(self): 430 n_classes = 5 431 n_samples = 3 432 433 # First test gradient when getting a gradient on the NLL output. 434 def grad_on_nll_dtype(dtype): 435 def grad_on_nll(x, b): 436 y_idx = np.random.randint(low=0, high=n_classes, size=n_samples).astype(dtype) 437 return self.op(x, b, y_idx=y_idx)[0] 438 return grad_on_nll 439 for dtype in ['uint8', 'int8', 'uint64', 'int64']: 440 utt.verify_grad(grad_on_nll_dtype(dtype), 441 [np.random.rand(n_samples, n_classes), 442 np.random.rand(n_classes)]) 443 444 # Then test gradient when getting a gradient on the softmax output. 445 def grad_on_softmax(x, b): 446 return self.op(x, b, y_idx=np.random.randint( 447 low=0, high=n_classes, size=n_samples))[1] 448 utt.verify_grad( 449 grad_on_softmax, 450 [np.random.rand(n_samples, n_classes), 451 np.random.rand(n_classes)]) 452 453 def test_infer_shape(self): 454 admat = matrix() 455 advec = vector() 456 alvec = lvector() 457 rng = np.random.RandomState(utt.fetch_seed()) 458 admat_val = rng.rand(3, 5).astype(config.floatX) 459 advec_val = rng.rand(5).astype(config.floatX) 460 alvec_val = rng.randint(low=0, high=5, size=3) 461 self._compile_and_check( 462 [admat, advec, alvec], 463 CrossentropySoftmaxArgmax1HotWithBias()(admat, advec, alvec), 464 [admat_val, advec_val, alvec_val], 465 CrossentropySoftmaxArgmax1HotWithBias) 466 467 def test_neg_idx(self): 468 admat = matrix() 469 advec = vector() 470 alvec = lvector() 471 rng = np.random.RandomState(utt.fetch_seed()) 472 admat_val = rng.rand(3, 5).astype(config.floatX) 473 advec_val = rng.rand(5).astype(config.floatX) 474 alvec_val = rng.randint(low=0, high=5, size=3) 475 alvec_val[1] = -1 476 out = CrossentropySoftmaxArgmax1HotWithBias()(admat, advec, alvec) 477 f = theano.function([admat, advec, alvec], out) 478 self.assertRaises(ValueError, f, admat_val, advec_val, alvec_val) 479 480 481class T_prepend(utt.InferShapeTester): 482 483 def test0(self): 484 x = tensor.matrix('x') 485 y = Prepend_scalar_constant_to_each_row(4.)(x) 486 f = theano.function([x], y) 487 m = np.random.rand(3, 5).astype(config.floatX) 488 my = f(m) 489 self.assertTrue(my.shape == (3, 6), my.shape) 490 self.assertTrue(np.all(my[:, 0] == 4.0)) 491 492 def test1(self): 493 "basic functionality" 494 x = tensor.matrix('x') 495 y = Prepend_scalar_to_each_row()(5., x) 496 f = theano.function([x], y) 497 m = np.ones((3, 5), dtype="float32") 498 my = f(m) 499 self.assertTrue(my.shape == (3, 6)) 500 self.assertTrue(np.all(my[:, 0] == 5.0)) 501 502 def test_infer_shape(self): 503 admat = matrix() 504 adscal = scalar() 505 rng = np.random.RandomState(utt.fetch_seed()) 506 admat_val = rng.rand(3, 5).astype(config.floatX) 507 adscal_val = np.asarray(rng.rand(), dtype=config.floatX).item() 508 self._compile_and_check( 509 [admat], 510 [Prepend_scalar_constant_to_each_row(adscal_val)(admat)], 511 [admat_val], 512 Prepend_scalar_constant_to_each_row) 513 514 self._compile_and_check( 515 [adscal, admat], 516 [Prepend_scalar_to_each_row()(adscal, admat)], 517 [adscal_val, admat_val], 518 Prepend_scalar_to_each_row) 519 520 521class T_CrossentropyCategorical1HotGrad(utt.InferShapeTester): 522 523 def test_infer_shape(self): 524 advec = vector() 525 admat = matrix() 526 alvec = lvector() 527 rng = np.random.RandomState(utt.fetch_seed()) 528 advec_val = rng.rand(3).astype(config.floatX) 529 admat_val = rng.rand(3, 2).astype(config.floatX) 530 alvec_val = [0, 1, 0] 531 self._compile_and_check( 532 [advec, admat, alvec], 533 [CrossentropyCategorical1HotGrad()(advec, admat, alvec)], 534 [advec_val, admat_val, alvec_val], 535 CrossentropyCategorical1HotGrad) 536 537 538class T_CrossentropyCategorical1Hot(utt.InferShapeTester): 539 540 def test_grad(self): 541 x = tensor.matrix('x') 542 one_of_n = tensor.lvector('one_of_n') 543 op = crossentropy_categorical_1hot 544 xe = op(x, one_of_n) 545 f = theano.function([x, one_of_n], xe) 546 x_val = np.asarray( 547 [[.4, .6, .0], [.1, .8, .1]], 548 dtype=config.floatX) 549 xe_val = f(x_val, [0, 1]) 550 assert np.allclose(xe_val, -np.log([.4, .8])) 551 552 def oplike(x): 553 return op(x, [0, 1]) 554 555 tensor.verify_grad(oplike, [x_val], rng=np.random) 556 557 def test_infer_shape(self): 558 admat = matrix() 559 alvec = lvector() 560 rng = np.random.RandomState(utt.fetch_seed()) 561 admat_val = rng.rand(3, 2).astype(config.floatX) 562 alvec_val = [0, 1, 0] 563 self._compile_and_check( 564 [admat, alvec], 565 [CrossentropyCategorical1Hot()(admat, alvec)], 566 [admat_val, alvec_val], 567 CrossentropyCategorical1Hot) 568 569 def test_softmax_optimizations(self): 570 x = tensor.matrix('x') 571 one_of_n = tensor.lvector('one_of_n') 572 op = crossentropy_categorical_1hot 573 # xe = op(x, one_of_n) 574 575 fgraph = gof.FunctionGraph( 576 [x, one_of_n], 577 [op(softmax_op(x), one_of_n)]) 578 assert fgraph.outputs[0].owner.op == op 579 580 theano.compile.mode.optdb.query( 581 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 582 assert (fgraph.outputs[0].owner.op == 583 crossentropy_softmax_argmax_1hot_with_bias) 584 585 def test_softmax_optimizations_vector(self): 586 x = tensor.vector('x') 587 one_of_n = tensor.lvector('one_of_n') 588 op = crossentropy_categorical_1hot 589 fgraph = gof.FunctionGraph( 590 [x, one_of_n], 591 [op(softmax_op(x), one_of_n)]) 592 assert fgraph.outputs[0].owner.op == op 593 594 theano.compile.mode.optdb.query( 595 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 596 assert (fgraph.outputs[0].owner.op == 597 crossentropy_softmax_argmax_1hot_with_bias) 598 599 def test_softmax_optimizations_w_bias(self): 600 x = tensor.matrix('x') 601 b = tensor.vector('b') 602 one_of_n = tensor.lvector('one_of_n') 603 op = crossentropy_categorical_1hot 604 # xe = op(x, one_of_n) 605 606 fgraph = gof.FunctionGraph( 607 [x, b, one_of_n], 608 [op(softmax_op(x + b), one_of_n)]) 609 assert fgraph.outputs[0].owner.op == op 610 611 # print 'BEFORE' 612 # for node in fgraph.toposort(): 613 # print node.op 614 # print printing.pprint(node.outputs[0]) 615 # print '----' 616 617 theano.compile.mode.optdb.query( 618 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 619 620 # print 'AFTER' 621 # for node in fgraph.toposort(): 622 # print node.op 623 # print printing.pprint(node.outputs[0]) 624 # print '====' 625 assert len(fgraph.toposort()) == 1 626 assert (fgraph.outputs[0].owner.op == 627 crossentropy_softmax_argmax_1hot_with_bias) 628 629 def test_softmax_optimizations_w_bias2(self): 630 x = tensor.matrix('x') 631 b = tensor.vector('b') 632 c = tensor.vector('c') 633 one_of_n = tensor.lvector('one_of_n') 634 op = crossentropy_categorical_1hot 635 636 fgraph = gof.FunctionGraph( 637 [x, b, c, one_of_n], 638 [op(softmax_op(T.add(x, b, c)), one_of_n)]) 639 assert fgraph.outputs[0].owner.op == op 640 641 # print 'BEFORE' 642 # for node in fgraph.toposort(): 643 # print node.op 644 # print '----' 645 646 theano.compile.mode.optdb.query( 647 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 648 649 # print 'AFTER' 650 # for node in fgraph.toposort(): 651 # print node.op 652 # print '====' 653 assert len(fgraph.toposort()) == 2 654 assert (fgraph.outputs[0].owner.op == 655 crossentropy_softmax_argmax_1hot_with_bias) 656 657 def test_softmax_optimizations_w_bias_vector(self): 658 x = tensor.vector('x') 659 b = tensor.vector('b') 660 one_of_n = tensor.lvector('one_of_n') 661 op = crossentropy_categorical_1hot 662 fgraph = gof.FunctionGraph( 663 [x, b, one_of_n], 664 [op(softmax_op(x + b), one_of_n)]) 665 assert fgraph.outputs[0].owner.op == op 666 # print 'BEFORE' 667 # for node in fgraph.toposort(): 668 # print node.op 669 # print printing.pprint(node.outputs[0]) 670 # print '----' 671 672 theano.compile.mode.optdb.query( 673 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 674 # print 'AFTER' 675 # for node in fgraph.toposort(): 676 # print node.op 677 # print '====' 678 assert len(fgraph.toposort()) == 2 679 assert (fgraph.outputs[0].owner.op == 680 crossentropy_softmax_argmax_1hot_with_bias) 681 682 def test_softmax_grad_optimizations(self): 683 x = tensor.matrix('x') 684 one_of_n = tensor.lvector('one_of_n') 685 op = crossentropy_categorical_1hot 686 xe = op(softmax_op(x), one_of_n) 687 sum_xe = tensor.sum(xe) 688 g_x = tensor.grad(sum_xe, x) 689 fgraph = gof.FunctionGraph( 690 [x, one_of_n], 691 [g_x]) 692 assert check_stack_trace( 693 fgraph, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, 694 softmax_op]) 695 696 # print 'BEFORE' 697 # for node in fgraph.toposort(): 698 # print node.op, node.inputs 699 # print '----' 700 theano.compile.mode.optdb.query( 701 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 702 703 # print 'AFTER' 704 # for node in fgraph.toposort(): 705 # print node.op, node.inputs 706 707 has_cx1hot = False 708 has_cx1hotdx = False 709 has_softmax = False 710 has_softmaxdx = False 711 for node in fgraph.toposort(): 712 if node.op == crossentropy_softmax_argmax_1hot_with_bias: 713 has_cx1hot = True 714 if node.op == crossentropy_softmax_1hot_with_bias_dx: 715 has_cx1hotdx = True 716 if node.op == softmax_op: 717 has_softmax = True 718 if node.op == softmax_grad: 719 has_softmaxdx = True 720 assert not has_cx1hot 721 assert has_cx1hotdx 722 assert has_softmax 723 assert not has_softmaxdx 724 725 def test_softmax_grad_optimizations_vector(self): 726 x = tensor.vector('x') 727 one_of_n = tensor.lvector('one_of_n') 728 op = crossentropy_categorical_1hot 729 xe = op(softmax_op(x), one_of_n) 730 sum_xe = tensor.sum(xe) 731 g_x = tensor.grad(sum_xe, x) 732 fgraph = gof.FunctionGraph( 733 [x, one_of_n], 734 [g_x]) 735 736 # print 'BEFORE' 737 # for node in fgraph.toposort(): 738 # print node.op, node.inputs 739 # print '----' 740 theano.compile.mode.optdb.query( 741 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 742 743 # print 'AFTER' 744 # for node in fgraph.toposort(): 745 # print node.op, node.inputs 746 747 has_cx1hot = False 748 has_cx1hotdx = False 749 has_softmax = False 750 has_softmaxdx = False 751 for node in fgraph.toposort(): 752 if node.op == crossentropy_softmax_argmax_1hot_with_bias: 753 has_cx1hot = True 754 if node.op == crossentropy_softmax_1hot_with_bias_dx: 755 has_cx1hotdx = True 756 if node.op == softmax_op: 757 has_softmax = True 758 if node.op == softmax_grad: 759 has_softmaxdx = True 760 assert not has_cx1hot 761 assert has_cx1hotdx 762 assert has_softmax 763 assert not has_softmaxdx 764 765 def test_get_rid_of_advanced_indexing_version_of_xent(self): 766 verbose = 0 767 # TODO: add the optimization in FAST_COMPILE? 768 # In the mean time, run it as 'FAST_RUN' instead 769 mode = theano.compile.mode.get_default_mode() 770 if mode == theano.compile.mode.get_mode('FAST_COMPILE'): 771 mode = 'FAST_RUN' 772 rng = np.random.RandomState(utt.fetch_seed()) 773 x_val = rng.randn(3, 5).astype(config.floatX) 774 b_val = rng.randn(5).astype(config.floatX) 775 y_val = np.asarray([2, 4, 1]) 776 x = T.matrix('x') 777 b = T.vector('b') 778 y = T.lvector('y') 779 780 # Basic case 781 expressions = [ 782 T.sum( 783 -T.log(softmax(x)[T.arange(y.shape[0]), y])), 784 -T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y])), 785 -T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y]), 786 T.sum(-T.log(softmax(x))[T.arange(y.shape[0]), y])] 787 for expr in expressions: 788 # Verify the optimizer worked on the expressions 789 f = theano.function([x, y], expr, mode=mode) 790 # todo: only the first output of the op has a stack trace 791 # assert check_stack_trace( 792 # f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) 793 if verbose: 794 theano.printing.debugprint(f) 795 try: 796 ops = [node.op for node in f.maker.fgraph.toposort()] 797 assert len(ops) == 4 798 assert crossentropy_softmax_argmax_1hot_with_bias in ops 799 assert not [1 for o in ops 800 if isinstance(o, T.AdvancedSubtensor)] 801 f(x_val, y_val) 802 except Exception: 803 theano.printing.debugprint(f) 804 raise 805 806 # Also verify the gradient wrt x 807 g = theano.function([x, y], T.grad(expr, x), mode=mode) 808 assert check_stack_trace( 809 g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, 810 softmax_op]) 811 if verbose: 812 theano.printing.debugprint(g) 813 try: 814 ops = [node.op for node in g.maker.fgraph.toposort()] 815 assert len(ops) == 2 816 assert crossentropy_softmax_1hot_with_bias_dx in ops 817 assert softmax_op in ops 818 assert softmax_grad not in ops 819 g(x_val, y_val) 820 except Exception: 821 theano.printing.debugprint(g) 822 raise 823 824 # Test that a biased softmax is optimized correctly 825 bias_expressions = [ 826 T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), 827 -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), 828 -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), 829 T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] 830 831 for expr in bias_expressions: 832 f = theano.function([x, b, y], expr, mode=mode) 833 # todo: only the first output of the op has a stack trace 834 # assert check_stack_trace( 835 # f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) 836 if verbose: 837 theano.printing.debugprint(f) 838 try: 839 ops = [node.op for node in f.maker.fgraph.toposort()] 840 assert len(ops) == 2 # [big_op, sum] 841 assert crossentropy_softmax_argmax_1hot_with_bias in ops 842 f(x_val, b_val, y_val) 843 except Exception: 844 theano.printing.debugprint(f) 845 raise 846 g = theano.function([x, b, y], T.grad(expr, x), mode=mode) 847 assert check_stack_trace( 848 g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, 849 softmax_with_bias]) 850 if verbose: 851 theano.printing.debugprint(g) 852 try: 853 ops = [node.op for node in g.maker.fgraph.toposort()] 854 assert len(ops) == 2 855 assert crossentropy_softmax_1hot_with_bias_dx in ops 856 assert softmax_with_bias in ops 857 assert softmax_grad not in ops 858 g(x_val, b_val, y_val) 859 except Exception: 860 theano.printing.debugprint(g) 861 raise 862 863 # Test that using "mean" instead of sum works, too 864 mean_expressions = [ 865 T.mean(-T.log(softmax(x)[T.arange(y.shape[0]), y])), 866 -T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y])), 867 -T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y]), 868 T.mean(-T.log(softmax(x))[T.arange(y.shape[0]), y])] 869 870 for expr in mean_expressions: 871 f = theano.function([x, y], expr, mode=mode) 872 # todo: only the first output of the op has a stack trace 873 # assert check_stack_trace( 874 # f, ops_to_check=[crossentropy_softmax_argmax_1hot_with_bias]) 875 if verbose: 876 theano.printing.debugprint(f) 877 try: 878 ops = [node.op for node in f.maker.fgraph.toposort()] 879 assert len(ops) == 6 880 assert crossentropy_softmax_argmax_1hot_with_bias in ops 881 assert not [1 for o in ops 882 if isinstance(o, T.AdvancedSubtensor)] 883 f(x_val, y_val) 884 except Exception: 885 theano.printing.debugprint(f) 886 raise 887 888 g = theano.function([x, y], T.grad(expr, x), mode=mode) 889 assert check_stack_trace( 890 g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, 891 softmax_op]) 892 if verbose: 893 theano.printing.debugprint(g) 894 try: 895 ops = [node.op for node in g.maker.fgraph.toposort()] 896 assert len(ops) == 5 897 # there's an extra dimshuffle in there 898 # but I can't think of a good rule to get rid of it 899 assert crossentropy_softmax_1hot_with_bias_dx in ops 900 assert softmax_op in ops 901 assert softmax_grad not in ops 902 g(x_val, y_val) 903 except Exception: 904 theano.printing.debugprint(g) 905 raise 906 907 mean_bias_expressions = [ 908 T.mean(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), 909 -T.mean(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), 910 -T.mean(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), 911 T.mean(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] 912 913 for expr in mean_bias_expressions: 914 f = theano.function([x, b, y], expr, mode=mode) 915 # todo: only the first output of the op has a stack trace 916 # assert check_stack_trace( 917 # f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) 918 if verbose: 919 theano.printing.debugprint(f) 920 try: 921 ops = [node.op for node in f.maker.fgraph.toposort()] 922 assert len(ops) == 4 923 assert crossentropy_softmax_argmax_1hot_with_bias in ops 924 assert not [1 for o in ops 925 if isinstance(o, T.AdvancedSubtensor)] 926 except Exception: 927 theano.printing.debugprint(f) 928 raise 929 g = theano.function([x, b, y], T.grad(expr, x), mode=mode) 930 assert check_stack_trace( 931 g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, 932 softmax_with_bias]) 933 if verbose: 934 theano.printing.debugprint(g) 935 try: 936 ops = [node.op for node in g.maker.fgraph.toposort()] 937 assert len(ops) == 5 938 assert crossentropy_softmax_1hot_with_bias_dx in ops 939 assert softmax_with_bias in ops 940 assert softmax_grad not in ops 941 g(x_val, b_val, y_val) 942 except Exception: 943 theano.printing.debugprint(g) 944 raise 945 946 def test_xent_thing_int32(self): 947 verbose = 0 948 mode = theano.compile.mode.get_default_mode() 949 if mode == theano.compile.mode.get_mode('FAST_COMPILE'): 950 mode = 'FAST_RUN' 951 rng = np.random.RandomState(utt.fetch_seed()) 952 x_val = rng.randn(3, 5).astype(config.floatX) 953 y_val = np.asarray([2, 4, 1], dtype='int64') 954 x = T.matrix('x') 955 y = T.lvector('y') 956 yi = T.cast(y, 'int32') 957 expressions = [ 958 T.sum(-T.log(softmax(x)[T.arange(yi.shape[0]), yi])), 959 -T.sum(T.log(softmax(x)[T.arange(yi.shape[0]), yi])), 960 -T.sum(T.log(softmax(x))[T.arange(yi.shape[0]), yi]), 961 T.sum(-T.log(softmax(x))[T.arange(yi.shape[0]), yi])] 962 963 for expr in expressions: 964 # Verify the optimizer worked on the expressions 965 f = theano.function([x, y], expr, mode=mode) 966 if verbose: 967 theano.printing.debugprint(f) 968 try: 969 ops = [node.op for node in f.maker.fgraph.toposort()] 970 assert len(ops) == 5 971 assert crossentropy_softmax_argmax_1hot_with_bias in ops 972 assert not [1 for o in ops 973 if isinstance(o, T.AdvancedSubtensor)] 974 f(x_val, y_val) 975 except Exception: 976 theano.printing.debugprint(f) 977 raise 978 979 # Also verify the gradient wrt x 980 g = theano.function([x, y], T.grad(expr, x), mode=mode) 981 if verbose: 982 theano.printing.debugprint(g) 983 try: 984 ops = [node.op for node in g.maker.fgraph.toposort()] 985 assert len(ops) == 3 986 assert crossentropy_softmax_1hot_with_bias_dx in ops 987 assert softmax_op in ops 988 assert softmax_grad not in ops 989 g(x_val, y_val) 990 except Exception: 991 theano.printing.debugprint(g) 992 raise 993 994 def test_optimize_xent_vector(self): 995 verbose = 0 996 mode = theano.compile.mode.get_default_mode() 997 if mode == theano.compile.mode.get_mode('FAST_COMPILE'): 998 mode = 'FAST_RUN' 999 rng = np.random.RandomState(utt.fetch_seed()) 1000 x_val = rng.randn(5).astype(config.floatX) 1001 y_val = np.asarray([2]) 1002 1003 x = T.vector('x') 1004 y = T.lvector('y') 1005 1006 # Test that a biased softmax is optimized correctly 1007 bias_expressions = [ 1008 T.sum(-T.log(softmax(x)[T.arange(y.shape[0]), y])), 1009 -T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y]))] 1010 1011 for expr in bias_expressions: 1012 f = theano.function([x, y], expr, mode=mode) 1013 if verbose: 1014 printing.debugprint(f) 1015 try: 1016 ops = [node.op for node in f.maker.fgraph.toposort()] 1017 assert len(ops) == 5 1018 assert crossentropy_softmax_argmax_1hot_with_bias in ops 1019 assert not [1 for o in ops 1020 if isinstance(o, T.AdvancedSubtensor)] 1021 f(x_val, y_val) 1022 except Exception: 1023 theano.printing.debugprint(f) 1024 raise 1025 g = theano.function([x, y], T.grad(expr, x), mode=mode) 1026 if verbose: 1027 printing.debugprint(g) 1028 try: 1029 ops = [node.op for node in g.maker.fgraph.toposort()] 1030 assert len(ops) == 4 1031 assert crossentropy_softmax_1hot_with_bias_dx in ops 1032 assert softmax_op in ops 1033 assert softmax_grad not in ops 1034 g(x_val, y_val) 1035 except Exception: 1036 theano.printing.debugprint(g) 1037 raise 1038 1039 def test_optimize_xent_vector2(self): 1040 verbose = 0 1041 mode = theano.compile.mode.get_default_mode() 1042 if mode == theano.compile.mode.get_mode('FAST_COMPILE'): 1043 mode = 'FAST_RUN' 1044 rng = np.random.RandomState(utt.fetch_seed()) 1045 x_val = rng.randn(5).astype(config.floatX) 1046 b_val = rng.randn(5).astype(config.floatX) 1047 y_val = np.asarray([2]) 1048 1049 x = T.vector('x') 1050 b = T.vector('b') 1051 y = T.lvector('y') 1052 1053 # Test that a biased softmax is optimized correctly 1054 bias_expressions = [ 1055 T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), 1056 -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), 1057 -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), 1058 T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] 1059 1060 for expr in bias_expressions: 1061 f = theano.function([x, b, y], expr, mode=mode) 1062 if verbose: 1063 printing.debugprint(f) 1064 try: 1065 ops = [node.op for node in f.maker.fgraph.toposort()] 1066 # [big_op, sum, dim_shuffle] 1067 assert len(ops) == 3 1068 assert crossentropy_softmax_argmax_1hot_with_bias in ops 1069 assert not [1 for o in ops 1070 if isinstance(o, T.AdvancedSubtensor)] 1071 f(x_val, b_val, y_val) 1072 except Exception: 1073 theano.printing.debugprint(f) 1074 raise 1075 1076 backup = config.warn.sum_div_dimshuffle_bug 1077 config.warn.sum_div_dimshuffle_bug = False 1078 try: 1079 g = theano.function([x, b, y], T.grad(expr, x), mode=mode) 1080 finally: 1081 config.warn.sum_div_dimshuffle_bug = backup 1082 1083 if verbose: 1084 printing.debugprint(g) 1085 try: 1086 ops = [node.op for node in g.maker.fgraph.toposort()] 1087 assert len(ops) <= 6 1088 assert crossentropy_softmax_1hot_with_bias_dx in ops 1089 assert softmax_with_bias in ops 1090 assert softmax_grad not in ops 1091 g(x_val, b_val, y_val) 1092 except Exception: 1093 theano.printing.debugprint(g) 1094 raise 1095 1096 def test_optimize_xent_vector3(self): 1097 # Same as test_optimize_xent_vector2, but y is the result of 1098 # a "flatten", and it used to make the constant-folding 1099 # of arange(y.shape[0]) happen before the xent optimization 1100 verbose = 0 1101 mode = theano.compile.mode.get_default_mode() 1102 if mode == theano.compile.mode.get_mode('FAST_COMPILE'): 1103 mode = 'FAST_RUN' 1104 rng = np.random.RandomState(utt.fetch_seed()) 1105 x_val = rng.randn(5).astype(config.floatX) 1106 b_val = rng.randn(5).astype(config.floatX) 1107 y_val = np.asarray([2]) 1108 1109 x = T.vector('x') 1110 b = T.vector('b') 1111 y_ = T.lvector('y_') 1112 y = y_.flatten() 1113 1114 # Test that a biased softmax is optimized correctly 1115 bias_expressions = [ 1116 T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), 1117 -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), 1118 -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), 1119 T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] 1120 1121 for expr in bias_expressions: 1122 f = theano.function([x, b, y_], expr, mode=mode) 1123 if verbose: 1124 printing.debugprint(f) 1125 try: 1126 ops = [node.op for node in f.maker.fgraph.toposort()] 1127 # [big_op, sum, dim_shuffle, flatten] 1128 assert len(ops) <= 4 1129 assert crossentropy_softmax_argmax_1hot_with_bias in ops 1130 assert not [1 for o in ops 1131 if isinstance(o, T.AdvancedSubtensor)] 1132 f(x_val, b_val, y_val) 1133 except Exception: 1134 theano.printing.debugprint(f) 1135 raise 1136 1137 backup = config.warn.sum_div_dimshuffle_bug 1138 config.warn.sum_div_dimshuffle_bug = False 1139 try: 1140 g = theano.function([x, b, y], T.grad(expr, x), mode=mode) 1141 finally: 1142 config.warn.sum_div_dimshuffle_bug = backup 1143 1144 if verbose: 1145 printing.debugprint(g) 1146 try: 1147 ops = [node.op for node in g.maker.fgraph.toposort()] 1148 assert len(ops) <= 6 1149 assert crossentropy_softmax_1hot_with_bias_dx in ops 1150 assert softmax_with_bias in ops 1151 assert softmax_grad not in ops 1152 g(x_val, b_val, y_val) 1153 except Exception: 1154 theano.printing.debugprint(g) 1155 raise 1156 1157 def test_optimize_xent_vector4(self): 1158 # Same as test_optimize_xent_vector2, but y is the result of 1159 # a "specify_shape" that indicates its length is 1, so the 1160 # constant-folding of arange(y.shape[0]) happen before the xent 1161 # optimization 1162 verbose = 0 1163 mode = theano.compile.mode.get_default_mode() 1164 if mode == theano.compile.mode.get_mode('FAST_COMPILE'): 1165 mode = 'FAST_RUN' 1166 rng = np.random.RandomState(utt.fetch_seed()) 1167 x_val = rng.randn(5).astype(config.floatX) 1168 b_val = rng.randn(5).astype(config.floatX) 1169 y_val = np.asarray([2]) 1170 1171 x = T.vector('x') 1172 b = T.vector('b') 1173 y_ = T.lvector('y_') 1174 y = T.specify_shape(y_, (1,)) 1175 1176 # Test that a biased softmax is optimized correctly 1177 bias_expressions = [ 1178 T.sum(-T.log(softmax(x + b)[T.arange(y.shape[0]), y])), 1179 -T.sum(T.log(softmax(b + x)[T.arange(y.shape[0]), y])), 1180 -T.sum(T.log(softmax(x + b))[T.arange(y.shape[0]), y]), 1181 T.sum(-T.log(softmax(b + x))[T.arange(y.shape[0]), y])] 1182 1183 for expr in bias_expressions: 1184 f = theano.function([x, b, y_], expr, mode=mode) 1185 if verbose: 1186 printing.debugprint(f) 1187 try: 1188 ops = [node.op for node in f.maker.fgraph.toposort()] 1189 # [big_op, sum, dim_shuffle, specify_shape] 1190 assert len(ops) <= 4 1191 assert crossentropy_softmax_argmax_1hot_with_bias in ops 1192 assert not [1 for o in ops 1193 if isinstance(o, T.AdvancedSubtensor)] 1194 f(x_val, b_val, y_val) 1195 except Exception: 1196 theano.printing.debugprint(f) 1197 raise 1198 1199 backup = config.warn.sum_div_dimshuffle_bug 1200 config.warn.sum_div_dimshuffle_bug = False 1201 try: 1202 g = theano.function([x, b, y], T.grad(expr, x), mode=mode) 1203 finally: 1204 config.warn.sum_div_dimshuffle_bug = backup 1205 1206 if verbose: 1207 printing.debugprint(g) 1208 try: 1209 ops = [node.op for node in g.maker.fgraph.toposort()] 1210 assert len(ops) <= 6 1211 assert crossentropy_softmax_1hot_with_bias_dx in ops 1212 assert softmax_with_bias in ops 1213 assert softmax_grad not in ops 1214 g(x_val, b_val, y_val) 1215 except Exception: 1216 theano.printing.debugprint(g) 1217 raise 1218 1219 def test_crossentropy_softmax_1hot_with_bias_dxcale_cost(self): 1220 # TODO: add the optimization in FAST_COMPILE? 1221 # In the mean time, run it as 'FAST_RUN' instead 1222 mode = theano.compile.mode.get_default_mode() 1223 if mode == theano.compile.mode.get_mode('FAST_COMPILE'): 1224 mode = 'FAST_RUN' 1225 rng = np.random.RandomState(utt.fetch_seed()) 1226 x_val = rng.randn(3, 5).astype(config.floatX) 1227 y_val = np.asarray([2, 4, 1]) 1228 x = T.matrix('x') 1229 y = T.lvector('y') 1230 a = T.scalar('a') 1231 1232 def validate_fn_graph(func): 1233 # The graph of the function should not have softmax anymore 1234 has_cx1hot = False 1235 has_softmax = False 1236 for node in func.maker.fgraph.toposort(): 1237 if node.op == crossentropy_softmax_argmax_1hot_with_bias: 1238 has_cx1hot = True 1239 if node.op == softmax_op: 1240 has_softmax = True 1241 1242 assert has_cx1hot 1243 assert not has_softmax 1244 1245 def validate_grad_graph(func): 1246 # The graph of the gradient should not have softmaxgrad anymore 1247 has_cx1hotdx = False 1248 has_softmax = False 1249 has_softmaxdx = False 1250 for node in func.maker.fgraph.toposort(): 1251 if node.op == crossentropy_softmax_1hot_with_bias_dx: 1252 has_cx1hotdx = True 1253 if node.op == softmax_op: 1254 has_softmax = True 1255 if node.op == softmax_grad: 1256 has_softmaxdx = True 1257 1258 assert has_cx1hotdx 1259 assert has_softmax 1260 assert not has_softmaxdx 1261 1262 # Cases to test 1263 expressions = [ 1264 a * T.sum(-T.log(softmax(x)[T.arange(y.shape[0]), y])), 1265 -a * T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y])), 1266 a * (-T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y]))), 1267 a * T.sum(T.log(softmax(x)[T.arange(y.shape[0]), y])), 1268 1269 a * T.sum(-T.log(softmax(x))[T.arange(y.shape[0]), y]), 1270 -a * T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y]), 1271 a * (-T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y])), 1272 a * T.sum(T.log(softmax(x))[T.arange(y.shape[0]), y]), 1273 1274 a * T.mean(-T.log(softmax(x)[T.arange(y.shape[0]), y])), 1275 -a * T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y])), 1276 a * (-T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y]))), 1277 a * T.mean(T.log(softmax(x)[T.arange(y.shape[0]), y])), 1278 1279 a * T.mean(-T.log(softmax(x))[T.arange(y.shape[0]), y]), 1280 -a * T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y]), 1281 a * (-T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y])), 1282 a * T.mean(T.log(softmax(x))[T.arange(y.shape[0]), y]), ] 1283 1284 for expr in expressions: 1285 # Verify the optimizer worked on the expressions 1286 f = theano.function([x, y, a], expr, mode=mode) 1287 try: 1288 assert 5 <= len(f.maker.fgraph.toposort()) <= 10 1289 validate_fn_graph(f) 1290 f(x_val, y_val, 0.1) 1291 except Exception: 1292 theano.printing.debugprint(f) 1293 raise 1294 1295 # Verify the gradient wrt x 1296 g = theano.function([x, y, a], T.grad(expr, x), mode=mode) 1297 try: 1298 assert 3 <= len(g.maker.fgraph.toposort()) <= 6 1299 validate_grad_graph(g) 1300 g(x_val, y_val, 0.1) 1301 except Exception: 1302 theano.printing.debugprint(g) 1303 raise 1304 1305 # Verify the gradient when providing output gradient 1306 h = theano.function( 1307 [x, y, a], T.grad(expr, x, known_grads={expr: a * x.sum()}), 1308 mode=mode) 1309 try: 1310 assert 6 <= len(h.maker.fgraph.toposort()) <= 8 1311 validate_grad_graph(h) 1312 h(x_val, y_val, 0.1) 1313 except Exception: 1314 theano.printing.debugprint(h) 1315 raise 1316 1317 1318def test_argmax_pushdown(): 1319 x = tensor.matrix() 1320 for sm in [softmax_graph, softmax_op]: 1321 # test that the max_and_argmax is pushed down if the max is not used 1322 out = tensor.max_and_argmax( 1323 sm(tensor.exp(tensor.tanh(sigmoid(x)))), 1324 axis=-1)[1] 1325 fgraph = gof.FunctionGraph( 1326 [x], 1327 [out]) 1328 theano.compile.mode.optdb.query( 1329 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 1330 1331 # print 'AFTER' 1332 # for node in fgraph.toposort(): 1333 # print node.op 1334 assert len(fgraph.toposort()) == 1 1335 assert isinstance(fgraph.toposort()[0].op, tensor.basic.Argmax) 1336 assert check_stack_trace( 1337 fgraph, ops_to_check=tensor.basic.Argmax) 1338 x = tensor.matrix() 1339 # test that the max_and_argmax is not pushed down if the max is used 1340 out = tensor.max_and_argmax( 1341 sm(tensor.exp(tensor.tanh(sigmoid(x)))), 1342 axis=-1)[0] 1343 fgraph = gof.FunctionGraph( 1344 [x], 1345 [out]) 1346 1347 assert hasattr(fgraph.outputs[0].tag, 'trace') 1348 backup = config.warn.argmax_pushdown_bug 1349 config.warn.argmax_pushdown_bug = False 1350 try: 1351 theano.compile.mode.optdb.query( 1352 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 1353 finally: 1354 config.warn.argmax_pushdown_bug = backup 1355 1356 # print 'AFTER' 1357 # for node in fgraph.toposort(): 1358 # print node.op 1359 assert len(fgraph.toposort()) == 3 1360 assert isinstance(fgraph.toposort()[0].op, tensor.Elemwise) 1361 assert isinstance(fgraph.toposort()[1].op, Softmax) 1362 assert isinstance(fgraph.toposort()[2].op, tensor.CAReduce) 1363 assert isinstance(fgraph.toposort()[2].op.scalar_op, theano.scalar.Maximum) 1364 1365 1366def test_argmax_pushdown_bias(): 1367 x = tensor.matrix() 1368 b = tensor.vector() 1369 1370 out = tensor.argmax(softmax_with_bias(x, b), axis=-1) 1371 fgraph = gof.FunctionGraph( 1372 [x, b], 1373 [out]) 1374 1375 theano.compile.mode.optdb.query( 1376 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 1377 1378 # print 'AFTER' 1379 # for node in fgraph.toposort(): 1380 # print node.op 1381 types_to_check = (tensor.DimShuffle, tensor.Elemwise, tensor.Argmax) 1382 assert len(fgraph.toposort()) == 3 1383 1384 for i, type in enumerate(types_to_check): 1385 assert isinstance(fgraph.toposort()[i].op, type) 1386 assert check_stack_trace(fgraph, ops_to_check=types_to_check) 1387 1388 x = tensor.matrix() 1389 b = tensor.vector() 1390 out = tensor.max_and_argmax(softmax_with_bias(x, b), axis=-1)[0] 1391 fgraph = gof.FunctionGraph( 1392 [x, b], 1393 [out]) 1394 1395 backup = config.warn.argmax_pushdown_bug 1396 config.warn.argmax_pushdown_bug = False 1397 try: 1398 theano.compile.mode.optdb.query( 1399 theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) 1400 finally: 1401 config.warn.argmax_pushdown_bug = backup 1402 1403 # print 'AFTER' 1404 # for node in fgraph.toposort(): 1405 # print node.op 1406 assert len(fgraph.toposort()) == 2 1407 assert isinstance(fgraph.toposort()[0].op, SoftmaxWithBias) 1408 assert isinstance(fgraph.toposort()[1].op, tensor.CAReduce) 1409 assert isinstance(fgraph.toposort()[1].op.scalar_op, theano.scalar.Maximum) 1410 assert check_stack_trace( 1411 fgraph, ops_to_check=(SoftmaxWithBias, tensor.CAReduce)) 1412 1413 1414def test_asymptotic_32(): 1415 # This test makes sure that our functions behave sensibly when 1416 # huge values are present 1417 1418 # TODO: consider adding the optimization of crossentropy into the current 1419 # mode for the purpose of running this test 1420 1421 for dtype in 'float32', 'float64': 1422 if dtype == 'float32': 1423 x = tensor.fmatrix() 1424 x2 = tensor.fvector() 1425 else: 1426 x = tensor.dmatrix() 1427 x2 = tensor.dvector() 1428 y = tensor.lvector() 1429 1430 c = categorical_crossentropy(softmax(x + x2), y) 1431 f = theano.function([x, y, x2], [c.sum(), 1432 tensor.grad(c.sum(), x)], mode='FAST_RUN') 1433 1434 xval = np.zeros((5, 5), dtype=dtype).astype(dtype) 1435 x2val = np.zeros(5, dtype=xval.dtype).astype(dtype) 1436 for i in xrange(100): 1437 cval, gxval = f(xval, np.arange(5), x2val) 1438 xval -= 100.3 * gxval 1439 # print cval, gxval 1440 assert cval == 0 # no problem going to zero error 1441 1442 # what about when x gets really big? 1443 1444 xval = np.zeros((5, 5), dtype=dtype) 1445 x2val = np.zeros(5, dtype=xval.dtype) 1446 for i in xrange(100): 1447 1448 cval, gxval = f(xval, np.arange(5), x2val) 1449 xval += 100000.3 * gxval 1450 # print cval, gxval 1451 1452 assert cval > 61750000 1453 assert gxval[0, 0] == -1.0 1454 assert gxval[0, 1] == 0.25 1455 1456 1457class Test_softmax_opt: 1458 # Test that expressions of softmax in terms of exponentiated things 1459 # divided by row sums are replaced by softmax expressions. 1460 # 1461 # Softmax_grad isn't that interesting as an Op, but it has the signature 1462 # we look for when trying to insert CrossEntropySoftmax... grad. So, for 1463 # now, we add softmax_grad to graphs. In the future, we may modify the 1464 # CrossEntropySoftmax...grad to look for the more basic pattern. 1465 # 1466 1467 def setUp(self): 1468 utt.seed_rng() 1469 self.rng = np.random.RandomState(utt.fetch_seed()) 1470 self.mode = theano.compile.mode.get_default_mode() 1471 self.mode = self.mode.including('canonicalize') 1472 1473 def test_basic(self): 1474 c = T.matrix() 1475 p_y = T.exp(c) / T.exp(c).sum(axis=1).dimshuffle(0, 'x') 1476 1477 # test that function contains softmax and no div. 1478 f = theano.function([c], p_y, mode=self.mode) 1479 1480 assert check_stack_trace(f, ops_to_check=softmax_op) 1481 1482 f_ops = [n.op for n in f.maker.fgraph.toposort()] 1483 # print '--- f =' 1484 # printing.debugprint(f) 1485 # print '===' 1486 assert len(f_ops) == 1 1487 assert softmax_op in f_ops 1488 f(self.rng.rand(3, 4).astype(config.floatX)) 1489 1490 def test_basic_keepdims(self): 1491 c = T.matrix() 1492 p_y = T.exp(c) / T.exp(c).sum(axis=1, keepdims=True) 1493 1494 # test that function contains softmax and no div. 1495 f = theano.function([c], p_y, mode=self.mode) 1496 1497 assert check_stack_trace(f, ops_to_check=softmax_op) 1498 1499 f_ops = [n.op for n in f.maker.fgraph.toposort()] 1500 # print '--- f =' 1501 # printing.debugprint(f) 1502 # print '===' 1503 assert len(f_ops) == 1 1504 assert softmax_op in f_ops 1505 f(self.rng.rand(3, 4).astype(config.floatX)) 1506 1507 def test_grad(self): 1508 c = T.matrix() 1509 p_y = T.exp(c) / T.exp(c).sum(axis=1).dimshuffle(0, 'x') 1510 1511 # test that function contains softmax and softmaxgrad 1512 w = T.matrix() 1513 backup = config.warn.sum_div_dimshuffle_bug 1514 config.warn.sum_div_dimshuffle_bug = False 1515 try: 1516 g = theano.function([c, w], T.grad((p_y * w).sum(), c)) 1517 finally: 1518 config.warn.sum_div_dimshuffle_bug = backup 1519 g_ops = [n.op for n in g.maker.fgraph.toposort()] 1520 # print '--- g =' 1521 # printing.debugprint(g) 1522 # print '===' 1523 1524 raise SkipTest('Optimization not enabled for the moment') 1525 assert len(g_ops) == 2 1526 assert softmax_op in g_ops 1527 assert softmax_grad in g_ops 1528 g(self.rng.rand(3, 4), self.rng.uniform(.5, 1, (3, 4))) 1529 1530 def test_transpose_basic(self): 1531 # this should be a transposed softmax 1532 c = T.matrix() 1533 p_y = T.exp(c) / T.exp(c).sum(axis=0) 1534 1535 # test that function contains softmax and no div. 1536 theano.function([c], p_y) 1537 # printing.debugprint(f) 1538 1539 # test that function contains softmax and no div. 1540 backup = config.warn.sum_div_dimshuffle_bug 1541 config.warn.sum_div_dimshuffle_bug = False 1542 try: 1543 theano.function([c], T.grad(p_y.sum(), c)) 1544 finally: 1545 config.warn.sum_div_dimshuffle_bug = backup 1546 # printing.debugprint(g) 1547 raise SkipTest('Optimization not enabled for the moment') 1548 1549 def test_1d_basic(self): 1550 # this should be a softmax, but of a one-row matrix 1551 c = T.vector() 1552 p_y = T.exp(c) / T.exp(c).sum() 1553 1554 # test that function contains softmax and no div. 1555 theano.function([c], p_y) 1556 # printing.debugprint(f) 1557 1558 # test that function contains softmax and no div. 1559 backup = config.warn.sum_div_dimshuffle_bug 1560 config.warn.sum_div_dimshuffle_bug = False 1561 try: 1562 theano.function([c], T.grad(p_y.sum(), c)) 1563 finally: 1564 config.warn.sum_div_dimshuffle_bug = backup 1565 # printing.debugprint(g) 1566 raise SkipTest('Optimization not enabled for the moment') 1567 1568 # REPEAT 3 CASES in presence of log(softmax) with the advanced indexing 1569 # etc. 1570 1571 1572def test_softmax_graph(): 1573 rng = np.random.RandomState(utt.fetch_seed()) 1574 x = theano.shared(rng.normal(size=(3, 4))) 1575 1576 def f(inputs): 1577 y = softmax_graph(x) 1578 return theano.grad(None, x, known_grads={y: inputs}) 1579 1580 utt.verify_grad(f, [rng.rand(3, 4)]) 1581 1582 1583def test_grad_softmax_grad(): 1584 rng = np.random.RandomState(utt.fetch_seed()) 1585 x = theano.shared(rng.normal(size=(3, 4))) 1586 1587 def f(inputs): 1588 y = softmax_op(x) 1589 return theano.grad(None, x, known_grads={y: inputs}) 1590 utt.verify_grad(f, [rng.rand(3, 4)]) 1591 1592 1593def test_stabilize_log_softmax(): 1594 mode = theano.compile.mode.get_default_mode() 1595 mode = mode.including('local_log_softmax', 'specialize') 1596 1597 x = matrix() 1598 y = softmax(x) 1599 z = theano.tensor.log(y) 1600 1601 f = theano.function([x], z, mode=mode) 1602 assert check_stack_trace(f, ops_to_check='all') 1603 1604 # check that the softmax has been optimized out 1605 for node in f.maker.fgraph.toposort(): 1606 assert not isinstance(node.op, y.owner.op.__class__) 1607 1608 # call the function so debug mode can verify the optimized 1609 # version matches the unoptimized version 1610 rng = np.random.RandomState([2012, 8, 22]) 1611 f(np.cast[config.floatX](rng.randn(2, 3))) 1612 1613 1614def test_relu(): 1615 x = matrix('x') 1616 seed = theano.tests.unittest_tools.fetch_seed() 1617 rng = np.random.RandomState(seed) 1618 X = rng.randn(20, 30).astype(config.floatX) 1619 1620 # test the base case, without custom alpha value 1621 y = relu(x).eval({x: X}) 1622 assert np.allclose(y, np.maximum(X, 0)) 1623 1624 # test for different constant alpha values (also outside of [0, 1]) 1625 for alpha in 0, 0.3, 1, 2, -0.3, -1, -2: 1626 y = relu(x, alpha).eval({x: X}) 1627 assert np.allclose(y, np.where(X > 0, X, alpha * X)) 1628 1629 # test for variable alpha (scalar, vector and matrix) 1630 for alpha in scalar(), vector(), matrix(): 1631 # create value for alpha (correct ndim and broadcastable against X) 1632 A = np.array(rng.randn(*X.shape[::-1][:alpha.ndim][::-1]), 1633 dtype=config.floatX) 1634 y = relu(x, alpha).eval({x: X, alpha: A}) 1635 assert np.allclose(y, np.where(X > 0, X, A * X), rtol=3e-5) 1636 # test that for alpha of ndarray don't cause upcast. 1637 x = matrix('x', dtype='float32') 1638 rng = np.random.RandomState(seed) 1639 X = rng.randn(20, 30).astype('float32') 1640 alpha = np.asarray(.123, dtype='float32') 1641 y = relu(x, alpha).eval({x: X}) 1642 assert np.allclose(y, np.where(X > 0, X, alpha * X)) 1643 assert y.dtype == 'float32' 1644 1645 1646def test_h_softmax(): 1647 # Tests the output dimensions of the h_softmax when a target is provided or 1648 # not. 1649 1650 ############# 1651 # Config 1652 ############# 1653 1654 input_size = 4 1655 batch_size = 2 1656 h_softmax_level1_size = 5 1657 h_softmax_level2_size = 3 1658 output_size = h_softmax_level1_size * h_softmax_level2_size 1659 1660 ############# 1661 # Initialize shared variables 1662 ############# 1663 1664 floatX = theano.config.floatX 1665 shared = theano.shared 1666 1667 # First level of h_softmax 1668 W1 = np.asarray(np.random.normal( 1669 size=(input_size, h_softmax_level1_size)), dtype=floatX) 1670 W1 = shared(W1) 1671 b1 = shared(np.asarray(np.zeros((h_softmax_level1_size,)), 1672 dtype=floatX)) 1673 1674 # Second level of h_softmax 1675 W2 = np.asarray(np.random.normal( 1676 size=(h_softmax_level1_size, input_size, h_softmax_level2_size)), 1677 dtype=floatX) 1678 W2 = shared(W2) 1679 b2 = shared( 1680 np.asarray(np.zeros((h_softmax_level1_size, 1681 h_softmax_level2_size)), dtype=floatX)) 1682 1683 ############# 1684 # Build graph 1685 ############# 1686 x = tensor.matrix('x') 1687 y = tensor.ivector('y') 1688 1689 # This only computes the output corresponding to the target 1690 y_hat_tg = h_softmax(x, batch_size, output_size, h_softmax_level1_size, 1691 h_softmax_level2_size, W1, b1, W2, b2, y) 1692 1693 # This computes all the outputs 1694 y_hat_all = h_softmax(x, batch_size, output_size, h_softmax_level1_size, 1695 h_softmax_level2_size, W1, b1, W2, b2) 1696 1697 ############# 1698 # Compile functions 1699 ############# 1700 fun_output_tg = theano.function([x, y], y_hat_tg) 1701 fun_output = theano.function([x], y_hat_all) 1702 1703 ############# 1704 # Test 1705 ############# 1706 x_mat = np.random.normal(size=(batch_size, input_size)).astype(floatX) 1707 y_mat = np.random.randint(0, output_size, batch_size).astype('int32') 1708 tg_output = fun_output_tg(x_mat, y_mat) 1709 all_outputs = fun_output(x_mat) 1710 1711 assert(tg_output.shape == (batch_size,)) 1712 assert(all_outputs.shape == (batch_size, output_size)) 1713 1714 # Verifies that the outputs computed by fun_output_tg are the same as those 1715 # computed by fun_output. 1716 utt.assert_allclose( 1717 all_outputs[np.arange(0, batch_size), y_mat], tg_output) 1718 1719 1720def test_elu(): 1721 x = matrix('x') 1722 seed = theano.tests.unittest_tools.fetch_seed() 1723 rng = np.random.RandomState(seed) 1724 X = rng.randn(20, 30).astype(config.floatX) 1725 1726 # test the base case, without custom alpha value 1727 y = elu(x).eval({x: X}) 1728 utt.assert_allclose(y, np.where(X > 0, X, np.exp(X) - 1)) 1729 1730 # test for different constant alpha values 1731 for alpha in 1.5, 2, -1, -1.5, -2: 1732 y = elu(x, alpha).eval({x: X}) 1733 utt.assert_allclose(y, np.where(X > 0, X, alpha * (np.exp(X) - 1))) 1734 1735 1736def test_selu(): 1737 alpha = 1.6732632423543772848170429916717 1738 scale = 1.0507009873554804934193349852946 1739 1740 x = matrix('x') 1741 seed = theano.tests.unittest_tools.fetch_seed() 1742 rng = np.random.RandomState(seed) 1743 X = rng.randn(20, 30).astype(config.floatX) 1744 1745 y = selu(x).eval({x: X}) 1746 utt.assert_allclose(y, np.where(X > 0, scale * X, scale * alpha * (np.exp(X) - 1))) 1747 1748 1749def test_binary_crossentropy_reshape(): 1750 # Reported as https://github.com/Theano/Theano/issues/4086 1751 a = tensor.tensor4('a') 1752 for c in (binary_crossentropy(sigmoid(a.reshape((-1, 1))), 1).sum(), 1753 binary_crossentropy(sigmoid(a).reshape((-1, 1)), 1).sum()): 1754 1755 ga = theano.grad(c, a) 1756 # This only works when "specialize" options are included 1757 mode = theano.compile.get_default_mode().including('fast_run') 1758 fga = theano.function([a], ga, mode=mode) 1759 utt.assert_allclose(fga(np.array([[[[30.]]]], dtype=config.floatX)), 1760 np.zeros((1, 1, 1, 1), dtype=config.floatX)) 1761 1762SoftsignTester = makeBroadcastTester( 1763 op=softsign, 1764 expected=upcast_int8_nfunc(lambda inputs: check_floatX( 1765 inputs, inputs / (1.0 + np.fabs(inputs)))), 1766 good=_good_broadcast_unary_normal_float_no_complex, 1767 name='SoftsignTester', 1768) 1769 1770 1771class T_sigmoid_binary_crossentropy(unittest.TestCase): 1772 1773 def setUp(self): 1774 utt.seed_rng() 1775 1776 def _get_test_inputs(self, n=50): 1777 pred, target = np.random.randn(2, n).astype(config.floatX) 1778 # apply sigmoid to target, but not pred 1779 return [pred, 1 / (1 + np.exp(-target))] 1780 1781 def test_matches_binary_crossentropy(self): 1782 # Test sigmoid_binary_crossentropy(p, t) == 1783 # binary_crossentropy(sigmoid(p), t). 1784 1785 pred, target = inputs = tensor.vectors('pt') 1786 1787 reference_val = binary_crossentropy(sigmoid(pred), target) 1788 f_reference = theano.function(inputs, reference_val) 1789 1790 test_val = sigmoid_binary_crossentropy(pred, target) 1791 f_test = theano.function(inputs, test_val) 1792 1793 test_inputs = self._get_test_inputs() 1794 utt.assert_allclose(f_reference(*test_inputs), f_test(*test_inputs)) 1795 1796 def test_grad(self): 1797 utt.verify_grad(sigmoid_binary_crossentropy, self._get_test_inputs()) 1798 1799 1800def test_confusion_matrix(): 1801 # Defining numpy implementation of confusion matrix 1802 def numpy_conf_mat(actual, pred): 1803 order = np.union1d(actual, pred) 1804 colA = np.matrix(actual).T 1805 colP = np.matrix(pred).T 1806 oneHotA = colA.__eq__(order).astype('int64') 1807 oneHotP = colP.__eq__(order).astype('int64') 1808 conf_mat = np.dot(oneHotA.T, oneHotP) 1809 conf_mat = np.asarray(conf_mat) 1810 return [conf_mat, order] 1811 1812 x = tensor.vector() 1813 y = tensor.vector() 1814 f = theano.function([x, y], confusion_matrix(x, y)) 1815 list_inputs = [[[0, 1, 2, 1, 0], [0, 0, 2, 1, 2]], 1816 [[2, 0, 2, 2, 0, 1], [0, 0, 2, 2, 0, 2]]] 1817 1818 for case in list_inputs: 1819 a = np.asarray(case[0]) 1820 b = np.asarray(case[1]) 1821 out_exp = numpy_conf_mat(a, b) 1822 outs = f(case[0], case[1]) 1823 for exp, out in zip(out_exp, outs): 1824 utt.assert_allclose(exp, out) 1825