1 // 2 // nnGradTest.cpp 3 // MNN 4 // 5 // Created by MNN on 2019/11/27. 6 // Copyright © 2018, Alibaba Group Holding Limited 7 // 8 9 #include <string.h> 10 #include "ADAM.hpp" 11 #include "DemoUnit.hpp" 12 #include "NN.hpp" 13 #include "SGD.hpp" 14 using namespace MNN::Express; 15 using namespace MNN::Train; 16 #include <random> 17 std::random_device gDevice; 18 class NNGrad : public DemoUnit { 19 public: run(int argc,const char * argv[])20 virtual int run(int argc, const char* argv[]) override { 21 MNN_PRINT("Test grad for convolution, pool, concat\n"); 22 int ic = 13; 23 int oc = 11; 24 int kw = 3; 25 int kh = 4; 26 int iw = 100; 27 int ih = 120; 28 int weightSize = ic * oc * kw * kh; 29 std::vector<float> targetVecs(weightSize); 30 for (int i = 0; i < weightSize; ++i) { 31 auto v = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 32 targetVecs[i] = v; 33 } 34 auto weightTarget = _Const(targetVecs.data(), {oc, ic, kh, kw}, NCHW); 35 std::vector<float> targetVecsBias(oc); 36 for (int i = 0; i < oc; ++i) { 37 targetVecsBias[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 38 } 39 auto biasTarget = _Const(targetVecsBias.data(), {oc}, NCHW); 40 41 NN::ConvOption convOption; 42 convOption.channel = {ic, oc}; 43 convOption.kernelSize = {kw, kh}; 44 convOption.stride = {2, 2}; 45 convOption.dilate = {1, 2}; 46 convOption.padMode = SAME; 47 std::shared_ptr<Module> convModule(NN::Conv(convOption)); 48 49 std::shared_ptr<SGD> sgd(new SGD(convModule)); 50 sgd->setLearningRate(0.01f); 51 std::vector<float> randomInputs(1 * ic * ih * iw); 52 for (int i = 0; i < randomInputs.size(); ++i) { 53 randomInputs[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 54 } 55 56 for (int i = 0; i < 100; ++i) { 57 auto input = _Input({1, ic, ih, iw}, NCHW); 58 auto inputPtr = input->writeMap<float>(); 59 ::memcpy(inputPtr, randomInputs.data(), randomInputs.size() * sizeof(float)); 60 61 auto targetValue = _Conv(weightTarget, biasTarget, _Convert(input, NC4HW4), convOption.padMode, 62 convOption.stride, convOption.dilate); 63 auto predictValue = convModule->forward(input); 64 65 auto targetValue1 = _MaxPool(targetValue, {2, 2}, {2, 2}); 66 auto targetValue2 = _AvePool(targetValue, {2, 2}, {2, 2}); 67 auto predictValue1 = _MaxPool(predictValue, {2, 2}, {2, 2}); 68 auto predictValue2 = _AvePool(predictValue, {2, 2}, {2, 2}); 69 targetValue = _Concat({targetValue1, targetValue2}, 1); 70 predictValue = _Concat({predictValue1, predictValue2}, 1); 71 targetValue = _Convert(targetValue, NCHW); 72 predictValue = _Convert(predictValue, NCHW); 73 auto loss = _ReduceMax(_Square(_Subtract(targetValue, predictValue)), {}); 74 MNN_PRINT("Loss = %f\n", loss->readMap<float>()[0]); 75 sgd->step(loss); 76 } 77 return 0; 78 } 79 }; 80 class NNGradV2 : public DemoUnit { 81 public: run(int argc,const char * argv[])82 virtual int run(int argc, const char* argv[]) override { 83 MNN_PRINT("Test grad for concat, split, transpose\n"); 84 int ic = 7; 85 int oc = 7; 86 int kw = 3; 87 int kh = 4; 88 int iw = 100; 89 int ih = 120; 90 int weightSize = ic * oc * kw * kh; 91 std::vector<float> targetVecs(weightSize); 92 for (int i = 0; i < weightSize; ++i) { 93 auto v = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 94 targetVecs[i] = v; 95 } 96 auto weightTarget = _Const(targetVecs.data(), {ic, 1, kh, kw}, NCHW); 97 std::vector<float> targetVecsBias(oc); 98 for (int i = 0; i < oc; ++i) { 99 targetVecsBias[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 100 } 101 auto biasTarget = _Const(targetVecsBias.data(), {oc}, NCHW); 102 103 NN::ConvOption convOption; 104 convOption.channel = {ic, oc}; 105 convOption.kernelSize = {kw, kh}; 106 convOption.stride = {2, 2}; 107 convOption.dilate = {1, 2}; 108 convOption.depthwise = true; 109 std::shared_ptr<Module> convModule(NN::Conv(convOption)); 110 111 std::shared_ptr<SGD> sgd(new SGD(convModule)); 112 sgd->setLearningRate(0.1f); 113 sgd->setWeightDecay(0.0f); 114 sgd->setMomentum(0.0f); 115 116 std::vector<float> randomInputs(1 * ic * ih * iw); 117 for (int i = 0; i < randomInputs.size(); ++i) { 118 randomInputs[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 119 } 120 121 for (int i = 0; i < 100; ++i) { 122 auto input = _Input({1, ic, ih, iw}, NCHW); 123 auto inputPtr = input->writeMap<float>(); 124 ::memcpy(inputPtr, randomInputs.data(), randomInputs.size() * sizeof(float)); 125 auto targetValue = _Conv(weightTarget, biasTarget, _Convert(input, NC4HW4), convOption.padMode, 126 convOption.stride, convOption.dilate, ic); 127 auto predictValue = convModule->forward(input); 128 129 auto targetValue1 = _MaxPool(targetValue, {2, 2}, {2, 2}); 130 auto targetValue2 = _AvePool(targetValue, {2, 2}, {2, 2}); 131 auto predictValue1 = _MaxPool(predictValue, {2, 2}, {2, 2}); 132 auto predictValue2 = _AvePool(predictValue, {2, 2}, {2, 2}); 133 targetValue = _Concat({targetValue1, targetValue2}, 1); 134 predictValue = _Concat({predictValue1, predictValue2}, 1); 135 136 auto slicetarget = _Split(targetValue, {2}, 2); 137 auto slicePredict = _Split(predictValue, {2}, 2); 138 targetValue = slicetarget[0]; 139 predictValue = slicePredict[0]; 140 targetValue = _Convert(targetValue, NCHW); 141 targetValue = _Transpose(targetValue, {1, 3, 2, 0}); 142 predictValue = _Convert(predictValue, NCHW); 143 predictValue = _Transpose(predictValue, {1, 3, 2, 0}); 144 auto loss = _ReduceMean(_Square(_Subtract(targetValue, predictValue)), {}); 145 MNN_PRINT("Loss = %f\n", loss->readMap<float>()[0]); 146 sgd->step(loss); 147 } 148 return 0; 149 } 150 }; 151 class NNGradV3 : public DemoUnit { 152 public: run(int argc,const char * argv[])153 virtual int run(int argc, const char* argv[]) override { 154 MNN_PRINT("Test grad for Deconvolution(+dw), Resize\n"); 155 int ic = 13; 156 int oc = 11; 157 int kw = 3; 158 int kh = 4; 159 int iw = 100; 160 int ih = 120; 161 int weightSize = ic * oc * kw * kh; 162 std::vector<float> targetVecs(weightSize); 163 for (int i = 0; i < weightSize; ++i) { 164 auto v = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 165 targetVecs[i] = v; 166 } 167 auto weightTarget = _Const(targetVecs.data(), {ic, oc, kh, kw}, NCHW); 168 std::vector<float> targetVecsBias(oc); 169 for (int i = 0; i < oc; ++i) { 170 targetVecsBias[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 171 } 172 auto biasTarget = _Const(targetVecsBias.data(), {oc}, NCHW); 173 174 NN::ConvOption convOption; 175 convOption.channel = {ic, oc}; 176 convOption.kernelSize = {kw, kh}; 177 convOption.stride = {2, 2}; 178 convOption.dilate = {1, 2}; 179 std::shared_ptr<Module> convModule(NN::ConvTranspose(convOption)); 180 181 convOption.depthwise = true; 182 convOption.channel = {oc, oc}; 183 std::shared_ptr<Module> convModule2(NN::ConvTranspose(convOption, false)); 184 VARP weightTarget2; 185 { 186 int weightSize = oc * kw * kh; 187 std::vector<float> targetVecs(weightSize); 188 for (int i = 0; i < weightSize; ++i) { 189 auto v = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 190 targetVecs[i] = v; 191 } 192 weightTarget2 = _Const(targetVecs.data(), {oc, 1, kh, kw}, NCHW); 193 } 194 195 std::shared_ptr<ADAM> sgd(new ADAM(convModule)); 196 sgd->setLearningRate(0.01f); 197 std::vector<float> randomInputs(1 * ic * ih * iw); 198 for (int i = 0; i < randomInputs.size(); ++i) { 199 randomInputs[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 200 } 201 202 for (int i = 0; i < 1000; ++i) { 203 auto input = _Input({1, ic, ih, iw}, NCHW); 204 auto inputPtr = input->writeMap<float>(); 205 ::memcpy(inputPtr, randomInputs.data(), randomInputs.size() * sizeof(float)); 206 207 auto targetValue = _Deconv(weightTarget, biasTarget, _Convert(input, NC4HW4), convOption.padMode, 208 convOption.stride, convOption.dilate); 209 auto predictValue = convModule->forward(input); 210 targetValue = _Deconv(weightTarget2, nullptr, targetValue, convOption.padMode, convOption.stride, 211 convOption.dilate, oc); 212 predictValue = convModule2->forward(predictValue); 213 214 auto targetValue1 = _MaxPool(targetValue, {2, 2}, {2, 2}); 215 auto targetValue2 = _AvePool(targetValue, {2, 2}, {2, 2}); 216 auto predictValue1 = _MaxPool(predictValue, {2, 2}, {2, 2}); 217 auto predictValue2 = _AvePool(predictValue, {2, 2}, {2, 2}); 218 targetValue = _Concat({targetValue1, targetValue2}, 1); 219 predictValue = _Concat({predictValue1, predictValue2}, 1); 220 targetValue = _Interp({targetValue}, 2.15f, 0.5f, 0, 0, 2, true); 221 predictValue = _Interp({predictValue}, 2.15f, 0.5f, 0, 0, 2, true); 222 223 targetValue = _Convert(targetValue, NCHW); 224 predictValue = _Convert(predictValue, NCHW); 225 auto loss = _ReduceMean(_Square(_Subtract(targetValue, predictValue)), {}); 226 MNN_PRINT("Loss = %f\n", loss->readMap<float>()[0]); 227 sgd->step(loss); 228 } 229 return 0; 230 } 231 }; 232 class MatMulGradTest : public DemoUnit { 233 public: run(int argc,const char * argv[])234 virtual int run(int argc, const char* argv[]) override { 235 MNN_PRINT("Test grad for MatMul, BatchMatMul\n"); 236 { 237 int e = 13; 238 int l = 11; 239 int h = 30; 240 int weightSize = l * h; 241 std::vector<float> targetVecs(weightSize); 242 for (int i = 0; i < weightSize; ++i) { 243 auto v = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 244 targetVecs[i] = v; 245 } 246 auto weightTarget = _Const(targetVecs.data(), {l, h}, NCHW); 247 auto weightOrigin = _TrainableParam(0.01f, {l, h}, NCHW); 248 std::shared_ptr<Module> _m(Module::createEmpty({weightOrigin})); 249 std::shared_ptr<SGD> sgd(new SGD(_m)); 250 sgd->setLearningRate(0.01f); 251 std::vector<float> randomInputs(e * l); 252 for (int i = 0; i < randomInputs.size(); ++i) { 253 randomInputs[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 254 } 255 256 for (int i = 0; i < 1000; ++i) { 257 auto input = _Input({e, l}, NCHW); 258 auto inputPtr = input->writeMap<float>(); 259 ::memcpy(inputPtr, randomInputs.data(), randomInputs.size() * sizeof(float)); 260 261 auto targetValue = _MatMul(input, weightTarget); 262 auto predictValue = _MatMul(input, weightOrigin); 263 auto loss = _ReduceMean(_Square(_Subtract(targetValue, predictValue)), {}); 264 if (i % 100 == 0) { 265 MNN_PRINT("Loss = %f\n", loss->readMap<float>()[0]); 266 } 267 sgd->step(loss); 268 } 269 } 270 MNN_PRINT("Test for BatchMatMul\n"); 271 { 272 int e = 13; 273 int l = 11; 274 int h = 30; 275 int b = 5; 276 int weightSize = b * l * h; 277 std::vector<float> targetVecs(weightSize); 278 for (int i = 0; i < weightSize; ++i) { 279 auto v = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 280 targetVecs[i] = v; 281 } 282 auto weightTarget = _Const(targetVecs.data(), {b, l, h}, NCHW); 283 auto weightOrigin = _TrainableParam(0.01f, {b, l, h}, NCHW); 284 std::shared_ptr<Module> _m(Module::createEmpty({weightOrigin})); 285 std::shared_ptr<ADAM> sgd(new ADAM(_m)); 286 sgd->setLearningRate(0.01f); 287 std::vector<float> randomInputs(b * e * l); 288 for (int i = 0; i < randomInputs.size(); ++i) { 289 randomInputs[i] = ((float)(gDevice() % 2000) - 1000.0f) / 1000.0f; 290 } 291 292 for (int i = 0; i < 10000; ++i) { 293 auto input = _Input({b, e, l}, NCHW); 294 auto inputPtr = input->writeMap<float>(); 295 ::memcpy(inputPtr, randomInputs.data(), randomInputs.size() * sizeof(float)); 296 297 auto targetValue = _BatchMatMul(input, weightTarget); 298 auto predictValue = _BatchMatMul(input, weightOrigin); 299 targetValue = _Relu6(targetValue); 300 predictValue = _Relu6(predictValue); 301 auto loss = _ReduceMean(_Square(_Subtract(targetValue, predictValue)), {}); 302 if (i % 1000 == 0) { 303 MNN_PRINT("Loss = %f\n", loss->readMap<float>()[0]); 304 } 305 sgd->step(loss); 306 } 307 } 308 return 0; 309 } 310 }; 311 class GatherGradTest : public DemoUnit { 312 public: run(int argc,const char * argv[])313 virtual int run(int argc, const char* argv[]) override { 314 MNN_PRINT("Test grad for Gather\n"); 315 { 316 // set input data 317 const float inpudata[] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 318 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21, 0, 22.0, 23.0, 24.0}; 319 std::vector<float> inputDataRaw(0.0f, sizeof(inpudata) / sizeof(float)); 320 auto params = _TrainableParam(inputDataRaw.data(), {4, 3, 2}, NCHW, halide_type_of<float>()); 321 const int indices_data[] = {1, 0, 1, 0}; 322 const std::vector<float> expectedOutput = {7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 323 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; 324 std::shared_ptr<Module> _m(Module::createEmpty({params})); 325 std::shared_ptr<SGD> sgd(new SGD(_m)); 326 sgd->setLearningRate(0.01f); 327 for (int i = 0; i < 1000; ++i) { 328 auto indices = _Const(indices_data, {4}, NCHW, halide_type_of<int>()); 329 auto output = _GatherV2(params, indices, nullptr); 330 output = _Reshape(output, {-1}); 331 auto predictValue = _Const(expectedOutput.data(), {(int)expectedOutput.size()}, NCHW); 332 auto loss = _ReduceMean(_Square(_Subtract(output, predictValue)), {}); 333 if (i % 100 == 0) { 334 MNN_PRINT("Loss = %f\n", loss->readMap<float>()[0]); 335 } 336 sgd->step(loss); 337 } 338 } 339 return 0; 340 } 341 }; 342 343 DemoUnitSetRegister(NNGrad, "NNGrad"); 344 DemoUnitSetRegister(NNGradV2, "NNGradV2"); 345 DemoUnitSetRegister(NNGradV3, "NNGradV3"); 346 DemoUnitSetRegister(MatMulGradTest, "MatMulGradTest"); 347 DemoUnitSetRegister(GatherGradTest, "GatherGradTest"); 348