1# Licensed to the Apache Software Foundation (ASF) under one
2# or more contributor license agreements.  See the NOTICE file
3# distributed with this work for additional information
4# regarding copyright ownership.  The ASF licenses this file
5# to you under the Apache License, Version 2.0 (the
6# "License"); you may not use this file except in compliance
7# with the License.  You may obtain a copy of the License at
8#
9#   http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing,
12# software distributed under the License is distributed on an
13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14# KIND, either express or implied.  See the License for the
15# specific language governing permissions and limitations
16# under the License.
17
18use strict;
19use warnings;
20use Test::More tests => 232;
21use AI::MXNet qw(mx);
22use AI::MXNet::Gluon qw(gluon);
23use AI::MXNet::Gluon::NN qw(nn);
24use AI::MXNet::TestUtils qw(almost_equal dies_ok);
25use Scalar::Util qw(refaddr);
26use AI::MXNet::Base;
27
28sub test_parameter
29{
30    my $p = gluon->Parameter('weight', shape=>[10, 10]);
31    $p->initialize(init=>'xavier', ctx=>[mx->cpu(0), mx->cpu(1)]);
32    ok(@{$p->list_data} == 2);
33    ok(@{$p->list_grad} == 2);
34    ok($p->data(mx->cpu(1))->context eq mx->cpu(1));
35    is_deeply($p->data(mx->cpu(0))->shape, [10, 10]);
36    ok($p->var->name eq  'weight');
37    ok($p->grad(mx->cpu(0))->stype eq 'default');
38    ok($p->data(mx->cpu(0))->stype eq 'default');
39
40    $p->reset_ctx(ctx=>[mx->cpu(1), mx->cpu(2)]);
41    is_deeply($p->list_ctx, [mx->cpu(1), mx->cpu(2)]);
42}
43
44test_parameter();
45
46sub test_invalid_parameter_stype
47{
48    dies_ok(sub { gluon->Parameter('weight', shape=>[10, 10], stype=>'invalid') });
49}
50
51test_invalid_parameter_stype();
52
53sub test_invalid_parameter_grad_stype
54{
55    dies_ok(sub { gluon->Parameter('weight', shape=>[10, 10], grad_stype=>'invalid') });
56}
57
58test_invalid_parameter_grad_stype();
59
60sub test_sparse_parameter
61{
62    my $p = gluon->Parameter('weight', shape=>[10, 10], stype=>'row_sparse', grad_stype=>'row_sparse');
63    $p->initialize(init=>'xavier', ctx=>[mx->cpu(0), mx->cpu(1)]);
64    my $row_id = mx->nd->arange(start => 0, stop => 10, ctx=>mx->cpu(1));
65    ok(@{ $p->list_grad } == 2);
66    # getting row_sparse data without trainer throws an exception
67    dies_ok(sub { $p->list_row_sparse_data($row_id) });
68    my $trainer = gluon->Trainer([$p], 'sgd');
69    ok(@{ $p->list_row_sparse_data($row_id) } == 2);
70    my $weight = $p->row_sparse_data($row_id);
71    ok($weight->context eq mx->cpu(1));
72    is_deeply($weight->shape, [10, 10]);
73    ok($weight->stype eq 'row_sparse');
74    ok($p->var->name eq 'weight');
75    ok($p->var->attr('__storage_type__') eq STORAGE_TYPE_STR_TO_ID->{row_sparse});
76    ok($p->grad(mx->cpu(0))->stype eq 'row_sparse');
77
78    $p->reset_ctx(ctx=>[mx->cpu(1), mx->cpu(2)]);
79    is_deeply($p->list_ctx, [mx->cpu(1), mx->cpu(2)]);
80}
81
82test_sparse_parameter();
83
84sub test_parameter_invalid_access
85{
86    # cannot call data on row_sparse parameters
87    my $p0 = gluon->Parameter('weight', shape=>[10, 10], stype=>'row_sparse', grad_stype=>'row_sparse');
88    $p0->initialize(init=>'xavier', ctx=>[mx->cpu(0), mx->cpu(1)]);
89    dies_ok(sub { $p0->data });
90    dies_ok(sub { $p0->list_data });
91    my $row_id = mx->nd->arange(start => 0, stop => 10);
92    # cannot call row_sparse_data on dense parameters
93    my $p1 = gluon->Parameter('weight', shape=>[10, 10]);
94    $p1->initialize(init=>'xavier', ctx=>[mx->cpu(0), mx->cpu(1)]);
95    dies_ok(sub { $p1->row_sparse_data($row_id->copyto(mx->cpu(0))) });
96    dies_ok(sub { $p1->list_row_sparse_data($row_id) });
97}
98
99test_parameter_invalid_access();
100
101sub test_paramdict
102{
103    my $ctx = mx->cpu(1);
104    my $params0 = gluon->ParameterDict('net_');
105    $params0->get('w0', shape=>[10, 10]);
106    $params0->get('w1', shape=>[10, 10], stype=>'row_sparse');
107    my $all_row_ids = mx->nd->arange(start => 0, stop => 10, ctx=>$ctx);
108    # check param names
109    is_deeply([$params0->keys()], ['net_w0', 'net_w1']);
110    $params0->initialize(ctx=>$ctx);
111    my $trainer0 = gluon->Trainer($params0, 'sgd');
112    my $prev_w0 = $params0->get('w0')->data($ctx);
113    my $prev_w1 = $params0->get('w1')->row_sparse_data($all_row_ids);
114    # save params
115    $params0->save('test_paramdict.params');
116
117    # load params
118    my $params1 = gluon->ParameterDict('net_');
119    $params1->get('w0', shape=>[10, 10]);
120    $params1->get('w1', shape=>[10, 10], stype=>'row_sparse');
121    $params1->load('test_paramdict.params', ctx=>$ctx);
122    my $trainer1 = gluon->Trainer($params1, 'sgd');
123
124    # compare the values before and after save/load
125    my $cur_w0 = $params1->get('w0')->data($ctx);
126    my $cur_w1 = $params1->get('w1')->row_sparse_data($all_row_ids);
127    ok(almost_equal($prev_w0->aspdl, $cur_w0->aspdl));
128    ok(almost_equal($prev_w1->aspdl, $cur_w1->aspdl));
129
130    # create a new param dict with dense params, and load from the checkpoint
131    # of sparse & dense params
132    my $params2 = gluon->ParameterDict('net_');
133    $params2->get('w0', shape=>[10, 10]);
134    $params2->get('w1', shape=>[10, 10]);
135    $params2->load('test_paramdict.params', ctx=>$ctx);
136
137    # compare the values before and after save/load
138    $cur_w0 = $params2->get('w0')->data($ctx);
139    $cur_w1 = $params2->get('w1')->data($ctx);
140    ok(almost_equal($prev_w0->aspdl, $cur_w0->aspdl));
141    ok(almost_equal($prev_w1->aspdl, $cur_w1->aspdl));
142}
143
144test_paramdict();
145
146sub test_parameter_row_sparse_data
147{
148    my $ctx0 = mx->cpu(1);
149    my $ctx1 = mx->cpu(2);
150    my $dim0 = 4;
151    my $x = gluon->Parameter('x', shape=>[$dim0, 2], stype=>'row_sparse');
152    $x->initialize(init=>'xavier', ctx=>[$ctx0, $ctx1]);
153    my $trainer = gluon->Trainer([$x], 'sgd');
154    my $x_param = $x->_data->[0]->copy();
155    is($x_param->stype, 'row_sparse');
156    my $row_id_0 = mx->nd->array([0,1], ctx=>$ctx0);
157    my $retained_0 = $x->row_sparse_data($row_id_0);
158    my $retained_target_0 = mx->nd->sparse->retain($x_param, $row_id_0->as_in_context($ctx0));
159    ok(almost_equal($retained_0->aspdl, $retained_target_0->aspdl));
160    is($retained_0->context, $ctx0);
161    my $row_id_1 = mx->nd->arange(start => 0, stop => $dim0, ctx=>$ctx1);
162    my $retained_1 = $x->row_sparse_data($row_id_1);
163    my $retained_target_1 = $x_param;
164    ok(almost_equal($retained_1->aspdl, $retained_target_1->aspdl));
165    is($retained_1->context, $ctx1);
166    my $row_id_2 = mx->nd->array([0,1,2]);
167    my $retained_2 = $x->list_row_sparse_data($row_id_2);
168    my $retained_target_2 = mx->nd->sparse->retain($x_param, $row_id_2->as_in_context($ctx0));
169    ok(almost_equal($retained_2->[0]->aspdl, $retained_target_2->aspdl));
170}
171
172test_parameter_row_sparse_data();
173
174sub test_constant
175{
176    package Test {
177        use AI::MXNet::Gluon::Mouse;
178        extends 'AI::MXNet::Gluon::HybridBlock';
179        sub BUILD
180        {
181            my $self = shift;
182            $self->value(mx->nd->array([[1,2], [3,4]])->aspdl);
183            $self->const($self->params->get_constant('const', $self->value));
184        }
185        sub hybrid_forward
186        {
187            my ($self, $F, $x, $name, $const) = @_;
188            return $x + $const;
189        }
190    };
191
192    my $test = Test->new();
193    $test->initialize();
194    my $trainer = gluon->Trainer(
195        $test->collect_params(), 'sgd',
196        {learning_rate => 1.0, momentum => 0.5}
197    );
198
199    my ($x, $y);
200    mx->autograd->record(sub {
201        $x = mx->nd->ones([2,2]);
202        $x->attach_grad();
203        $y = $test->($x);
204        $y->backward();
205    });
206
207    $trainer->step(1);
208
209    ok(($test->const->data->aspdl == $test->value)->all);
210    ok(($x->grad->aspdl == 1)->all);
211}
212
213test_constant();
214
215package Net;
216use AI::MXNet::Gluon::Mouse;
217use AI::MXNet::Function::Parameters;
218extends 'AI::MXNet::Gluon::Block';
219has 'in_units' => (is => 'rw', default => 0);
220
221sub BUILD
222{
223    my $self = shift;
224    $self->name_scope(sub {
225        $self->dense0(nn->Dense(5, in_units=>$self->in_units));
226        $self->dense1(nn->Dense(5, in_units=>$self->in_units));
227    });
228}
229
230method forward($x)
231{
232    return $self->dense1->($self->dense0->($x));
233}
234
235package main;
236
237sub test_parameter_sharing
238{
239    my $net1 = Net->new(prefix=>'net1_', in_units => 5);
240    my $net2 = Net->new(prefix=>'net2_', params=>$net1->collect_params());
241    $net1->collect_params()->initialize();
242    $net2->(mx->nd->zeros([3, 5]));
243    $net1->save_parameters('net1.params');
244    my $net3 = Net->new(prefix=>'net3_');
245    $net3->load_parameters('net1.params', ctx => mx->cpu());
246    my $net4 = Net->new(prefix=>'net4_');
247    my $net5 = Net->new(prefix=>'net5_', in_units=>5, params=>$net4->collect_params());
248    $net4->collect_params()->initialize();
249    $net5->(mx->nd->zeros([3, 5]));
250    $net4->save_parameters('net4.params');
251    my $net6 = Net->new(prefix=>'net6_');
252    $net6->load_parameters('net4.params', ctx => mx->cpu());
253}
254
255test_parameter_sharing();
256
257sub test_parameter_str
258{
259    package Net1 {
260        use AI::MXNet::Gluon::Mouse;
261        extends 'AI::MXNet::Gluon::Block';
262        sub BUILD
263        {
264            my $self = shift;
265            $self->name_scope(sub {
266                $self->dense0(nn->Dense(10, in_units=>5, use_bias=>0));
267            });
268        }
269    };
270    my $net = Net1->new(prefix=>'net1_');
271    my @lines = split(/\n/, $net->collect_params());
272    ok($lines[0] eq 'net1_ (');
273    ok($lines[1] =~ /net1_dense0_weight/);
274    ok($lines[1] =~ /\(10, 5\)/);
275    ok($lines[1] =~ /float32/);
276    ok($lines[2] eq ')');
277}
278
279test_parameter_str();
280
281sub test_collect_parameters
282{
283    my $net = nn->HybridSequential(prefix=>"test_");
284    $net->name_scope(sub {
285        $net->add(nn->Conv2D(10, 3));
286        $net->add(nn->Dense(10, activation=>'relu'));
287    });
288    is_deeply(
289        [$net->collect_params->keys],
290        ['test_conv0_weight', 'test_conv0_bias','test_dense0_weight','test_dense0_bias']
291    );
292    is_deeply(
293        [$net->collect_params('.*weight')->keys],
294        ['test_conv0_weight', 'test_dense0_weight']
295    );
296    is_deeply(
297        [$net->collect_params('test_conv0_bias|test_dense0_bias')->keys],
298        ['test_conv0_bias', 'test_dense0_bias']
299    )
300};
301
302test_collect_parameters();
303
304sub test_basic
305{
306    my $model = nn->Sequential();
307    $model->add(nn->Dense(128, activation=>'tanh', in_units=>10, flatten=>0));
308    $model->add(nn->Dropout(0.5));
309    $model->add(nn->Dense(64, activation=>'tanh', in_units=>256));
310    $model->add(nn->Dense(32, in_units=>64));
311    $model->add(nn->Activation('relu'));
312
313    # symbol
314    my $x = mx->sym->var('data');
315    my $y = $model->($x);
316    ok(@{ $y->list_arguments } == 7);
317
318    # ndarray
319    $model->collect_params()->initialize(init => mx->init->Xavier(magnitude=>2.24));
320    $x = $model->(mx->nd->zeros([32, 2, 10]));
321    is_deeply($x->shape, [32, 32]);
322    $x->wait_to_read;
323
324    $model->collect_params()->setattr(grad_req => 'null');
325    ok(not defined( ($model->collect_params()->values())[0]->_grad));
326    $model->collect_params()->setattr(grad_req => 'write');
327    ok(defined (($model->collect_params()->values())[0]->_grad));
328}
329
330test_basic();
331
332sub test_dense
333{
334    my $model = nn->Dense(128, activation=>'tanh', in_units=>10, flatten=>0, prefix=>'test_');
335    my $inputs = mx->sym->Variable('data');
336    my $outputs = $model->($inputs);
337    is_deeply({map { $_ => 1 } $model->collect_params()->keys()}, {'test_weight', 1, 'test_bias', 1});
338    is_deeply($outputs->list_outputs(), ['test_tanh_fwd_output']);
339    my ($args, $outs, $auxs) = $outputs->infer_shape(data=>[2, 3, 10]);
340    is_deeply($outs, [[2, 3, 128]]);
341
342    $model = nn->Dense(128, activation=>'relu', in_units=>30, flatten=>1, prefix=>'test2_');
343    $inputs = mx->sym->Variable('data');
344    $outputs = $model->($inputs);
345    is_deeply({map { $_ => 1 } $model->collect_params()->keys()}, {'test2_weight', 1, 'test2_bias', 1});
346    is_deeply($outputs->list_outputs(), ['test2_relu_fwd_output']);
347    ($args, $outs, $auxs) = $outputs->infer_shape(data=>[17, 2, 5, 3]);
348    is_deeply($outs, [[17, 128]]);
349}
350
351test_dense();
352
353package Net2;
354use AI::MXNet::Gluon::Mouse;
355use AI::MXNet::Function::Parameters;
356extends 'AI::MXNet::Gluon::HybridBlock';
357has 'model' => (is => 'rw');
358
359method hybrid_forward($F, $x)
360{
361    my $out = $self->model->($x);
362    return $F->add_n(map { $_->sum } @{ $out });
363}
364
365package main;
366
367sub test_symbol_block
368{
369    my $model = nn->HybridSequential();
370    $model->add(nn->Dense(128, activation=>'tanh'));
371    $model->add(nn->Dropout(0.5));
372    $model->add(nn->Dense(64, activation=>'tanh'));
373    $model->add(nn->Dense(32, in_units=>64));
374    $model->add(nn->Activation('relu'));
375
376    $model->initialize();
377
378    my $inputs = mx->sym->var('data');
379    my $outputs = $model->($inputs)->get_internals();
380    my $smodel = gluon->SymbolBlock($outputs, $inputs, params=>$model->collect_params);
381
382    ok($smodel->(mx->nd->zeros([16, 10])) == 14);
383    my $out = $smodel->(mx->sym->var('in'));
384    ok(@{ $out } == @{ $outputs->list_outputs() });
385
386    my $net = Net2->new(model => $smodel);
387    $net->hybridize();
388    ok(ref $net->(mx->nd->zeros([16, 10])) eq 'AI::MXNet::NDArray');
389
390    $inputs = mx->sym->var('data');
391    $outputs = $model->($inputs);
392    $smodel = gluon->SymbolBlock($outputs, $inputs, params=>$model->collect_params);
393    $net = Net2->new(model => $smodel);
394    $net->hybridize();
395    ok(ref $net->(mx->nd->zeros([16, 10])) eq 'AI::MXNet::NDArray');
396}
397
398test_symbol_block();
399
400sub test_sparse_symbol_block
401{
402    my $data = mx->sym->var('data');
403    my $weight = mx->sym->var('weight', stype=>'row_sparse');
404    my $bias = mx->sym->var('bias');
405    my $out = mx->sym->broadcast_add(mx->sym->dot($data, $weight), $bias);
406    # an exception is expected when creating a SparseBlock w/ sparse param
407    dies_ok(sub { gluon->SymbolBlock($out, $data) });
408}
409
410test_sparse_symbol_block();
411
412sub test_sparse_hybrid_block0
413{
414    my $params = gluon->ParameterDict('net_');
415    $params->get('weight', shape=>[5,5], stype=>'row_sparse', dtype=>'float32', allow_deferred_init => 1);
416    $params->get('bias', shape=>[5], dtype=>'float32', allow_deferred_init => 1);
417    my $net = nn->Dense(5, params=>$params);
418    $net->initialize();
419    my $x = mx->nd->ones([2,5]);
420    # an exception is expected when forwarding a HybridBlock w/ sparse param
421    dies_ok(sub { $net->($x) });
422}
423
424test_sparse_hybrid_block0();
425
426sub check_layer_forward
427{
428    my ($layer, $dshape) = @_;
429    $layer->collect_params()->initialize();
430    my $x = mx->nd->ones($dshape);
431    $x->attach_grad();
432    my $out;
433    mx->autograd->record(sub {
434        $out = $layer->($x);
435    });
436    $out->backward();
437    my $pdl_out = $out->aspdl;
438    my $pdl_dx  = $x->grad->aspdl;
439
440    $layer->hybridize();
441
442    $x = mx->nd->ones($dshape);
443    $x->attach_grad();
444    mx->autograd->record(sub {
445        $out = $layer->($x);
446    });
447    $out->backward();
448
449    ok(almost_equal($pdl_out, $out->aspdl, 1e-5));
450    ok(almost_equal($pdl_dx, $x->grad->aspdl, 1e-5));
451}
452
453sub test_conv
454{
455    my @layers1d = (
456        nn->Conv1D(16, 3, in_channels=>4),
457        nn->Conv1D(16, 3, groups=>2, in_channels=>4),
458        nn->Conv1D(16, 3, strides=>3, groups=>2, in_channels=>4),
459    );
460    for my $layer (@layers1d)
461    {
462        check_layer_forward($layer, [1, 4, 10]);
463    }
464
465    my @layers2d = (
466        nn->Conv2D(16, [3, 4], in_channels=>4),
467        nn->Conv2D(16, [5, 4], in_channels=>4),
468        nn->Conv2D(16, [3, 4], groups=>2, in_channels=>4),
469        nn->Conv2D(16, [3, 4], strides=>4, in_channels=>4),
470        nn->Conv2D(16, [3, 4], dilation=>4, in_channels=>4),
471        nn->Conv2D(16, [3, 4], padding=>4, in_channels=>4),
472    );
473    for my $layer (@layers2d)
474    {
475        check_layer_forward($layer, [1, 4, 20, 20]);
476    }
477
478    my @layers3d = (
479        nn->Conv3D(16, [1, 8, 4], in_channels=>4, activation=>'relu'),
480        nn->Conv3D(16, [5, 4, 3], in_channels=>4),
481        nn->Conv3D(16, [3, 3, 3], groups=>2, in_channels=>4),
482        nn->Conv3D(16, 4, strides=>4, in_channels=>4),
483        nn->Conv3D(16, [3, 3, 3], padding=>4, in_channels=>4),
484    );
485    for my $layer (@layers3d)
486    {
487        check_layer_forward($layer, [1, 4, 10, 10, 10]);
488    }
489
490    # These layouts only supported on GPU for now
491    my $layer = nn->Conv2D(16, [3, 3], layout=>'NHWC', in_channels=>4);
492    #check_layer_forward($layer, [1, 10, 10, 4]);
493
494    $layer = nn->Conv3D(16, [3, 3, 3], layout=>'NDHWC', in_channels=>4);
495    # check_layer_forward(layer, (1, 10, 10, 10, 4))
496}
497
498test_conv();
499
500
501sub test_deconv
502{
503    # commented out code is only supported on GPU for now
504    # my @layers1d = (
505    #     nn->Conv1DTranspose(16, 3, in_channels=>4),
506    #     nn->Conv1DTranspose(16, 3, groups=>2, in_channels=>4),
507    #     nn->Conv1DTranspose(16, 3, strides=>3, groups=>2, in_channels=>4),
508    # );
509    # for my $layer (@layers1d)
510    # {
511    #     check_layer_forward($layer, [1, 4, 10]);
512    # }
513
514
515    my @layers2d = (
516        nn->Conv2DTranspose(16, [3, 4], in_channels=>4),
517        nn->Conv2DTranspose(16, [5, 4], in_channels=>4),
518        nn->Conv2DTranspose(16, [3, 4], groups=>2, in_channels=>4),
519        nn->Conv2DTranspose(16, [3, 4], strides=>4, in_channels=>4),
520        nn->Conv2DTranspose(16, [3, 4], dilation=>4, in_channels=>4),
521        nn->Conv2DTranspose(16, [3, 4], padding=>4, in_channels=>4),
522        nn->Conv2DTranspose(16, [3, 4], strides=>4, output_padding=>3, in_channels=>4),
523    );
524    for my $layer (@layers2d)
525    {
526        check_layer_forward($layer, [1, 4, 20, 20]);
527    }
528
529    # @layers3d = (
530    #     nn->Conv3DTranspose(16, [1, 8, 4], in_channels=>4),
531    #     nn->Conv3DTranspose(16, [5, 4, 3], in_channels=>4),
532    #     nn->Conv3DTranspose(16, [3, 3, 3], groups=>2, in_channels=>4),
533    #     nn->Conv3DTranspose(16, 4, strides=>4, in_channels=>4),
534    #     nn->Conv3DTranspose(16, [3, 3, 3], padding=>4, in_channels=>4),
535    # );
536    # for my $layer (@layers3d)
537    # {
538    #     check_layer_forward($layer, [1, 4, 10, 10, 10]);
539    # }
540    #
541    my $layer = nn->Conv2DTranspose(16, [3, 3], layout=>'NHWC', in_channels=>4);
542    # check_layer_forward($layer, [1, 10, 10, 4]);
543    #
544    # $layer = nn->Conv3DTranspose(16, [3, 3, 3], layout=>'NDHWC', in_channels=>4);
545    # check_layer_forward(layer, [1, 10, 10, 10, 4]);
546}
547
548test_deconv();
549
550sub test_pool
551{
552    my @layers1d = (
553        nn->MaxPool1D(),
554        nn->MaxPool1D(3),
555        nn->MaxPool1D(3, 2),
556        nn->AvgPool1D(),
557        nn->AvgPool1D(count_include_pad=>0),
558        nn->GlobalAvgPool1D(),
559    );
560    for my $layer (@layers1d)
561    {
562        check_layer_forward($layer, [1, 2, 10]);
563    }
564
565    my @layers2d = (
566        nn->MaxPool2D(),
567        nn->MaxPool2D([3, 3]),
568        nn->MaxPool2D(3, 2),
569        nn->AvgPool2D(),
570        nn->AvgPool2D(count_include_pad=>0),
571        nn->GlobalAvgPool2D(),
572    );
573    for my $layer (@layers2d)
574    {
575        check_layer_forward($layer, [1, 2, 10, 10]);
576    }
577
578    my @layers3d = (
579        nn->MaxPool3D(),
580        nn->MaxPool3D([3, 3, 3]),
581        nn->MaxPool3D(3, 2),
582        nn->AvgPool3D(),
583        nn->AvgPool3D(count_include_pad=>0),
584        nn->GlobalAvgPool3D(),
585    );
586    for my $layer (@layers3d)
587    {
588        check_layer_forward($layer, [1, 2, 10, 10, 10]);
589    }
590
591    # test ceil_mode
592    my $x = mx->nd->zeros([2, 2, 10, 10]);
593
594    my $layer = nn->MaxPool2D(3, ceil_mode=>0);
595    $layer->collect_params()->initialize();
596    is_deeply($layer->($x)->shape, [2, 2, 3, 3]);
597
598    $layer = nn->MaxPool2D(3, ceil_mode=>1);
599    $layer->collect_params()->initialize();
600    is_deeply($layer->($x)->shape, [2, 2, 4, 4]);
601}
602
603test_pool();
604
605sub test_batchnorm
606{
607    my $layer = nn->BatchNorm(in_channels=>10);
608    check_layer_forward($layer, [2, 10, 10, 10]);
609}
610
611test_batchnorm();
612
613sub test_instancenorm
614{
615    my $layer = nn->InstanceNorm(in_channels=>10);
616    check_layer_forward($layer, [2, 10, 10, 10]);
617}
618
619test_instancenorm();
620
621sub test_layernorm
622{
623    my $layer = nn->LayerNorm(in_channels=>10);
624    check_layer_forward($layer, [2, 10, 10, 10]);
625}
626
627test_layernorm();
628
629sub test_reflectionpad
630{
631    my $layer = nn->ReflectionPad2D(3);
632    check_layer_forward($layer, [2, 3, 24, 24]);
633}
634
635test_reflectionpad();
636
637sub test_reshape
638{
639    my $x = mx->nd->ones([2, 4, 10, 10]);
640    my $layer = nn->Conv2D(10, 2, in_channels=>4);
641    $layer->collect_params()->initialize();
642    mx->autograd->record(sub {
643        $x = $layer->($x);
644        $x = $x->reshape([-1]);
645        $x = $x + 10;
646    });
647    $x->backward();
648}
649
650test_reshape();
651
652sub test_slice
653{
654    my $x = mx->nd->ones([5, 4, 10, 10]);
655    my $layer = nn->Conv2D(10, 2, in_channels=>4);
656    $layer->collect_params()->initialize();
657    mx->autograd->record(sub {
658        $x = $layer->($x);
659        $x = $x->slice([1,3]);
660        $x = $x + 10;
661    });
662    $x->backward();
663}
664
665test_slice();
666
667sub test_at
668{
669    my $x = mx->nd->ones([5, 4, 10, 10]);
670    my $layer = nn->Conv2D(10, 2, in_channels=>4);
671    $layer->collect_params()->initialize();
672    mx->autograd->record(sub {
673        $x = $layer->($x);
674        $x = $x->at(1);
675        $x = $x + 10;
676    });
677    $x->backward();
678}
679
680test_at();
681
682sub test_deferred_init
683{
684    my $x = mx->nd->ones([5, 4, 10, 10]);
685    my $layer = nn->Conv2D(10, 2);
686    $layer->collect_params()->initialize();
687    $layer->($x);
688}
689
690test_deferred_init();
691
692
693sub check_split_data
694{
695    my ($x, $num_slice, $batch_axis, %kwargs) = @_;
696    my $res = gluon->utils->split_data($x, $num_slice, $batch_axis, %kwargs);
697    ok(@{ $res } == $num_slice);
698    ok(almost_equal(mx->nd->concat(@$res, dim=>$batch_axis)->aspdl(), $x->aspdl()));
699}
700
701sub test_split_data
702{
703    my $x = mx->nd->random->uniform(shape=>[128, 33, 64]);
704
705    check_split_data($x, 8, 0);
706    check_split_data($x, 3, 1);
707    check_split_data($x, 4, 1, even_split=>0);
708    check_split_data($x, 15, 1, even_split=>0);
709    eval {
710        check_split_data($x, 4, 1);
711    };
712    ok($@);
713}
714
715test_split_data();
716
717sub test_flatten
718{
719    my $flatten = nn->Flatten();
720    my $x = mx->nd->zeros([3,4,5,6]);
721    is_deeply($flatten->($x)->shape, [3, 4*5*6]);
722    $x = mx->nd->zeros([3,6]);
723    is_deeply($flatten->($x)->shape, [3, 6]);
724    $x = mx->nd->zeros([3]);
725    is_deeply($flatten->($x)->shape, [3, 1]);
726}
727
728test_flatten();
729
730sub test_block_attr_hidden
731{
732    my $b = gluon->Block();
733    # regular attributes can change types
734    $b->a(undef);
735    $b->a(1);
736}
737
738test_block_attr_hidden();
739
740sub test_block_attr_block
741{
742    my $b = gluon->Block();
743    # regular variables can't change types
744    $b->b(gluon->Block());
745    eval { $b->b([2]); };
746    ok($@ =~ /not allowed/i);
747}
748
749test_block_attr_block();
750
751sub test_block_attr_param
752{
753    my $b = gluon->Block();
754    # regular variables can't change types
755    $b->b(gluon->Parameter(name => 'test'));
756    eval { $b->b([2]); };
757    ok($@ =~ /not allowed/i);
758}
759
760test_block_attr_param();
761
762sub test_block_attr_regular
763{
764    my $b = gluon->Block();
765
766    # set block attribute also sets _children
767    $b->c(gluon->Block());
768    my $c2 = gluon->Block();
769    $b->c($c2);
770    ok(refaddr($b->c) == refaddr($c2) and refaddr(($b->_children->values)[0]) == refaddr($c2));
771}
772
773test_block_attr_regular();
774
775sub test_block_attr_list_of_block
776{
777    package Model1 {
778        use AI::MXNet::Gluon::Mouse;
779        extends 'AI::MXNet::Gluon::Block';
780        sub BUILD
781        {
782            my $self = shift;
783            $self->name_scope(sub {
784                $self->layers([map { nn->Dense($_ * 10) } 0..5]);
785            });
786        }
787    };
788    package Model2 {
789        use AI::MXNet::Gluon::Mouse;
790        extends 'AI::MXNet::Gluon::Block';
791        sub BUILD
792        {
793            my $self = shift;
794            $self->name_scope(sub {
795                $self->layers({});
796                $self->layers->{a} = [map { nn->Dense($_ * 10) } 0..5];
797            });
798        }
799    };
800    package Model3 {
801        use AI::MXNet::Gluon::Mouse;
802        extends 'AI::MXNet::Gluon::Block';
803        sub BUILD
804        {
805            my $self = shift;
806            $self->name_scope(sub {
807                $self->layers(nn->Sequential());
808                $self->layers->add(map { nn->Dense($_ * 10) } 0..5);
809            });
810        }
811    };
812    package Model4 {
813        use AI::MXNet::Gluon::Mouse;
814        extends 'AI::MXNet::Gluon::Block';
815        sub BUILD
816        {
817            my $self = shift;
818            $self->name_scope(sub {
819                $self->data({a => '4', b => 123});
820            });
821        }
822    };
823    my $w = 0;
824    local($SIG{__WARN__}) = sub {
825        $w++;
826    };
827    Model1->new->collect_params;
828    ok($w > 0); $w = 0;
829    Model2->new->collect_params;
830    ok($w > 0); $w = 0;
831    Model3->new->collect_params;
832    ok($w == 0); $w = 0;
833    Model4->new->collect_params;
834    ok($w == 0);
835}
836
837test_block_attr_list_of_block();
838
839sub check_sequential
840{
841    my ($net) = @_;
842    my $dense1 = nn->Dense(10);
843    $net->add($dense1);
844    my $dense2 = nn->Dense(10);
845    $net->add($dense2);
846    my $dense3 = nn->Dense(10);
847    $net->add($dense3);
848
849    ok(refaddr($net->[1]) == refaddr($dense2));
850    ok(refaddr($net->[-1]) == refaddr($dense3));
851    my $slc = $net->slice([1,2]);
852    ok(@$slc == 2 and refaddr($slc->[0]) == refaddr($dense2) and refaddr($slc->[1]) == refaddr($dense3));
853    ok(ref $slc eq ref $net);
854}
855
856sub test_sequential
857{
858    check_sequential(nn->Sequential());
859    check_sequential(nn->HybridSequential());
860}
861
862test_sequential();
863
864sub test_global_norm_clip
865{
866    my @stypes = ('default', 'row_sparse');
867    my $check_global_norm_clip = sub { my ($stype) = @_;
868        my $x1 = mx->nd->ones([3,3])->tostype($stype);
869        my $x2 = mx->nd->ones([4,4])->tostype($stype);
870        my $norm = gluon->utils->clip_global_norm([$x1, $x2], 1.0);
871        ok($norm == 5);
872        ok(almost_equal($x1->aspdl, mx->nd->ones([3,3])->aspdl/5));
873        ok(almost_equal($x2->aspdl, mx->nd->ones([4,4])->aspdl/5));
874
875        my $x3 = mx->nd->array([1.0, 2.0, 'nan'])->tostype($stype);
876        my $w = 0;
877        local($SIG{__WARN__}) = sub {
878            $w++;
879        };
880        gluon->utils->clip_global_norm([$x1, $x3], 2.0);
881        ok($w == 1);
882    };
883    for my $stype (@stypes)
884    {
885        $check_global_norm_clip->($stype);
886    }
887}
888
889test_global_norm_clip();
890
891sub test_embedding
892{
893    local($ENV{MXNET_STORAGE_FALLBACK_LOG_VERBOSE}) = 0;
894    my $check_embedding = sub { my ($sparse_grad) = @_;
895        my $layer = nn->Embedding(10, 100, sparse_grad=>$sparse_grad);
896        $layer->initialize();
897        my $x = mx->nd->array([3,4,2,0,1]); my $y;
898        mx->autograd->record(sub {
899            $y = $layer->($x);
900            $y->backward();
901        });
902        ok(($layer->weight->grad->aspdl->slice('X', [0, 4]) == 1)->all);
903        ok(($layer->weight->grad->aspdl->slice('X', [5, -1]) == 0)->all);
904    };
905    my $check_embedding_large_input = sub { my ($sparse_grad) = @_;
906        my $embedding = nn->Embedding(10, 1, sparse_grad=>$sparse_grad);
907        $embedding->initialize();
908        $embedding->hybridize();
909        my $shape = [20481];
910        my ($emb_in, $loss);
911        mx->autograd->record(sub {
912            $emb_in = $embedding->(mx->nd->ones($shape));
913            $loss = $emb_in->sum;
914        });
915        $loss->backward;
916        ok($embedding->weight->grad->sum->asscalar == 20481);
917    };
918    $check_embedding->(1);
919    $check_embedding->(0);
920    $check_embedding_large_input->(1);
921    $check_embedding_large_input->(0);
922}
923
924test_embedding();
925
926sub test_hybrid_stale_cache
927{
928    my $net = nn->HybridSequential();
929    $net->name_scope(sub {
930        $net->add(nn->Dense(10, weight_initializer=>'zeros', bias_initializer=>'ones', flatten=>0));
931    });
932
933    $net->hybridize();
934    $net->initialize();
935    $net->(mx->nd->ones([2,3,5]));
936
937    $net->add(nn->Flatten());
938    is_deeply($net->(mx->nd->ones([2,3,5]))->shape, [2, 30]);
939
940    $net = nn->HybridSequential();
941    $net->name_scope(sub {
942        $net->fc1(nn->Dense(10, weight_initializer=>'zeros',
943                                    bias_initializer=>'ones', flatten=>0));
944        $net->fc2(nn->Dense(10, weight_initializer=>'zeros',
945                                    bias_initializer=>'ones', flatten=>0));
946    });
947    $net->hybridize();
948    $net->initialize();
949    $net->(mx->nd->ones([2,3,5]));
950
951    $net->fc2(nn->Dense(10, weight_initializer=>'zeros',
952                                bias_initializer=>'ones', flatten=>1));
953    $net->initialize();
954    is_deeply($net->(mx->nd->ones([2,3,5]))->shape, [2, 10]);
955}
956
957test_hybrid_stale_cache();
958
959sub test_lambda
960{
961    my $net1 = nn->HybridSequential();
962    $net1->add(nn->Activation('tanh'),
963             nn->LeakyReLU(0.1));
964
965    my $net2 = nn->HybridSequential();
966    my $op3 = sub { my ($F, $x, @args) = @_; $F->LeakyReLU($x, @args, slope=>0.1); };
967    $net2->add(nn->HybridLambda('tanh'),
968             nn->HybridLambda($op3));
969
970    my $op4 = sub { mx->nd->LeakyReLU($_[0], slope=>0.1); };
971    my $net3 = nn->Sequential();
972    $net3->add(nn->Lambda('tanh'),
973             nn->Lambda($op4));
974
975    my $input_data = mx->nd->random->uniform(shape=>[2, 3, 5, 7]);
976    my ($out1, $out2, $out3) = ($net1->($input_data), $net2->($input_data), $net3->($input_data));
977    ok(almost_equal($out1->aspdl, $out2->aspdl, 1e-3));
978    ok(almost_equal($out1->aspdl, $out3->aspdl, 1e-3));
979}
980
981test_lambda();
982
983sub test_fill_shape_deferred
984{
985    my $net = nn->HybridSequential();
986    $net->name_scope(sub {
987        $net->add(nn->Conv2D(64, kernel_size=>2, padding=>1),
988                nn->BatchNorm(),
989                nn->Dense(10));
990    });
991    $net->hybridize();
992    $net->initialize();
993    $net->(mx->nd->ones([2,3,5,7]));
994    ok($net->[0]->weight->shape->[1] == 3);
995    ok($net->[1]->gamma->shape->[0] == 64);
996    ok($net->[2]->weight->shape->[1] == 3072);
997}
998
999test_fill_shape_deferred();
1000
1001sub test_fill_shape_load
1002{
1003    my $ctx = mx->context->current_context();
1004    my $net1 = nn->HybridSequential();
1005    $net1->name_scope(sub {
1006        $net1->add(nn->Conv2D(64, kernel_size=>2, padding=>1),
1007                 nn->BatchNorm(),
1008                 nn->Dense(10))
1009    });
1010    $net1->hybridize();
1011    $net1->initialize(mx->init->Uniform, ctx => $ctx);
1012    $net1->(mx->nd->ones([2,3,5,7], ctx => $ctx));
1013    $net1->save_parameters('net_fill.params');
1014
1015    my $net2 = nn->HybridSequential();
1016    $net2->name_scope(sub {
1017        $net2->add(nn->Conv2D(64, kernel_size=>2, padding=>1),
1018                 nn->BatchNorm(),
1019                 nn->Dense(10))
1020    });
1021    $net2->hybridize();
1022    $net2->initialize();
1023    $net2->load_parameters('net_fill.params', ctx=>$ctx);
1024    ok($net2->[0]->weight->shape->[1] == 3);
1025    ok($net2->[1]->gamma->shape->[0] == 64);
1026    ok($net2->[2]->weight->shape->[1] == 3072);
1027}
1028
1029test_fill_shape_load();
1030
1031use JSON::PP qw(decode_json);
1032
1033sub test_inline
1034{
1035    my $y;
1036
1037    my $net = nn->HybridSequential();
1038    $net->name_scope(sub {
1039        $net->add(nn->Dense(10));
1040        $net->add(nn->Dense(10));
1041        $net->add(nn->Dense(10));
1042    });
1043    $net->initialize();
1044
1045    $net->hybridize(inline_limit=>3);
1046    mx->autograd->record(sub {
1047        $y = $net->(mx->nd->zeros([1,10]));
1048    });
1049    my $len_1 = @{ decode_json(mx->autograd->get_symbol($y)->tojson())->{nodes} };
1050    $y->backward();
1051
1052    $net->hybridize(inline_limit=>0);
1053    mx->autograd->record(sub {
1054        $y = $net->(mx->nd->zeros([1,10]));
1055    });
1056    my $len_2 = @{ decode_json(mx->autograd->get_symbol($y)->tojson())->{nodes} };
1057    $y->backward();
1058
1059    is($len_1, $len_2 + 2);
1060}
1061
1062test_inline();
1063
1064sub test_activations
1065{
1066    my $point_to_validate = mx->nd->array([(-0.1, 0.1) x 3]);
1067
1068    my $swish = nn->Swish();
1069    my $swish_test = sub { my ($x) = @_;
1070        return $x * mx->nd->sigmoid($x)
1071    };
1072
1073    for(zip($swish_test->($point_to_validate), $swish->($point_to_validate)))
1074    {
1075        my ($test_point, $ref_point) = @$_;
1076        ok($test_point == $ref_point);
1077    }
1078
1079    my $elu = nn->ELU();
1080    my $elu_test = sub { my ($x) = @_;
1081        my $elu = sub { my ($x) = @_;
1082            return $x < 0 ? 1.0 * (mx->nd->exp($x) - 1) : $x;
1083        };
1084        return [map { $elu->($_) } @{ $x }];
1085    };
1086
1087    for(zip($elu_test->($point_to_validate), $elu->($point_to_validate)))
1088    {
1089        my ($test_point, $ref_point) = @$_;
1090        ok($test_point == $ref_point);
1091    }
1092
1093    my $selu = nn->SELU();
1094    my $selu_test = sub { my ($x) = @_;
1095        my $selu = sub { my ($x) = @_;
1096            my ($scale, $alpha) = (1.0507009873554804934193349852946, 1.6732632423543772848170429916717);
1097            return $x => 0 ? $scale * $x : $alpha * mx->nd->exp($x) - $alpha;
1098        };
1099        return [map { $selu->($_) } @{ $x }];
1100    };
1101
1102    for(zip($selu_test->($point_to_validate), $selu->($point_to_validate)))
1103    {
1104        my ($test_point, $ref_point) = @$_;
1105        ok($test_point == $ref_point);
1106    }
1107
1108    my $prelu = nn->PReLU();
1109    $prelu->initialize();
1110    my $x = $point_to_validate->reshape([1, 3, 2]);
1111    ok(almost_equal($prelu->($x)->aspdl, mx->nd->where($x >= 0, $x, 0.25 * $x)->aspdl));
1112}
1113
1114test_activations();
1115
1116sub test_req
1117{
1118    my $data = mx->nd->random->uniform(shape=>[1,3,224,224]);
1119    my $label = mx->nd->array([1]);
1120    my $loss = gluon->loss->SoftmaxCrossEntropyLoss();
1121
1122    my $net = nn->HybridSequential();
1123    my $net1 = nn->HybridSequential();
1124    $net1->add(nn->Dense(4));
1125    my $net2 = nn->HybridSequential();
1126    $net2->add(nn->Dense(3));
1127    $net2->add(nn->Dense(2));
1128    $net->add($net1);
1129    $net->add($net2);
1130    $net->initialize();
1131
1132    $net->hybridize();
1133
1134    for my $v ($net->collect_params->values)
1135    {
1136        $v->grad_req('add');
1137    }
1138
1139    $net->collect_params->zero_grad();
1140    my $grad;
1141    mx->autograd->record(sub {
1142        my $pred = $net->($data);
1143        my $l = $loss->($pred, $label);
1144        $l->backward();
1145        $grad = $net->[0][0]->weight->grad->mean->aspdl;
1146        # run twice to check req = add
1147        $pred = $net->($data);
1148        $l = $loss->($pred, $label);
1149        $l->backward;
1150    });
1151
1152    my $grad_double = $net->[0][0]->weight->grad->mean->aspdl;
1153    ok(almost_equal($grad * 2, $grad_double));
1154}
1155
1156test_req();
1157
1158sub test_zero_grad
1159{
1160    my $data = mx->nd->random->uniform(shape=>[3,3]);
1161    my $net = nn->Embedding(3, 4, sparse_grad=>1, prefix=>'test_zero_grad_');
1162    $net->initialize();
1163    mx->autograd->record(sub {
1164        $net->($data)->backward;
1165    });
1166    $net->collect_params->zero_grad;
1167    my $grad = $net->collect_params->params->get('test_zero_grad_weight')->grad;
1168    ok(almost_equal($grad->aspdl, $grad->aspdl * 0));
1169}
1170
1171test_zero_grad();
1172
1173sub test_hook
1174{
1175    my $hook_call_count = 0;
1176    my $pre_hook_call_count = 0;
1177
1178    my $call_hook = sub { my ($block, $x, $y) = @_;
1179        $hook_call_count += 1;
1180    };
1181
1182    my $call_pre_hook = sub { my ($block, $x) = @_;
1183        $pre_hook_call_count += 1;
1184    };
1185
1186    my $block = nn->Dense(10);
1187    $block->initialize();
1188    my $handle = $block->register_forward_hook($call_hook);
1189    my $pre_handle = $block->register_forward_pre_hook($call_pre_hook);
1190    $block->(mx->nd->ones([3, 5]));
1191
1192    ok($hook_call_count == 1);
1193    ok($pre_hook_call_count == 1);
1194
1195    $handle->detach();
1196    $block->(mx->nd->ones([3, 5]));
1197
1198    ok($hook_call_count == 1);
1199    ok($pre_hook_call_count == 2);
1200
1201    $pre_handle->detach();
1202    $block->(mx->nd->ones([3, 5]));
1203
1204    ok($hook_call_count == 1);
1205    ok($pre_hook_call_count == 2);
1206}
1207
1208test_hook();
1209
1210sub test_apply
1211{
1212    my @called_blocks;
1213
1214    my $record_name = sub { my ($block) = @_;
1215        push @called_blocks, $block->name;
1216    };
1217    my $block = nn->HybridSequential(prefix=>'test_');
1218    $block->name_scope(sub {
1219        $block->add(nn->Dense(10));
1220        $block->add(nn->Dropout(0.5));
1221    });
1222    $block->apply($record_name);
1223
1224    is_deeply(\@called_blocks, ['test_dense0', 'test_dropout0', 'test']);
1225}
1226
1227test_apply();
1228
1229sub test_sparse_hybrid_block_grad
1230{
1231    package Embedding {
1232        use AI::MXNet::Gluon::Mouse;
1233        use AI::MXNet::Function::Parameters;
1234        extends 'AI::MXNet::Gluon::HybridBlock';
1235        has ['num_tokens', 'embedding_size'] => (is => 'rw');
1236        method python_constructor_arguments() { ['num_tokens', 'embedding_size'] }
1237        sub BUILD {
1238            my $self = shift;
1239            $self->name_scope(sub {
1240                $self->embedding(nn->Embedding(
1241                    $self->num_tokens, $self->embedding_size, sparse_grad=>1
1242                ));
1243            });
1244        }
1245
1246        method hybrid_forward($F, $words)
1247        {
1248            my $emb = $self->embedding->($words);
1249            return $emb + $F->ones_like($emb);
1250        }
1251    };
1252    my $embedding = Embedding->new(20, 3);
1253    $embedding->initialize();
1254    $embedding->hybridize();
1255
1256    my $loss;
1257    mx->autograd->record(sub {
1258        my $emb0 = $embedding->(mx->nd->arange(stop => 10))->sum;
1259        my $emb1 = $embedding->(mx->nd->arange(stop => 10))->sum;
1260        $loss = $emb0 + $emb1;
1261    });
1262    $loss->backward();
1263    my $grad = $embedding->embedding->weight->grad->aspdl;
1264    ok(($grad->slice('X', ':9') == 2)->all);
1265    ok(($grad->slice('X', '10:') == 0)->all);
1266}
1267
1268test_sparse_hybrid_block_grad();
1269
1270sub test_sparse_hybrid_block
1271{
1272    package Linear {
1273        use AI::MXNet::Gluon::Mouse;
1274        use AI::MXNet::Function::Parameters;
1275        extends 'AI::MXNet::Gluon::HybridBlock';
1276        has ['units'] => (is => 'rw');
1277        method python_constructor_arguments() { ['units'] }
1278        sub BUILD {
1279            my $self = shift;
1280            $self->name_scope(sub {
1281                $self->w($self->params->get(
1282                    'w', shape => [$self->units, $self->units]
1283                ));
1284            });
1285        }
1286        method hybrid_forward($F, $x, :$w)
1287        {
1288            return $F->dot($x, $w);
1289        }
1290    };
1291    package SparseBlock {
1292        use AI::MXNet::Gluon::Mouse;
1293        use AI::MXNet::Function::Parameters;
1294        extends 'AI::MXNet::Gluon::HybridBlock';
1295        has ['units'] => (is => 'rw');
1296        method python_constructor_arguments() { ['units'] }
1297        sub BUILD {
1298            my $self = shift;
1299            $self->name_scope(sub {
1300                $self->net(Linear->new($self->units));
1301            });
1302        }
1303        method hybrid_forward($F, $x)
1304        {
1305            return $self->net->($x) * $x;
1306        }
1307    };
1308    my $block = SparseBlock->new(2);
1309    $block->initialize();
1310    $block->hybridize();
1311    my $x = mx->nd->ones([2,2])->tostype('csr');
1312    my $z;
1313    mx->autograd->record(sub {
1314        $z = $block->($x) + $block->($x);
1315    });
1316    $z->backward;
1317    ok(($block->net->w->grad->aspdl == 4)->all);
1318}
1319
1320test_sparse_hybrid_block();
1321