AI/MXNet/Optimizer.pm

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

package AI::MXNet::Optimizer;
use strict;
use warnings;
use AI::MXNet::NS;
use AI::MXNet::Base;
use AI::MXNet::NDArray;
use AI::MXNet::Random;
use List::Util qw(max);

=head1 NAME

    AI::MXNet::Optimizer - Common Optimization algorithms with regularizations.

=head1  DESCRIPTION

    Common Optimization algorithms with regularizations.
=cut

use Mouse;
use AI::MXNet::Function::Parameters;
my %opt_registry;
method get_opt_registry()
{
    return \%opt_registry;
}

method register()
{
    my $name = $self;
    ($name) = $name =~ /::(\w+)$/;
    {  no strict 'refs'; *{__PACKAGE__."::$name"} = sub { shift; $self->new(@_)  }; }
    $name = lc $name;
    if(exists $opt_registry{ $name })
    {
        my $existing = $opt_registry{ $name };
        warn(
            "WARNING: New optimizer $self.$name"
            ."is overriding existing optimizer $existing.$name"
        );
    }
    $opt_registry{ $name } = $self;
}

=head2 create_optimizer

        Create an optimizer with specified name.

        Parameters
        ----------
        $name: Str
            Name of required optimizer. Should be the name
            of a subclass of Optimizer. Case insensitive.

        :$rescale_grad : Num
            Rescaling factor on gradient. Normally should be 1/batch_size.

        %kwargs: Hash
            Parameters for optimizer

        Returns
        -------
        opt : Optimizer
            The result optimizer.
=cut

method create_optimizer(Str $name, %kwargs)
{
    if(exists $opt_registry{ lc $name })
    {
        my $rescale_grad = delete($kwargs{rescale_grad})//1;
        return $opt_registry{ lc $name }->new(
            rescale_grad => $rescale_grad,
            %kwargs
        );
    }
    confess("Cannot find optimizer $name");
}

*create = \&create_optimizer;

has 'rescale_grad'        => (is => "rw", isa => "Num", default=>1);
has 'lr'                  => (is => "rw", isa => "Num");
has 'learning_rate'       => (is => "rw", isa => "Num", default => 0.01);
has 'lr_scheduler'        => (is => "rw", isa => "Maybe[AI::MXNet::LRScheduler]");
has 'wd'                  => (is => "rw", isa => "Num", default => 0);
has 'lr_mult'             => (is => "rw", isa => "HashRef", default => sub { +{} });
has 'wd_mult'             => (is => "rw", isa => "HashRef", , default => sub { +{} });
has 'num_update'          => (is => "rw", isa => "Int");
has 'begin_num_update'    => (is => "rw", isa => "Int", default => 0);
has '_index_update_count' => (is => "rw", isa => "HashRef", default => sub { +{} });
has 'clip_gradient'       => (is => "rw", isa => "Maybe[Num]");
has 'param_idx2name'      => (is => "rw", isa => "HashRef[Str]", default => sub { +{} });
has 'idx2name'            => (is => "rw", isa => "HashRef[Str]");
has 'sym'                 => (is => "rw", isa => "Maybe[AI::MXNet::Symbol]");
has 'param_dict'          => (is => "rw", isa => "HashRef", default => sub { +{} });

sub BUILD
{
    my $self = shift;
    if($self->lr_scheduler)
    {
        $self->lr_scheduler->base_lr($self->learning_rate);
    }
    $self->lr($self->learning_rate);
    $self->num_update($self->begin_num_update);
    $self->idx2name({ %{ $self->param_idx2name } });
    $self->set_lr_mult({});
    $self->set_wd_mult({});
}
# Create additional optimizer state such as momentum.
# override in implementations.
method create_state($index, $weight){}

# Update the parameters. override in implementations
method update($index, $weight, $grad, $state){}

# set lr scale is deprecated. Use set_lr_mult instead.
method set_lr_scale($args_lrscale)
{
    Carp::cluck("set lr scale is deprecated. Use set_lr_mult instead.");
}

=head2 set_lr_mult

        Set individual learning rate multipler for parameters

        Parameters
        ----------
        args_lr_mult : dict of string/int to float
            set the lr multipler for name/index to float.
            setting multipler by index is supported for backward compatibility,
            but we recommend using name and symbol.
=cut

method set_lr_mult(HashRef[Num] $args_lr_mult)
{
    $self->lr_mult({});
    if($self->sym)
    {
        my $attr = $self->sym->attr_dict();
        for my $name (@{ $self->sym->list_arguments() })
        {
            if(exists $attr->{ $name } and exists $attr->{ $name }{ __lr_mult__ })
            {
                $self->lr_mult->{ $name } = $attr->{ $name }{ __lr_mult__ };
            }
        }
    }
    $self->lr_mult({ %{ $self->lr_mult }, %{ $args_lr_mult } });
}

=head2 set_wd_mult

        Set individual weight decay multipler for parameters.
        By default wd multipler is 0 for all params whose name doesn't
        end with _weight, if param_idx2name is provided.

        Parameters
        ----------
        args_wd_mult : dict of string/int to float
            set the wd multipler for name/index to float.
            setting multipler by index is supported for backward compatibility,
            but we recommend using name and symbol.
=cut

method set_wd_mult(HashRef[Num] $args_wd_mult)
{
    $self->wd_mult({});
    for my $n (values %{ $self->idx2name })
    {
        if(not $n =~ /(?:_weight|_gamma)$/)
        {
            $self->wd_mult->{ $n } = 0;
        }
    }
    if($self->sym)
    {
        my $attr = $self->sym->attr_dict();
        for my $name (@{ $self->sym->list_arguments() })
        {
            if(exists $attr->{ $name } and exists $attr->{ $name }{ __wd_mult__ })
            {
                $self->wd_mult->{ $name } = $attr->{ $name }{ __wd_mult__ };
            }
        }
    }
    $self->wd_mult({ %{ $self->wd_mult }, %{ $args_wd_mult } });
}

method _update_count(Index $index)
{
    if(not exists $self->_index_update_count->{ $index })
    {
        $self->_index_update_count->{ $index } = $self->begin_num_update;
    }
    $self->_index_update_count->{ $index } += 1;
    $self->num_update(max($self->_index_update_count->{ $index }, $self->num_update));
}

method _get_lr(Index $index)
{
    my $lr;
    if($self->lr_scheduler)
    {
        $lr = $self->lr_scheduler->($self->num_update);
    }
    else
    {
        $lr = $self->lr;
    }

    if(exists $self->param_dict->{ $index })
    {
        $lr *= $self->param_dict->{ $index }->lr_mult;
    }
    elsif(exists $self->lr_mult->{ $index })
    {
        $lr *= $self->lr_mult->{ $index };
    }
    elsif(exists $self->idx2name->{ $index })
    {
        $lr *= $self->lr_mult->{ $self->idx2name->{ $index } }//1;
    }
    return $lr;
}

method _get_wd(Index $index)
{
    my $wd = $self->wd;
    if(exists $self->param_dict->{ $index })
    {
        $wd *= $self->param_dict->{ $index }->wd_mult;
    }
    elsif(exists $self->wd_mult->{ $index })
    {
        $wd *= $self->wd_mult->{ $index };
    }
    elsif(exists $self->idx2name->{ $index })
    {
        $wd *= $self->wd_mult->{ $self->idx2name->{ $index } }//1;
    }
    return $wd;
}

=head1 NAME

    AI::MXNet::SGD - A very simple SGD optimizer with momentum and weight regularization.
=cut

=head1 DESCRIPTION

    A very simple SGD optimizer with momentum and weight regularization.

    If the storage types of weight and grad are both 'row_sparse', and 'lazy_update' is True,
    **lazy updates** are applied by

        for row in grad.indices:
            rescaled_grad[row] = lr * rescale_grad * clip(grad[row], clip_gradient) + wd * weight[row]
            state[row] = momentum[row] * state[row] + rescaled_grad[row]
            weight[row] = weight[row] - state[row]

    The sparse update only updates the momentum for the weights whose row_sparse
    gradient indices appear in the current batch, rather than updating it for all
    indices. Compared with the original update, it can provide large
    improvements in model training throughput for some applications. However, it
    provides slightly different semantics than the original update, and
    may lead to different empirical results.

    Otherwise, **standard updates** are applied by::

        rescaled_grad = lr * rescale_grad * clip(grad, clip_gradient) + wd * weight
        state = momentum * state + rescaled_grad
        weight = weight - state

    Parameters
    ----------
    learning_rate : Num, optional
        learning_rate of SGD

    momentum : Num, optional
       momentum value

    wd : Num, optional
        L2 regularization coefficient add to all the weights

    rescale_grad : Num, optional
        rescaling factor of gradient. Normally should be 1/batch_size.

    clip_gradient : Num, optional
        clip gradient in range [-clip_gradient, clip_gradient]

    param_idx2name : hash ref of Str/Int to Num, optional
        special treat weight decay in parameter ends with bias, gamma, and beta

    multi_precision: Bool, optional
        Flag to control the internal precision of the optimizer.
        False results in using the same precision as the weights (default),
        True makes internal 32-bit copy of the weights and applies gradients
        in 32-bit precision even if actual weights used in the model have lower precision.
        Turning this on can improve convergence and accuracy when training with float16.

    lazy_update: Bool, optional, default true
=cut

package AI::MXNet::SGD;
use Mouse;
extends 'AI::MXNet::Optimizer';

has 'kwargs'          => (is => "rw", isa => "HashRef[Num]");
has 'momentum'        => (is => "rw", isa => "Num", default => 0);
has 'multi_precision' => (is => "ro", isa => "Bool", default => 0);
has 'lazy_update'     => (is => "ro", isa => "Bool", default => 1);

sub BUILD
{
    my $self = shift;
    $self->kwargs({});
    if($self->momentum)
    {
        $self->kwargs->{momentum} = $self->momentum;
    }
    if($self->clip_gradient)
    {
        $self->kwargs->{clip_gradient} = $self->clip_gradient;
    }
}

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    my $momentum;
    my $weight_master_copy;
    my $stype = $self->lazy_update ? $weight->stype : 'default';
    if($self->multi_precision and $weight->dtype eq 'float16')
    {
        my $weight_master_copy = AI::MXNet::NDArray->array($weight, ctx => $weight->context, dtype => 'float32');
        if($self->momentum != 0)
        {
            $momentum = AI::MXNet::NDArray->zeros($weight->shape, stype => $stype, ctx => $weight->context, dtype => 'float32');
        }
        return [$momentum, $weight_master_copy];
    }
    if($weight->dtype eq 'float16' and not $self->multi_precision)
    {
        AI::MXNet::Logging->warning(
            "Accumulating with float16 in optimizer can lead to ".
            "poor accuracy or slow convergence. ".
            "Consider using multi_precision=True option of the ".
            "SGD optimizer"
        );
    }
    if($self->momentum != 0)
    {
        $momentum = AI::MXNet::NDArray->zeros($weight->shape, stype => $stype, ctx => $weight->context, dtype => $weight->dtype);
    }
    return $momentum;
}

method update(
    Index                     $index,
    AI::MXNet::NDArray        $weight,
    AI::MXNet::NDArray        $grad,
    Maybe[AI::MXNet::NDArray|ArrayRef[Maybe[AI::MXNet::NDArray]]] $state
)
{
    $self->_update_count($index);
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    my $kwargs = {
        out => $weight,
        lr  => $lr,
        wd  => $wd,
        rescale_grad => $self->rescale_grad,
        %{ $self->kwargs }
    };
    my $use_multi_precision = ref($state) eq 'ARRAY';
    if(not $use_multi_precision)
    {
        if(defined $state)
        {
            AI::MXNet::NDArray->sgd_mom_update(
                $weight, $grad, $state, $kwargs
            );
        }
        else
        {
            AI::MXNet::NDArray->sgd_update(
                $weight, $grad, $kwargs
            );
        }
    }
    else
    {
        if(defined $state->[0])
        {
            AI::MXNet::NDArray->mp_sgd_mom_update(
                $weight, $grad, $state->[0], $state->[1], $kwargs
            );
        }
        else
        {
            AI::MXNet::NDArray->mp_sgd_update(
                $weight, $grad, $state->[1], $kwargs
            );
        }
    }
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::Signum - The Signum optimizer that takes the sign of gradient or momentum.
=cut

=head1 DESCRIPTION

    The optimizer updates the weight by:

        rescaled_grad = rescale_grad * clip(grad, clip_gradient) + wd * weight
        state = momentum * state + (1-momentum)*rescaled_grad
        weight = (1 - lr * wd_lh) * weight - lr * sign(state)

    See the original paper at: https://jeremybernste.in/projects/amazon/signum.pdf

    This optimizer accepts the following parameters in addition to those accepted
    by AI::MXNet::Optimizer

    Parameters
    ----------
    momentum : Num, optional
       The momentum value.
    wd_lh : Num, optional
       The amount of decoupled weight decay regularization, see details in the original paper at:
       https://arxiv.org/abs/1711.05101
=cut

package AI::MXNet::Signum;
use Mouse;
extends 'AI::MXNet::Optimizer';

has 'momentum' => (is => "rw", isa => "Num", default => 0.9);
has 'wd_lh'    => (is => "rw", isa => "Num", default => 0);

method create_state(Index $index, AI::MXNet::NDArray $weight)
{

    my $momentum;
    if($self->momentum != 0)
    {
        $momentum = AI::MXNet::NDArray->zeros(
            $weight->shape,
            ctx => $weight->context,
            dtype=>$weight->dtype,
            stype=>$weight->stype
        );
    }
    return $momentum;
}

method update(
    Index                     $index,
    AI::MXNet::NDArray        $weight,
    AI::MXNet::NDArray        $grad,
    Maybe[AI::MXNet::NDArray|ArrayRef[Maybe[AI::MXNet::NDArray]]] $state
)
{
    $self->_update_count($index);
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    my %kwargs = (
        out => $weight,
        lr  => $lr,
        wd  => $wd,
        rescale_grad => $self->rescale_grad,
    );
    if($self->momentum > 0)
    {
        $kwargs{momentum} = $self->momentum;
    }
    if($self->clip_gradient)
    {
        $kwargs{clip_gradient} = $self->clip_gradient;
    }
    if($self->wd_lh)
    {
        $kwargs{wd_lh} = $self->wd_lh;
    }
    if(defined $state)
    {
        AI::MXNet::NDArray->signum_update(
            $weight, $grad, $state, %kwargs
        );
    }
    else
    {
        AI::MXNet::NDArray->signsgd_update(
            $weight, $grad, %kwargs
        );
    }
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::FTML - The FTML optimizer.
=cut

=head1 DESCRIPTION

    This class implements the optimizer described in
    *FTML - Follow the Moving Leader in Deep Learning*,
    available at http://proceedings.mlr.press/v70/zheng17a/zheng17a.pdf.

    This optimizer accepts the following parameters in addition to those accepted
    by AI::MXNet::Optimizer

    Parameters
    ----------
    beta1 : Num, optional
        0 < beta1 < 1. Generally close to 0.5.
    beta2 : Num, optional
        0 < beta2 < 1. Generally close to 1.
    epsilon : Num, optional
        Small value to avoid division by 0.
=cut

package AI::MXNet::FTML;
use Mouse;
extends 'AI::MXNet::Optimizer';

has 'beta1'   => (is => "rw", isa => "Num", default => 0.6);
has 'beta2'   => (is => "rw", isa => "Num", default => 0.999);
has 'epsilon' => (is => "rw", isa => "Num", default => 1e-8);

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return [
        AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>$weight->dtype), # d_0
        AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>$weight->dtype), # v_0
        AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>$weight->dtype), # z_0
    ];
}

method update(
    Index                     $index,
    AI::MXNet::NDArray        $weight,
    AI::MXNet::NDArray        $grad,
    Maybe[AI::MXNet::NDArray|ArrayRef[Maybe[AI::MXNet::NDArray]]] $state
)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    my $t = $self->_update_count($index);
    my %kwargs = (
        out => $weight,
        lr  => $lr,
        wd  => $wd,
        t   => $t,
        beta1 => $self->beta1,
        beta2 => $self->beta2,
        epsilon => $self->epsilon,
        rescale_grad => $self->rescale_grad
    );
    if($self->clip_gradient)
    {
        $kwargs{clip_grad} = $self->clip_gradient;
    }
    AI::MXNet::NDArray->ftml_update($weight, $grad, @{ $state }, \%kwargs);
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::LBSGD - The Large Batch SGD optimizer with momentum and weight decay.
=cut

=head1 DESCRIPTION

    The optimizer updates the weight by::

        state = momentum * state + lr * rescale_grad * clip(grad, clip_gradient) + wd * weight
        weight = weight - state

    Parameters
    ----------
    momentum : Num, optional
       The momentum value.
    multi_precision: Bool, optional
       Flag to control the internal precision of the optimizer.
       0 results in using the same precision as the weights (default),
       1 makes internal 32-bit copy of the weights and applies gradients
                in 32-bit precision even if actual weights used in the model have lower precision.`<
                Turning this on can improve convergence and accuracy when training with float16.
    warmup_strategy: string ('linear', 'power2', 'sqrt'. , 'lars'   default : 'linear')
    warmup_epochs: unsigned, default: 5
    batch_scale:   unsigned, default: 1 (same as batch size*numworkers)
    updates_per_epoch: updates_per_epoch (default: 32, Default might not reflect true number batches per epoch. Used for warmup.)
    begin_epoch: unsigned, default 0, starting epoch.
=cut

package AI::MXNet::LBSGD;
use Mouse;
extends 'AI::MXNet::Optimizer';

has 'momentum'          => (is => 'rw', isa => 'Num', default => 0);
has 'multi_precision'   => (is => 'rw', isa => 'Bool', default => 0);
has 'warmup_startegy'   => (is => 'rw', isa => 'Str', default => 'linear');
has 'warmup_epochs'     => (is => 'rw', isa => 'Int', default => 5);
has 'batch_scale'       => (is => 'rw', isa => 'Num', default => 1);
has 'updates_per_epoch' => (is => 'rw', isa => 'Int', default => 32);
has 'begin_epoch'       => (is => 'rw', isa => 'Int', default => 0);
has 'num_epochs'        => (is => 'rw', isa => 'Int', default => 60);
has 'beta2'             => (is => 'rw', isa => 'Num', default => 0.999);
has 'epsilon'           => (is => 'rw', isa => 'Num', default => 1e-8);
has 'init_updates'      => (is => 'rw', init_arg => undef);
has [qw/lbmult
        cumgrads
        adaptive
        init_updates
        admult/]        => (is => 'rw', init_arg => undef);

sub BUILD
{
    my $self = shift;
    AI::MXNet::Logging->info('Running Large-Batch SGD Algorithm');
    AI::MXNet::Logging->info(
        '(Batch_scale=%f, warmup_epochs=%d, warmup_strategy=%s, updates_per_epoch=%d)',
        map { $self->$_ } qw/batch_scale warmup_epochs warmup_strategy updates_per_epoch/
    );
    $self->init_updates($self->begin_epoch * $self->updates_per_epoch);
    $self->lbmult(1);
    $self->cumgrads({});
    $self->adaptive(0);
    $self->admult(1);
}

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return [
        AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>$weight->dtype), # d_0
        AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>$weight->dtype), # v_0
        AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>$weight->dtype), # z_0
    ];
    my $momentum;
    my $weight_master_copy;
    if($self->multi_precision and $weight->dtype eq 'float16')
    {
        $weight_master_copy = AI::MXNet::NDArray->array($weight, ctx=>$weight->context, dtype=>'float32');
        if($self->momentum != 0)
        {
            $momentum = AI::MXNet::NDArray->zeros(
                $weight->shape, ctx => $weight->context, dtype => 'float32',
                stype => $weight->stype
            );
        }
        return [$momentum, $weight_master_copy];
    }
    if($weight->dtype eq 'float16' and not $self->multi_precision)
    {
        AI::MXNet::Logging->warning(
            "Accumulating with float16 in optimizer can lead to "
            ."poor accuracy or slow convergence. "
            ."Consider using multi_precision=True option of the "
            ."LBSGD optimizer"
        );
    }
    if($self->momentum != 0)
    {
        $momentum = AI::MXNet::NDArray->zeros(
            $weight->shape, ctx => $weight->context, dtype => $weight->dtype,
            stype => $weight->stype
        );
    }
    return $momentum;
}

method _get_lbmult($nup)
{
    my $nwup = $self->warmup_epochs * $self->updates_per_epoch;
    my $strategy = $self->warmup_strategy;
    my $maxmult = $self->batch_scale;
    my $mult;
    if($nup >= $nwup)
    {
        $mult = $maxmult;
    }
    elsif($nwup <= 1)
    {
        $mult = 1;
    }
    else
    {
        if ($strategy eq 'linear')
        {
            $mult = 1 + ($maxmult - 1) * $nup / $nwup;
        }
        elsif($strategy eq 'power2')
        {
            $mult = 1 + ($maxmult-1) * ($nup*$nup)/($nwup*$nwup);
        }
        elsif($strategy eq 'sqrt')
        {
            $mult = 1 + ($maxmult - 1) * sqrt($nup / $nwup);
        }
        else
        {
            $mult = 1;
        }
    }
    return $mult;
}


method _get_lars($weight, $g, $wd)
{
    my $weight2 = $self->_l2norm($weight);
    my $grad2 = $self->_l2norm($g);
    my $lars = sqrt($weight2 / ($grad2 + $wd * $weight2 + 1e-18));
    if($lars < 0.01)
    {
        $lars = 0.01;
    }
    elsif($lars > 100)
    {
        $lars = 100;
    }
    return $lars;
}

method _l2norm($v)
{
    my $norm = AI::MXNet::NDArray->multiply($v, $v)->aspdl->sum;
    return $norm;
}

method  _reset_cum_gradient($index)
{
    $self->cumgrads->{$index}{cum_grad} = 0;
}

method _get_cum_gradient($index)
{
    if(exists $self->cumgrads->{$index})
    {
        return $self->cumgrads->{$index};
    }
    else
    {
        return {}
    }
}

method _put_cum_gradient($index, $cgrad)
{
    $self->cumgrads->{$index} = $cgrad;
}

method _cumulate_gradient($grad, $index)
{
    my $cgrad = $self->_get_cum_gradient($index);
    my ($num_cums, $cum_grad);
    if(%{ $cgrad })
    {
        my $num_cums = $cgrad->{num_cums};
        if($num_cums > 0)
        {
            $cum_grad = $cgrad->{cum_grad} + $grad;
            $num_cums += 1;
        }
        else
        {
            $cum_grad = $grad;
            $num_cums = $self->init_updates + 1;
        }
    }
    else
    {
        $cum_grad = $grad;
        $num_cums = $self->init_updates + 1;
    }
    $cgrad = {cum_grad => $cum_grad, num_cums => $num_cums};
    $self->_put_cum_gradient($index, $cgrad);
    return $cgrad;
}


method update(
    Index                     $index,
    AI::MXNet::NDArray        $weight,
    AI::MXNet::NDArray        $grad,
    Maybe[AI::MXNet::NDArray|ArrayRef[Maybe[AI::MXNet::NDArray]]] $state
)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    my $t = $self->_update_count($index);
    my $cgrad = $self->_cumulate_gradient($grad, $index);
    if(($cgrad->{num_cums} % $self->batch_scale) == 0)
    {
        my $lbmult;
        $grad = $cgrad->{cum_grad} / $self->batch_scale;
        if($self->warmup_strategy eq 'lars')
        {
            $lbmult = $self->_get_lars($weight, $grad, $wd);
        }
        else
        {
            $lbmult = $self->_get_lbmult($cgrad->{num_cums});
        }
        $lr = $lr * $lbmult;
        my %kwargs = (
            out => $weight,
            lr  => $lr,
            wd  => $wd,
            rescale_grad => $self->rescale_grad
        );
        if($self->clip_gradient)
        {
            $kwargs{clip_gradient} = $self->clip_gradient;
        }
        if($self->momentum > 0)
        {
            $kwargs{momentum} = $self->momentum;
        }
        my $use_multi_precision = ref($state) eq 'ARRAY';
        if(not $use_multi_precision)
        {
            if(defined $state)
            {
                AI::MXNet::NDArray->sgd_mom_update($weight, $grad, $state, %kwargs);
            }
            else
            {
                AI::MXNet::NDArray->sgd_update($weight, $grad, %kwargs);
            }
        }
        else
        {
            if(defined $state->[0])
            {
                AI::MXNet::NDArray->mp_sgd_mom_update($weight, $grad, @{ $state }, %kwargs);
            }
            else
            {
                AI::MXNet::NDArray->mp_sgd_update($weight, $grad, $state->[1], %kwargs);
            }
        }
        $self->_reset_cum_gradient($index);
    }
    else
    {
        AI::MXNet::NDArray->sgd_update($weight, $grad, out => $weight, lr => 0, wd => $wd);
    }
}

__PACKAGE__->register;

package AI::MXNet::DCASGD;
use Mouse;
use AI::MXNet::Base;
extends 'AI::MXNet::Optimizer';

=head1 NAME

    AI::MXNet::DCASGD - DCASGD optimizer with momentum and weight regularization.
=cut

=head1 DESCRIPTION

    DCASGD optimizer with momentum and weight regularization.

    Implements paper "Asynchronous Stochastic Gradient Descent with
                    Delay Compensation for Distributed Deep Learning"

    Parameters
    ----------
    learning_rate : Num, optional
        learning_rate of SGD

    momentum : Num, optional
       momentum value

    lamda : NUm, optional
       scale DC value

    wd : Num, optional
        L2 regularization coefficient add to all the weights

    rescale_grad : Num, optional
        rescaling factor of gradient. Normally should be 1/batch_size.

    clip_gradient : Num, optional
        clip gradient in range [-clip_gradient, clip_gradient]

    param_idx2name : hash ref of Str/Int to Num, optional
        special threating of weight decay for parameters that end with bias, gamma, and beta
=cut
has 'momentum'        => (is => 'ro', isa => 'Num', default => 0);
has 'lamda'           => (is => 'ro', isa => 'Num', default => 0.04);
has 'weight_previous' => (is => 'rw', init_arg => undef);

sub BUILD
{
    my $self = shift;
    $self->weight_previous({});
}

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
        return [
            $self->momentum ? AI::MXNet::NDArray->zeros(
                $weight->shape, ctx => $weight->context, dtype => $weight->dtype
            ) : undef,
            $weight->copy
        ];
}

method update(
    Index                     $index,
    AI::MXNet::NDArray        $weight,
    AI::MXNet::NDArray        $grad,
    Maybe[AI::MXNet::NDArray|ArrayRef[Maybe[AI::MXNet::NDArray]]] $state
)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    $self->_update_count($index);
    $grad *= $self->rescale_grad;
    if($self->clip_gradient)
    {
        $grad = AI::MXNet::NDArray->clip(
            $grad,
            -$self->clip_gradient,
            $self->clip_gradient
        );
    }
    my ($mom, $weight_previous) = @{ $state };
    if(defined $mom)
    {
        $mom *= $self->momentum;
        $mom += -$lr * (
                $grad + $wd * $weight
                    +
                $self->lamda * $grad * $grad * ($weight - $weight_previous)
        );
    }
    else
    {
        assert($self->momentum == 0);
        $mom = -$lr * (
                $grad + $wd * $weight
                    +
                $self->lamda * $grad * $grad * ($weight - $weight_previous)
        );
    }
    $weight_previous .= $weight;
    $weight += $mom;
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::NAG - SGD with Nesterov weight handling.
=cut

=head1 DESCRIPTION

    It is implemented according to
    https://github.com/torch/optim/blob/master/sgd.lua
=cut

package AI::MXNet::NAG;
use Mouse;
extends 'AI::MXNet::SGD';

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    my $momentum;
    my $weight_master_copy;
    my $do_multi_precision = ($self->multi_precision and $weight->dtype eq 'float16');
    if($do_multi_precision)
    {
        if($self->momentum != 0)
        {
            $momentum = AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>'float32');
        }
        $weight_master_copy = AI::MXNet::NDArray->array($weight, ctx=>$weight->context, dtype=>'float32');
        return [$weight_master_copy, $momentum];
    }
    else
    {
        if($self->momentum != 0)
        {
            $momentum = AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype=>$weight->dtype);
        }
        return $momentum;
    }
}

method update($index, $weight, $grad, $state)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    $self->_update_count($index);
    my $use_multi_precision = (defined $state and not Scalar::Util::blessed($state) and ref($state eq 'ARRAY'));
    if(not $use_multi_precision)
    {
        $grad *= $self->rescale_grad;
        if(defined $self->clip_gradient)
        {
            $grad = AI::MXNet::NDArray->clip($grad, -$self->clip_gradient, $self->clip_gradient);
        }
        if($self->momentum == 0)
        {
            $weight += -$lr * ($grad + $wd * $weight);
        }
        else
        {
            my $mom = $state;
            $mom *= $self->momentum;
            $grad += $wd * $weight;
            $mom += $grad;
            $grad += $self->momentum * $mom;
            $weight += -$lr * $grad;
        }
    }
    else
    {
        my $grad32 = AI::MXNet::NDArray->array($grad, ctx=>$grad->context, dtype=>'float32');
        $grad32 *= $self->rescale_grad;
        if(defined $self->clip_gradient)
        {
            $grad32 = AI::MXNet::NDArray->clip($grad32, -$self->clip_gradient, $self->clip_gradient);
        }
        my $mom = $state->[1];
        my $weight32 = $state->[0];
        if($self->momentum == 0)
        {
            $weight32 += -$lr * ($grad32 + $wd * $weight32);
        }
        else
        {
            $mom *= $self->momentum;
            $grad32 += $wd * $weight32;
            $mom += $grad32;
            $grad32 += $self->momentum * $mom;
            $weight32 += -$lr * $grad32;
        }
        my $tmp = $weight32->astype($weight->dtype);
        $tmp->copyto($weight);
    }
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::SGLD - Stochastic Gradient Riemannian Langevin Dynamics.
=cut

=head1 DESCRIPTION

    Stochastic Gradient Riemannian Langevin Dynamics.

    This class implements the optimizer described in the paper *Stochastic Gradient
    Riemannian Langevin Dynamics on the Probability Simplex*, available at
    https://papers.nips.cc/paper/4883-stochastic-gradient-riemannian-langevin-dynamics-on-the-probability-simplex.pdf.

    Parameters
    ----------
    learning_rate : Num, optional
        learning_rate of SGD

    wd : Num, optional
        L2 regularization coefficient add to all the weights

    rescale_grad : Num, optional
        rescaling factor of gradient. Normally should be 1/batch_size.

    clip_gradient : Num, optional
        clip gradient in range [-clip_gradient, clip_gradient]
=cut

package AI::MXNet::SGLD;
use Mouse;

extends 'AI::MXNet::Optimizer';

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return undef;
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    AI::MXNet::NDArray|Undef $state
)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    $self->_update_count($index);
    $grad *= $self->rescale_grad;
    if($self->clip_gradient)
    {
        $grad = AI::MXNet::NDArray->clip(
            $grad,
            -$self->clip_gradient,
             $self->clip_gradient
        );
    }
    $weight +=  - $lr/2 * ($grad + $wd * $weight)
                    +
                AI::MXNet::Random->normal(
                        0, sqrt($lr),
                        shape => $weight->shape,
                        ctx   => $weight->context,
                        dtype => $weight->dtype
                );
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::Adam - Adam optimizer as described in [King2014]_.
=cut

=head1 DESCRIPTION

    Adam optimizer as described in [King2014]_.

    .. [King2014] Diederik Kingma, Jimmy Ba,
       *Adam: A Method for Stochastic Optimization*,
       http://arxiv.org/abs/1412.6980

    Parameters
    ----------
    learning_rate : Num, optional
        Step size.
        Default value is set to 0.001.
    beta1 : Num, optional
        Exponential decay rate for the first moment estimates.
        Default value is set to 0.9.
    beta2 : Num, optional
        Exponential decay rate for the second moment estimates.
        Default value is set to 0.999.
    epsilon : Num, optional
        Default value is set to 1e-8.

    wd : NUm, optional
        L2 regularization coefficient add to all the weights
    rescale_grad : Num, optional
        rescaling factor of gradient. Normally should be 1/batch_size.

    clip_gradient : Num, optional
        clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::Adam;
use Mouse;

extends 'AI::MXNet::Optimizer';

has 'kwargs'   => (is => "rw", isa => "HashRef[Num]");
has '+learning_rate' => (default => 0.001);
has 'beta1'    => (is => "rw", isa => "Num", default => 0.9);
has 'beta2'    => (is => "rw", isa => "Num", default => 0.999);
has 'epsilon'  => (is => "rw", isa => "Num", default => 1e-8);
has 'lazy_update' => (is => 'rw', isa => 'Bool', default => 1);

sub BUILD
{
    my $self = shift;
    $self->kwargs({
        beta1   => $self->beta1,
        beta2   => $self->beta2,
        epsilon => $self->epsilon
    });
    if($self->clip_gradient)
    {
        $self->kwargs->{clip_gradient} = $self->clip_gradient;
    }
}

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    my $stype = $self->lazy_update ? $weight->stype : 'default';
    return [AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                dtype => $weight->dtype,
                stype => $stype
            ),  # mean
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                dtype => $weight->dtype,
                stype => $stype
            )  # variance
    ];
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    ArrayRef[AI::MXNet::NDArray] $state
)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    $self->_update_count($index);
    my $t = $self->_index_update_count->{$index};
    my $coef1 = 1 - $self->beta1**$t;
    my $coef2 = 1 - $self->beta2**$t;
    $lr *= sqrt($coef2)/$coef1;
    my ($mean, $var) = @{ $state };
    AI::MXNet::NDArray->adam_update(
        $weight, $grad, $mean, $var,
        {
            out => $weight,
            lr  => $lr,
            wd  => $wd,
            rescale_grad => $self->rescale_grad,
            %{ $self->kwargs }
        }
    );
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::AdaGrad - AdaGrad optimizer of Duchi et al., 2011
=cut

=head1 DESCRIPTION

    AdaGrad optimizer of Duchi et al., 2011,

    This code follows the version in http://arxiv.org/pdf/1212.5701v1.pdf  Eq(5)
    by Matthew D. Zeiler, 2012. AdaGrad will help the network to converge faster
    in some cases.

    Parameters
    ----------
    learning_rate : Num, optional
        Step size.
        Default value is set to 0.05.

    wd : Num, optional
        L2 regularization coefficient add to all the weights

    rescale_grad : Num, optional
        rescaling factor of gradient. Normally should be 1/batch_size.

    eps: Num, optional
        A small float number to make the updating processing stable
        Default value is set to 1e-7.

    clip_gradient : Num, optional
        clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::AdaGrad;
use Mouse;

extends 'AI::MXNet::Optimizer';

has 'eps'    => (is => "rw", isa => "Num", default => 1e-7);

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                stype => $weight->stype
    );  # history
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    AI::MXNet::NDArray $state
)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    $self->_update_count($index);
    my $is_sparse = $grad->stype eq 'row_sparse' ? 1 : 0;
    my $history = $state;
    if($is_sparse)
    {
        my %kwargs = (
            epsilon => $self->eps,
            rescale_grad => $self->rescale_grad
        );
        if($self->clip_gradient)
        {
            $kwargs{clip_gradient} = $self->clip_gradient;
        }
        AI::MXNet::NDArray::Sparse->adagrad_update($weight, $grad, $history, { out=>$weight, lr=>$lr, wd=>$wd, %kwargs });
    }
    else
    {
        $grad *= $self->rescale_grad;
        if(defined $self->clip_gradient)
        {
            $grad = AI::MXNet::NDArray->clip($grad, -$self->clip_gradient, $self->clip_gradient);
        }
        $history += $grad->square;
        my $div = $grad / ($history + $self->eps)->sqrt;
        $weight += ($div + $weight * $wd) * -$lr;
    }
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::RMSProp - RMSProp optimizer of Tieleman & Hinton, 2012.
=cut

=head1 DESCRIPTION

    RMSProp optimizer of Tieleman & Hinton, 2012,

    For centered=False, the code follows the version in
    http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf by
    Tieleman & Hinton, 2012

    For centered=True, the code follows the version in
    http://arxiv.org/pdf/1308.0850v5.pdf Eq(38) - Eq(45) by Alex Graves, 2013.

    Parameters
    ----------
    learning_rate : Num, optional
        Step size.
        Default value is set to 0.001.
    gamma1: Num, optional
        decay factor of moving average for gradient^2.
        Default value is set to 0.9.
    gamma2: Num, optional
        "momentum" factor.
        Default value if set to 0.9.
        Only used if centered=True
    epsilon : Num, optional
        Default value is set to 1e-8.
    centered : Bool, optional
        Use Graves or Tielemans & Hintons version of RMSProp
    wd : Num, optional
        L2 regularization coefficient add to all the weights
    rescale_grad : Num, optional
        rescaling factor of gradient.
    clip_gradient : Num, optional
        clip gradient in range [-clip_gradient, clip_gradient]
    clip_weights : Num, optional
        clip weights in range [-clip_weights, clip_weights]
=cut

package AI::MXNet::RMSProp;
use Mouse;

extends 'AI::MXNet::Optimizer';

has '+learning_rate' => (default => 0.001);
has 'gamma1'         => (is => "ro", isa => "Num",  default => 0.9);
has 'gamma2'         => (is => "ro", isa => "Num",  default => 0.9);
has 'epsilon'        => (is => "ro", isa => "Num",  default => 1e-8);
has 'centered'       => (is => "ro", isa => "Bool", default => 0);
has 'clip_weights'   => (is => "ro", isa => "Num");
has 'kwargs'         => (is => "rw", init_arg => undef);

sub BUILD
{
    my $self = shift;
    $self->kwargs({
        gamma1       => $self->gamma1,
        epsilon      => $self->epsilon
    });
    if($self->centered)
    {
        $self->kwargs->{gamma2} = $self->gamma2;
    }
    if($self->clip_gradient)
    {
        $self->kwargs->{clip_gradient} = $self->clip_gradient;
    }
    if($self->clip_weights)
    {
        $self->kwargs->{clip_weights} = $self->clip_weights;
    }
}

# For centered=False: n
# For centered=True: n, g, delta
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return [
            $self->centered
            ? (
                AI::MXNet::NDArray->zeros(
                    $weight->shape,
                    ctx => $weight->context,
                    stype => $weight->stype
                ),  # n
                AI::MXNet::NDArray->zeros(
                    $weight->shape,
                    ctx => $weight->context,
                    stype => $weight->stype
                ),  # g
                AI::MXNet::NDArray->zeros(
                    $weight->shape,
                    ctx => $weight->context,
                    stype => $weight->stype
                )
            )   # delta
            : (
                AI::MXNet::NDArray->zeros(
                    $weight->shape,
                    ctx => $weight->context,
                    stype => $weight->stype
                ),  # n
            )
    ];
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    ArrayRef[AI::MXNet::NDArray] $state
)
{
    my $lr = $self->_get_lr($index);
    my $wd = $self->_get_wd($index);
    $self->_update_count($index);
    my ($n, $g, $delta) = @{ $state };
    if($self->centered)
    {
        AI::MXNet::NDArray->rmspropalex_update(
            $weight, $grad, $n, $g, $delta,
            {
                out => $weight,
                lr  => $lr,
                wd  => $wd,
                rescale_grad => $self->rescale_grad,
                %{ $self->kwargs }
            }
        );
    }
    else
    {
        AI::MXNet::NDArray->rmsprop_update(
            $weight, $grad, $n,
            {
                out => $weight,
                lr  => $lr,
                wd  => $wd,
                rescale_grad => $self->rescale_grad,
                %{ $self->kwargs }
            }
        );
    }
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::AdaDelta - AdaDelta optimizer.
=cut

=head1 DESCRIPTION

    AdaDelta optimizer as described in
    Zeiler, M. D. (2012).
    *ADADELTA: An adaptive learning rate method.*

    http://arxiv.org/abs/1212.5701

    Parameters
    ----------
    rho: Num
        Decay rate for both squared gradients and delta x
    epsilon : Num
        The constant as described in the thesis
    wd : Num
        L2 regularization coefficient add to all the weights
    rescale_grad : Num, optional
        rescaling factor of gradient. Normally should be 1/batch_size.
    clip_gradient : Num, optional
        clip gradient in range [-clip_gradient, clip_gradient]
=cut
package AI::MXNet::AdaDelta;
use Mouse;

extends 'AI::MXNet::Optimizer';

has 'rho'    => (is => "rw", isa => "Num", default => 0.9);
has 'epsilon'    => (is => "rw", isa => "Num", default => 1e-5);

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return [
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context
            ),  # accumulated g
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context
            )   # accumulated delta
    ];
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    ArrayRef[AI::MXNet::NDArray] $state
)
{
    my $wd = $self->_get_wd($index);
    $self->_update_count($index);
    $grad *= $self->rescale_grad;
    if($self->clip_gradient)
    {
        $grad = AI::MXNet::NDArray->clip(
            $grad,
            -$self->clip_gradient,
             $self->clip_gradient
        );
    }
    my ($acc_g, $acc_delta) = @{ $state };
    $acc_g .= $self->rho * $acc_g + (1 - $self->rho) * $grad * $grad;
    my $current_delta = ($acc_delta + $self->epsilon)->sqrt
                            /
                        ($acc_g + $self->epsilon)->sqrt
                            *
                        $grad;
    $acc_delta .= $self->rho * $acc_delta + (1 - $self->rho) * $current_delta * $current_delta;
    $weight -= $current_delta + $wd * $weight;
}

__PACKAGE__->register;

# For test use
package AI::MXNet::Test;
use Mouse;

extends 'AI::MXNet::Optimizer';

# Create a state to duplicate weight
method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context
    );
}

# performs w += rescale_grad * grad
method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    AI::MXNet::NDArray $state
)
{
    $weight += $grad * $self->rescale_grad;
    $state .= $weight;
}

__PACKAGE__->register;

package AI::MXNet::Ftrl;


=head1 NAME

    AI::MXNet::Ftrl
=cut

=head1 DESCRIPTION

    Referenced from *Ad Click Prediction: a View from the Trenches*, available at
    http://dl.acm.org/citation.cfm?id=2488200.

    The optimizer updates the weight by:

        rescaled_grad = clip(grad * rescale_grad, clip_gradient)
        z += rescaled_grad - (sqrt(n + rescaled_grad**2) - sqrt(n)) * weight / learning_rate
        n += rescaled_grad**2
        w = (sign(z) * lamda1 - z) / ((beta + sqrt(n)) / learning_rate + wd) * (abs(z) > lamda1)

    If the storage types of weight, state and grad are all row_sparse,
    **sparse updates** are applied by::

        for row in grad.indices:
            rescaled_grad[row] = clip(grad[row] * rescale_grad, clip_gradient)
            z[row] += rescaled_grad[row] - (sqrt(n[row] + rescaled_grad[row]**2) - sqrt(n[row])) * weight[row] / learning_rate
            n[row] += rescaled_grad[row]**2
            w[row] = (sign(z[row]) * lamda1 - z[row]) / ((beta + sqrt(n[row])) / learning_rate + wd) * (abs(z[row]) > lamda1)

    The sparse update only updates the z and n for the weights whose row_sparse
    gradient indices appear in the current batch, rather than updating it for all
    indices. Compared with the original update, it can provide large
    improvements in model training throughput for some applications. However, it
    provides slightly different semantics than the original update, and
    may lead to different empirical results.

    This optimizer accepts the following parameters in addition to those accepted
    by AI::MXNet::Optimizer

    Parameters
    ----------
    lamda1 : Num, optional
        L1 regularization coefficient.
    learning_rate : Num, optional
        The initial learning rate.
    beta : Num, optional
        Per-coordinate learning rate correlation parameter.
=cut

use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.1);
has 'beta'           => (is => "ro", isa => "Num",  default => 1);
has 'lamda1'         => (is => "ro", isa => "Num",  default => 0.01);

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return [
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                stype => $weight->stype
            ),  # z
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                stype => $weight->stype
            )   # n
    ];
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    ArrayRef[AI::MXNet::NDArray] $state
)
{
    $self->_update_count($index);
    my $wd = $self->_get_wd($index);
    my $lr = $self->_get_lr($index);
    my %kwargs = (lamda1 => $self->lamda1, beta => $self->beta, rescale_grad => $self->rescale_grad);
    if($self->clip_gradient)
    {
        $kwargs{clip_gradient} = $self->clip_gradient;
    }
    # accumulated g and delta initialization
    my ($z, $n) = @{ $state };
    AI::MXNet::NDArray->ftrl_update(
        $weight, $grad, $z, $n,
        { lr => $lr, wd => $wd, %kwargs, out => $weight }
    );
}

__PACKAGE__->register;

package AI::MXNet::Adamax;

=head1 NAME

    AI::MXNet::Adamax
=cut

=head1 DESCRIPTION

    It is a variant of Adam based on the infinity norm
    available at http://arxiv.org/abs/1412.6980 Section 7.

    This optimizer accepts the following parameters in addition to those accepted
    AI::MXNet::Optimizer.

    Parameters
    ----------
    beta1 : Num, optional
        Exponential decay rate for the first moment estimates.
    beta2 : Num, optional
        Exponential decay rate for the second moment estimates.
=cut

use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.002);
has 'beta1'          => (is => "ro", isa => "Num",  default => 0.9);
has 'beta2'          => (is => "ro", isa => "Num",  default => 0.999);

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return [
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                dtype => $weight->dtype
            ),  # mean
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                dtype => $weight->dtype
            )   # variance
    ];
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    ArrayRef[AI::MXNet::NDArray] $state
)
{
    my $wd = $self->_get_wd($index);
    my $lr = $self->_get_lr($index);
    $self->_update_count($index);
    my $t = $self->_index_update_count->{$index};
    $lr /= (1 - $self->beta1**$t);

    $grad = $grad * $self->rescale_grad + $wd * $weight;
    if($self->clip_gradient)
    {
        $grad = AI::MXNet::NDArray->clip(
            $grad,
            -$self->clip_gradient,
             $self->clip_gradient
        );
    }

    # update m_t and u_t
    my($m_t, $u_t) = @{ $state };
    $m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad;
    $u_t .= AI::MXNet::NDArray->maximum($self->beta2 * $u_t, $grad->abs);

    # update weight
    $weight -= $lr * $m_t / $u_t;
}

__PACKAGE__->register;

package AI::MXNet::Nadam;

=head1 NAME

    AI::MXNet::Nadam
=cut

=head1 DESCRIPTION

    The Nesterov Adam optimizer.

    Much like Adam is essentially RMSprop with momentum,
    Nadam is Adam RMSprop with Nesterov momentum available
    at http://cs229.stanford.edu/proj2015/054_report.pdf.

    This optimizer accepts the following parameters in addition to those accepted
    by AI::MXNet::Optimizer.

    Parameters
    ----------
    beta1 : Num, optional
        Exponential decay rate for the first moment estimates.
    beta2 : Num, optional
        Exponential decay rate for the second moment estimates.
    epsilon : Num, optional
        Small value to avoid division by 0.
    schedule_decay : Num, optional
        Exponential decay rate for the momentum schedule
=cut

use Mouse;
extends 'AI::MXNet::Optimizer';
has '+learning_rate' => (default => 0.001);
has 'beta1'          => (is => "ro", isa => "Num",  default => 0.9);
has 'beta2'          => (is => "ro", isa => "Num",  default => 0.999);
has 'epsilon'        => (is => "ro", isa => "Num",  default => 1e-8);
has 'schedule_decay' => (is => "ro", isa => "Num",  default => 0.004);
has 'm_schedule'     => (is => "rw", default => 1, init_arg => undef);

method create_state(Index $index, AI::MXNet::NDArray $weight)
{
    return [
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                dtype => $weight->dtype
            ),  # mean
            AI::MXNet::NDArray->zeros(
                $weight->shape,
                ctx => $weight->context,
                dtype => $weight->dtype
            )   # variance
    ];
}

method update(
    Index $index,
    AI::MXNet::NDArray $weight,
    AI::MXNet::NDArray $grad,
    ArrayRef[AI::MXNet::NDArray] $state
)
{
    my $wd = $self->_get_wd($index);
    my $lr = $self->_get_lr($index);
    $self->_update_count($index);
    my $t = $self->_index_update_count->{$index};
    $grad = $grad * $self->rescale_grad + $wd * $weight;
    if($self->clip_gradient)
    {
        $grad = AI::MXNet::NDArray->clip(
            $grad,
            -$self->clip_gradient,
             $self->clip_gradient
        );
    }
    # warming momentum schedule
    my $momentum_t    = $self->beta1 * (1 - 0.5 * (0.96**($t * $self->schedule_decay)));
    my $momentum_t_1  = $self->beta1 * (1 - 0.5 * (0.96**(($t + 1) * $self->schedule_decay)));
    $self->m_schedule = $self->m_schedule * $momentum_t;
    my $m_schedule_next  = $self->m_schedule * $momentum_t_1;

    # update m_t and v_t
    my ($m_t, $v_t) = @{ $state };
    $m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad;
    $v_t .= $self->beta2 * $v_t + (1 - $self->beta2) * $grad * $grad;

    my $grad_prime = $grad / (1 - $self->m_schedule);
    my $m_t_prime  = $m_t  / (1 - $m_schedule_next);
    my $v_t_prime  = $v_t  / (1 - $self->beta2**$t);
    my $m_t_bar    = (1 - $momentum_t) * $grad_prime + $momentum_t_1 * $m_t_prime;

    # update weight
    $weight -= $lr * $m_t_bar / (sqrt($v_t_prime) + $self->epsilon);
}

__PACKAGE__->register;

=head1 NAME

    AI::MXNet::Updater - Updater for kvstore
=cut

package AI::MXNet::Updater;
use Mouse;
use Storable qw(thaw freeze);
use overload "&{}" => sub { my $self = shift; sub { $self->call(@_) } },
             fallback => 1;

has "optimizer"     => (is => "rw", isa => "AI::MXNet::Optimizer");
has "states"        => (is => "rw", isa => "HashRef", default => sub { +{} });
has "states_synced" => (is => "rw", isa => "HashRef", default => sub { +{} });

method call(Index $index, AI::MXNet::NDArray $grad, AI::MXNet::NDArray $weight)
{
    if(not exists $self->states->{ $index })
    {
        $self->states->{ $index } = $self->optimizer->create_state($index, $weight);
        $self->states_synced->{ $index } = 1;
    }
    elsif(not $self->states_synced->{ $index })
    {
        $self->states->{ $index } = $self->sync_state_context($self->states->{ $index }, $weight->context);
        $self->states_synced->{ $index } = 1;
    }
    $self->optimizer->update($index, $weight, $grad, $self->states->{ $index });
}
*slice = *call;

method sync_state_context(Maybe[AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]] $state, AI::MXNet::Context $context)
{
    if(blessed $state)
    {
        return $state->as_in_context($context);
    }
    elsif(ref $state)
    {
        return [map { $self->sync_state_context($_, $context) } @{ $state }];
    }
    return $state;
}

=head2 set_states

    Sets updater states.
=cut

method set_states($states)
{
    my $thawed_states = thaw($states);
    my ($optimizer);
    if(ref $thawed_states eq 'ARRAY')
    {
        ($thawed_states, $optimizer) = @{ $thawed_states };
        $self->optimizer($optimizer);
    }
    $self->states($thawed_states);
    %{ $self->states_synced } = map { $_ => 0 } keys %{ $thawed_states };
}

=head2 get_states

        Gets updater states.

        Parameters
        ----------
        dump_optimizer : bool, default False
            Whether to also save the optimizer itself. This would also save optimizer
            information such as learning rate and weight decay schedules.
=cut

method get_states(Bool $dump_optimizer=0)
{
    if($dump_optimizer)
    {
        my $param_dict = $self->optimizer->param_dict;
        $self->optimizer->param_dict({});
        my $freezed = freeze([$self->states, $self->optimizer]);
        $self->optimizer->param_dict($param_dict);
        return $freezed;
    }
    return freeze($self->states);
}

package AI::MXNet::Optimizer;

method get_updater(AI::MXNet::Optimizer $optimizer)
{
    return AI::MXNet::Updater->new(optimizer => $optimizer);
}

1;