1"""
2Holds common functions for l1 solvers.
3"""
4
5import numpy as np
6
7from statsmodels.tools.sm_exceptions import ConvergenceWarning
8
9
10def qc_results(params, alpha, score, qc_tol, qc_verbose=False):
11    """
12    Theory dictates that one of two conditions holds:
13        i) abs(score[i]) == alpha[i]  and  params[i] != 0
14        ii) abs(score[i]) <= alpha[i]  and  params[i] == 0
15    qc_results checks to see that (ii) holds, within qc_tol
16
17    qc_results also checks for nan or results of the wrong shape.
18
19    Parameters
20    ----------
21    params : ndarray
22        model parameters.  Not including the added variables x_added.
23    alpha : ndarray
24        regularization coefficients
25    score : function
26        Gradient of unregularized objective function
27    qc_tol : float
28        Tolerance to hold conditions (i) and (ii) to for QC check.
29    qc_verbose : bool
30        If true, print out a full QC report upon failure
31
32    Returns
33    -------
34    passed : bool
35        True if QC check passed
36    qc_dict : Dictionary
37        Keys are fprime, alpha, params, passed_array
38
39    Prints
40    ------
41    Warning message if QC check fails.
42    """
43    ## Check for fatal errors
44    assert not np.isnan(params).max()
45    assert (params == params.ravel('F')).min(), \
46        "params should have already been 1-d"
47
48    ## Start the theory compliance check
49    fprime = score(params)
50    k_params = len(params)
51
52    passed_array = np.array([True] * k_params)
53    for i in range(k_params):
54        if alpha[i] > 0:
55            # If |fprime| is too big, then something went wrong
56            if (abs(fprime[i]) - alpha[i]) / alpha[i] > qc_tol:
57                passed_array[i] = False
58    qc_dict = dict(
59        fprime=fprime, alpha=alpha, params=params, passed_array=passed_array)
60    passed = passed_array.min()
61    if not passed:
62        num_failed = (~passed_array).sum()
63        message = 'QC check did not pass for %d out of %d parameters' % (
64            num_failed, k_params)
65        message += '\nTry increasing solver accuracy or number of iterations'\
66            ', decreasing alpha, or switch solvers'
67        if qc_verbose:
68            message += _get_verbose_addon(qc_dict)
69
70        import warnings
71        warnings.warn(message, ConvergenceWarning)
72
73    return passed
74
75
76def _get_verbose_addon(qc_dict):
77    alpha = qc_dict['alpha']
78    params = qc_dict['params']
79    fprime = qc_dict['fprime']
80    passed_array = qc_dict['passed_array']
81
82    addon = '\n------ verbose QC printout -----------------'
83    addon = '\n------ Recall the problem was rescaled by 1 / nobs ---'
84    addon += '\n|%-10s|%-10s|%-10s|%-10s|' % (
85        'passed', 'alpha', 'fprime', 'param')
86    addon += '\n--------------------------------------------'
87    for i in range(len(alpha)):
88        addon += '\n|%-10s|%-10.3e|%-10.3e|%-10.3e|' % (
89                passed_array[i], alpha[i], fprime[i], params[i])
90    return addon
91
92
93def do_trim_params(params, k_params, alpha, score, passed, trim_mode,
94        size_trim_tol, auto_trim_tol):
95    """
96    Trims (set to zero) params that are zero at the theoretical minimum.
97    Uses heuristics to account for the solver not actually finding the minimum.
98
99    In all cases, if alpha[i] == 0, then do not trim the ith param.
100    In all cases, do nothing with the added variables.
101
102    Parameters
103    ----------
104    params : ndarray
105        model parameters.  Not including added variables.
106    k_params : Int
107        Number of parameters
108    alpha : ndarray
109        regularization coefficients
110    score : Function.
111        score(params) should return a 1-d vector of derivatives of the
112        unpenalized objective function.
113    passed : bool
114        True if the QC check passed
115    trim_mode : 'auto, 'size', or 'off'
116        If not 'off', trim (set to zero) parameters that would have been zero
117            if the solver reached the theoretical minimum.
118        If 'auto', trim params using the Theory above.
119        If 'size', trim params if they have very small absolute value
120    size_trim_tol : float or 'auto' (default = 'auto')
121        For use when trim_mode === 'size'
122    auto_trim_tol : float
123        For sue when trim_mode == 'auto'.  Use
124    qc_tol : float
125        Print warning and do not allow auto trim when (ii) in "Theory" (above)
126        is violated by this much.
127
128    Returns
129    -------
130    params : ndarray
131        Trimmed model parameters
132    trimmed : ndarray of booleans
133        trimmed[i] == True if the ith parameter was trimmed.
134    """
135    ## Trim the small params
136    trimmed = [False] * k_params
137
138    if trim_mode == 'off':
139        trimmed = np.array([False] * k_params)
140    elif trim_mode == 'auto' and not passed:
141        import warnings
142        msg = "Could not trim params automatically due to failed QC check. " \
143              "Trimming using trim_mode == 'size' will still work."
144        warnings.warn(msg, ConvergenceWarning)
145        trimmed = np.array([False] * k_params)
146    elif trim_mode == 'auto' and passed:
147        fprime = score(params)
148        for i in range(k_params):
149            if alpha[i] != 0:
150                if (alpha[i] - abs(fprime[i])) / alpha[i] > auto_trim_tol:
151                    params[i] = 0.0
152                    trimmed[i] = True
153    elif trim_mode == 'size':
154        for i in range(k_params):
155            if alpha[i] != 0:
156                if abs(params[i]) < size_trim_tol:
157                    params[i] = 0.0
158                    trimmed[i] = True
159    else:
160        raise ValueError(
161            "trim_mode == %s, which is not recognized" % (trim_mode))
162
163    return params, np.asarray(trimmed)
164