1from pyomo.common.tempfiles import TempfileManager
2from pyomo.common.fileutils import Executable
3from pyomo.contrib.appsi.base import PersistentSolver, Results, TerminationCondition, SolverConfig, PersistentSolutionLoader
4from pyomo.contrib.appsi.writers import NLWriter
5from pyomo.common.log import LogStream
6import logging
7import subprocess
8from pyomo.core.kernel.objective import minimize
9import math
10from pyomo.common.collections import ComponentMap
11from pyomo.core.expr.numvalue import value
12from pyomo.core.expr.visitor import replace_expressions
13from typing import Optional, Sequence, NoReturn, List, Mapping
14from pyomo.core.base.var import _GeneralVarData
15from pyomo.core.base.constraint import _GeneralConstraintData
16from pyomo.core.base.block import _BlockData
17from pyomo.core.base.param import _ParamData
18from pyomo.core.base.objective import _GeneralObjectiveData
19from pyomo.common.timing import HierarchicalTimer
20from pyomo.common.tee import TeeStream
21import sys
22from typing import Dict
23from pyomo.common.config import ConfigValue, NonNegativeInt
24from pyomo.common.errors import PyomoException
25
26
27logger = logging.getLogger(__name__)
28
29
30class IpoptConfig(SolverConfig):
31    def __init__(self,
32                 description=None,
33                 doc=None,
34                 implicit=False,
35                 implicit_domain=None,
36                 visibility=0):
37        super(IpoptConfig, self).__init__(description=description,
38                                          doc=doc,
39                                          implicit=implicit,
40                                          implicit_domain=implicit_domain,
41                                          visibility=visibility)
42
43        self.declare('executable', ConfigValue())
44        self.declare('filename', ConfigValue(domain=str))
45        self.declare('keepfiles', ConfigValue(domain=bool))
46        self.declare('solver_output_logger', ConfigValue())
47        self.declare('log_level', ConfigValue(domain=NonNegativeInt))
48
49        self.executable = Executable('ipopt')
50        self.filename = None
51        self.keepfiles = False
52        self.solver_output_logger = logger
53        self.log_level = logging.INFO
54
55
56ipopt_command_line_options = {'acceptable_compl_inf_tol',
57                              'acceptable_constr_viol_tol',
58                              'acceptable_dual_inf_tol',
59                              'acceptable_tol',
60                              'alpha_for_y',
61                              'bound_frac',
62                              'bound_mult_init_val',
63                              'bound_push',
64                              'bound_relax_factor',
65                              'compl_inf_tol',
66                              'constr_mult_init_max',
67                              'constr_viol_tol',
68                              'diverging_iterates_tol',
69                              'dual_inf_tol',
70                              'expect_infeasible_problem',
71                              'file_print_level',
72                              'halt_on_ampl_error',
73                              'hessian_approximation',
74                              'honor_original_bounds',
75                              'linear_scaling_on_demand',
76                              'linear_solver',
77                              'linear_system_scaling',
78                              'ma27_pivtol',
79                              'ma27_pivtolmax',
80                              'ma57_pivot_order',
81                              'ma57_pivtol',
82                              'ma57_pivtolmax',
83                              'max_cpu_time',
84                              'max_iter',
85                              'max_refinement_steps',
86                              'max_soc',
87                              'maxit',
88                              'min_refinement_steps',
89                              'mu_init',
90                              'mu_max',
91                              'mu_oracle',
92                              'mu_strategy',
93                              'nlp_scaling_max_gradient',
94                              'nlp_scaling_method',
95                              'obj_scaling_factor',
96                              'option_file_name',
97                              'outlev',
98                              'output_file',
99                              'pardiso_matching_strategy',
100                              'print_level',
101                              'print_options_documentation',
102                              'print_user_options',
103                              'required_infeasibility_reduction',
104                              'slack_bound_frac',
105                              'slack_bound_push',
106                              'tol',
107                              'wantsol',
108                              'warm_start_bound_push',
109                              'warm_start_init_point',
110                              'warm_start_mult_bound_push',
111                              'watchdog_shortened_iter_trigger'}
112
113
114class Ipopt(PersistentSolver):
115    def __init__(self):
116        self._config = IpoptConfig()
117        self._solver_options = dict()
118        self._writer = NLWriter()
119        self._filename = None
120        self._dual_sol = dict()
121        self._primal_sol = ComponentMap()
122        self._reduced_costs = ComponentMap()
123        self._last_results_object: Optional[Results] = None
124
125    def available(self):
126        if self.config.executable.path() is None:
127            return self.Availability.NotFound
128        return self.Availability.FullLicense
129
130    def version(self):
131        results = subprocess.run([str(self.config.executable), '--version'],
132                                 timeout=1,
133                                 stdout=subprocess.PIPE,
134                                 stderr=subprocess.STDOUT,
135                                 universal_newlines=True)
136        version = results.stdout.splitlines()[0]
137        version = version.split(' ')[1]
138        version = version.strip()
139        version = tuple(int(i) for i in version.split('.'))
140        return version
141
142    def nl_filename(self):
143        if self._filename is None:
144            return None
145        else:
146            return self._filename + '.nl'
147
148    def sol_filename(self):
149        if self._filename is None:
150            return None
151        else:
152            return self._filename + '.sol'
153
154    def options_filename(self):
155        if self._filename is None:
156            return None
157        else:
158            return self._filename + '.opt'
159
160    @property
161    def config(self):
162        return self._config
163
164    @config.setter
165    def config(self, val):
166        self._config = val
167
168    @property
169    def ipopt_options(self):
170        """
171        Returns
172        -------
173        ipopt_options: dict
174            A dictionary mapping solver options to values for those options. These
175            are solver specific.
176        """
177        return self._solver_options
178
179    @ipopt_options.setter
180    def ipopt_options(self, val: Dict):
181        self._solver_options = val
182
183    @property
184    def update_config(self):
185        return self._writer.update_config
186
187    @property
188    def writer(self):
189        return self._writer
190
191    @property
192    def symbol_map(self):
193        return self._writer.symbol_map
194
195    def set_instance(self, model):
196        self._writer.set_instance(model)
197
198    def add_variables(self, variables: List[_GeneralVarData]):
199        self._writer.add_variables(variables)
200
201    def add_params(self, params: List[_ParamData]):
202        self._writer.add_params(params)
203
204    def add_constraints(self, cons: List[_GeneralConstraintData]):
205        self._writer.add_constraints(cons)
206
207    def add_block(self, block: _BlockData):
208        self._writer.add_block(block)
209
210    def remove_variables(self, variables: List[_GeneralVarData]):
211        self._writer.remove_variables(variables)
212
213    def remove_params(self, params: List[_ParamData]):
214        self._writer.remove_params(params)
215
216    def remove_constraints(self, cons: List[_GeneralConstraintData]):
217        self._writer.remove_constraints(cons)
218
219    def remove_block(self, block: _BlockData):
220        self._writer.remove_block(block)
221
222    def set_objective(self, obj: _GeneralObjectiveData):
223        self._writer.set_objective(obj)
224
225    def update_variables(self, variables: List[_GeneralVarData]):
226        self._writer.update_variables(variables)
227
228    def update_params(self):
229        self._writer.update_params()
230
231    def _write_options_file(self):
232        f = open(self._filename + '.opt', 'w')
233        for k, val in self.ipopt_options.items():
234            if k not in ipopt_command_line_options:
235                f.write(str(k) + ' ' + str(val) + '\n')
236        f.close()
237
238    def solve(self, model, timer: HierarchicalTimer = None):
239        avail = self.available()
240        if not avail:
241            raise PyomoException(f'Solver {self.__class__} is not available ({avail}).')
242        if self._last_results_object is not None:
243            self._last_results_object.solution_loader.invalidate()
244        if timer is None:
245            timer = HierarchicalTimer()
246        try:
247            TempfileManager.push()
248            if self.config.filename is None:
249                nl_filename = TempfileManager.create_tempfile(suffix='.nl')
250                self._filename = nl_filename.split('.')[0]
251            else:
252                self._filename = self.config.filename
253                TempfileManager.add_tempfile(self._filename + '.nl', exists=False)
254            TempfileManager.add_tempfile(self._filename + '.sol', exists=False)
255            TempfileManager.add_tempfile(self._filename + '.opt', exists=False)
256            self._write_options_file()
257            timer.start('write nl file')
258            self._writer.write(model, self._filename+'.nl', timer=timer)
259            timer.stop('write nl file')
260            res = self._apply_solver(timer)
261            self._last_results_object = res
262            if self.config.report_timing:
263                logger.info('\n' + str(timer))
264            return res
265        finally:
266            # finally, clean any temporary files registered with the
267            # temp file manager, created/populated *directly* by this
268            # plugin.
269            TempfileManager.pop(remove=not self.config.keepfiles)
270            if not self.config.keepfiles:
271                self._filename = None
272
273    def _parse_sol(self):
274        solve_vars = self._writer.get_ordered_vars()
275        solve_cons = self._writer.get_ordered_cons()
276        results = Results()
277
278        f = open(self._filename + '.sol', 'r')
279        all_lines = list(f.readlines())
280        f.close()
281
282        termination_line = all_lines[1]
283        if 'Optimal Solution Found' in termination_line:
284            results.termination_condition = TerminationCondition.optimal
285        elif 'Problem may be infeasible' in termination_line:
286            results.termination_condition = TerminationCondition.infeasible
287        elif 'problem might be unbounded' in termination_line:
288            results.termination_condition = TerminationCondition.unbounded
289        elif 'Maximum Number of Iterations Exceeded' in termination_line:
290            results.termination_condition = TerminationCondition.maxIterations
291        elif 'Maximum CPU Time Exceeded' in termination_line:
292            results.termination_condition = TerminationCondition.maxTimeLimit
293        else:
294            results.termination_condition = TerminationCondition.unknown
295
296        n_cons = len(solve_cons)
297        n_vars = len(solve_vars)
298        dual_lines = all_lines[12:12+n_cons]
299        primal_lines = all_lines[12+n_cons:12+n_cons+n_vars]
300
301        rc_upper_info_line = all_lines[12+n_cons+n_vars+1]
302        assert rc_upper_info_line.startswith('suffix')
303        n_rc_upper = int(rc_upper_info_line.split()[2])
304        assert 'ipopt_zU_out' in all_lines[12+n_cons+n_vars+2]
305        upper_rc_lines = all_lines[12+n_cons+n_vars+3:12+n_cons+n_vars+3+n_rc_upper]
306
307        rc_lower_info_line = all_lines[12+n_cons+n_vars+3+n_rc_upper]
308        assert rc_lower_info_line.startswith('suffix')
309        n_rc_lower = int(rc_lower_info_line.split()[2])
310        assert 'ipopt_zL_out' in all_lines[12+n_cons+n_vars+3+n_rc_upper+1]
311        lower_rc_lines = all_lines[12+n_cons+n_vars+3+n_rc_upper+2:12+n_cons+n_vars+3+n_rc_upper+2+n_rc_lower]
312
313        self._dual_sol = dict()
314        self._primal_sol = ComponentMap()
315        self._reduced_costs = ComponentMap()
316
317        for ndx, dual in enumerate(dual_lines):
318            dual = float(dual)
319            con = solve_cons[ndx]
320            self._dual_sol[con] = dual
321
322        for ndx, primal in enumerate(primal_lines):
323            primal = float(primal)
324            var = solve_vars[ndx]
325            self._primal_sol[var] = primal
326
327        for rcu_line in upper_rc_lines:
328            split_line = rcu_line.split()
329            var_ndx = int(split_line[0])
330            rcu = float(split_line[1])
331            var = solve_vars[var_ndx]
332            self._reduced_costs[var] = rcu
333
334        for rcl_line in lower_rc_lines:
335            split_line = rcl_line.split()
336            var_ndx = int(split_line[0])
337            rcl = float(split_line[1])
338            var = solve_vars[var_ndx]
339            if var in self._reduced_costs:
340                if abs(rcl) > abs(self._reduced_costs[var]):
341                    self._reduced_costs[var] = rcl
342            else:
343                self._reduced_costs[var] = rcl
344
345        for var in solve_vars:
346            if var not in self._reduced_costs:
347                self._reduced_costs[var] = 0
348
349        if results.termination_condition == TerminationCondition.optimal and self.config.load_solution:
350            for v, val in self._primal_sol.items():
351                v.value = val
352
353            if self._writer.get_active_objective() is None:
354                results.best_feasible_objective = None
355            else:
356                results.best_feasible_objective = value(self._writer.get_active_objective().expr)
357        elif results.termination_condition == TerminationCondition.optimal:
358            if self._writer.get_active_objective() is None:
359                results.best_feasible_objective = None
360            else:
361                obj_expr_evaluated = replace_expressions(self._writer.get_active_objective().expr,
362                                                         substitution_map={id(v): val for v, val in self._primal_sol.items()},
363                                                         descend_into_named_expressions=True,
364                                                         remove_named_expressions=True)
365                results.best_feasible_objective = value(obj_expr_evaluated)
366        elif self.config.load_solution:
367            raise RuntimeError('A feasible solution was not found, so no solution can be loaded.'
368                               'Please set opt.config.load_solution=False and check '
369                               'results.termination_condition and '
370                               'resutls.best_feasible_objective before loading a solution.')
371
372        results.solution_loader = PersistentSolutionLoader(solver=self)
373
374        return results
375
376    def _apply_solver(self, timer: HierarchicalTimer):
377        config = self.config
378
379        if config.time_limit is not None:
380            timeout = config.time_limit + min(max(1, 0.01 * config.time_limit), 100)
381        else:
382            timeout = None
383
384        ostreams = [LogStream(level=self.config.log_level, logger=self.config.solver_output_logger)]
385        if self.config.stream_solver:
386            ostreams.append(sys.stdout)
387
388        cmd = [str(config.executable),
389               self._filename + '.nl',
390               '-AMPL',
391               'option_file_name=' + self._filename + '.opt']
392        if 'option_file_name' in self.ipopt_options:
393            raise ValueError('Use Ipopt.config.filename to specify the name of the options file. '
394                             'Do not use Ipopt.ipopt_options["option_file_name"].')
395        for k, v in self.ipopt_options.items():
396            cmd.append(str(k) + '=' + str(v))
397
398        with TeeStream(*ostreams) as t:
399            timer.start('subprocess')
400            cp = subprocess.run(cmd,
401                                timeout=timeout,
402                                stdout=t.STDOUT,
403                                stderr=t.STDERR,
404                                universal_newlines=True)
405            timer.stop('subprocess')
406
407        if cp.returncode != 0:
408            if self.config.load_solution:
409                raise RuntimeError('A feasible solution was not found, so no solution can be loaded.'
410                                   'Please set opt.config.load_solution=False and check '
411                                   'results.termination_condition and '
412                                   'results.best_feasible_objective before loading a solution.')
413            results = Results()
414            results.termination_condition = TerminationCondition.error
415            results.best_feasible_objective = None
416            self._primal_sol = None
417            self._dual_sol = None
418        else:
419            timer.start('parse solution')
420            results = self._parse_sol()
421            timer.stop('parse solution')
422
423        if self._writer.get_active_objective() is None:
424            results.best_objective_bound = None
425        else:
426            if self._writer.get_active_objective().sense == minimize:
427                results.best_objective_bound = -math.inf
428            else:
429                results.best_objective_bound = math.inf
430
431        return results
432
433    def get_primals(self, vars_to_load: Optional[Sequence[_GeneralVarData]] = None) -> Mapping[_GeneralVarData, float]:
434        res = ComponentMap()
435        if vars_to_load is None:
436            for v, val in self._primal_sol.items():
437                res[v] = val
438        else:
439            for v in vars_to_load:
440                res[v] = self._primal_sol[v]
441        return res
442
443    def get_duals(self, cons_to_load = None):
444        if cons_to_load is None:
445            return {k: v for k, v in self._dual_sol.items()}
446        else:
447            return {c: self._dual_sol[c] for c in cons_to_load}
448
449    def get_reduced_costs(self, vars_to_load: Optional[Sequence[_GeneralVarData]] = None) -> Mapping[_GeneralVarData, float]:
450        if vars_to_load is None:
451            return ComponentMap((k, v) for k, v in self._reduced_costs.items())
452        else:
453            return ComponentMap((v, self._reduced_costs[v]) for v in vars_to_load)
454