1 /*
2 
3  HyPhy - Hypothesis Testing Using Phylogenies.
4 
5  Copyright (C) 1997-now
6  Core Developers:
7  Sergei L Kosakovsky Pond (sergeilkp@icloud.com)
8  Art FY Poon    (apoon42@uwo.ca)
9  Steven Weaver (sweaver@temple.edu)
10 
11  Module Developers:
12  Lance Hepler (nlhepler@gmail.com)
13  Martin Smith (martin.audacis@gmail.com)
14 
15  Significant contributions from:
16  Spencer V Muse (muse@stat.ncsu.edu)
17  Simon DW Frost (sdf22@cam.ac.uk)
18 
19  Permission is hereby granted, free of charge, to any person obtaining a
20  copy of this software and associated documentation files (the
21  "Software"), to deal in the Software without restriction, including
22  without limitation the rights to use, copy, modify, merge, publish,
23  distribute, sublicense, and/or sell copies of the Software, and to
24  permit persons to whom the Software is furnished to do so, subject to
25  the following conditions:
26 
27  The above copyright notice and this permission notice shall be included
28  in all copies or substantial portions of the Software.
29 
30  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
31  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
32  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
33  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
34  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
35  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
36  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
37 
38  */
39 
40 #include <math.h>
41 #include <float.h>
42 #include <limits.h>
43 #include "string.h"
44 #include <stdio.h>
45 #include <time.h>
46 #include <ctype.h>
47 
48 #include "likefunc.h"
49 #include "parser.h"
50 #include "matrix.h"
51 #include "polynoml.h"
52 #include "batchlan.h"
53 #include "global_things.h"
54 
55 using namespace hy_global;
56 
57 
58 
59 
60 extern
61 _SimpleList     simpleOperationCodes,
62                 simpleOperationFunctions;
63 
64 _List           hyReservedWords,
65                 varNamesSupportList,
66                 variablePtrs;   // stores all the variables declared so far
67 
68 _AVLList        *lookAside = nil;
69 _AVLListX       variableNames (&varNamesSupportList);
70 
71 
72 
73 
74 // indices of all independent variables
75 
76 _Trie           FunctionNameList;
77 _List           BuiltInFunctions;
78 
79 
80 
81 _SimpleList     freeSlots,
82                 deferIsConstant,
83                 *deferSetFormula = nil;
84 
85 
86 _AVLList       *deferClearConstraint = nil;
87 
88 bool            useGlobalUpdateFlag = false;
89 
90 
91 _String         HalfOps (":<>=!&|");
92 
93 _Trie           UnOps;
94 
95 _SimpleList opPrecedence,
96             BinOps,
97             associativeOps;
98 
99 hyFloat pi_const = 3.141592653589793,
100 long_max = (hyFloat)LONG_MAX;
101 
102 /**********************************/
103 /* Defining Globals here for now */
104 /********************************/
105 
106 //Used in formula, and constant
107 
108 hyFloat  tolerance  = DBL_EPSILON;
109 
110 
111 
112 /*********************************/
113 /*          End Globals         */
114 /*******************************/
115 
116 //__________________________________________________________________________________
117 //SW: Helper functions
118 
119 
120 
121 
122 //__________________________________________________________________________________
parameterToCharBuffer(hyFloat value,char * dump,long length,bool json)123 void     parameterToCharBuffer (hyFloat value, char* dump, long length, bool json)
124 {
125     if (json) {
126       if (isnan (value)) {
127         snprintf (dump, length, "null");
128         return;
129       }
130       if (isinf(value)) {
131         snprintf (dump, length, value < 0 ? "-1e9999" : "1e9999");
132         return;
133       }
134     }
135 
136     long digs = print_digit_specification;
137     if (digs<=0 || digs>15) {
138         if (round(value) == value && fabs (value) < long_max) {
139             snprintf (dump,length, "%ld",lrint (value));
140         } else {
141             snprintf (dump,length, PRINTF_FORMAT_STRING,value);
142         }
143     } else {
144         _String format("%-");
145 #ifdef __USE_LONG_DOUBLE__
146         format = format&_String(digs)&"Lg";
147 #else
148         format = format&_String(digs)&'g';
149 #endif
150         snprintf (dump,length,(const char*)format,value);
151     }
152 }
153 
154 
155 //__________________________________________________________________________________
parameterToString(hyFloat value)156 BaseRef     parameterToString (hyFloat value)
157 {
158     char dump [256];
159     parameterToCharBuffer (value, dump, 256);
160     return new _String (dump);
161 }
162 
163 //__________________________________________________________________________________
SplitVariableIDsIntoLocalAndGlobal(const _SimpleList & theList,_List & splitStorage)164 void SplitVariableIDsIntoLocalAndGlobal (const _SimpleList& theList, _List& splitStorage)
165 {
166     splitStorage.Clear();
167     splitStorage.AppendNewInstance(new _SimpleList);
168     splitStorage.AppendNewInstance(new _SimpleList);
169 
170     for (unsigned long k=0; k<theList.lLength; k++) {
171         long varID = theList.list_data[k];
172         (*(_SimpleList*)splitStorage(1-LocateVar (varID)->IsGlobal())) << varID;
173     }
174 }
175 
176 //__________________________________________________________________________________
FetchObjectNameFromType(const unsigned long objectClass)177 _String FetchObjectNameFromType (const unsigned long objectClass) {
178 
179 
180     if (objectClass == HY_ANY_OBJECT) {
181         return "Any object";
182     }
183     _StringBuffer result;
184 
185     auto push_with_spacer = [&result] (const char * v) -> void {
186         if (result.nonempty()) {
187             result << " or ";
188         }
189         result << v;
190     };
191 
192     if (objectClass & HY_UNDEFINED) {
193         result << "Undefined";
194     }
195     if (objectClass & NUMBER) {
196         push_with_spacer ("Number");
197     }
198     if (objectClass & MATRIX) {
199         push_with_spacer ("Matrix");
200     }
201     if (objectClass & CONTAINER) {
202         push_with_spacer ("Variable container");
203     }
204     if (objectClass & TREE_NODE) {
205         push_with_spacer ("Tree Node");
206     }
207     if (objectClass & TREE) {
208         push_with_spacer ("Tree");
209     }
210     if (objectClass & STRING) {
211         push_with_spacer ("String");
212     }
213     if (objectClass & ASSOCIATIVE_LIST) {
214         push_with_spacer ("Associative Array");
215     }
216     if (objectClass & TOPOLOGY) {
217         push_with_spacer ("Topology");
218     }
219     if (objectClass & POLYNOMIAL) {
220         push_with_spacer ("Polynomial");
221     }
222 
223     return result;
224 }
225 
226 //__________________________________________________________________________________
FetchMathObjectNameOfTypeByIndex(const unsigned long objectClass,const long objectIndex)227 _String*   FetchMathObjectNameOfTypeByIndex (const unsigned long objectClass, const long objectIndex)
228 {
229     if (objectIndex >=0 && objectIndex < variableNames.countitems()) {
230             long tc = 0;
231             _SimpleList nts;
232             long        rt,
233                         vi = variableNames.Traverser (nts, rt, variableNames.GetRoot());
234 
235             for (; vi >= 0; vi = variableNames.Traverser (nts, rt))
236                 if (FetchVar(variableNames.GetXtra (vi))->ObjectClass () == objectClass) {
237                     if (tc==objectIndex) {
238                         return (_String*)variableNames.Retrieve(vi);
239                         break;
240                     } else {
241                         tc++;
242                     }
243                 }
244     }
245     return nil;
246 }
247 
248 //__________________________________________________________________________________
FetchObjectFromFormulaByType(_Formula & f,const unsigned long objectClass,long command_id,_String * errMsg)249 HBLObjectRef   FetchObjectFromFormulaByType (_Formula& f, const unsigned long objectClass, long command_id, _String *errMsg) {
250   HBLObjectRef v = f.Compute();
251   if (v) {
252     if (objectClass == HY_ANY_OBJECT || v->ObjectClass () == objectClass) {
253       return v;
254     }
255     if (command_id >= 0 || errMsg) {
256       if (command_id >= 0) {
257         HandleApplicationError (_String ((_String*)f.toStr(kFormulaStringConversionNormal)).Enquote() & (" must evaluate to a ") & FetchObjectNameFromType (objectClass) & " in call to "
258                    &_HY_ValidHBLExpressions.RetrieveKeyByPayload(command_id) & '.');
259       }
260     }
261   }
262   return nil;
263 }
264 
265 //__________________________________________________________________________________
FetchObjectFromVariableByType(_String const * id,const unsigned long objectClass,long command_id,_String * errMsg)266 HBLObjectRef   FetchObjectFromVariableByType (_String const* id, const unsigned long objectClass, long command_id, _String *errMsg) {
267     if (id) {
268         _Variable * v = FetchVar (LocateVarByName (*id));
269         if (v && (objectClass == HY_ANY_OBJECT || (v->ObjectClass () & objectClass))) {
270             return v->Compute();
271         }
272         if (command_id >= 0 || errMsg) {
273             if (command_id >= 0) {
274                 HandleApplicationError (id->Enquote() & (" must refer to a ") & FetchObjectNameFromType (objectClass) & " in call to "
275                                          &_HY_ValidHBLExpressions.RetrieveKeyByPayload(command_id) & '.');
276             } else {
277                 HandleApplicationError (errMsg->Replace ("_VAR_NAME_ID_", *id, true));
278             }
279         }
280     }
281     return nil;
282 }
283 
284 
285 //__________________________________________________________________________________
FetchObjectFromVariableByTypeIndex(long idx,const unsigned long objectClass,long command_id,_String * errMsg)286 HBLObjectRef   FetchObjectFromVariableByTypeIndex (long idx, const unsigned long objectClass, long command_id, _String *errMsg) {
287     _Variable * v = FetchVar (idx);
288     if (v) {
289         if (objectClass == HY_ANY_OBJECT || v->ObjectClass () == objectClass) {
290             return v->Compute ();
291         }
292         if (command_id >= 0 || errMsg) {
293             if (command_id >= 0) {
294                 HandleApplicationError (v->GetName()->Enquote() & (" must refer to a ") & FetchObjectNameFromType (objectClass) & " in call to "
295                                          &_HY_ValidHBLExpressions.RetrieveKeyByPayload(command_id) & '.');
296             } else {
297                 HandleApplicationError (errMsg->Replace ("_VAR_NAME_ID_", *v->GetName(), true));
298             }
299         }
300     }
301     return nil;
302 }
303 
304 //__________________________________________________________________________________
LocateVarByName(_String const & name)305 long LocateVarByName (_String const& name) {
306     return variableNames.Find (&name);
307 }
308 
309 //__________________________________________________________________________________
FetchVar(long index,unsigned long type_check)310 _Variable* FetchVar (long index, unsigned long type_check) {
311     if (index >= 0) {
312         _Variable * var = (_Variable *)variablePtrs.GetItemRangeCheck(variableNames.GetXtra(index));
313         if (var) {
314             if (type_check == HY_ANY_OBJECT || (var->ObjectClass() & type_check)) {
315                 return var;
316             }
317         }
318     }
319     return nil;
320 }
321 
322 //__________________________________________________________________________________
UpdateChangingFlas(long vN)323 void       UpdateChangingFlas (long vN) {
324     // check to see if formulae contain a reference to this var
325     // if so: "decompile" them
326 
327     long topLimit = compiledFormulaeParameters.countitems();
328 
329     _SimpleList * toDelete = nil;
330 
331     for (long k = 0L; k<topLimit; k++) {
332         long g = ((_SimpleList*)compiledFormulaeParameters.list_data[k])->BinaryFind (vN,0);
333 
334         if (g>=0) {
335             ((_ElementaryCommand*)listOfCompiledFormulae.list_data[k])->DecompileFormulae();
336 
337 
338             if (!toDelete) {
339                 toDelete = new _SimpleList;
340             }
341 
342             *toDelete << k;
343         }
344     }
345 
346     if (toDelete) {
347         listOfCompiledFormulae.DeleteList       (*toDelete);
348         compiledFormulaeParameters.DeleteList   (*toDelete);
349         DeleteObject (toDelete);
350     }
351 }
352 
353 //__________________________________________________________________________________
UpdateChangingFlas(_SimpleList const & involvedVariables)354 void       UpdateChangingFlas (_SimpleList const & involvedVariables) {
355     long          topLimit         = compiledFormulaeParameters.lLength;
356     _SimpleList * toDelete         = nil;
357 
358     for (long k = 0; k<topLimit; k++) {
359         long g = ((_SimpleList*)compiledFormulaeParameters.list_data[k])->CountCommonElements (involvedVariables,true);
360 
361         if (g>0) {
362             ((_ElementaryCommand*)listOfCompiledFormulae.list_data[k])->DecompileFormulae();
363 
364             if (!toDelete) {
365                 toDelete = new _SimpleList;
366             }
367 
368             *toDelete << k;
369         }
370     }
371 
372     if (toDelete) {
373         listOfCompiledFormulae.DeleteList       (*toDelete);
374         compiledFormulaeParameters.DeleteList   (*toDelete);
375         DeleteObject (toDelete);
376     }
377 }
378 
379 //__________________________________________________________________________________
DeleteVariable(long dv,bool deleteself,bool do_checks)380 void DeleteVariable (long dv, bool deleteself, bool do_checks) {
381     if (dv>=0L) {
382 
383 
384         _String *name   = (_String*)variableNames.Retrieve (dv);
385         _String my_name = *name&'.';
386 
387         long    vidx    = variableNames.GetXtra (dv);
388 
389 
390         UpdateChangingFlas (vidx);
391 
392         _SimpleList recCache;
393         variableNames.Find (name,recCache);
394         _String     nextVarID;// = *(_String*)variableNames.Retrieve(variableNames.Next (dv,recCache));
395         long        nvid;
396         if ((nvid = variableNames.Next (dv,recCache))>=0L) {
397             nextVarID = *(_String*)variableNames.Retrieve(nvid);
398         }
399 
400         if (deleteself) {
401 
402 
403             _Variable * self_variable = FetchVar(dv);
404 
405             if (do_checks) {
406                 for (AVLListXIteratorKeyValue variable_iterator : AVLListXIterator (&variableNames)) {
407                     _Variable * check_variable = FetchVar(variable_iterator.get_value());
408                     if (self_variable != check_variable && check_variable->CheckFForDependence (vidx,false)) {
409                         HBLObjectRef current_variable_value = check_variable->Compute();
410                         current_variable_value->AddAReference(); // if this isn't done; the object will be deleted when the formula is cleared in SetValue
411                         check_variable->SetValue (current_variable_value,true,true,NULL);
412                         DeleteObject (current_variable_value);
413                     }
414                 }
415             }
416 
417 
418             variableNames.Delete (variableNames.Retrieve(dv),true);
419             variablePtrs[vidx] = nil;
420             DeleteObject (self_variable);
421             freeSlots<<vidx;
422         } else {
423             _Variable* delvar = (FetchVar(dv));
424             if (delvar->IsContainer()) {
425                 _VariableContainer* dc = (_VariableContainer*)delvar;
426                 dc->Clear();
427             }
428         }
429 
430         _List       toDelete;
431 
432         recCache.Clear();
433         long nextVar = variableNames.Find (&nextVarID,recCache);
434 
435         for (; nextVar>=0; nextVar = variableNames.Next (nextVar, recCache)) {
436             _String dependent = *(_String*)variableNames.Retrieve (nextVar);
437             if (dependent.BeginsWith(my_name)) {
438                 toDelete && & dependent;
439             } else {
440                 break;
441             }
442         }
443 
444         for (unsigned long k=0; k< toDelete.lLength; k++) {
445             DeleteVariable (*(_String*)toDelete(k), true, do_checks);
446         }
447     }
448 }
449 
450 //__________________________________________________________________________________
DeleteTreeVariable(long tree_variable_index,_SimpleList & parms,_String * prefix,bool leftovers)451 void DeleteTreeVariable (long tree_variable_index, _SimpleList & parms, _String * prefix, bool leftovers) {
452     if (tree_variable_index>=0) {
453 
454         /*
455             1. Indentify all the variables that are in the tree.* and tree.*.* domain
456             2. Scan all the variables that are dependend on the list here and "batch" unconstrain them
457             3. Delete the tree and tree nodes; store the remainign variables in parms
458         */
459 
460 
461         _String const *tree_name  = prefix ? prefix : (_String const*)variableNames.Retrieve (tree_variable_index),
462                        kDot = ".",
463                        tree_prefix = *tree_name&kDot;
464 
465         _SimpleList traversal_stack,
466                     indices_to_delete,
467                     all_subdomain_indices;
468 
469         _AVLList    indices_to_delete_avl (&indices_to_delete),
470                     all_subdomain_indices_avl (&all_subdomain_indices),
471                     will_keep_these (&parms);
472 
473 
474         long tree_array_index = tree_variable_index;
475         variableNames.Find (tree_name,traversal_stack);
476 
477         while (long next_var = variableNames.Next (tree_array_index,traversal_stack)) {
478             if (next_var < 0) {
479                 break;
480             }
481             _String const * next_var_name = (_String const *)variableNames.Retrieve(next_var);
482             if (next_var_name->BeginsWith(tree_prefix)) {
483                 all_subdomain_indices_avl.InsertNumber(next_var);
484 
485                 _Variable * secondary_variable = FetchVar (next_var);
486 
487                 if (next_var_name->FindBackwards(kDot, tree_prefix.length() + 1, kStringEnd) != kNotFound && !secondary_variable->IsContainer()) {
488                     // 20200220: deleting all containers here; only "plain" variables will be kept
489                     // subnode level
490                     will_keep_these.InsertNumber(next_var);
491                  } else {
492                     //printf ("Deleting variable %s\n" , next_var_name->get_str());
493                     indices_to_delete_avl.InsertNumber (next_var);
494                 }
495             } else {
496                 break;
497             }
498             tree_array_index = next_var;
499         }
500 
501         //if ((nvid = variableNames.Next (dv,recCache))>=0)
502 
503         all_subdomain_indices_avl.ReorderList();
504         indices_to_delete_avl.ReorderList();
505         will_keep_these.ReorderList();
506 
507         _SimpleList _touched_dependent_variables,
508                     touched_containers;
509 
510         _AVLList    touched_dependent_variables (&_touched_dependent_variables);
511 
512         DoForEachVariable([&touched_dependent_variables, &indices_to_delete_avl, &all_subdomain_indices_avl] (_Variable* v, long idx) -> void {
513             // only care about variables that are not going in the subdomain
514             if (all_subdomain_indices_avl.FindLong(idx) < 0) {
515                 if (v->IsContainer()) {
516                     //_VariableContainer * vc = (_VariableContainer*)vc;
517                     /* TODO, it is theoretically possible that a node / tree variable is a template variable
518                        in another container, but there are no use cases of this;
519                        need to check, but later
520                      */
521                 } else {
522                     if (v->CheckFForDependence(indices_to_delete_avl)) {
523                         //printf ("Touched dependent variable %s\n" , v->GetName()->get_str());
524                         touched_dependent_variables.InsertNumber(v->get_index());
525                     }
526                 }
527             }
528         });
529 
530 
531 
532         //SetVariableToOwn
533 
534         UpdateChangingFlas(all_subdomain_indices);
535         // reset compiled formulas that may contain variable indices being deleted
536 
537         if (leftovers) {
538             SetVariablesToOwnValues (will_keep_these);
539         } else {
540             indices_to_delete << parms;
541         }
542 
543         if (touched_dependent_variables.countitems()) {
544             SetVariablesToOwnValues (touched_dependent_variables);
545         }
546 
547         /*if (prefix) {
548             printf ("CLEARING %d variables\n", indices_to_delete.countitems() );
549         }*/
550 
551         indices_to_delete.Each ([] (long var_idx, unsigned long) -> void {
552             _Variable * delvar = LocateVar (var_idx);
553             //printf ("DeleteTree::Deleting variable %s\n" , delvar->GetName()->get_str());
554             if (delvar->ObjectClass() != TREE) {
555                 variableNames.Delete (delvar->GetName(),true);
556                 (*((_SimpleList*)&variablePtrs))[delvar->get_index()]=0;
557                 freeSlots<<delvar->get_index();
558                 DeleteObject (delvar);
559             } else {
560                 ((_VariableContainer*)delvar)->Clear();
561             }
562         });
563 
564 
565         return;
566 
567     }
568 }
569 //__________________________________________________________________________________
DeleteVariable(_String const & name,bool deleteself,bool do_checks)570 void DeleteVariable (_String const &name, bool deleteself, bool do_checks) {
571     DeleteVariable(LocateVarByName (name), deleteself, do_checks);
572 }
573 
574 
575 //__________________________________________________________________________________
CheckReceptacle(_String const * name,_String const & fID,bool checkValid,bool isGlobal,bool clear_trees)576 _Variable* CheckReceptacle (_String const * name, _String const & fID, bool checkValid, bool isGlobal, bool clear_trees) {
577     if (checkValid && (!name->IsValidIdentifier(fIDAllowCompound))) {
578         HandleApplicationError(name->Enquote() & " is not a valid variable identifier");
579         return nil;
580     }
581 
582     long    f = LocateVarByName (*name);
583 
584     if (f>=0L) {
585       _Variable * existing = FetchVar (f);
586 
587       if (clear_trees && (existing->ObjectClass() == TREE || existing->ObjectClass() == TOPOLOGY)) {
588         DeleteVariable (*existing->GetName());
589       }
590       else {
591         return existing;
592       }
593     }
594 
595     _Variable dummy (*name, isGlobal);
596     f = LocateVarByName (*name);
597 
598     return FetchVar(f);
599 }
600 
601 //__________________________________________________________________________________
CheckReceptacleCommandIDException(_String const * name,const long id,bool checkValid,bool isGlobal,_ExecutionList * context)602 _Variable* CheckReceptacleCommandIDException (_String const* name, const long id, bool checkValid, bool isGlobal, _ExecutionList* context) {
603     // TODO: allow ^name and such to constitute valid run-time references
604     if (checkValid && (!name->IsValidIdentifier(fIDAllowCompound))) {
605         throw  (name->Enquote('\'') & " is not a valid variable identifier");
606     }
607 
608     long    f = LocateVarByName (*name);
609 
610     if (f>=0L) {
611         _Variable * existing = FetchVar (f);
612         if (existing->ObjectClass() == TREE) {
613           DeleteVariable (*existing->GetName());
614         }
615         else {
616           return existing;
617         }
618     }
619 
620     _Variable (*name, isGlobal);
621     f = LocateVarByName (*name);
622 
623     return FetchVar(f);
624 }
625 
626 
627 //__________________________________________________________________________________
CheckReceptacleCommandID(_String const * name,const long id,bool checkValid,bool isGlobal,_ExecutionList * context)628 _Variable* CheckReceptacleCommandID (_String const* name, const long id, bool checkValid, bool isGlobal, _ExecutionList* context) {
629   try {
630     return CheckReceptacleCommandIDException (name, id, checkValid, isGlobal, context);
631   } catch (_String const& err_msg) {
632     if (context) {
633       context->ReportAnExecutionError(err_msg);
634     } else {
635       HandleApplicationError (err_msg);
636     }
637   }
638   return nil;
639 }
640 
641 //__________________________________________________________________________________
CheckReceptacleCommandIDAndStore(_String const * name,const long id,bool checkValid,HBLObjectRef v,bool dup,bool isGlobal)642 bool CheckReceptacleCommandIDAndStore (_String const* name, const long id, bool checkValid, HBLObjectRef v, bool dup, bool isGlobal)
643 {
644     _Variable *theV = CheckReceptacleCommandID (name, id, checkValid, isGlobal);
645     if (theV) {
646         theV->SetValue (v, dup, true, NULL);
647         return true;
648     }
649     if (!dup) {
650         DeleteObject (v);
651     }
652     return false;
653 }
654 
655 
656 //__________________________________________________________________________________
CheckReceptacleAndStore(_String const * name,_String fID,bool checkValid,HBLObjectRef v,bool dup)657 bool CheckReceptacleAndStore (_String const* name, _String fID, bool checkValid, HBLObjectRef v, bool dup) {
658     _Variable * theV = CheckReceptacle(name, fID, checkValid);
659     if (theV) {
660         theV->SetValue (v, dup, true, NULL);
661         return true;
662     }
663     if (!dup) {
664         DeleteObject (v);
665     }
666     return false;
667 }
668 
669 //__________________________________________________________________________________
CheckReceptacleAndStore(_String name,_String fID,bool checkValid,HBLObjectRef v,bool dup)670 bool CheckReceptacleAndStore (_String name, _String fID, bool checkValid, HBLObjectRef v, bool dup)
671 {
672     return CheckReceptacleAndStore (&name, fID, checkValid, v, dup);
673 }
674 
675 //__________________________________________________________________________________
InsertVar(_Variable * theV)676 void  InsertVar (_Variable* theV) {
677 
678 
679     long pos = variableNames.Insert (theV->theName);
680 
681     if (pos < 0 && isDefiningATree == kTreeNodeBeingCreated) {
682         // automatically fix duplicate autogenerated tree node name
683         long trySuffix  = 1;
684         _String * tryName = new _String;
685         do {
686             *tryName = *theV->theName & "_" & trySuffix;
687             pos      = variableNames.Insert (tryName);
688             trySuffix ++;
689         } while (pos < 0);
690         DeleteObject(theV->theName);
691         theV->theName = tryName;
692     }
693 
694     if (pos < 0) {
695         if (isDefiningATree == kTreeIsBeingParsed) {
696             HandleApplicationError(_String("Error while creating a tree: duplicate node name ") & *theV->GetName()->Enquote());
697             return;
698         }
699         theV->theIndex = variableNames.GetXtra(-pos-1);
700         return;
701     } else {
702         theV->theName->AddAReference();
703     }
704 
705     if (freeSlots.lLength) {
706         theV->theIndex = freeSlots.list_data[freeSlots.lLength-1];
707         variablePtrs[theV->theIndex]=theV->makeDynamic();
708         freeSlots.Delete(freeSlots.lLength-1);
709     } else {
710         theV->theIndex = variablePtrs.lLength;
711         variablePtrs&&theV;
712     }
713     variableNames.SetXtra (pos, theV->theIndex);
714 }
715 
716 //__________________________________________________________________________________
AppendContainerName(_String const & inString,_VariableContainer const * theP)717 _String const&  AppendContainerName (_String const& inString, _VariableContainer const* theP) {
718     return AppendContainerName (inString, theP?theP->GetName():nil);
719 }
720 
721 //__________________________________________________________________________________
AppendContainerName(_String const & inString,_String const * namescp)722 _String const&  AppendContainerName (_String const& inString, _String const* namescp) {
723     static _String returnMe;
724 
725     if (_hy_application_globals.Find (&inString) >= 0) {
726         return inString;
727     }
728 
729     hy_reference_type reference_type = inString.ProcessVariableReferenceCases (returnMe, namescp && !namescp -> empty() ? namescp : nil);
730 
731     if (reference_type != kStringInvalidReference) {
732         return returnMe;
733     }
734     return inString;
735 }
736 
737 
738 //__________________________________________________________________________________
RenameVariable(_String * oldName,_String * newName)739 void  RenameVariable (_String* oldName, _String* newName)
740 {
741     _String     oldNamePrefix (*oldName&'.'),
742                 newNamePrefix (*newName&'.');
743 
744     _List           toRename;
745     _SimpleList     xtras,
746                     traverser;
747 
748     long f = variableNames.Find (oldName, traverser);
749     if (f>=0) {
750         toRename << oldName;
751         xtras    << variableNames.GetXtra (f);
752         f = variableNames.Next (f, traverser);
753 
754         for  (; f>=0 && ((_String*)variableNames.Retrieve (f))->BeginsWith (oldNamePrefix); f = variableNames.Next (f, traverser)) {
755             toRename << variableNames.Retrieve (f);
756             xtras << variableNames.GetXtra (f);
757         }
758     }
759 
760     for (unsigned long k = 0; k < toRename.lLength; k++) {
761         _Variable * thisVar = FetchVar (xtras.list_data[k]);
762         thisVar->GetName()->RemoveAReference();
763         if (k) {
764             thisVar->theName = new _String(thisVar->GetName()->Replace(oldNamePrefix,newNamePrefix,true));
765         } else {
766             thisVar->theName = new _String(*newName);
767         }
768 
769         variableNames.Delete (toRename (k), true);
770         variableNames.Insert (thisVar->GetName(),xtras.list_data[k]);
771         thisVar->GetName()->AddAReference();
772     }
773 }
774 
775 //__________________________________________________________________________________
ReplaceVar(_Variable * theV)776 void  ReplaceVar (_Variable* theV) {
777     long pos = variableNames.Find (theV->theName);
778     if (pos>=0) {
779         pos = variableNames.GetXtra(pos);
780         UpdateChangingFlas   (pos);
781         variablePtrs.Replace (pos,theV,true);
782     } else {
783         InsertVar (theV);
784     }
785 }
786 
787 
788 //__________________________________________________________________________________
SetupOperationLists(void)789 void    SetupOperationLists (void) {
790 
791     auto package_ops = [] (long op1, long op2) -> long {
792         if (op1 < op2) {
793             return (op1 << 16) + op2;
794         }
795         return (op2 << 16) + op1;
796     };
797 
798     UnOps  < "-" <
799              "!" <
800              "+" <
801              "*" <
802              "^" <
803              "&" <
804              "Abs" <
805              "Sin" <
806              "Cos" <
807              "Tan" <
808              "Exp" <
809              "Log" <
810              "Arctan" <
811              "Time" <
812              "Gamma" <
813              "Transpose" <
814              "Sqrt" <
815              "Erf" <
816              "Rows" <
817              "Columns" <
818              "LUDecompose" <
819              "Inverse" <
820              "BranchCount" <
821              "TipCount" <
822              "ZCDF" <
823              "Eigensystem" <
824              "Simplex" <
825              "Type" <
826              "Eval" <
827              "LnGamma";
828 
829 
830     BinOps<<'|'*256+'|';
831     opPrecedence<<1;
832     associativeOps << opPrecedence.lLength;
833 
834     BinOps<<'&'*256+'&';
835     opPrecedence<<2;
836     associativeOps << opPrecedence.lLength;
837 
838     BinOps<<'='*256+'=';
839     opPrecedence<<3;
840     BinOps<<'!'*256+'=';
841     opPrecedence<<3;
842 
843     BinOps<<'<';
844     opPrecedence<<4;
845     BinOps<<'>';
846     opPrecedence<<4;
847     BinOps<<'<'*256+'=';
848     opPrecedence<<4;
849     BinOps<<'>'*256+'=';
850     opPrecedence<<4;
851 
852     BinOps<<'+';
853     associativeOps << opPrecedence.lLength;
854     opPrecedence<<5;
855     BinOps<<'-';
856     opPrecedence<<5;
857     BinOps<<'*';
858     associativeOps << opPrecedence.lLength;
859 
860     opPrecedence<<6;
861     BinOps<<'/';
862     opPrecedence<<6;
863     BinOps<<'%';
864     opPrecedence<<6;
865     BinOps<<'$';
866     opPrecedence<<6;
867 
868     BinOps<<'^';
869     opPrecedence<<7;
870 
871     BinOps<<'+'*256+'=';
872     opPrecedence<<8;
873 
874     _Operation::ListOfInverseOps
875     << package_ops(HY_OP_CODE_NOT, HY_OP_CODE_NOT)
876     << package_ops (HY_OP_CODE_EXP, HY_OP_CODE_LOG)
877     << package_ops (HY_OP_CODE_TAN, HY_OP_CODE_ARCTAN);
878 
879     if (BuiltInFunctions.empty()) {
880         // construct a list of operations
881         // don't forget to update SimplifyConstants, simpleOperationCodes, InternalDifferentiate, InternalSimplify, Formula::HasChanged and all Execute commands
882         // also MAccess and MCoord codes are used in Parse to merge multiple matrix access operations
883 
884         //HY_OP_CODE_NOT
885         BuiltInFunctions.AppendNewInstance (new _String ('!'));
886 
887         //HY_OP_CODE_NEQ
888         BuiltInFunctions.AppendNewInstance (new _String ("!="));
889 
890         //HY_OP_CODE_IDIV
891         BuiltInFunctions.AppendNewInstance (new _String ('$'));
892 
893         //HY_OP_CODE_MOD
894         BuiltInFunctions.AppendNewInstance (new _String ('%'));
895 
896         //HY_OP_CODE_REF
897         BuiltInFunctions.AppendNewInstance (new _String ('&'));
898 
899         //HY_OP_CODE_AND
900         BuiltInFunctions.AppendNewInstance (new _String ("&&"));
901         simpleOperationCodes    << HY_OP_CODE_AND;
902         simpleOperationFunctions<< (long)AndNumbers;
903 
904         //HY_OP_CODE_MUL
905         BuiltInFunctions.AppendNewInstance (new _String ('*'));
906         simpleOperationCodes    << HY_OP_CODE_MUL;
907         simpleOperationFunctions<< (long)MultNumbers;
908 
909         //HY_OP_CODE_ADD
910         BuiltInFunctions.AppendNewInstance (new _String ('+'));
911         simpleOperationCodes<< HY_OP_CODE_ADD;
912         simpleOperationFunctions<<(long)AddNumbers;
913 
914         //HY_OP_CODE_SUB
915         BuiltInFunctions.AppendNewInstance (new _String ('-'));
916         simpleOperationCodes<<HY_OP_CODE_SUB;
917         simpleOperationFunctions<<(long)SubNumbers;
918 
919         //HY_OP_CODE_DIV
920         BuiltInFunctions.AppendNewInstance (new _String ('/'));
921         simpleOperationCodes<<HY_OP_CODE_DIV;
922         simpleOperationFunctions<<(long)DivNumbers;
923 
924         //HY_OP_CODE_LESS
925         BuiltInFunctions.AppendNewInstance (new _String ('<'));
926         simpleOperationCodes<<HY_OP_CODE_LESS;
927         simpleOperationFunctions<<(long)LessThan;
928 
929         //HY_OP_CODE_LEQ
930         BuiltInFunctions.AppendNewInstance (new _String ("<="));
931         simpleOperationCodes<<HY_OP_CODE_LEQ;
932         simpleOperationFunctions<<(long)LessThanE;
933 
934         //HY_OP_CODE_EQ
935         BuiltInFunctions.AppendNewInstance (new _String ("=="));
936         simpleOperationCodes<<HY_OP_CODE_EQ;
937         simpleOperationFunctions<<(long)EqualNumbers;
938 
939         //HY_OP_CODE_GREATER
940         BuiltInFunctions.AppendNewInstance (new _String ('>'));
941         simpleOperationCodes<<HY_OP_CODE_GREATER;
942         simpleOperationFunctions<<(long)GreaterThan;
943 
944         //HY_OP_CODE_GEQ
945         BuiltInFunctions.AppendNewInstance (new _String (">="));
946         simpleOperationCodes<<HY_OP_CODE_GEQ;
947         simpleOperationFunctions<<(long)GreaterThanE;
948 
949         //HY_OP_CODE_ABS
950         BuiltInFunctions.AppendNewInstance (new _String ("Abs"));
951         simpleOperationCodes<<HY_OP_CODE_ABS;
952         simpleOperationFunctions<<(long)AbsNumber;
953 
954         //HY_OP_CODE_ARCTAN
955         BuiltInFunctions.AppendNewInstance (new _String ("Arctan"));
956 
957         //HY_OP_CODE_BETA
958         BuiltInFunctions.AppendNewInstance (new _String ("Beta"));
959         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_BETA), 2L);
960 
961         //HY_OP_CODE_BRANCHCOUNT
962         BuiltInFunctions.AppendNewInstance (new _String ("BranchCount"));
963         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_BRANCHCOUNT), 2L);
964 
965         //HY_OP_CODE_BRANCHLENGTH
966         BuiltInFunctions.AppendNewInstance (new _String ("BranchLength"));
967         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_BRANCHLENGTH), 2L);
968 
969         //HY_OP_CODE_BRANCHNAME
970         BuiltInFunctions.AppendNewInstance (new _String ("BranchName"));
971         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_BRANCHNAME), 2L);
972 
973 
974         //HY_OP_CODE_CCHI2
975         BuiltInFunctions.AppendNewInstance (new _String ("CChi2"));
976         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_CCHI2), 2L);
977 
978         //HY_OP_CODE_CGAMMADIST
979         BuiltInFunctions.AppendNewInstance (new _String ("CGammaDist"));
980         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_CGAMMADIST), 3L);
981 
982         //HY_OP_CODE_CALL
983         BuiltInFunctions.AppendNewInstance (new _String ("Call"));
984         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_CALL), 1L + (0x7fff << 16));
985             // mimimum of 1 argument
986 
987         //HY_OP_CODE_COLUMNS
988         BuiltInFunctions.AppendNewInstance (new _String ("Columns"));
989 
990         //HY_OP_CODE_COS
991         BuiltInFunctions.AppendNewInstance (new _String ("Cos"));
992 
993         //HY_OP_CODE_DIFF
994         BuiltInFunctions.AppendNewInstance (new _String ("Differentiate"));
995         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_DIFF), 2L);
996 
997         //HY_OP_CODE_EIGENSYSTEM
998         BuiltInFunctions.AppendNewInstance (new _String ("Eigensystem"));
999 
1000         //HY_OP_CODE_ERF
1001         BuiltInFunctions.AppendNewInstance (new _String ("Erf"));
1002 
1003         //HY_OP_CODE_EVAL
1004         BuiltInFunctions.AppendNewInstance (new _String ("Eval"));
1005 
1006         //HY_OP_CODE_EXP
1007         BuiltInFunctions.AppendNewInstance (new _String ("Exp"));
1008         simpleOperationCodes<<HY_OP_CODE_EXP;
1009         simpleOperationFunctions<<(long)ExpNumbers;
1010 
1011         //HY_OP_CODE_FORMAT
1012         BuiltInFunctions.AppendNewInstance (new _String ("Format"));
1013         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_FORMAT), 3L);
1014 
1015         //HY_OP_CODE_GAMMA
1016         BuiltInFunctions.AppendNewInstance (new _String ("Gamma"));
1017 
1018         //HY_OP_CODE_GAMMADIST
1019         BuiltInFunctions.AppendNewInstance (new _String ("GammaDist"));
1020         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_GAMMADIST), 3L);
1021 
1022         //HY_OP_CODE_IBETA
1023         BuiltInFunctions.AppendNewInstance (new _String ("IBeta"));
1024         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_IBETA), 3L);
1025 
1026         //HY_OP_CODE_IGAMMA
1027         BuiltInFunctions.AppendNewInstance (new _String ("IGamma"));
1028         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_IGAMMA), 2L);
1029 
1030         //HY_OP_CODE_INVCHI2
1031         BuiltInFunctions.AppendNewInstance (new _String ("InvChi2"));
1032         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_INVCHI2), 2L);
1033 
1034         //HY_OP_CODE_INVERSE
1035         BuiltInFunctions.AppendNewInstance (new _String ("Inverse"));
1036 
1037         //HY_OP_CODE_JOIN
1038         BuiltInFunctions.AppendNewInstance (new _String ("Join"));
1039         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_JOIN), 2L);
1040 
1041         //HY_OP_CODE_LUDECOMPOSE
1042         BuiltInFunctions.AppendNewInstance (new _String ("LUDecompose"));
1043 
1044         //HY_OP_CODE_LUSOLVE
1045         BuiltInFunctions.AppendNewInstance (new _String ("LUSolve"));
1046         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_LUSOLVE), 2L);
1047 
1048         //HY_OP_CODE_LOG_GAMMA
1049         BuiltInFunctions.AppendNewInstance (new _String ("LnGamma"));
1050 
1051 
1052         //HY_OP_CODE_LOG
1053         BuiltInFunctions.AppendNewInstance (new _String ("Log"));
1054         simpleOperationCodes<<HY_OP_CODE_LOG;
1055         simpleOperationFunctions<<(long)LogNumbers;
1056 
1057         //HY_OP_CODE_MACCESS
1058         BuiltInFunctions.AppendNewInstance (new _String ("MAccess"));
1059         simpleOperationCodes<<HY_OP_CODE_MACCESS;
1060         simpleOperationFunctions<<(long)FastMxAccess;
1061 
1062         //HY_OP_CODE_MCOORD
1063         BuiltInFunctions.AppendNewInstance (new _String ("MCoord"));
1064         simpleOperationCodes<<HY_OP_CODE_MCOORD;
1065         simpleOperationFunctions<<(long)FastMxWrite;
1066 
1067         //HY_OP_CODE_MAX
1068         BuiltInFunctions.AppendNewInstance (new _String ("Max"));
1069         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_MAX), 1L + ((2L) << 16)); // (1 or 2 arguments)
1070 
1071         simpleOperationCodes<<HY_OP_CODE_MAX;
1072         simpleOperationFunctions<<(long)MaxNumbers;
1073 
1074         //HY_OP_CODE_MIN
1075         BuiltInFunctions.AppendNewInstance (new _String ("Min"));
1076         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_MIN), 1L + ((2L) << 16)); // (1 or 2 arguments)
1077 
1078         simpleOperationCodes<<HY_OP_CODE_MIN;
1079         simpleOperationFunctions<<(long)MinNumbers;
1080 
1081         //HY_OP_CODE_PSTREESTRING
1082         BuiltInFunctions.AppendNewInstance (new _String ("PSTreeString"));
1083         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_PSTREESTRING), 3L);
1084 
1085 
1086         //HY_OP_CODE_RANDOM
1087         BuiltInFunctions.AppendNewInstance (new _String ("Random"));
1088         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_RANDOM), 2L);
1089 
1090         simpleOperationCodes<<HY_OP_CODE_RANDOM;
1091         simpleOperationFunctions<<(long)RandomNumber;
1092 
1093         //HY_OP_CODE_REROOTTREE
1094         BuiltInFunctions.AppendNewInstance (new _String ("RerootTree"));
1095         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_REROOTTREE), 2L);
1096 
1097         //HY_OP_CODE_ROWS
1098         BuiltInFunctions.AppendNewInstance (new _String ("Rows"));
1099 
1100         //HY_OP_CODE_SIMPLEX
1101         BuiltInFunctions.AppendNewInstance (new _String ("Simplex"));
1102 
1103         //HY_OP_CODE_EXPRESSION
1104         BuiltInFunctions.AppendNewInstance (new _String ("Simplify"));
1105         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_SIMPLIFY), 2L);
1106 
1107         //HY_OP_CODE_SIN
1108         BuiltInFunctions.AppendNewInstance (new _String ("Sin"));
1109 
1110         //HY_OP_CODE_SQRT
1111         BuiltInFunctions.AppendNewInstance (new _String ("Sqrt"));
1112 
1113         //HY_OP_CODE_TEXTREESTRING
1114         BuiltInFunctions.AppendNewInstance (new _String ("TEXTreeString"));
1115         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_TEXTREESTRING), 2L);
1116 
1117         //HY_OP_CODE_TAN
1118         BuiltInFunctions.AppendNewInstance (new _String ("Tan"));
1119 
1120         //HY_OP_CODE_TIME
1121         BuiltInFunctions.AppendNewInstance (new _String ("Time"));
1122 
1123         //HY_OP_CODE_TIPCOUNT
1124         BuiltInFunctions.AppendNewInstance (new _String ("TipCount"));
1125 
1126         //HY_OP_CODE_TIPNAME
1127         BuiltInFunctions.AppendNewInstance (new _String ("TipName"));
1128         FunctionNameList.Insert (*(_String*)BuiltInFunctions (HY_OP_CODE_TIPNAME), 2L);
1129 
1130         //HY_OP_CODE_TRANSPOSE
1131         BuiltInFunctions.AppendNewInstance (new _String ("Transpose"));
1132 
1133         //HY_OP_CODE_TYPE
1134         BuiltInFunctions.AppendNewInstance (new _String ("Type"));
1135 
1136         //HY_OP_CODE_ZCDF
1137         BuiltInFunctions.AppendNewInstance (new _String ("ZCDF"));
1138 
1139         //HY_OP_CODE_POWER
1140         BuiltInFunctions.AppendNewInstance (new _String ('^'));
1141         simpleOperationCodes<<HY_OP_CODE_POWER;
1142         simpleOperationFunctions<<(long)Power;
1143 
1144         //HY_OP_CODE_OR
1145         BuiltInFunctions.AppendNewInstance (new _String ("||"));
1146 
1147         hyReservedWords << BuiltInFunctions;
1148         hyReservedWords.AppendNewInstance (new _String("global"));
1149 
1150 
1151         hyReservedWords.Sort();
1152         /*hyReservedWords.ForEach ([] (BaseRef item, unsigned long index) -> void {
1153             printf ("%d %s\n", index, ((_String*)item)->get_str());
1154         });*/
1155     }
1156 
1157 
1158 
1159 }
1160 
1161 
1162 
1163 //__________________________________________________________________________________
1164 
FinishDeferredSF(void)1165 void  FinishDeferredSF (void) {
1166     if (deferSetFormula->nonempty()) {
1167         SortLists (deferSetFormula, &deferIsConstant);
1168 
1169         for (AVLListXIteratorKeyValue variable_record : AVLListXIterator (&variableNames)) {
1170             _Variable * theV = LocateVar(variable_record.get_value());
1171             if (theV->IsContainer()) {
1172                 ((_VariableContainer*)theV)->SetMDependance (*deferSetFormula);
1173             }
1174         }
1175 
1176         likeFuncList.ForEach([] (BaseRef lf_object, unsigned long idx) -> void {
1177             if (((_String*)likeFuncNamesList(idx))->nonempty()) {
1178                 _LikelihoodFunction * lf = (_LikelihoodFunction*)lf_object;
1179                 for (long k = 0L; k < deferSetFormula->countitems(); k++) {
1180                     lf->UpdateIndependent(deferSetFormula->get(k),deferIsConstant.get(k));
1181                 }
1182             }
1183         });
1184     }
1185     if (deferClearConstraint && deferClearConstraint->countitems()) {
1186         DoForEachVariable ([](_Variable* v, long v_idx) -> void {
1187             if (v->IsContainer()) {
1188                 ((_VariableContainer*)v)->RemoveDependance (*deferClearConstraint);
1189             }
1190         });
1191 
1192         for (unsigned long i = 0UL; i<likeFuncList.lLength; i++)
1193             if (((_String*)likeFuncNamesList(i))->nonempty()) {
1194                 ((_LikelihoodFunction*)likeFuncList(i))->UpdateDependent(*deferClearConstraint);
1195             }
1196     }
1197 
1198     if (deferClearConstraint) {
1199         delete deferClearConstraint->dataList;
1200         DeleteAndZeroObject(deferClearConstraint);
1201     }
1202     DeleteObject (deferSetFormula);
1203     deferSetFormula = nil;
1204     deferIsConstant.Clear();
1205 }
1206