1 /*
2
3 HyPhy - Hypothesis Testing Using Phylogenies.
4
5 Copyright (C) 1997-now
6 Core Developers:
7 Sergei L Kosakovsky Pond (sergeilkp@icloud.com)
8 Art FY Poon (apoon42@uwo.ca)
9 Steven Weaver (sweaver@temple.edu)
10
11 Module Developers:
12 Lance Hepler (nlhepler@gmail.com)
13 Martin Smith (martin.audacis@gmail.com)
14
15 Significant contributions from:
16 Spencer V Muse (muse@stat.ncsu.edu)
17 Simon DW Frost (sdf22@cam.ac.uk)
18
19 Permission is hereby granted, free of charge, to any person obtaining a
20 copy of this software and associated documentation files (the
21 "Software"), to deal in the Software without restriction, including
22 without limitation the rights to use, copy, modify, merge, publish,
23 distribute, sublicense, and/or sell copies of the Software, and to
24 permit persons to whom the Software is furnished to do so, subject to
25 the following conditions:
26
27 The above copyright notice and this permission notice shall be included
28 in all copies or substantial portions of the Software.
29
30 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
31 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
32 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
33 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
34 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
35 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
36 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
37
38 */
39
40
41 #include <string.h>
42 #include <ctype.h>
43 #include <time.h>
44
45
46 #include "likefunc.h"
47 #include "batchlan.h"
48 #include "polynoml.h"
49 #include "scfg.h"
50 #include "bayesgraph.h"
51 #include "avllistx.h"
52 #include "global_object_lists.h"
53 #include "global_things.h"
54 #include "time_difference.h"
55 #include "global_things.h"
56 #include "hy_string_buffer.h"
57 #include "tree_iterator.h"
58
59
60
61
62 using namespace hy_global;
63
64 using namespace hyphy_global_objects;
65
66
67 //____________________________________________________________________________________
68 // global variables
69
70 _List
71 dataSetList,
72 dataSetNamesList,
73 likeFuncList, // list of all datasets
74 likeFuncNamesList, // list of all dataset filters
75 pathNames,
76 theModelList,
77 batchLanguageFunctions,
78 batchLanguageFunctionNames,
79 batchLanguageFunctionParameterLists,
80 batchLanguageFunctionParameterTypes,
81 compiledFormulaeParameters,
82 modelNames,
83 executionStack,
84 loadedLibraryPathsBackend;
85
86
87 #ifdef __MAC__
88 _String volumeName;
89 #endif
90
91
92 // retrieval functions
93
94 _SimpleList
95 returnlist,
96 batchLanguageFunctionClassification,
97 modelMatrixIndices,
98 modelTypeList,
99 modelFrequenciesIndices,
100 listOfCompiledFormulae;
101
102 _String
103 globalPolynomialCap ("GLOBAL_POLYNOMIAL_CAP"),
104 enforceGlobalPolynomialCap ("ENFORCE_GLOBAL_POLYNOMIAL_CAP"),
105 dropPolynomialTerms ("DROP_POLYNOMIAL_TERMS"),
106 maxPolyTermsPerVariable ("MAX_POLY_TERMS_PER_VARIABLE"),
107 maxPolyExpIterates ("MAX_POLYNOMIAL_EXP_ITERATES"),
108 polyExpPrecision ("POLYNOMIAL_EXP_PRECISION"),
109 explicitFormMExp ("EXPLICIT_FORM_MATRIX_EXPONENTIAL"),
110 multByFrequencies ("MULTIPLY_BY_FREQUENCIES"),
111 defFileString ("DEFAULT_FILE_SAVE_NAME"),
112 useLastDefinedMatrix ("USE_LAST_DEFINED_MATRIX"),
113 dataPanelSourcePath ("DATA_PANEL_SOURCE_PATH"),
114 windowTypeTree ("TREEWINDOW"),
115 windowTypeClose ("CLOSEWINDOW"),
116 windowTypeTable ("CHARTWINDOW"),
117 windowTypeDistribTable ("DISTRIBUTIONWINDOW"),
118 windowTypeDatabase ("DATABASEWINDOW"),
119 screenWidthVar ("SCREEN_WIDTH"),
120 screenHeightVar ("SCREEN_HEIGHT"),
121 useNexusFileData ("USE_NEXUS_FILE_DATA"),
122 mpiMLELFValue ("MPI_MLE_LF_VALUE"),
123 lf2SendBack ("LIKE_FUNC_NAME_TO_SEND_BACK"),
124 simulationFilter ("_SIM_INTERNAL_FILTER_"),
125 prefixDS ("DataSet_"),
126 prefixDF ("Partition_"),
127 prefixLF ("LF_"),
128 replaceTreeStructure ("REPLACE_TREE_STRUCTURE"),
129 matrixEvalCount ("MATRIX_EXPONENTIATION_COUNTS"),
130 _hyLastExecutionError ("LAST_HBL_EXECUTION_ERROR"),
131
132 kBGMData ("BGM_DATA_MATRIX"),
133
134 gdiDFAtomSize ("ATOM_SIZE"),
135
136 dialogPrompt,
137 defFileNameValue;
138
139
140 //____________________________________________________________________________________
141
142
143 _String
144 blWhile ("while("), // moved
145 blFunction ("function "), // moved
146 blFFunction ("ffunction "), // moved
147 blLFunction ("lfunction "), // moved
148 blCFunction ("cfunction "), // moved
149 blNameSpace ("namespace "),
150 blReturn ("return "), // moved
151 blReturnPrefix ("return"),
152 blIf ("if("), // moved
153 blElse ("else"), // moved
154 blDo ("do{"), // moved
155 blBreak ("break;"), // moved
156 blContinue ("continue;"), // moved
157 blInclude ("#include"), // moved
158 blDataSet ("DataSet "), // moved
159 blDataSetFilter ("DataSetFilter "),
160 blTree ("Tree "),
161 blLF ("LikelihoodFunction "),
162 blLF3 ("LikelihoodFunction3 "),
163 blGetString ("GetString("),
164 blExport ("Export("),
165 blReplicate ("ReplicateConstraint("),
166 blImport ("Import"),
167 blCategory ("category "),
168 blClearConstraints ("ClearConstraints("),
169 blSetDialogPrompt ("SetDialogPrompt("),
170 blModel ("Model "),
171 blChoiceList ("ChoiceList("),
172 blTopology ("Topology "),
173 blHBLProfile ("#profile"),
174 blSCFG ("SCFG "),
175 blBGM ("BayesianGraphicalModel "),
176 blSimulateDataSet ("SimulateDataSet");
177
178 _Trie _HY_HBL_KeywordsPreserveSpaces ;
179
180 #ifdef __HYPHYMPI__
181
182 void ReportMPIError (int, bool);
183
184 #endif
185
186 _hy_nested_check isInFunction = _HY_NO_FUNCTION;
187
188 hyFloat explicitFormMatrixExponential = 0.0,
189 messageLogFlag = 1.0;
190
191
192 extern _String MATRIX_AGREEMENT,
193 ANAL_COMP_FLAG;
194
195
196 extern _SimpleList freeSlots;
197
198
199 _AVLList loadedLibraryPaths (&loadedLibraryPathsBackend);
200
201 _ExecutionList
202 *currentExecutionList = nil;
203
204 _List const _ElementaryCommand::fscanf_allowed_formats (new _String ("Number"),
205 new _String ("Matrix"),
206 new _String ("Tree"),
207 new _String ("String"),
208 new _String ("NMatrix"),
209 new _String ("Raw"),
210 new _String ("Lines")
211 );
212
213 //____________________________________________________________________________________
214 // Function prototypes
215
216 #ifdef __HYPHYMPI__
217
218 //____________________________________________________________________________________
219
ReportMPIError(int code,bool send)220 void ReportMPIError (int code, bool send) {
221 if (code != MPI_SUCCESS) {
222 _String errMsg = "MPI Error while ";
223 if (send) {
224 errMsg = errMsg & "sending";
225 } else {
226 errMsg = errMsg & "receiving";
227 }
228
229 errMsg = errMsg & _String(" with code ") & (long)code;
230 HandleApplicationError (errMsg);
231 }
232 }
233
234 #define MPI_SEND_CHUNK 0xFFFFFFL
235
236 //____________________________________________________________________________________
237
MPISendString(_String const & theMessage,long destID,bool isError)238 void MPISendString (_String const& theMessage, long destID, bool isError)
239 {
240
241 long messageLength = theMessage.length(),
242 transferCount = 0L;
243
244 if (isError) {
245 messageLength = -messageLength;
246 }
247
248 ReportMPIError(MPI_Send(&messageLength, 1, MPI_LONG, destID, HYPHY_MPI_SIZE_TAG, MPI_COMM_WORLD),true);
249
250 if (messageLength == 0L) {
251 return;
252 }
253
254 if (isError) {
255 messageLength = -messageLength;
256 }
257
258 while (messageLength-transferCount>MPI_SEND_CHUNK) {
259 //printf("%s",theMessage.get_str());
260 ReportMPIError(MPI_Send((void*)(theMessage.get_str()+transferCount), MPI_SEND_CHUNK, MPI_CHAR, destID, HYPHY_MPI_STRING_TAG, MPI_COMM_WORLD),true);
261 transferCount += MPI_SEND_CHUNK;
262 }
263
264 if (messageLength-transferCount) {
265 ReportMPIError(MPI_Send((void*)(theMessage.get_str()+transferCount), messageLength-transferCount, MPI_CHAR, destID, HYPHY_MPI_STRING_TAG, MPI_COMM_WORLD),true);
266 }
267
268 //ReportMPIError(MPI_Send(&messageLength, 1, MPI_LONG, destID, HYPHY_MPI_DONE_TAG, MPI_COMM_WORLD),true);
269
270 _FString* sentVal = new _FString ((_String*)theMessage.makeDynamic());
271 _Variable * mpiMsgVar = CheckReceptacle (&hy_env::mpi_last_sent_message, kEmptyString, false);
272 mpiMsgVar->SetValue (sentVal, false, true, NULL);
273 //setParameter (mpiLastSentMsg, &sentVal);
274
275 }
276
277 //____________________________________________________________________________________
MPIRecvString(long senderT,long & senderID)278 _String* MPIRecvString (long senderT, long& senderID) {
279 _String* theMessage = nil;
280 long messageLength = 0L,
281 transferCount = 0L;
282
283 int actualReceived = 0;
284 bool isError = false;
285
286 if (senderT<0) {
287 senderT = MPI_ANY_SOURCE;
288 }
289
290 MPI_Status status;
291
292 // nonagressive polling mode
293
294 //ReportWarning ("Step 1");
295
296 int message_received = 0;
297 while (! message_received) {
298 MPI_Iprobe (senderT, HYPHY_MPI_SIZE_TAG, MPI_COMM_WORLD, &message_received, MPI_STATUS_IGNORE);
299 usleep (100);
300 }
301
302 //ReportWarning ("Step 2");
303 // nonagressive polling mode
304
305
306 ReportMPIError(MPI_Recv(&messageLength, 1, MPI_LONG, senderT, HYPHY_MPI_SIZE_TAG, MPI_COMM_WORLD,&status),false);
307
308 if (messageLength < 0) {
309 isError = true;
310 messageLength = -messageLength;
311 }
312
313 //ReportWarning ("Step 3");
314 //printf ("MPIRecvString size tag %ld (size chunk %ld) \n",messageLength, MPI_SEND_CHUNK);
315
316 if (!isError) {
317 //MPI_Get_count (&status,MPI_CHAR,&actualReceived);
318
319 if (messageLength==0) {
320 return new _String;
321 }
322
323 theMessage = new _String ((unsigned long)messageLength);
324
325 senderT = senderID = status.MPI_SOURCE;
326
327 while (messageLength-transferCount>MPI_SEND_CHUNK) {
328 ReportMPIError(MPI_Recv((void*)(theMessage->get_str()+transferCount), MPI_SEND_CHUNK, MPI_CHAR, senderT, HYPHY_MPI_STRING_TAG, MPI_COMM_WORLD,&status),false);
329 MPI_Get_count (&status,MPI_CHAR,&actualReceived);
330 if (actualReceived!=MPI_SEND_CHUNK) {
331 HandleApplicationError ("Failed in MPIRecvString - some data was not properly received\n");
332 }
333 //return nil;
334 transferCount += MPI_SEND_CHUNK;
335 }
336
337 if (messageLength-transferCount) {
338 //printf ("Clause 2 %d %d\n", messageLength-transferCount, theMessage->length());
339 ReportMPIError(MPI_Recv((void*)(theMessage->get_str()+transferCount), messageLength-transferCount, MPI_CHAR, senderT, HYPHY_MPI_STRING_TAG, MPI_COMM_WORLD,&status),false);
340 MPI_Get_count (&status,MPI_CHAR,&actualReceived);
341 if (actualReceived!=messageLength-transferCount) {
342 HandleApplicationError ("Failed in MPIRecvString - some data was not properly received\n");
343 }
344 //return nil;
345 }
346
347 //ReportWarning ("Step 4");
348
349 //ReportMPIError(MPI_Recv(&messageLength, 1, MPI_LONG, senderT, HYPHY_MPI_DONE_TAG, MPI_COMM_WORLD,&status),false);
350
351 if (isError) {
352 HandleApplicationError (theMessage);
353 }
354 }
355 return theMessage;
356 }
357 #endif
358
359
360 //____________________________________________________________________________________
361
GetStringFromFormula(_String const * data,_VariableContainer * theP)362 const _String GetStringFromFormula (_String const* data,_VariableContainer* theP) {
363
364 _Formula nameForm (*data,theP);
365 HBLObjectRef formRes = nameForm.Compute();
366
367 if (formRes&& formRes->ObjectClass()==STRING) {
368 return ((_FString*)formRes)->get_str();
369 }
370
371 return *data;
372 }
373
374 //____________________________________________________________________________________
375
376 bool numericalParameterSuccessFlag = true;
377
_ProcessNumericArgumentWithExceptions(_String & data,_VariableContainer const * theP)378 hyFloat _ProcessNumericArgumentWithExceptions (_String& data, _VariableContainer const* theP) {
379
380 /*if (data == _String("unit")) {
381 printf ("%s\n", AppendContainerName (data, theP).get_str());
382 }*/
383
384 HBLObjectRef simple_var = FetchObjectFromVariableByType (&AppendContainerName (data, theP), NUMBER | STRING);
385 if (simple_var) {
386 hyFloat res;
387 if (simple_var->ObjectClass() == NUMBER) {
388 res = simple_var->Value();
389 } else {
390 res = _String((_String*)((_FString*)simple_var)->toStr()).to_float();
391 }
392 //DeleteObject (simple_var);
393 return res;
394 }
395
396
397 _String errMsg;
398 _Formula nameForm (data,theP, &errMsg);
399
400 if (errMsg.empty()) {
401 HBLObjectRef formRes = nameForm.Compute();
402 if (formRes&& formRes->ObjectClass()==NUMBER) {
403 return formRes->Value();
404 } else {
405 if (formRes&& formRes->ObjectClass()==STRING) {
406 return _String((_String*)((_FString*)formRes)->toStr()).to_float();
407 } else {
408 throw (data.Enquote() & " was expected to be a numerical argument.");
409 }
410 }
411 } else {
412 throw (errMsg);
413 }
414
415 return 0.0;
416
417 }
418
419
ProcessNumericArgument(_String * data,_VariableContainer const * theP,_ExecutionList * currentProgram)420 hyFloat ProcessNumericArgument (_String* data, _VariableContainer const* theP, _ExecutionList* currentProgram) {
421
422 numericalParameterSuccessFlag = true;
423
424 try {
425 return _ProcessNumericArgumentWithExceptions (*data, theP);
426 } catch (_String const& err) {
427 if (currentProgram) {
428 currentProgram->ReportAnExecutionError (err);
429 } else {
430 HandleApplicationError(err);
431 }
432 }
433
434 numericalParameterSuccessFlag = false;
435 return 0.0;
436 }
437
438 //____________________________________________________________________________________
439
440 /** This function returns expects that the caller will handle reference counting on the returned object;
441 it will be returned with +1 reference, i.e. it needs to be deleted / managed by the caller */
442
ProcessAnArgumentByType(_String const * expression,_VariableContainer const * theP,long objectType,_ExecutionList * currentProgram)443 HBLObjectRef ProcessAnArgumentByType (_String const* expression, _VariableContainer const* theP, long objectType, _ExecutionList* currentProgram) {
444 _String errMsg;
445 _Formula expressionProcessor (*expression, theP, currentProgram?&errMsg:nil);
446
447 if (errMsg.nonempty() && currentProgram) {
448 currentProgram->ReportAnExecutionError (errMsg);
449 }
450 else {
451 HBLObjectRef expressionResult = expressionProcessor.Compute(0,theP);
452 if (expressionResult && (expressionResult->ObjectClass() & objectType)) {
453 expressionResult->AddAReference();
454 return expressionResult;
455 }
456 }
457
458 return nil;
459 }
460
461
462 //____________________________________________________________________________________
463
ProcessLiteralArgument(_String const * data,_VariableContainer const * theP,_ExecutionList * currentProgram)464 const _String ProcessLiteralArgument (_String const* data, _VariableContainer const* theP, _ExecutionList* currentProgram) {
465 //NLToConsole(); BufferToConsole("ProcessLiteralArgument:"); StringToConsole(*data); NLToConsole();
466 HBLObjectRef getString = ProcessAnArgumentByType (data, theP, STRING, currentProgram);
467
468 if (getString) {
469 _String result (((_FString*)getString)->get_str());
470 DeleteObject(getString);
471 return result;
472 }
473
474 return kEmptyString;
475 }
476
477 //____________________________________________________________________________________
478
ProcessDictionaryArgument(_String * data,_VariableContainer * theP,_ExecutionList * currentProgram)479 _AssociativeList* ProcessDictionaryArgument (_String* data, _VariableContainer* theP, _ExecutionList* currentProgram) {
480 return (_AssociativeList* )ProcessAnArgumentByType (data, theP, ASSOCIATIVE_LIST, currentProgram);
481 }
482
483 //____________________________________________________________________________________
GetBFFunctionNameByIndex(long idx)484 const _String& GetBFFunctionNameByIndex (long idx) {
485 return *GetObjectNameByType (HY_BL_HBL_FUNCTION, idx, false);
486 }
487
488 //____________________________________________________________________________________
GetBFFunctionArgumentCount(long idx)489 long GetBFFunctionArgumentCount (long idx) {
490 return ((_List*)batchLanguageFunctionParameterLists.Element (idx))->countitems();
491 }
492
493 //____________________________________________________________________________________
GetBFFunctionArgumentList(long idx)494 _List& GetBFFunctionArgumentList (long idx) {
495 return *(_List*)batchLanguageFunctionParameterLists.Element (idx);
496 }
497
498 //____________________________________________________________________________________
GetBFFunctionArgumentTypes(long idx)499 _SimpleList& GetBFFunctionArgumentTypes (long idx) {
500 return *(_SimpleList*)batchLanguageFunctionParameterTypes.Element (idx);
501 }
502
503 //____________________________________________________________________________________
GetBFFunctionBody(long idx)504 _ExecutionList& GetBFFunctionBody (long idx) {
505 return *(_ExecutionList*)batchLanguageFunctions.GetItem (idx);
506 }
507
508 //____________________________________________________________________________________
GetBFFunctionType(long idx)509 hyBLFunctionType GetBFFunctionType (long idx) {
510 return (hyBLFunctionType) batchLanguageFunctionClassification.Element (idx);
511 }
512
513 //____________________________________________________________________________________
ExportBFFunction(long idx,bool recursive)514 _String const ExportBFFunction (long idx, bool recursive) {
515
516
517 _StringBuffer bf (8192UL);
518 if (IsBFFunctionIndexValid(idx)) {
519
520 _String hbf_name = GetBFFunctionNameByIndex (idx);
521 _ExecutionList * body = &GetBFFunctionBody(idx);
522
523 if (body->enclosingNamespace.nonempty()) {
524 bf << "namespace " << body->enclosingNamespace << " {\n";
525 }
526
527 switch (GetBFFunctionType (idx)) {
528 case kBLFunctionSkipUpdate:
529 bf << blFFunction;
530 break;
531 case kBLFunctionLocal:
532 bf << blLFunction;
533 break;
534 default:
535 bf << blFunction;
536 }
537
538
539 bf << hbf_name << '(';
540
541 long argument_count = GetBFFunctionArgumentCount (idx);
542 _List * argument_list = &GetBFFunctionArgumentList (idx);
543 for (long argument_id = 0; argument_id < argument_count; argument_id ++) {
544 if (argument_id) {
545 bf << ',';
546 }
547
548
549 bf << body->TrimNameSpaceFromID(*(_String*)argument_list->Element (argument_id));
550 if (GetBFFunctionArgumentTypes (idx).GetElement(argument_id) == kBLFunctionArgumentReference) {
551 bf << '&';
552 }
553 }
554 bf << ") {\n" << body->sourceText << "\n}";
555
556 if (body->enclosingNamespace.nonempty()) {
557 bf << "\n}";
558 }
559
560 if (recursive) {
561 _List hbl_functions;
562 _AVLListX other_functions (&hbl_functions);
563
564 other_functions.Insert (new _String (hbf_name), HY_BL_HBL_FUNCTION, false, false);
565
566 body->BuildListOfDependancies (other_functions, true);
567
568 for (long i = 0; i < hbl_functions.lLength; i++) {
569 _String * a_name = (_String*)hbl_functions (i);
570 if (! a_name -> Equal( hbf_name)) {
571 bf << "\n/*----- Called function '"
572 << *a_name
573 << "' ------*/\n"
574 << ExportBFFunction (FindBFFunctionName(*a_name), false)
575 << "\n\n";
576 }
577 }
578 }
579 }
580
581 return bf;
582
583 }
584
585 //____________________________________________________________________________________
ClearBFFunctionLists(long start_here)586 void ClearBFFunctionLists (long start_here) {
587 if (start_here >= 0L && start_here < batchLanguageFunctionNames.countitems()) {
588
589 _SimpleList delete_me (batchLanguageFunctionNames.countitems()-start_here, start_here, 1L);
590
591 for (unsigned long k = 0UL; k < delete_me.countitems(); k++) {
592 batchLanguageFunctionNamesIndexed.Delete (batchLanguageFunctionNames.GetItem (delete_me.get (k)), true);
593 }
594
595 batchLanguageFunctionNames.DeleteList (delete_me);
596 batchLanguageFunctions.DeleteList (delete_me);
597 batchLanguageFunctionClassification.DeleteList (delete_me);
598 batchLanguageFunctionParameterLists.DeleteList (delete_me);
599 batchLanguageFunctionParameterTypes.DeleteList (delete_me);
600 } else {
601 if (start_here < 0) {
602 batchLanguageFunctionNames.Clear ();
603 batchLanguageFunctions.Clear ();
604 batchLanguageFunctionClassification.Clear ();
605 batchLanguageFunctionParameterLists.Clear ();
606 batchLanguageFunctionParameterTypes.Clear ();
607 batchLanguageFunctionNamesIndexed.Clear (true);
608 }
609 }
610 }
611
612 //____________________________________________________________________________________
IsBFFunctionIndexValid(long index)613 bool IsBFFunctionIndexValid (long index) {
614 if (index >= 0L && index < batchLanguageFunctionNames.countitems()) {
615 return batchLanguageFunctions.GetItem(index) != nil;
616 }
617 return false;
618 }
619
620 //____________________________________________________________________________________
GetBFFunctionCount(void)621 long GetBFFunctionCount (void) {
622 return batchLanguageFunctions.countitems();
623 }
624
625 //____________________________________________________________________________________
FindBFFunctionName(_String const & s,_VariableContainer const * theP)626 long FindBFFunctionName (_String const&s, _VariableContainer const* theP) {
627 if (theP) {
628 _String prefix = *(theP->GetName());
629
630 //ReportWarning (_String ("Looking for ") & s.Enquote() & " in " & prefix.Enquote());
631
632 while (1) {
633 _String test_id = prefix & '.' & s;
634 long idx = batchLanguageFunctionNamesIndexed.GetDataByKey(&test_id);
635 if (idx >= 0) {
636 //s = test_id;
637 return idx;
638 }
639 long cut_at = prefix.FindBackwards ('.', 0, -1);
640 if (cut_at > 0) {
641 prefix.Trim (0, cut_at - 1);
642 } else {
643 break;
644 }
645 };
646 }
647
648 //ReportWarning (_String ("Looking for ") & s.Enquote() & " in global context");
649 return batchLanguageFunctionNamesIndexed.GetDataByKey (&s);
650 }
651
652
653 //____________________________________________________________________________________
FindBgmName(_String const & s)654 long FindBgmName (_String const&s) {
655 return bgmNamesList.FindObject (&s);
656 }
657
658
659
660 //__________________________________________________________
AddDataSetToList(_String & theName,_DataSet * theDS)661 long AddDataSetToList (_String& theName,_DataSet* theDS) {
662 theName = GenerateUniqueObjectIDByType(theName, HY_BL_DATASET);
663 long k = dataSetNamesList.FindObject (&kEmptyString);
664 if (k==-1) {
665 dataSetList.AppendNewInstance (theDS);
666 dataSetNamesList&& & theName;
667 k = dataSetNamesList.lLength-1;
668 } else {
669 dataSetNamesList.Replace (k, &theName, true);
670 dataSetList.list_data[k] = (long)theDS;
671 }
672 return k;
673 }
674
675
676
677 //__________________________________________________________
678
KillLFRecord(long lfID,bool completeKill)679 void KillLFRecord (long lfID, bool completeKill) {
680 /* compile the list of variables which will no longer be referenced */
681
682
683
684 if (lfID>=0) {
685 //printf ("\n****\nKillLFRecord\n%s\n****\n", (char const*) * (_String*)likeFuncNamesList.GetItem (lfID));
686 _LikelihoodFunction *me = (_LikelihoodFunction*)likeFuncList (lfID);
687
688 if (completeKill) {
689 _SimpleList wastedVars,
690 otherVars,
691 myVars,
692 otherModels,
693 wastedModels;
694
695
696
697 myVars << me->GetIndependentVars();
698 myVars << me->GetDependentVars();
699
700
701
702 for (unsigned long k=0UL; k<likeFuncList.lLength; k++) {
703 if (k!=lfID) {
704 if (((_String*)likeFuncNamesList(k))->nonempty()) {
705 _LikelihoodFunction *lf = (_LikelihoodFunction*)likeFuncList (k);
706 otherVars << lf->GetIndependentVars();
707 otherVars << lf->GetDependentVars();
708
709 unsigned long component_count = lf->CountObjects(kLFCountPartitions);
710
711 for (long tree_index = 0UL; tree_index < component_count; tree_index++) {
712 lf->GetIthTree(tree_index)->CompileListOfModels(otherModels);
713 }
714
715 }
716 }
717 }
718
719 myVars.Sort ();
720 otherVars.Sort();
721 otherModels.Sort();
722
723 wastedVars.Subtract(myVars, otherVars);
724
725 for (unsigned long k=0UL; k<myVars.lLength; k++)
726 if (otherVars.BinaryFind(myVars.list_data[k])<0) {
727 wastedVars << myVars.list_data[k];
728 }
729
730 myVars.Clear();
731
732 unsigned long component_count = me->CountObjects(kLFCountPartitions);
733
734 for (long tree_index = 0UL; tree_index < component_count; tree_index++) {
735 _TheTree* thisTree = me->GetIthTree(tree_index);
736 thisTree->CompileListOfModels (myVars);
737 _TreeIterator ti (thisTree, _HY_TREE_TRAVERSAL_POSTORDER);
738 while (_CalcNode* tNode = ti.Next()) {
739 tNode->SetValue (new _Constant (tNode->ComputeBranchLength()),false,true,NULL);
740 }
741 thisTree->RemoveModel();
742 }
743
744 for (unsigned long k=0UL; k<myVars.lLength; k++)
745 if (otherModels.BinaryFind (myVars.list_data[k])<0) {
746 KillModelRecord (myVars.list_data[k]);
747 }
748
749 for (unsigned long k=0UL; k<wastedVars.lLength; k++) {
750 //printf ("Deleting %ld (%s)\n", wastedVars.list_data[k], ->GetName()->getStr());
751 _Variable * check_me = LocateVar(wastedVars.list_data[k]);
752 if (check_me) {
753 DeleteVariable (*check_me->GetName());
754 }
755 }
756
757 }
758
759 me->UnregisterListeners();
760
761 if (lfID<likeFuncList.lLength-1) {
762 DeleteObject(likeFuncList(lfID));
763 likeFuncList.list_data[lfID] = 0L;
764 likeFuncNamesList.Replace(lfID,new _String,false);
765 } else {
766 likeFuncList.Delete(lfID);
767 likeFuncNamesList.Delete(lfID);
768 if (lfID)
769 while (((_String*)likeFuncNamesList (--lfID))->empty()) {
770 likeFuncList.Delete(lfID);
771 likeFuncNamesList.Delete(lfID);
772 if (lfID==0) {
773 break;
774 }
775 }
776 }
777 }
778 }
779
780 //__________________________________________________________
781
KillLFRecordFull(long lfID)782 void KillLFRecordFull (long lfID) {
783 _LikelihoodFunction* lf = (_LikelihoodFunction*) likeFuncList (lfID);
784
785 _SimpleList l;
786 lf->GetGlobalVars (l);
787
788 for (unsigned long k=0UL; k<l.lLength; k++) {
789 DeleteVariable (*LocateVar(l.list_data[k])->GetName());
790 }
791
792 l.Clear ();
793
794 unsigned long partition_count = lf->CountObjects(kLFCountPartitions);
795
796 for (unsigned long k=0UL; k<partition_count; k++) {
797 _TheTree * thisTree = lf->GetIthTree(k);
798 thisTree->CompileListOfModels (l);
799 DeleteVariable (*thisTree->GetName());
800 }
801
802 for (unsigned long k=0UL; k<l.lLength; k++) {
803 KillModelRecord (l.list_data[k]);
804 }
805
806 KillLFRecord (lfID);
807 }
808
809 //__________________________________________________________
810
KillDataSetRecord(long dsID)811 void KillDataSetRecord (long dsID)
812 {
813 if (dsID<dataSetList.lLength-1) {
814 DeleteObject(dataSetList(dsID));
815 dataSetList.list_data[dsID] = 0;
816 dataSetNamesList.Replace(dsID,new _String,false);
817 } else {
818 dataSetList.Delete(dsID);
819 dataSetNamesList.Delete(dsID);
820 if (dsID)
821 while (((_String*)dataSetNamesList (--dsID))->empty()) {
822 dataSetList.Delete(dsID);
823 dataSetNamesList.Delete(dsID);
824 if (dsID==0) {
825 break;
826 }
827 }
828 }
829 }
830
831 //__________________________________________________________
832
KillExplicitModelFormulae(void)833 void KillExplicitModelFormulae (void)
834 {
835 for (long i = 0; i < modelTypeList.lLength; i++)
836 if (modelTypeList.list_data[i]) {
837 delete (_Formula*)(modelMatrixIndices.list_data[i]);
838 }
839 }
840
841
842 //__________________________________________________________
843
KillModelRecord(long mdID)844 void KillModelRecord (long mdID)
845 {
846 if (lastMatrixDeclared == mdID) {
847 lastMatrixDeclared = -1;
848 }
849
850 // SLKP 20110816
851 // can't delete matrices before checking that no other model depends on the
852
853
854 if (modelTypeList.list_data[mdID]) {
855 delete (_Formula*)(modelMatrixIndices.list_data[mdID]);
856 } else {
857 _Variable * modelMatrix = nil,
858 * freqMatrix = nil;
859
860 bool multByFreqs = false;
861
862
863
864 _SimpleList saveTheseVariablesAux;
865 _AVLList saveTheseVariables (&saveTheseVariablesAux);
866
867 for (long k = 0; k < modelNames.lLength; k++) {
868 if (k != mdID && ((_String*)modelNames(k))->nonempty()) {
869 if (modelTypeList.list_data[k]) {
870 _SimpleList dependantMatrices;
871 ((_Formula*)(modelMatrixIndices.list_data[k]))->ScanFForType (dependantMatrices, MATRIX);
872 for (long k2 = 0; k2 < dependantMatrices.lLength; k2++) {
873 saveTheseVariables.Insert((BaseRef)dependantMatrices.list_data[k2]);
874 }
875 } else {
876 RetrieveModelComponents(k, modelMatrix, freqMatrix, multByFreqs);
877
878 if (modelMatrix) {
879 saveTheseVariables.Insert((BaseRef)modelMatrix->get_index());
880 }
881 if (freqMatrix) {
882 saveTheseVariables.Insert((BaseRef)freqMatrix->get_index());
883 }
884 }
885 }
886 }
887
888 RetrieveModelComponents(mdID, modelMatrix, freqMatrix, multByFreqs);
889 if (modelMatrix && saveTheseVariables.Find ((BaseRef)modelMatrix->get_index()) < 0) {
890 DeleteVariable (*modelMatrix->GetName());
891 }
892 if (freqMatrix && saveTheseVariables.Find ((BaseRef)freqMatrix->get_index()) < 0) {
893 DeleteVariable (*freqMatrix->GetName());
894 }
895 }
896
897 if (mdID<modelNames.lLength-1) {
898 modelMatrixIndices.list_data[mdID] = -1;
899 modelTypeList.list_data[mdID] = 0;
900 modelFrequenciesIndices.list_data[mdID] = -1;
901 modelNames.Replace(mdID,new _String,false);
902 } else {
903 modelNames.Delete(mdID);
904 modelMatrixIndices.Delete (modelMatrixIndices.lLength-1);
905 modelFrequenciesIndices.Delete (modelFrequenciesIndices.lLength-1);
906 modelTypeList.Delete (modelTypeList.lLength-1);
907
908 if (mdID)
909 while (((_String*)modelNames (--mdID))->empty()) {
910 modelNames.Delete(mdID);
911 modelMatrixIndices.Delete (mdID);
912 modelFrequenciesIndices.Delete (mdID);
913 modelTypeList.Delete (mdID);
914 if (mdID == 0) {
915 break;
916 }
917 }
918 }
919 }
920
921 //____________________________________________________________________________________
922 //____________________________________________________________________________________
_ExecutionList()923 _ExecutionList::_ExecutionList () {
924 Init();
925 } // doesn't do much
926
927 //____________________________________________________________________________________
_ExecutionList(_StringBuffer & source,_String * namespaceID,bool copySource,bool * successFlag)928 _ExecutionList::_ExecutionList (_StringBuffer& source, _String* namespaceID , bool copySource, bool* successFlag) {
929 Init (namespaceID);
930
931 if (copySource) {
932 sourceText.Duplicate (&source);
933 }
934
935 bool result = BuildList (source, nil, false, true);
936 if (successFlag) {
937 *successFlag = result;
938 }
939 }
940
941
942
943 //____________________________________________________________________________________
Init(_String * namespaceID)944 void _ExecutionList::Init (_String* namespaceID) {
945 result = nil;
946 currentCommand = 0;
947 cli = nil;
948 profileCounter = nil;
949 stdinRedirect = nil;
950 stdinRedirectAux = nil;
951 doProfile = 0;
952 nameSpacePrefix = nil;
953 kwargs = nil;
954 kwarg_tags = nil;
955 currentKwarg = 0;
956
957 if (currentExecutionList) {
958 errorHandlingMode = currentExecutionList->errorHandlingMode;
959 errorState = currentExecutionList->errorState;
960 } else {
961 errorHandlingMode = HY_BL_ERROR_HANDLING_DEFAULT;
962 errorState = false;
963 }
964
965 if (namespaceID) {
966 SetNameSpace (*namespaceID);
967 }
968
969 }
970
971 //____________________________________________________________________________________
SetKWArgs(_AssociativeList * kwarg_list)972 void _ExecutionList::SetKWArgs(_AssociativeList* kwarg_list) {
973 DeleteAndZeroObject(kwargs);
974 if (kwarg_list && kwarg_list->countitems()) {
975 kwargs = (_AssociativeList*)kwarg_list->makeDynamic();
976 }
977 }
978
979 //____________________________________________________________________________________
ClearExecutionList(void)980 void _ExecutionList::ClearExecutionList (void) {
981 if (cli) {
982 delete [] cli->values;
983 delete [] cli->stack;
984 delete [] cli->is_compiled;
985 delete cli;
986 cli = nil;
987 }
988
989 DeleteAndZeroObject (profileCounter);
990 DeleteAndZeroObject (stdinRedirect);
991 DeleteAndZeroObject (stdinRedirectAux);
992 DeleteAndZeroObject (nameSpacePrefix);
993 DeleteAndZeroObject (kwargs);
994 DeleteAndZeroObject (kwarg_tags);
995
996 ResetFormulae();
997 DeleteAndZeroObject (result);
998 }
999
1000 //____________________________________________________________________________________
1001
~_ExecutionList(void)1002 _ExecutionList::~_ExecutionList (void) {
1003 ClearExecutionList();
1004 }
1005
1006 //____________________________________________________________________________________
1007
makeDynamic(void) const1008 BaseRef _ExecutionList::makeDynamic (void) const {
1009 _ExecutionList * Res = new _ExecutionList;
1010
1011 //memcpy ((char*)Res, (char*)this, sizeof (_ExecutionList));
1012
1013 Res->_List::Duplicate (this);
1014 Res->Duplicate (this);
1015 Res->cli = nil;
1016 Res->profileCounter = nil;
1017 Res->doProfile = doProfile;
1018 Res->errorHandlingMode = errorHandlingMode;
1019 Res->errorState = errorState;
1020
1021 if(result) {
1022 Res->result = (HBLObjectRef)result->makeDynamic();
1023 }
1024
1025 return Res;
1026 }
1027
1028 //____________________________________________________________________________________
1029
Duplicate(BaseRefConst source)1030 void _ExecutionList::Duplicate (BaseRefConst source) {
1031 _List::Duplicate (source);
1032
1033 _ExecutionList const* s = (_ExecutionList const*)source;
1034
1035 if (s->result) {
1036 result=(HBLObjectRef)s->result->makeDynamic();
1037 }
1038
1039 errorHandlingMode = s->errorHandlingMode;
1040 errorState = s->errorState;
1041 }
1042
1043
1044 //____________________________________________________________________________________
ReportAnExecutionError(_String errMsg,bool doCurrentCommand,bool appendToExisting)1045 void _ExecutionList::ReportAnExecutionError (_String errMsg, bool doCurrentCommand, bool appendToExisting) {
1046 if (doCurrentCommand) {
1047 _ElementaryCommand *theCommand = FetchLastCommand();
1048 if (theCommand) {
1049 errMsg = errMsg & " in call to " & _String ((_String*)theCommand->toStr());
1050 }
1051 }
1052 errorState = true;
1053 switch (errorHandlingMode) {
1054 case HY_BL_ERROR_HANDLING_SOFT:
1055 if (appendToExisting) {
1056 _FString * existing = (_FString*) FetchObjectFromVariableByType(&_hyLastExecutionError, STRING);
1057 if (existing) {
1058 errMsg = existing->get_str() & '\n' & errMsg;
1059 }
1060 }
1061 setParameter(_hyLastExecutionError, new _FString (errMsg, false), nil, false);
1062
1063 break;
1064 default:
1065 HandleApplicationError (errMsg);
1066 }
1067 }
1068
1069 //____________________________________________________________________________________
StartProfile(void)1070 void _ExecutionList::StartProfile (void) {
1071 DeleteObject (profileCounter);
1072 profileCounter= new _Matrix (lLength, 2, false, true);
1073 doProfile = 1;
1074 }
1075
1076 //____________________________________________________________________________________
CollectProfile(void)1077 _AssociativeList* _ExecutionList::CollectProfile (void) {
1078 _AssociativeList * profileDump = new _AssociativeList;
1079
1080 if (profileCounter) {
1081 _SimpleList instructions;
1082 _List descriptions;
1083
1084 for (unsigned long k=1UL; k<2*lLength; k+=2UL) {
1085 if (profileCounter->theData[k] > 0.0) {
1086 instructions << k/2;
1087 descriptions.AppendNewInstance((_String*)((_ElementaryCommand*)(*this)(k/2))->toStr());
1088 }
1089 }
1090
1091 _Matrix * execProfile = new _Matrix (instructions.lLength,2,false,true),
1092 * instCounter = new _Matrix (instructions),
1093 * descList = new _Matrix (descriptions);
1094
1095 unsigned long k2 = 0UL;
1096 for (unsigned long m=1UL; m<2*lLength; m+=2UL) {
1097 if (profileCounter->theData[m] > 0.0) {
1098 execProfile->theData[k2++] = profileCounter->theData[m];
1099 execProfile->theData[k2++] = profileCounter->theData[m-1];
1100 }
1101 }
1102
1103 profileDump->MStore ("INSTRUCTION INDEX", instCounter, false);
1104 profileDump->MStore ("INSTRUCTION", descList, false);
1105 profileDump->MStore ("STATS", execProfile, false);
1106 doProfile = 0;
1107 DeleteAndZeroObject (profileCounter);
1108 }
1109
1110 return profileDump;
1111 }
1112
1113 //____________________________________________________________________________________
FetchFromStdinRedirect(_String const * dialog_tag,bool handle_multi_choice,bool do_echo)1114 _String* _ExecutionList::FetchFromStdinRedirect (_String const * dialog_tag, bool handle_multi_choice, bool do_echo) {
1115 // grab a string from the front of the input queue
1116 // complain if nothing is left
1117 if (! (has_stdin_redirect() || has_keyword_arguments())) {
1118 throw _String ("No input buffer / keyword arguments was given for a redirected standard input read.");
1119 return new _String;
1120 }
1121
1122 /**
1123 if keyword arguments are present, try them first
1124 */
1125
1126 HBLObjectRef user_argument = nil;
1127 _List ref_manager;
1128 _String* kwarg_used = nil;
1129
1130 auto echo_argument = [&do_echo] (_String const * kwarg, _String const& value) -> void {
1131 if (do_echo && kwarg) {
1132 bool do_markdown = hy_env :: EnvVariableTrue(hy_env :: produce_markdown_output);
1133 if (do_markdown) {
1134 NLToConsole(); BufferToConsole(">"); StringToConsole(*kwarg); BufferToConsole( " –> "); BufferToConsole(value); NLToConsole();
1135 } else {
1136 StringToConsole(*kwarg); BufferToConsole( ": "); BufferToConsole(value); NLToConsole();
1137 }
1138 }
1139 };
1140
1141 try {
1142 if (has_keyword_arguments()) {
1143 if (kwarg_tags && kwarg_tags->countitems() > currentKwarg) {
1144 _List* current_tag = (_List*)kwarg_tags->GetItem(currentKwarg++);
1145
1146 // check to see if we need to match with the current dialog prompt
1147 if (current_tag -> countitems() > 3UL) {
1148 _String check_against = dialog_tag ? *dialog_tag : dialogPrompt;
1149 if (check_against != *(_String*)current_tag->GetItem(3)) {
1150 throw (1L);
1151 }
1152 }
1153
1154 if (kwargs) {
1155 user_argument = kwargs->GetByKey(*(_String*)current_tag->GetItem(0));
1156 }
1157
1158 if (user_argument) { // user argument provided
1159 user_argument -> AddAReference();
1160 kwarg_used = (_String*)current_tag->GetItem(0);
1161 kwargs->DeleteByKey(*kwarg_used);
1162 ref_manager < user_argument;
1163 } else { // see if there are defaults
1164 if (current_tag->countitems() > 2 && ! ignore_kw_defaults) {
1165 _String * default_value = (_String*)current_tag->GetItem(2);
1166 kwarg_used = (_String*)current_tag->GetItem(0);
1167 if (default_value) {
1168 user_argument = new _FString (*(_String*)current_tag->GetItem(2));
1169 ref_manager < user_argument;
1170 }
1171 }
1172 }
1173
1174 //printf ("%ld => %s\n", currentKwarg - 1, user_argument ? _String ((_String*)user_argument->toStr()).get_str() : "''");
1175 }
1176 }
1177 } catch (long) {
1178 return FetchFromStdinRedirect (dialog_tag, handle_multi_choice, do_echo);
1179 }
1180
1181 if (user_argument) {
1182 if (user_argument->ObjectClass() == STRING) {
1183 echo_argument (kwarg_used, ((_FString*)user_argument)->get_str());
1184 return new _String (((_FString*)user_argument)->get_str());
1185 } else {
1186 if (handle_multi_choice) {
1187 echo_argument (kwarg_used, _String ((_String*)user_argument->toStr()));
1188 user_argument->AddAReference();
1189 throw (user_argument);
1190 } else {
1191 throw _String ("Multi-choice keyword argument not supported in this context");
1192 }
1193 }
1194 }
1195
1196 if (has_stdin_redirect()) {
1197 long d = stdinRedirect->First();
1198 if (d<0) {
1199 throw _String ("Ran out of input in buffer during a redirected standard input read.");
1200 }
1201
1202 _String *sendBack = (_String*)stdinRedirect->GetXtra (d);
1203 //printf ("Consumed stdin redrect %ld => %s\n", d, sendBack->get_str());
1204 sendBack->AddAReference();
1205 if (do_echo) {
1206 StringToConsole(*sendBack);
1207 NLToConsole();
1208 }
1209 stdinRedirect->Delete ((*(_List*)stdinRedirect->dataList)(d),true);
1210 return sendBack;
1211 }
1212
1213 throw kNoKWMatch;
1214 }
1215
1216 //____________________________________________________________________________________
1217
GetFileName(void) const1218 _String const _ExecutionList::GetFileName (void) const {
1219 if (sourceFile.nonempty()) {
1220 return sourceFile;
1221 } else {
1222 _String const *top_path = PeekFilePath();
1223 if (top_path)
1224 return *top_path;
1225 }
1226 return kEmptyString;
1227 }
1228
1229 //____________________________________________________________________________________
1230
BuildListOfDependancies(_AVLListX & collection,bool recursive)1231 void _ExecutionList::BuildListOfDependancies (_AVLListX & collection, bool recursive) {
1232 for (unsigned long step = 0UL; step < lLength; step++) {
1233 ((_ElementaryCommand*)GetItem(step))->BuildListOfDependancies (collection, recursive, *this);
1234 }
1235 }
1236
1237 //____________________________________________________________________________________
1238
GenerateHelpMessage(_AVLList * scanned_functions) const1239 _StringBuffer const _ExecutionList::GenerateHelpMessage(_AVLList * scanned_functions) const {
1240 _StringBuffer help_message;
1241
1242 _List ref_manager;
1243 bool nested = true;
1244 if (!scanned_functions) {
1245 _List * _aux_list = new _List;
1246 scanned_functions = new _AVLList (_aux_list);
1247 ref_manager < _aux_list < scanned_functions;
1248 nested = false;
1249 }
1250
1251 auto simplify_string = [] (_String const * s) -> const _String {
1252 _String sc (*s);
1253 if (sc.IsALiteralArgument(true)) {
1254 return sc;
1255 }
1256 return sc & " [computed at run time]";
1257 };
1258
1259 ForEach ([&help_message, simplify_string, this, scanned_functions] (BaseRef command, unsigned long index) -> void {
1260 _ElementaryCommand * this_command = (_ElementaryCommand * )command;
1261 if (this_command->code == HY_HBL_COMMAND_KEYWORD_ARGUMENT) {
1262 _String * def_value = this_command->GetIthParameter(2L, false),
1263 * applies_to = this_command->GetIthParameter(3L, false);
1264
1265 if (def_value && (*def_value == kNoneToken || *def_value == kNullToken)) {
1266 def_value = nil;
1267 }
1268
1269 help_message << simplify_string(this_command->GetIthParameter(0L)) << (def_value == nil ? (applies_to ? " [conditionally required]" : " [required]"): "") << '\n'
1270 << '\t' << simplify_string(this_command->GetIthParameter(1L)) << '\n';
1271
1272
1273 if (def_value) {
1274 help_message << "\tdefault value: " << simplify_string(def_value) << '\n';
1275 }
1276 if (applies_to) {
1277 help_message << "\tapplies to: " << simplify_string(applies_to) << '\n';
1278 }
1279 help_message << '\n';
1280 } else {
1281 if (this_command->code == HY_HBL_COMMAND_FORMULA) {
1282 _List hbl_functions;
1283 _AVLListX other_functions (&hbl_functions);
1284 this_command->BuildListOfDependancies(other_functions, true, *this, true);
1285
1286 for (AVLListXIteratorKeyValue function_iterator : AVLListXIterator (&other_functions)) {
1287 _String * function_name = (_String *)other_functions.Retrieve (function_iterator.get_index());
1288 if (scanned_functions->Insert (new _String (*function_name),0,true, true) >= 0) {
1289 long idx = FindBFFunctionName(*function_name);
1290 if (idx >= 0) {
1291 help_message << GetBFFunctionBody(idx).GenerateHelpMessage(scanned_functions);
1292 }
1293 }
1294 }
1295 }
1296 }
1297
1298 });
1299
1300 if (help_message.empty() && !nested) {
1301 help_message << "No annotated keyword arguments are available for this analysis\n";
1302 }
1303
1304 return help_message;
1305 }
1306
1307 //____________________________________________________________________________________
1308
Execute(_ExecutionList * parent,bool ignore_CEL_kwargs)1309 HBLObjectRef _ExecutionList::Execute (_ExecutionList* parent, bool ignore_CEL_kwargs) {
1310
1311 //setParameter(_hyLastExecutionError, new _MathObject, nil, false);
1312 try{
1313
1314 _ExecutionList* stashCEL = currentExecutionList;
1315 callPoints << currentCommand;
1316 executionStack << this;
1317
1318 _AVLListXL * stash1 = nil;
1319 _List* stash2 = nil,
1320 * stash_kw_tags = nil;// recursion
1321 _AssociativeList * stash_kw = nil;
1322
1323 currentKwarg = (stashCEL && !ignore_CEL_kwargs) ? stashCEL->currentKwarg : 0;
1324
1325 if (parent && (stdinRedirect == nil || kwargs == nil)) {
1326 if (!stdinRedirect) {
1327 stash1 = stdinRedirect = parent->stdinRedirect;
1328 stash2 = stdinRedirectAux = parent->stdinRedirectAux;
1329 if (stash1) {
1330 stash1->AddAReference();
1331 stash2->AddAReference();
1332 }
1333 }
1334 if (!kwargs) {
1335 stash_kw_tags = kwarg_tags = parent->kwarg_tags;
1336 stash_kw = kwargs = parent->kwargs;
1337 if (stash_kw_tags) stash_kw_tags->AddAReference();
1338 if (stash_kw) stash_kw->AddAReference();
1339 currentKwarg = parent->currentKwarg;
1340 }
1341 } else {
1342 parent = nil;
1343 }
1344
1345 _FString cfp (PeekFilePath() ? *PeekFilePath () :kEmptyString),
1346 *stashed = (_FString*)hy_env::EnvVariableGet(hy_env::path_to_current_bf, STRING);
1347
1348 if (stashed) {
1349 stashed = (_FString*)stashed->makeDynamic();
1350 }
1351
1352 hy_env::EnvVariableSet(hy_env::path_to_current_bf, &cfp, true);
1353
1354 DeleteAndZeroObject (result);
1355 currentExecutionList = this;
1356 currentCommand = 0;
1357
1358 terminate_execution = false;
1359
1360
1361 bool is_c = is_compiled();
1362 if (is_c) {
1363 //PopulateArraysForASimpleFormula (cli->varList, cli->values);
1364 cli->is_compiled[0] = false;
1365 }
1366
1367 while (currentCommand<lLength) {
1368
1369 if (is_c) {
1370 if ( cli->is_compiled [currentCommand+1] == false) {
1371 if (cli->is_compiled[0]) {
1372 CopyCLIToVariables();
1373 cli->is_compiled [0] = false;
1374 }
1375
1376 } else {
1377 if (cli->is_compiled[0] == false) {
1378 PopulateArraysForASimpleFormula (cli->varList, cli->values);
1379 cli->is_compiled[0] = true;
1380 }
1381 }
1382 }
1383
1384 if (doProfile == 1 && profileCounter) {
1385 long instCounter = currentCommand;
1386 hyFloat timeDiff = 0.0;
1387
1388 TimeDifference timer;
1389 (((_ElementaryCommand**)list_data)[currentCommand])->Execute(*this);
1390 timeDiff = timer.TimeSinceStart();
1391
1392
1393 if (profileCounter) {
1394 // a call to _hyphy_profile_dump can set this to NULL
1395 profileCounter->theData[instCounter*2] += timeDiff;
1396 profileCounter->theData[instCounter*2+1] += 1.0;
1397 }
1398 } else {
1399 (((_ElementaryCommand**)list_data)[currentCommand])->Execute(*this);
1400 }
1401
1402 if (terminate_execution) {
1403 break;
1404 }
1405 }
1406 currentCommand = callPoints.list_data[callPoints.lLength-1];
1407 callPoints.Delete (callPoints.lLength-1);
1408 currentExecutionList = stashCEL;
1409
1410 if (currentExecutionList && !ignore_CEL_kwargs) {
1411 currentExecutionList->currentKwarg = currentKwarg;
1412 }
1413
1414 if (stashed) {
1415 hy_env::EnvVariableSet(hy_env::path_to_current_bf, stashed, false);
1416 }
1417
1418 executionStack.Delete (executionStack.lLength-1);
1419 if (result == nil) {
1420 result = new _MathObject();
1421 }
1422
1423 if (parent) {
1424 stdinRedirect = nil;
1425 stdinRedirectAux = nil;
1426 kwargs = nil;
1427 kwarg_tags = nil;
1428 parent->currentKwarg = currentKwarg;
1429 BatchDeleteObject (stash1, stash2, stash_kw, stash_kw_tags);
1430 }
1431
1432 if (is_compiled(0)) {
1433 CopyCLIToVariables();
1434 }
1435
1436 } catch (const _String& err) {
1437 HandleApplicationError(err);
1438 }
1439
1440 return result;
1441 }
1442
1443 //____________________________________________________________________________________
1444
ExecuteAndClean(long g)1445 void _ExecutionList::ExecuteAndClean (long g) {
1446 Execute ();
1447 ClearBFFunctionLists (g);
1448 }
1449
1450 //____________________________________________________________________________________
1451
TryToMakeSimple(bool partial_ok)1452 bool _ExecutionList::TryToMakeSimple (bool partial_ok) {
1453 _SimpleList varListAux,
1454 formulaeToConvert,
1455 parseCodes;
1456
1457 _AVLList varList (&varListAux);
1458
1459 long stackDepth = 0L;
1460 bool status = true;
1461 unsigned long k = 0UL;
1462
1463 bool *is_compiled = new bool [countitems()+1];
1464 InitializeArray (is_compiled, countitems()+1, false);
1465
1466 for (; k<lLength && status; k++) {
1467 _ElementaryCommand * aStatement = (_ElementaryCommand*)(*this)(k);
1468 switch (aStatement->code) {
1469 case 0: { // expression
1470 _String * formulaString = (_String*)aStatement->parameters(0);
1471
1472 if ((*formulaString) (-1) != '}') {
1473 _Formula *f = new _Formula,
1474 *f2 = new _Formula;
1475
1476 _FormulaParsingContext fpc (nil, nameSpacePrefix);
1477
1478 long parseCode = Parse(f,*formulaString,fpc,f2);
1479
1480 if (parseCode == HY_FORMULA_EXPRESSION || parseCode == HY_FORMULA_VARIABLE_VALUE_ASSIGNMENT || parseCode == HY_FORMULA_FORMULA_VALUE_ASSIGNMENT) {
1481
1482 if (f->AmISimple(stackDepth,varList)) {
1483 try {
1484 if (parseCode == HY_FORMULA_FORMULA_VALUE_ASSIGNMENT) {
1485 if (!f2->AmISimple(stackDepth, varList)) throw 0;
1486 long assignment_length = f->NumberOperations();
1487 if (assignment_length < 3) throw 0;
1488 _Variable * mx = f->GetIthTerm(0)->RetrieveVar();
1489 if (! mx) throw 0;
1490 f->GetIthTerm (0)->SetAVariable(mx->get_index());
1491 _Operation * last = f->GetIthTerm(assignment_length-1);
1492 if (! (last->TheCode() == HY_OP_CODE_MCOORD && last->GetNoTerms() == 2)) throw 0;
1493
1494 f2->GetList() << f->GetList();
1495 f->Clear();
1496
1497 _Formula *t = f2;
1498 f2 = f;
1499 f = t;
1500
1501 }
1502
1503 } catch (int e) {
1504 if (partial_ok) {
1505 parseCodes << -1;
1506 continue;
1507 }
1508 status = false;
1509 break;
1510 }
1511
1512 aStatement->simpleParameters<<parseCode;
1513 aStatement->simpleParameters<<(long)f;
1514 aStatement->simpleParameters<<(long)f2;
1515 aStatement->simpleParameters<<fpc.assignmentRefID();
1516
1517
1518 formulaeToConvert << (long)f;
1519 is_compiled [k+1] = true;
1520
1521
1522 if (parseCode == HY_FORMULA_VARIABLE_VALUE_ASSIGNMENT) {
1523 varList.InsertNumber(fpc.assignmentRefID());
1524 parseCodes << fpc.assignmentRefID();
1525 } else {
1526 parseCodes << -1;
1527 }
1528 break;
1529
1530 }
1531 }
1532
1533 delete f;
1534 delete f2;
1535 }
1536
1537 parseCodes << -1;
1538 if (!partial_ok) {
1539 status = false;
1540 }
1541 break;
1542 }
1543
1544 case 4:
1545 parseCodes << -1;
1546 if (aStatement->simpleParameters.lLength == 3 || aStatement->parameters.lLength) {
1547 if (aStatement->parameters.lLength) {
1548 _Formula f;
1549 _FormulaParsingContext fpc (nil, nameSpacePrefix);
1550
1551 long status = Parse (&f, *(_String*)aStatement->parameters(0), fpc, nil);
1552
1553 if (status== HY_FORMULA_EXPRESSION) {
1554 aStatement->simpleParameters<<long(f.makeDynamic());
1555 }
1556 }
1557
1558
1559 _Formula *cf = ((_Formula*)aStatement->simpleParameters(2));
1560 if (cf->AmISimple(stackDepth,varList)) {
1561 formulaeToConvert << (long)cf;
1562 is_compiled [k+1] = true;
1563 } else {
1564 if (partial_ok) {
1565 parseCodes << -1;
1566 continue;
1567 }
1568 status = false;
1569 }
1570 } else {
1571 is_compiled[k+1] = true;
1572 }
1573 break;
1574
1575 case 14: // return statements are OK
1576 parseCodes << -1;
1577 break;
1578
1579 default:
1580 status = false;
1581 }
1582 if (status == false) {
1583 ReportWarning (_String ("Failed to compile an execution list: offending command was\n") & _String (((_String*)aStatement->toStr())));
1584 }
1585 }
1586
1587 if (status) {
1588 if (formulaeToConvert.nonempty()) {
1589 cli = new _CELInternals;
1590 //varList.ReorderList();
1591 cli->values = new _SimpleFormulaDatum[varList.countitems()+1];
1592 cli->stack = new _SimpleFormulaDatum[stackDepth+1];
1593 cli->is_compiled = is_compiled;
1594
1595 _SimpleList avlData;
1596 _AVLListX avlList (&avlData);
1597
1598 for (unsigned long fi = 0; fi < formulaeToConvert.lLength; fi++) {
1599 ((_Formula*)formulaeToConvert(fi))->ConvertToSimple (varList);
1600 }
1601
1602 for (unsigned long vi = 0; vi < varListAux.countitems(); vi++) {
1603 avlList.Insert ((BaseRef)varListAux.list_data[vi], vi);
1604 }
1605
1606 for (unsigned long ri = 0; ri<parseCodes.lLength; ri++) {
1607 if (parseCodes.list_data[ri] < 0) {
1608 cli->storeResults << -1;
1609 } else {
1610 cli->storeResults << avlList.GetXtra (avlList.Find ((BaseRef) parseCodes.list_data[ri]));
1611 }
1612 //printf ("\n%ld\n", cli->storeResults.list_data[ri]);
1613 }
1614 cli->varList.Duplicate(&varListAux);
1615 }
1616 } else {
1617 // clean up partially converted statements
1618 delete [] is_compiled;
1619 for (unsigned long k2 = 0UL; k2 < k; k2++) {
1620 GetIthCommand(k2)->simpleParameters.Pop(3UL);
1621
1622 }
1623 }
1624
1625 return status;
1626 }
1627
1628 //____________________________________________________________________________________
1629
CopyCLIToVariables(void)1630 void _ExecutionList::CopyCLIToVariables(void) {
1631
1632 cli->varList.Each ([this] (long index, unsigned long idx) -> void {
1633 _Variable * mv = LocateVar(index);
1634 if (mv->ObjectClass() == NUMBER) {
1635 mv->SetValue (new _Constant (this->cli->values[idx].value),false,true,NULL);
1636 }
1637 });
1638 }
1639
1640 //____________________________________________________________________________________
1641
ExecuteSimple(_ExecutionList * parent)1642 void _ExecutionList::ExecuteSimple (_ExecutionList * parent) {
1643 //PopulateArraysForASimpleFormula (cli->varList, cli->values);
1644 Execute(parent);
1645 //CopyCLIToVariables();
1646 }
1647
1648 //____________________________________________________________________________________
1649
ResetFormulae(void)1650 void _ExecutionList::ResetFormulae (void) {
1651 currentCommand = 0L;
1652 _SimpleList to_delete_aux;
1653 _AVLList to_delete (&to_delete_aux);
1654 while (currentCommand<lLength) {
1655 _ElementaryCommand* thisCommand = ((_ElementaryCommand**)list_data)[currentCommand];
1656 if (thisCommand->DecompileFormulae()) {
1657 to_delete.Insert(thisCommand);
1658 }
1659 currentCommand++;
1660 }
1661
1662 if (to_delete.countitems()) {
1663 _SimpleList batch_delete;
1664 for (unsigned long i = 0; i < listOfCompiledFormulae.lLength; i++) {
1665 if (to_delete.Find ((BaseRef)listOfCompiledFormulae.Element(i)) >= 0) {
1666 batch_delete << i;
1667 }
1668 }
1669 listOfCompiledFormulae.DeleteList(batch_delete);
1670 compiledFormulaeParameters.DeleteList(batch_delete);
1671 }
1672 }
1673 //____________________________________________________________________________________
1674
toStr(unsigned long)1675 BaseRef _ExecutionList::toStr (unsigned long) {
1676 _StringBuffer *result = new _StringBuffer (256);
1677 _String step ("\n\nStep ");
1678
1679 _ExecutionList* stash = currentExecutionList;
1680
1681 currentExecutionList = this;
1682
1683 for (unsigned long i=0UL; i<countitems(); i++) {
1684 (*result) << &step << _String((long)i) << '.';
1685 if (is_compiled(i+1)) {
1686 (*result) << "[compiled]";
1687 }
1688 result->AppendNewInstance ((_String*)GetItem(i)->toStr());
1689 }
1690
1691 currentExecutionList = stash;
1692 return result;
1693 }
1694
1695 //____________________________________________________________________________________
1696
ResetNameSpace(void)1697 void _ExecutionList::ResetNameSpace (void)
1698 {
1699 DeleteObject (nameSpacePrefix);
1700 nameSpacePrefix = nil;
1701 }
1702
1703 //____________________________________________________________________________________
1704
SetNameSpace(_String const & nID)1705 void _ExecutionList::SetNameSpace (_String const& nID) {
1706 ResetNameSpace ();
1707 nameSpacePrefix = new _VariableContainer(nID);
1708 }
1709
1710 //____________________________________________________________________________________
1711
GetNameSpace()1712 _String* _ExecutionList::GetNameSpace () {
1713 if (nameSpacePrefix) {
1714 return nameSpacePrefix->GetName();
1715 }
1716 return nil;
1717 }
1718
1719 //____________________________________________________________________________________
1720
AddNameSpaceToID(_String const & theID,_String const * extra)1721 _String const _ExecutionList::AddNameSpaceToID (_String const & theID, _String const * extra) {
1722 _String name_space;
1723
1724 if (extra && extra->nonempty()) {
1725 if (nameSpacePrefix) {
1726 name_space = (*nameSpacePrefix->GetName())&'.'& *extra;
1727 } else {
1728 name_space = *extra;
1729 }
1730 } else {
1731 if (nameSpacePrefix) {
1732 name_space = (*nameSpacePrefix->GetName());
1733 }
1734 }
1735
1736 return AppendContainerName (theID, &name_space);
1737 }
1738
1739 //____________________________________________________________________________________
1740
TrimNameSpaceFromID(_String & theID)1741 _String _ExecutionList::TrimNameSpaceFromID (_String& theID) {
1742 if (nameSpacePrefix) {
1743 _String * prefix_name = nameSpacePrefix->GetName();
1744 if (theID.BeginsWith(*prefix_name)) {
1745 return theID.Cut(prefix_name->length () +1,-1);
1746 }
1747 }
1748 return theID;
1749 }
1750
1751 //____________________________________________________________________________________
1752
BuildChoiceList(_List * pieces,long code)1753 void _ExecutionList::BuildChoiceList (_List * pieces, long code) {
1754 _ElementaryCommand * choice_list = new _ElementaryCommand (code);
1755
1756 choice_list->parameters << pieces->GetItem(0L)
1757 << pieces->GetItem(1L)
1758 << pieces->GetItem(2L)
1759 << pieces->GetItem(3L);
1760
1761 if (pieces->countitems() > 5UL) { // expliit list of choices
1762
1763 if (pieces->countitems() % 2 > 0) {
1764 DeleteObject(choice_list);
1765 throw (_String ("Must have an even number of arguments for explicitly enumerated choice - description pairs in ") & __PRETTY_FUNCTION__ & ". Had " & _String ((_String*)pieces->toStr()).Enquote());
1766 }
1767
1768 _List * choices = new _List;
1769
1770 for (unsigned long k = 4UL; k < pieces->countitems(); k+=2) {
1771 _String * selector = new _String (*(_String*)pieces->GetItem(k)),
1772 * desription = new _String (*(_String*)pieces->GetItem(k+1));
1773 selector->StripQuotes(); desription->StripQuotes();
1774 *choices < new _List (selector,desription);
1775 }
1776 choice_list->parameters < choices;
1777 choice_list->simpleParameters << 0L;
1778 } else {
1779 choice_list->parameters << pieces->GetItem(4L);
1780 choice_list->simpleParameters << 1L;
1781 }
1782
1783 choice_list->addAndClean (*this);
1784
1785 }
1786
1787 //____________________________________________________________________________________
1788
BuildExecuteCommandInstruction(_List * pieces,long code)1789 void _ExecutionList::BuildExecuteCommandInstruction (_List * pieces, long code) {
1790
1791 const _String kExecuteCompiled ("compiled"),
1792 kExecuteEncloseingNamespace ("enclosing_namespace");
1793
1794 _ElementaryCommand * run_source = new _ElementaryCommand (code);
1795 run_source->parameters<<pieces->GetItem(0);
1796
1797 if (PeekFilePath()) {
1798 run_source->parameters && *PeekFilePath();
1799 } else {
1800 run_source->parameters.AppendNewInstance(new _String);
1801 }
1802
1803 if (pieces->countitems() > 1UL) {
1804 if (*(_String*)pieces->GetItem (1UL) == kExecuteCompiled) {
1805 run_source->simpleParameters << 1;
1806 } else {
1807 if (*(_String*)pieces->GetItem (1UL) == kExecuteEncloseingNamespace) {
1808 run_source->parameters.Delete(1UL);
1809 run_source->parameters < new _String;
1810 } else {
1811 run_source->parameters << pieces->GetItem(1UL);
1812 }
1813 }
1814
1815 if (pieces->countitems () > 2UL) {
1816 run_source->parameters << pieces->GetItem(2UL);
1817 }
1818 }
1819 run_source->addAndClean (*this);
1820
1821 }
1822
1823 //____________________________________________________________________________________
1824
BuildFscanf(_List * pieces,long code)1825 void _ExecutionList::BuildFscanf(_List * pieces, long code) {
1826
1827 static const _String kFscanfRewind ("REWIND"),
1828 kScanfCreate ("CREATE_FILE");
1829
1830 long names_vs_types_offset = 0L;
1831
1832 _List local_object_manager;
1833
1834
1835 _ElementaryCommand * scanf = new _ElementaryCommand (code);
1836 scanf->parameters << pieces->GetItem(0);
1837
1838 bool has_rewind = *(_String*) pieces->GetItem (1) == kFscanfRewind;
1839 bool has_create = *(_String*) pieces->GetItem (1) == kScanfCreate;
1840
1841 // process argument types
1842
1843 local_object_manager < scanf;
1844 _List argument_types;
1845 _String* argument_type_spec = (_String*)pieces->GetItem(has_rewind || has_create ? 2L : 1L);
1846 argument_type_spec->StripQuotes();
1847 _ElementaryCommand::ExtractConditions(*argument_type_spec, 0, argument_types, ',');
1848
1849 argument_types.ForEach ([&] (BaseRefConst t, unsigned long) -> void {
1850 long argument_type = _ElementaryCommand :: fscanf_allowed_formats.FindObject(t);
1851 if (argument_type == kNotFound) {
1852 throw (((_String*)t)->Enquote() & " is not one of the supported argument types: " & _ElementaryCommand :: fscanf_allowed_formats.Join (", "));
1853 }
1854 scanf->simpleParameters << argument_type;
1855 }
1856 );
1857
1858 for (unsigned long index = has_rewind || has_create? 3L : 2L; index < pieces->countitems(); index ++) {
1859 scanf->parameters << pieces->GetItem(index);
1860 }
1861
1862 if (scanf->parameters.countitems() != scanf->simpleParameters.countitems() + 1UL) {
1863 throw (_String("The numbers of parameter type descriptors (")& _String((long)scanf->simpleParameters.countitems()) &") and arguments ("
1864 &_String((long)(scanf->parameters.countitems() - 1UL))& ") did not match");
1865 }
1866
1867 if (has_rewind) {
1868 scanf->simpleParameters << -1L;
1869 }
1870 if (has_create) {
1871 scanf->simpleParameters << -2L;
1872 }
1873
1874 local_object_manager.Pop();
1875 scanf->addAndClean (*this);
1876 }
1877
1878 //____________________________________________________________________________________
1879
1880
BuildList(_StringBuffer & s,_SimpleList * bc,bool processed,bool empty_is_success)1881 bool _ExecutionList::BuildList (_StringBuffer& s, _SimpleList* bc, bool processed, bool empty_is_success) {
1882 if (terminate_execution) {
1883 return false;
1884 }
1885
1886 //char const * savePointer = s.get_str();
1887
1888 _List local_object_manager;
1889 _StringBuffer currentLine (128UL);
1890
1891 try {
1892
1893 while (s.nonempty ()) { // repeat while there is stuff left in the buffer
1894 _ElementaryCommand::FindNextCommand (s,currentLine);
1895
1896 if (currentLine.get_char(0)=='}') {
1897 currentLine.Trim(1,kStringEnd);
1898 }
1899
1900 if (currentLine.empty()) {
1901 continue;
1902 }
1903
1904 long prefixTreeCode = _HY_ValidHBLExpressions.FindKey (currentLine, nil, true);
1905
1906 _List *pieces = nil;
1907 _HBLCommandExtras *commandExtraInfo = nil;
1908
1909 local_object_manager < (pieces = new _List);
1910 if (prefixTreeCode != kNotFound) {
1911 prefixTreeCode = _HY_ValidHBLExpressions.GetValue(prefixTreeCode);
1912 long commandExtra = _HY_HBLCommandHelper.FindLong (prefixTreeCode);
1913 if (commandExtra >= 0) { // pre-trim all strings as needed
1914 commandExtraInfo = (_HBLCommandExtras*)_HY_HBLCommandHelper.GetXtra (commandExtra);
1915 if (!commandExtraInfo->extract_conditions.empty()) {
1916
1917
1918 long upto = _ElementaryCommand::ExtractConditions (currentLine, commandExtraInfo->cut_string,*pieces,commandExtraInfo->extract_condition_separator),
1919 condition_index_match = commandExtraInfo->extract_conditions.Find(pieces->lLength);
1920 if (condition_index_match < 0) {
1921 // try to see if the command accepts a variable number of arguments (at least X)
1922 if (commandExtraInfo->extract_conditions.countitems() == 1 && commandExtraInfo->extract_conditions.get(0) < 0) {
1923 if (pieces->countitems() < -commandExtraInfo->extract_conditions.get(0)) {
1924 throw (_String("Incorrect number of arguments (") & (long) pieces->lLength & ") supplied: expected at least " & _String (-commandExtraInfo->extract_conditions.get(0)) & ", while processing '"& currentLine.Cut (0, upto) & "'. ");
1925 }
1926 } else {
1927 throw (_String("Incorrect number of arguments (") & (long) pieces->lLength & ") supplied: expected one of " & _String ((_String*)commandExtraInfo->extract_conditions.toStr()) & ", while processing '"& currentLine.Cut (0, upto) & "'. ");
1928 }
1929
1930 }
1931
1932 if (commandExtraInfo->do_trim) {
1933 currentLine.Trim (upto, kStringEnd);
1934 }
1935 }
1936 }
1937 }
1938
1939 bool handled = true;
1940
1941 switch (prefixTreeCode) {
1942 case HY_HBL_COMMAND_FOR:
1943 _ElementaryCommand::BuildFor (currentLine, *this, pieces);
1944 break;
1945 case HY_HBL_COMMAND_WHILE:
1946 _ElementaryCommand::BuildWhile (currentLine, *this, pieces);
1947 break;
1948 case HY_HBL_COMMAND_BREAK:
1949 case HY_HBL_COMMAND_CONTINUE:
1950 if (bc) {
1951 AppendNewInstance(new _ElementaryCommand);
1952 (*bc) << ((prefixTreeCode == HY_HBL_COMMAND_BREAK) ? (countitems()-1) : (-(long)countitems()+1));
1953 currentLine = kEmptyString;
1954 } else {
1955 throw (currentLine.Enquote() & " only makes sense in the context of a loop.");
1956 }
1957 break;
1958 case HY_HBL_COMMAND_SET_DIALOG_PROMPT:
1959 case HY_HBL_COMMAND_HARVEST_FREQUENCIES:
1960 case HY_HBL_COMMAND_OPTIMIZE:
1961 case HY_HBL_COMMAND_COVARIANCE_MATRIX:
1962 case HY_HBL_COMMAND_LFCOMPUTE:
1963 case HY_HBL_COMMAND_SELECT_TEMPLATE_MODEL:
1964 case HY_HBL_COMMAND_USE_MODEL:
1965 case HY_HBL_COMMAND_SET_PARAMETER:
1966 case HY_HBL_COMMAND_ASSERT:
1967 case HY_HBL_COMMAND_REQUIRE_VERSION:
1968 case HY_HBL_COMMAND_DELETE_OBJECT:
1969 case HY_HBL_COMMAND_CLEAR_CONSTRAINTS:
1970 case HY_HBL_COMMAND_MOLECULAR_CLOCK:
1971 case HY_HBL_COMMAND_GET_URL:
1972 case HY_HBL_COMMAND_GET_STRING:
1973 case HY_HBL_COMMAND_EXPORT:
1974 case HY_HBL_COMMAND_DIFFERENTIATE:
1975 case HY_HBL_COMMAND_FPRINTF:
1976 case HY_HBL_COMMAND_GET_DATA_INFO:
1977 case HY_HBL_COMMAND_GET_INFORMATION:
1978 case HY_HBL_COMMAND_REPLICATE_CONSTRAINT:
1979 case HY_HBL_COMMAND_MPI_SEND:
1980 case HY_HBL_COMMAND_MPI_RECEIVE:
1981 case HY_HBL_COMMAND_FIND_ROOT:
1982 case HY_HBL_COMMAND_INTEGRATE:
1983 case HY_HBL_COMMAND_ALIGN_SEQUENCES:
1984 case HY_HBL_COMMAND_CONSTRUCT_CATEGORY_MATRIX:
1985 case HY_HBL_COMMAND_KEYWORD_ARGUMENT:
1986 case HY_HBL_COMMAND_DO_SQL:
1987 {
1988 _ElementaryCommand::ExtractValidateAddHBLCommand (currentLine, prefixTreeCode, pieces, commandExtraInfo, *this);
1989 break;
1990 }
1991 case HY_HBL_COMMAND_EXECUTE_A_FILE:
1992 case HY_HBL_COMMAND_EXECUTE_COMMANDS:
1993 case HY_HBL_COMMAND_LOAD_FUNCTION_LIBRARY: {
1994 BuildExecuteCommandInstruction (pieces, prefixTreeCode);
1995 }
1996 break;
1997
1998 case HY_HBL_COMMAND_FSCANF:
1999 case HY_HBL_COMMAND_SSCANF: {
2000 BuildFscanf (pieces, prefixTreeCode);
2001 }
2002 break;
2003
2004 case HY_HBL_COMMAND_CHOICE_LIST: {
2005 BuildChoiceList(pieces, HY_HBL_COMMAND_CHOICE_LIST);
2006 }
2007 break;
2008
2009 default :
2010 handled = false;
2011
2012 }
2013
2014
2015 // TODO 20111212: this horrendous switch statement should be replaced with a
2016 // prefix tree lookup
2017 if (handled) {
2018 if (currentLine.length() > 1UL) {
2019 throw (currentLine.Enquote() & " contained syntax errors, possibly a missing semicolon. " );
2020 }
2021 } else {
2022 if (currentLine.BeginsWith (blFunction)||currentLine.BeginsWith (blFFunction)||currentLine.BeginsWith (blLFunction) || currentLine.BeginsWith (blNameSpace) || currentLine.BeginsWith (blCFunction)) { // function declaration
2023 _ElementaryCommand::ConstructFunction (currentLine, *this);
2024 } else if (currentLine.BeginsWithAndIsNotAnIdent (blReturnPrefix)) { // function return statement
2025 //StringToConsole(currentLine); NLToConsole();
2026 _ElementaryCommand::ConstructReturn (currentLine, *this);
2027 } else if (currentLine.BeginsWith (blIf)) { // if-then-else statement
2028 _ElementaryCommand::BuildIfThenElse (currentLine, *this, bc);
2029 } else if (currentLine.BeginsWith (blElse)) { // else clause of an if-then-else statement
2030 if (lastif.countitems()) {
2031 long temp = countitems(),
2032 lc = lastif.countitems(),
2033 lif = lastif.list_data[lc-1];
2034
2035 _ElementaryCommand * stuff = new _ElementaryCommand ();
2036 stuff->MakeJumpCommand (nil,0,0,*this);
2037 AppendNewInstance (stuff);
2038 currentLine.Trim (4,-1);
2039
2040 long index = currentLine.length ()-1L,
2041 scopeIn = 0;
2042
2043 while (currentLine.char_at (scopeIn) =='{' && currentLine.char_at (index)=='}') {
2044 scopeIn++;
2045 index--;
2046 }
2047
2048 if (scopeIn) {
2049 currentLine.Trim (scopeIn,index);
2050 }
2051
2052 BuildList (currentLine,bc,true);
2053
2054 if (lif<0 || lif>=lLength) {
2055 throw _String("'else' w/o an if to latch on to...");
2056 }
2057
2058
2059 ((_ElementaryCommand*)((*this)(lif)))->MakeJumpCommand(nil,-1,temp+1,*this);
2060 ((_ElementaryCommand*)(*this)(temp))->simpleParameters[0]=countitems();
2061
2062 while (lastif.countitems()>=lc) {
2063 lastif.Delete(lastif.countitems()-1);
2064 }
2065 } else {
2066 throw (_String ("'else' w/o an 'if' to latch on to..."));
2067 }
2068
2069 } else if (currentLine.BeginsWith (blDo)) { // do {} while statement
2070 _ElementaryCommand::BuildDoWhile (currentLine, *this);
2071 } else if (currentLine.BeginsWith (blInclude)) { // #include
2072 _ElementaryCommand::ProcessInclude (currentLine, *this);
2073 } else if (currentLine.BeginsWith (blDataSet)) { // data set definition
2074 _ElementaryCommand::ConstructDataSet (currentLine, *this);
2075 } else if (currentLine.BeginsWith (blDataSetFilter)) { // data set filter definition
2076 _ElementaryCommand::ConstructDataSetFilter (currentLine, *this);
2077 } else if (currentLine.BeginsWith (blTree) || currentLine.BeginsWith (blTopology)) { // tree definition
2078 _ElementaryCommand::ConstructTree (currentLine, *this);
2079 } else if (currentLine.BeginsWith (blLF) || currentLine.BeginsWith (blLF3)) { // LF definition
2080 _ElementaryCommand::ConstructLF (currentLine, *this);
2081 } else if (currentLine.BeginsWith (blCategory)) { // category variable declaration
2082 _ElementaryCommand::ConstructCategory (currentLine, *this);
2083 } else if (currentLine.BeginsWith (blModel)) { // Model declaration
2084 _ElementaryCommand::ConstructModel (currentLine, *this);
2085 } else if (currentLine.BeginsWith (blChoiceList)) { // choice list
2086 _ElementaryCommand::ConstructChoiceList (currentLine, *this);
2087 } else if (currentLine.BeginsWith (blHBLProfile)) { // #profile
2088 _ElementaryCommand::ConstructProfileStatement (currentLine, *this);
2089 } else if (currentLine.BeginsWith (blSCFG)) { // SCFG definition
2090 _ElementaryCommand::ConstructSCFG (currentLine, *this);
2091 } else if (currentLine.BeginsWith (blBGM)) { // Bayesian Graphical Model definition
2092 _ElementaryCommand::ConstructBGM (currentLine, *this);
2093 }
2094 // plain ol' formula - parse it as such!
2095 else {
2096 _StringBuffer checker (currentLine);
2097 _StringBuffer next_command;
2098 _ElementaryCommand::FindNextCommand (checker,next_command);
2099 if (next_command.length ()==currentLine.length()) {
2100 if (currentLine.length()>1)
2101 while (currentLine (-1L) ==';') {
2102 currentLine.Trim (0,currentLine.length()-2);
2103 }
2104 else {
2105 continue;
2106 }
2107 _ElementaryCommand* oddCommand = new _ElementaryCommand(currentLine);
2108 oddCommand->code = 0;
2109 oddCommand->parameters.AppendNewInstance (new _String (currentLine));
2110 AppendNewInstance (oddCommand);
2111 } else {
2112 while (currentLine.nonempty()) {
2113 _ElementaryCommand::FindNextCommand (currentLine,next_command);
2114 BuildList (next_command,bc,processed);
2115 }
2116 }
2117 }
2118 }
2119 }
2120 } catch (_String const & error) {
2121 if (currentExecutionList) {
2122 currentExecutionList->ReportAnExecutionError(error, false, true);
2123 } else {
2124 HandleApplicationError(error);
2125 }
2126 }
2127 // s.sData = savePointer;
2128 // TODO: SLKP 20170623 why is this here? 20170704 ; for the "soft trim" situation, which we won't be using any more
2129 s.Clear();
2130 return empty_is_success || countitems();
2131 }
2132
2133 //____________________________________________________________________________________
2134
_ElementaryCommand(void)2135 _ElementaryCommand::_ElementaryCommand (void) {
2136 code = -1;
2137 }
2138
2139 //____________________________________________________________________________________
2140
_ElementaryCommand(long ccode)2141 _ElementaryCommand::_ElementaryCommand (long ccode) {
2142 code = ccode;
2143 }
2144
2145 //____________________________________________________________________________________
2146
_ElementaryCommand(_String & s)2147 _ElementaryCommand::_ElementaryCommand (_String& s) {
2148 code = -1;
2149 _String::Duplicate (&s);
2150 }
2151
2152 //____________________________________________________________________________________
~_ElementaryCommand(void)2153 _ElementaryCommand::~_ElementaryCommand (void) {
2154 if (CanFreeMe()) {
2155 if (code==4) {
2156 if (simpleParameters.lLength>2) {
2157 _Formula* f = (_Formula*)simpleParameters(2);
2158 delete (f);
2159 }
2160 } else if (code==0) {
2161 if (simpleParameters.lLength) {
2162
2163 _Formula* f = (_Formula*)simpleParameters(2);
2164 delete (f);
2165 f = (_Formula*)simpleParameters(1);
2166 delete(f);
2167 simpleParameters.Clear();
2168 }
2169 } else if ((code==6)||(code==9)) {
2170 for (long i = 0; i<simpleParameters.lLength; i++) {
2171 _Formula* f = (_Formula*)simpleParameters(i);
2172 delete (f);
2173 }
2174 } else if (code == HY_HBL_COMMAND_INIT_ITERATOR) {
2175 if (simpleParameters.get (1) == ASSOCIATIVE_LIST && simpleParameters.countitems() > 2) {
2176 delete (AVLListXLIterator*)simpleParameters.get(2);
2177 } else {
2178 if ((simpleParameters.get (1) == TREE || simpleParameters.get (1) == TOPOLOGY) && simpleParameters.countitems() > 2) {
2179 delete (node_iterator<long>*)simpleParameters.get(2);
2180 }
2181 }
2182 }
2183 }
2184
2185 }
2186
2187 //____________________________________________________________________________________
makeDynamic(void) const2188 BaseRef _ElementaryCommand::makeDynamic (void) const
2189 {
2190 _ElementaryCommand * nec = new _ElementaryCommand;
2191 nec->code = code;
2192 nec->Duplicate (this);
2193 return nec;
2194 }
2195
2196 //____________________________________________________________________________________
2197
Duplicate(BaseRefConst source)2198 void _ElementaryCommand::Duplicate (BaseRefConst source)
2199 {
2200 _ElementaryCommand* sec = (_ElementaryCommand*)source;
2201 _String::Duplicate (source);
2202
2203 parameters.Duplicate(&sec->parameters);
2204 if (code != 0) {
2205 simpleParameters.Duplicate(&sec->simpleParameters);
2206 }
2207 }
2208
2209 //____________________________________________________________________________________
2210
_hblCommandAccessor(_ExecutionList * theList,long index)2211 const _String _hblCommandAccessor (_ExecutionList* theList, long index) {
2212 if (theList) {
2213 if (index >= 0) {
2214 if (index < theList->lLength) {
2215 _ElementaryCommand * aCommand = (_ElementaryCommand*)theList->GetItem (index);
2216 return _String ((_String*)aCommand->toStr());
2217 } else {
2218 return _String("<END EXECUTION>");
2219 }
2220 }
2221 }
2222 return _String ("command index ") & index;
2223 }
2224
2225 //____________________________________________________________________________________
2226
toStr(unsigned long)2227 BaseRef _ElementaryCommand::toStr (unsigned long) {
2228
2229 auto parameter_to_string = [&] (unsigned long i) -> _String const {
2230 return _String ((_String*)GetIthParameter(i)->toStr());
2231 };
2232
2233 auto procedure = [&] (long i) -> _String const {
2234
2235 _String command (_HY_ValidHBLExpressions.RetrieveKeyByPayload(i));
2236
2237 if (command.EndsWith('(')) {
2238 return _StringBuffer (command)
2239 << _String ((_String*)parameters.Join (", ")) << ");";
2240
2241 } else {
2242 return _StringBuffer (command)
2243 << '(' << _String ((_String*)parameters.Join (", ")) << ");";
2244 }
2245 };
2246
2247 auto assignment = [&] (long i, const _String& call) -> _String const {
2248 return _StringBuffer (_HY_ValidHBLExpressions.RetrieveKeyByPayload(i))
2249 << parameter_to_string (0)
2250 << " = "
2251 << call
2252 << _String ((_String*)parameters.Join (", ", 1)).Enquote('(', ')')
2253 << ";";
2254 };
2255
2256 auto hash_pragma = [&] (long i) {
2257 return _StringBuffer (_HY_ValidHBLExpressions.RetrieveKeyByPayload(i))
2258 << _String ((_String*)parameters.Join (", ")) << ";";
2259 };
2260
2261 _StringBuffer* string_form = new _StringBuffer (256);
2262
2263 switch (code) {
2264
2265 case HY_HBL_COMMAND_FORMULA: // formula reparser
2266 (*string_form) << parameter_to_string (0) << ";";
2267 break;
2268
2269 case 4: {
2270 if (simpleParameters.countitems()==3 || parameters.countitems() == 1) {
2271 (*string_form) << "Branch under condition "
2272 << parameter_to_string (0).Enquote()
2273 << "\n\tto\n\t\t"
2274 << _hblCommandAccessor (currentExecutionList,simpleParameters(0))
2275 << "\n\telse\n\t\t"
2276 << _hblCommandAccessor (currentExecutionList,simpleParameters(1));
2277 } else {
2278 (*string_form) << "Go to step " << _String (simpleParameters(0));
2279 }
2280 }
2281 break;
2282
2283 case 5: { // data set contruction
2284 (*string_form) << assignment (HY_HBL_COMMAND_DATA_SET, "ReadDataFile");
2285 }
2286 break;
2287
2288 case 6: { // data set filter
2289 (*string_form) << assignment (HY_HBL_COMMAND_DATA_SET_FILTER, "CreateFilter");
2290 }
2291 break;
2292
2293 case HY_HBL_COMMAND_HARVEST_FREQUENCIES:
2294 case HY_HBL_COMMAND_FPRINTF:
2295 case HY_HBL_COMMAND_OPTIMIZE: // optimize the likelihood function
2296 case HY_HBL_COMMAND_COVARIANCE_MATRIX: // compute the covariance matrix
2297 case HY_HBL_COMMAND_EXPORT:
2298 case HY_HBL_COMMAND_MOLECULAR_CLOCK:
2299 case HY_HBL_COMMAND_CLEAR_CONSTRAINTS:
2300 case HY_HBL_COMMAND_SET_DIALOG_PROMPT:
2301 case HY_HBL_COMMAND_USE_MODEL:
2302 case HY_HBL_COMMAND_GET_STRING:
2303 case HY_HBL_COMMAND_SET_PARAMETER:
2304 case HY_HBL_COMMAND_DIFFERENTIATE:
2305 case HY_HBL_COMMAND_LFCOMPUTE:
2306 case HY_HBL_COMMAND_GET_URL:
2307 case HY_HBL_COMMAND_DELETE_OBJECT:
2308 case HY_HBL_COMMAND_REQUIRE_VERSION:
2309 case HY_HBL_COMMAND_ASSERT:
2310 case HY_HBL_COMMAND_FIND_ROOT:
2311 case HY_HBL_COMMAND_INTEGRATE:
2312 case HY_HBL_COMMAND_GET_DATA_INFO:
2313 case HY_HBL_COMMAND_CONSTRUCT_CATEGORY_MATRIX:
2314 case HY_HBL_COMMAND_ALIGN_SEQUENCES:
2315 case HY_HBL_COMMAND_REPLICATE_CONSTRAINT:
2316 case HY_HBL_COMMAND_MPI_RECEIVE:
2317 case HY_HBL_COMMAND_MPI_SEND :
2318 case HY_HBL_COMMAND_EXECUTE_A_FILE :
2319 case HY_HBL_COMMAND_EXECUTE_COMMANDS :
2320 case HY_HBL_COMMAND_LOAD_FUNCTION_LIBRARY :
2321 case HY_HBL_COMMAND_DO_SQL:
2322 case HY_HBL_COMMAND_CHOICE_LIST:
2323 case HY_HBL_COMMAND_SELECT_TEMPLATE_MODEL:
2324 case HY_HBL_COMMAND_KEYWORD_ARGUMENT:
2325 case HY_HBL_COMMAND_GET_INFORMATION:
2326 case HY_HBL_COMMAND_SIMULATE_DATA_SET: {
2327 (*string_form) << procedure (code);
2328 }
2329
2330 break;
2331
2332 case 7: // build a tree
2333 case 54: { // build a tree
2334 (*string_form) << assignment (code == 7 ? HY_HBL_COMMAND_TREE : HY_HBL_COMMAND_TOPOLOGY, kEmptyString);
2335
2336 }
2337 break;
2338
2339
2340 case 11: { // build the likelihood function
2341 (*string_form) << assignment (HY_HBL_COMMAND_LIKELIHOOD_FUNCTION, kEmptyString);
2342 }
2343 break;
2344
2345
2346 case 13: { // a function
2347 (*string_form) << "function "
2348 << parameter_to_string(0)
2349 << " ( "
2350 << _String ((_String*)parameters.Join (", ", 1,parameters.countitems()-2L))
2351 << " ) {\n" << parameter_to_string (parameters.countitems()-1L) << "\n}";
2352 }
2353 break;
2354
2355 case 14: { // return statement
2356 (*string_form) << "return "
2357 << parameter_to_string(0)
2358 << ";";
2359 }
2360 break;
2361
2362 case 16: { // data set merger
2363 (*string_form) << assignment (HY_HBL_COMMAND_DATA_SET, labs(simpleParameters(0))==1 ? "Combine" : "Concatenate");
2364 /*if (simpleParameters (0)<0) {
2365 string_form << "(deleting arguments upon completion)";
2366 }*/
2367 }
2368 break;
2369
2370
2371 case 20: {// category variable construction
2372 (*string_form) << assignment (HY_HBL_COMMAND_CATEGORY, kEmptyString);
2373 }
2374 break;
2375
2376 case 24: { // select standard model
2377 (*string_form) << procedure (HY_HBL_COMMAND_SELECT_TEMPLATE_MODEL);
2378 }
2379 break;
2380
2381 case HY_HBL_COMMAND_FSCANF: // fscanf
2382 case HY_HBL_COMMAND_SSCANF: { // sscanf
2383 (*string_form) << (code == HY_HBL_COMMAND_FSCANF ? "fscanf(" : "sscanf(")
2384 << parameter_to_string (0)
2385 << ",\"";
2386
2387 long shift = 1L;
2388
2389 for (long p = 0; p<simpleParameters.lLength; p++) {
2390 long theFormat = simpleParameters(p);
2391 if (theFormat < 0) {
2392 (*string_form) << "REWIND";
2393 } else {
2394 (*string_form) << *((_String*)_ElementaryCommand::fscanf_allowed_formats.GetItem (theFormat));
2395 }
2396 if (p) {
2397 (*string_form) << ", ";
2398 }
2399 }
2400 (*string_form) << "\",";
2401 for (long p = 0; p<simpleParameters.lLength; p++) {
2402 long theFormat = simpleParameters(p);
2403 if (theFormat < 0) {
2404 shift++;
2405 } else {
2406 (*string_form) << parameter_to_string (p+shift);
2407 }
2408 if (p) {
2409 (*string_form) << ", ";
2410 }
2411 }
2412 (*string_form) << ");";
2413 }
2414 break;
2415
2416
2417 case 31: { // define a model
2418 (*string_form) << procedure (HY_HBL_COMMAND_MODEL);
2419 }
2420 break;
2421
2422
2423 case 38: { // reconsruct ancestors
2424 (*string_form) << assignment (HY_HBL_COMMAND_DATA_SET, "ReconstuctAncestors");
2425
2426 }
2427 break;
2428
2429 case 47: { //GetDataInfo
2430 (*string_form) << procedure (HY_HBL_COMMAND_STATE_COUNTER);
2431 }
2432 break;
2433
2434 case 52: { //Simulate
2435 (*string_form) << assignment (HY_HBL_COMMAND_DATA_SET, "Simulate");
2436 }
2437 break;
2438
2439
2440 case 58: {
2441 (*string_form) << hash_pragma (HY_HBL_COMMAND_PROFILE);
2442 }
2443 break;
2444
2445 case 61: {
2446 (*string_form) << assignment (HY_HBL_COMMAND_SCFG, kEmptyString);
2447 }
2448 break;
2449
2450 case 64: {
2451 (*string_form) << assignment (HY_HBL_COMMAND_BGM, kEmptyString);
2452 }
2453 break;
2454
2455 case HY_HBL_COMMAND_NESTED_LIST: {
2456 (*string_form) << "namespace " << parameter_to_string (0) << ";";
2457 break;
2458 }
2459
2460 case HY_HBL_COMMAND_INIT_ITERATOR: {
2461 (*string_form) << "Initialize iterator on " << parameter_to_string (0) << ";";
2462 break;
2463 }
2464
2465 case HY_HBL_COMMAND_ADVANCE_ITERATOR: {
2466 (*string_form) << "Advance iterator into ";
2467 (string_form -> AppendNewInstance(parameters.Join(",",0,simpleParameters.get (1)))) << ";";
2468 break;
2469 }
2470
2471 }
2472 return string_form;
2473 }
2474
2475 //____________________________________________________________________________________
2476
ExecuteCase0(_ExecutionList & chain)2477 void _ElementaryCommand::ExecuteCase0 (_ExecutionList& chain) {
2478 chain.currentCommand++;
2479
2480 _String * errMsg = nil;
2481
2482 try {
2483
2484 if (chain.is_compiled(chain.currentCommand)) {
2485 //if (chain.is_compiled(0) == false) {
2486 // PopulateArraysForASimpleFormula (chain.cli->varList, chain.cli->values);
2487 //}
2488 hyFloat result = ((_Formula*)simpleParameters.list_data[1])->ComputeSimple (chain.cli->stack, chain.cli->values);
2489 long sti = chain.cli->storeResults.list_data[chain.currentCommand-1];
2490 if (sti>=0) {
2491 chain.cli->values[sti].value = result;
2492 }
2493 return;
2494 }
2495
2496 if (!simpleParameters.lLength) { // not compiled yet
2497 _Formula f,
2498 f2;
2499
2500 _String* theFla = (_String*)parameters(0),
2501 err_msg;
2502
2503 _FormulaParsingContext fpc (&err_msg, chain.nameSpacePrefix);
2504
2505 long parseCode = Parse(&f,(*theFla),fpc,&f2);
2506
2507 //printf ("RHS = %s\n", _String ((_String*)f2.toStr(kFormulaStringConversionNormal)).get_str());
2508
2509 if (parseCode != HY_FORMULA_FAILED ) {
2510 if (fpc.isVolatile() == false) { // not a matrix constant
2511 simpleParameters <<parseCode
2512 <<long (f.makeDynamic())
2513 <<long (f2.makeDynamic())
2514 <<fpc.assignmentRefID ()
2515 <<fpc.assignmentRefType ();
2516
2517 appendCompiledFormulae (&f, &f2);
2518
2519 } else {
2520 ExecuteFormula(&f,&f2,parseCode,fpc.assignmentRefID(),chain.nameSpacePrefix,fpc.assignmentRefType());
2521 if (terminate_execution) {
2522 errMsg = new _String ("Error computing the compiled statement: ");
2523 throw 0;
2524 }
2525 return;
2526 }
2527 } else {
2528 errMsg = new _String (_String ("Parsing error ") & _String (err_msg.Enquote('(',')') & " while compiling the statement: "));
2529 throw 0;
2530 }
2531 }
2532
2533 ExecuteFormula ((_Formula*)simpleParameters.list_data[1],(_Formula*)simpleParameters.list_data[2],simpleParameters.list_data[0],simpleParameters.list_data[3], chain.nameSpacePrefix, simpleParameters.list_data[4]);
2534
2535 if (terminate_execution) {
2536 errMsg = new _String ("Error computing the interpreted statement: ");
2537 throw 0;
2538 }
2539
2540 } catch (int e) {
2541 if (errMsg) {
2542 HandleApplicationError (_String(errMsg) & *this);
2543 }
2544 }
2545 }
2546
2547
2548 //____________________________________________________________________________________
2549
ExecuteCase4(_ExecutionList & chain)2550 void _ElementaryCommand::ExecuteCase4 (_ExecutionList& chain) {
2551 chain.currentCommand++;
2552
2553 _Formula * expression = nil;
2554 _String * errMsg = nil;
2555
2556 try {
2557 if (simpleParameters.lLength==3 || parameters.lLength) {
2558
2559
2560 if ( parameters.lLength && simpleParameters.lLength < 3) {
2561 expression = new _Formula;
2562 //printf ("Namespace: %x\nCode: %s\n", chain.nameSpacePrefix, ((_String*)parameters(0))->sData);
2563
2564 _FormulaParsingContext fpc (nil, chain.nameSpacePrefix);
2565 long status = Parse (expression, *(_String*)parameters(0), fpc, nil);
2566
2567 //printf ("Print formula: %s\n", _String((_String*)expression->toStr()).sData);
2568
2569 if (status== HY_FORMULA_EXPRESSION) {
2570 if (fpc.isVolatile() == false) {
2571 simpleParameters << (long)expression;
2572 appendCompiledFormulae (expression);
2573 expression = nil;
2574 }
2575 } else {
2576 errMsg = new _String (" is not a valid conditional expression");
2577 throw (0);
2578 }
2579 }
2580
2581 if (chain.is_compiled(chain.currentCommand)) {
2582 //if (chain.is_compiled(0) == false) {
2583 // PopulateArraysForASimpleFormula (chain.cli->varList, chain.cli->values);
2584 //}
2585 if ( ((_Formula*)simpleParameters(2))->ComputeSimple(chain.cli->stack, chain.cli->values)==0.0) {
2586 chain.currentCommand = simpleParameters.list_data[1];
2587 return;
2588 }
2589 } else {
2590 HBLObjectRef result;
2591 if (expression) {
2592 //printf ("\n*** Interpreted condition\n");
2593 result = expression->Compute(0, nil, nil, nil, HY_ANY_OBJECT, false);
2594 } else {
2595 result = ((_Formula*)simpleParameters(2))->Compute(0, nil, nil, nil, HY_ANY_OBJECT, false);
2596 }
2597
2598 // printf ("\n*** %s\n", ((_String*)result->toStr())->sData);
2599
2600 if (terminate_execution && !result) {
2601 _String *s = (_String*)((_Formula*)simpleParameters(2))->toStr(kFormulaStringConversionSubstiteValues);
2602 errMsg = new _String(_String("Failed while evaluating: ") & _String((_String*)((_Formula*)simpleParameters(2))->toStr(kFormulaStringConversionNormal)) & " which expanded to " & s);
2603 throw (1);
2604 }
2605
2606 bool conditionFalse = false;
2607
2608 switch (result->ObjectClass()) {
2609 case NUMBER:
2610 conditionFalse = result->Value()==0.0;
2611 break;
2612 case STRING:
2613 conditionFalse = ((_FString*)result)->empty();
2614 break;
2615 case HY_UNDEFINED:
2616 conditionFalse = true;
2617 break;
2618 default:
2619 errMsg = new _String(_String(" did not evaluate to a number, a string, or a null (") & (_String*)result->toStr() & ")");
2620 throw (0);
2621 }
2622
2623 if (expression) {
2624 delete expression;
2625 }
2626
2627 if (conditionFalse) {
2628 chain.currentCommand = simpleParameters.list_data[1];
2629 return;
2630 }
2631 }
2632 }
2633 chain.currentCommand = simpleParameters.list_data[0];
2634
2635 if (chain.currentCommand == -1) {
2636 terminate_execution = true;
2637 chain.currentCommand = chain.lLength;
2638 }
2639 }
2640 catch (int e) {
2641 if (expression) {
2642 delete expression;
2643 }
2644 if (errMsg) {
2645 if (e == 0) {
2646 HandleApplicationError (_String ("'") & *(_String*)parameters(0) & "'" & errMsg);
2647 } else {
2648 HandleApplicationError (errMsg);
2649 }
2650 // note that errMsg will be deleted by _String (*_String) constructors
2651 }
2652 }
2653 }
2654
2655 //____________________________________________________________________________________
2656
ExecuteCase5(_ExecutionList & chain)2657 void _ElementaryCommand::ExecuteCase5 (_ExecutionList& chain) {
2658 chain.currentCommand++;
2659 hyFile* df;
2660 _String fName (*GetIthParameter(1));
2661 _DataSet*ds;
2662
2663
2664
2665 if (simpleParameters.lLength == 1) {
2666 fName = GetStringFromFormula ((_String*)parameters(1),chain.nameSpacePrefix);
2667 ds = ReadDataSetFile (nil,0,&fName,nil,chain.nameSpacePrefix?chain.nameSpacePrefix->GetName():nil);
2668 } else {
2669 if (fName == useNexusFileData) {
2670 if (!lastNexusDataMatrix) {
2671 HandleApplicationError (useNexusFileData & " was used in ReadDataFile, and no NEXUS data matrix was available.");
2672 return;
2673 }
2674 ds = lastNexusDataMatrix;
2675 } else {
2676 ProcessFileName(fName, false,true,(hyPointer)chain.nameSpacePrefix, false, &chain, true);
2677 if (terminate_execution) {
2678 return;
2679 }
2680 SetStatusLine ("Loading Data");
2681
2682 df = hyFile::openFile (fName.get_str(),"rb");
2683 if (df==nil) {
2684 // try reading this file as a string formula
2685 fName = GetStringFromFormula ((_String*)parameters(1),chain.nameSpacePrefix);
2686 ProcessFileName(fName, false,false,(hyPointer)chain.nameSpacePrefix, false, &chain, true);
2687
2688 if (terminate_execution) {
2689 return;
2690 }
2691
2692 df = hyFile::openFile (fName.get_str(),"rb");
2693 if (df==nil) {
2694 HandleApplicationError ((_String ("Could not find source dataset file ") & ((_String*)parameters(1))->Enquote('"')
2695 & " (resolved to '" & fName & "')\nPath stack:\n\t" & GetPathStack ("\n\t")));
2696 return;
2697 }
2698 }
2699 ds = ReadDataSetFile (df,0,nil,nil,chain.nameSpacePrefix?chain.nameSpacePrefix->GetName():nil);
2700 if (df) {
2701 df->close();
2702 delete df;
2703 }
2704 }
2705 }
2706
2707
2708 // 20110802: need to check that this data set is not empty
2709
2710 if (ds->NoOfSpecies() && ds->NoOfColumns()) {
2711 _String * dsID = new _String (chain.AddNameSpaceToID(*(_String*)parameters(0)));
2712 StoreADataSet (ds, dsID);
2713 DeleteObject (dsID);
2714 } else {
2715 DeleteObject (ds);
2716 HandleApplicationError ("The format of the sequence file has not been recognized and may be invalid");
2717 }
2718
2719 //StoreADataSet (ds, (_String*)parameters(0));
2720 }
2721
2722
2723 //____________________________________________________________________________________
2724
ExecuteCase11(_ExecutionList & chain)2725 void _ElementaryCommand::ExecuteCase11 (_ExecutionList& chain)
2726 /*
2727 code cleanup SLKP 20090316
2728 */
2729
2730 {
2731 chain.currentCommand++;
2732
2733 _String parm,
2734 errMsg;
2735
2736 bool explicitFreqs = simpleParameters.lLength,
2737 // if false then the likelihood function will be of the form (filter1,tree1,filter2,tree2,...,filterN,treeN)
2738 // if true then the likelihood function will be of the form (filter1,tree1,freq1,filter2,tree2,freq2,...,filterN,treeN,freqN)
2739 assumeList = parameters.lLength > 2;
2740 // if there is only one parameter to the function constructor, it is assumed to be a string matrix
2741 // otherwise it is expected to be a collection of literals
2742
2743 _List * likelihoodFunctionSpec = nil,
2744 passThisToLFConstructor;
2745
2746 if (assumeList) {
2747 likelihoodFunctionSpec = new _List (parameters, 1, - 1);
2748 } else {
2749 _Matrix * matrixOfStrings = (_Matrix*)ProcessAnArgumentByType ((_String*)parameters(1), chain.nameSpacePrefix, MATRIX);
2750 if (matrixOfStrings && matrixOfStrings->IsAStringMatrix()) {
2751 likelihoodFunctionSpec = new _List;
2752 matrixOfStrings->FillInList(*likelihoodFunctionSpec);
2753 if (likelihoodFunctionSpec->lLength == 0) {
2754 DeleteObject (likelihoodFunctionSpec);
2755 likelihoodFunctionSpec = nil;
2756 }
2757 DeleteObject (matrixOfStrings);
2758 }
2759 if (likelihoodFunctionSpec == nil) {
2760 HandleApplicationError (_String("Not a valid string matrix object passed to a _LikelihoodFunction constructor: ") & *(_String*)parameters(1));
2761 return;
2762 }
2763 }
2764
2765 long i = 0,
2766 stepper = explicitFreqs?3:2;
2767
2768 for (; i<=(long)likelihoodFunctionSpec->lLength-stepper; i+=stepper) {
2769 _String *dataset = (_String*)(*likelihoodFunctionSpec)(i),
2770 *tree = (_String*)(*likelihoodFunctionSpec)(i+1),
2771 *freq = explicitFreqs?(_String*)(*likelihoodFunctionSpec)(i+2):nil;
2772
2773 if(GetDataFilter (AppendContainerName(*dataset,chain.nameSpacePrefix))) {
2774 _TheTree* thisTree = (_TheTree*)FetchObjectFromVariableByType(&AppendContainerName(*tree,chain.nameSpacePrefix),TREE);
2775 if (thisTree) {
2776 _TreeIterator ti (thisTree, _HY_TREE_TRAVERSAL_POSTORDER);
2777 if (!freq) { // no explicit frequency parameter; grab one from the tree
2778 long theFreqID = -1,
2779 theModelID = -1,
2780 finalFreqID = -1;
2781 bool done = false;
2782
2783 while (1) {
2784 _CalcNode *thisNode = ti.Next();
2785
2786 if ((theModelID = thisNode->GetModelIndex()) == HY_NO_MODEL) { // this node has no model
2787 done = false;
2788 break;
2789 }
2790 theFreqID = modelFrequenciesIndices.list_data[theModelID];
2791
2792 while((thisNode = ti.Next()) && !ti.IsAtRoot()) {
2793 theModelID = thisNode->GetModelIndex();
2794 if (theModelID == HY_NO_MODEL) { // no model
2795 done = false;
2796 break;
2797 }
2798 if (modelFrequenciesIndices.list_data[theModelID]!=theFreqID) {
2799 done = true;
2800 break;
2801 }
2802 }
2803 if (theFreqID<0) {
2804 finalFreqID = -theFreqID-1;
2805 } else {
2806 finalFreqID = theFreqID;
2807 }
2808 break;
2809 }
2810
2811 if (finalFreqID>=0) {
2812 _String freqID = chain.TrimNameSpaceFromID(*LocateVar(finalFreqID)->GetName());
2813 passThisToLFConstructor && dataset;
2814 passThisToLFConstructor && tree;
2815 passThisToLFConstructor && freqID;
2816 continue;
2817 } else {
2818 if (!done) {
2819 errMsg = (((_String)("LF: Not a well-defined tree/model combination: ")&*tree));
2820 } else {
2821 errMsg = (((_String)("LF: All models in the tree: ")&*tree&_String(" must share the same frequencies vector")));
2822 }
2823 }
2824 } else {
2825 if (FetchObjectFromVariableByType(&AppendContainerName(*freq,chain.nameSpacePrefix),MATRIX)) {
2826 passThisToLFConstructor && dataset;
2827 passThisToLFConstructor && tree;
2828 passThisToLFConstructor && freq;
2829 continue;
2830 }
2831 errMsg = (((_String)("LF: Not a valid frequency matrix ID: ")& *freq));
2832 }
2833 } else {
2834 errMsg = (((_String)("LF: Not a valid tree ID: `")& *tree & "`"));
2835 }
2836
2837 } else {
2838 errMsg = (((_String)("LF: Not a valid dataset filter `")& *dataset & "`"));
2839 }
2840
2841 if (errMsg.nonempty()) {
2842 DeleteObject (likelihoodFunctionSpec);
2843 HandleApplicationError (errMsg);
2844 return;
2845 }
2846 }
2847
2848 if (i==likelihoodFunctionSpec->lLength-1) { // computing template
2849 passThisToLFConstructor && *((_String*)(*likelihoodFunctionSpec)(i));
2850 }
2851
2852
2853 DeleteObject (likelihoodFunctionSpec);
2854
2855
2856 _String lfID = chain.AddNameSpaceToID(*(_String*)parameters(0)); // the ID of the likelihood function
2857 long likeFuncObjectID = FindLikeFuncName (lfID);
2858 if (likeFuncObjectID==-1)
2859 // not an existing LF ID
2860 {
2861 _LikelihoodFunction * lkf = new _LikelihoodFunction ();
2862 if (! lkf->Construct(passThisToLFConstructor,chain.nameSpacePrefix))
2863 // constructor failed
2864 {
2865 DeleteObject (lkf);
2866 } else {
2867 likeFuncObjectID = likeFuncNamesList.FindObject(&kEmptyString);
2868 // see if there are any vacated spots in the list
2869
2870 if (likeFuncObjectID < 0) {
2871 likeFuncList << lkf;
2872 likeFuncNamesList&&(&lfID);
2873 DeleteObject (lkf);
2874 } else {
2875 likeFuncNamesList.Replace(likeFuncObjectID,&lfID,true);
2876 likeFuncList.list_data[likeFuncObjectID] = (long)lkf;
2877 }
2878 }
2879 } else {
2880 _LikelihoodFunction* lkf = (_LikelihoodFunction*)likeFuncList(likeFuncObjectID);
2881 if (!lkf->Construct(passThisToLFConstructor,chain.nameSpacePrefix)) {
2882 KillLFRecord (likeFuncObjectID,false);
2883 }
2884 }
2885
2886 }
2887
2888
2889
2890 //____________________________________________________________________________________
2891
ExecuteCase12(_ExecutionList & chain)2892 void _ElementaryCommand::ExecuteCase12 (_ExecutionList& chain)
2893 {
2894 chain.currentCommand++;
2895 SetStatusLine ("Simulating Data");
2896
2897 _String likefID = chain.AddNameSpaceToID(*(_String*)parameters(1)),
2898 errMsg;
2899
2900
2901 long f = FindLikeFuncName (likefID),
2902 s2 = FindSCFGName (likefID);
2903
2904 if (f==-1 && s2==-1) {
2905 HandleApplicationError (_String("Likelihood Function (or SCFG)")&likefID& " has not been initialized" );
2906 return ;
2907 }
2908
2909 if (f>=0) {
2910 _DataSet * ds = new _DataSet;
2911
2912 _List theExclusions;
2913
2914 if (parameters.lLength>2) // there is a list of exclusions there
2915 // ';'-sep for different partititons
2916 // ','-sep for states in a given partition
2917 {
2918 // SLKP mod 20070622 to allow string expressions as well
2919 _String theExc (ProcessLiteralArgument((_String*)parameters(2),chain.nameSpacePrefix));
2920 if (theExc.nonempty()) {
2921 long f = theExc.Find(';'),
2922 g = 0;
2923
2924 while(1) {
2925 _String subExc (theExc,g,(f==-1)?(-1):(f-1));
2926 long h = subExc.Find(','),
2927 l = 0;
2928 _List myExc;
2929
2930 while(1) {
2931 _String excludeMe (subExc,l,(h==-1)?(-1):(h-1));
2932 myExc && & excludeMe;
2933 if (h==-1) {
2934 break;
2935 }
2936 l = h+1;
2937 h = subExc.Find(',',h+1,-1);
2938 }
2939 theExclusions&& &myExc;
2940 if (f==-1) {
2941 break;
2942 }
2943 g = f+1;
2944 f = theExc.Find(';',f+1,-1);
2945 }
2946 }
2947
2948 }
2949
2950 _Matrix * catValues = nil,
2951 * catNames = nil;
2952
2953 _Variable* catValVar = nil,
2954 * catNameVar = nil;
2955
2956
2957
2958
2959 if (parameters.lLength>3) {
2960 // a matrix to store simulated category values
2961 _String matrixName (chain.AddNameSpaceToID(*(_String*)parameters(3)));
2962
2963 if (!(catValVar = CheckReceptacle(&matrixName,blSimulateDataSet,true))) {
2964 DeleteObject (ds);
2965 return;
2966 } else {
2967 catValues = new _Matrix (1,1,false,true);
2968 }
2969 }
2970
2971 if (parameters.lLength>4) {
2972 // a matrix to store simulated category values
2973 _String matrixName (chain.AddNameSpaceToID(*(_String*)parameters(4)));
2974 if (!(catNameVar = CheckReceptacle(&matrixName,blSimulateDataSet,true))) {
2975 DeleteObject (catValues);DeleteObject (catNames); DeleteObject (ds);
2976 return;
2977 } else {
2978 catNames = new _Matrix (1,1,false,true);
2979 }
2980 }
2981
2982 _String * resultingDSName = new _String (chain.AddNameSpaceToID(*(_String*)parameters(0)));
2983
2984 if (!resultingDSName->IsValidIdentifier(fIDAllowCompound)) {
2985 errMsg = *resultingDSName & " is not a valid receptacle identifier in call to " & blSimulateDataSet;
2986 DeleteObject (resultingDSName);DeleteObject (catValues);DeleteObject (catNames); DeleteObject (ds);
2987 HandleApplicationError (errMsg);
2988 return;
2989 }
2990
2991 ((_LikelihoodFunction*)likeFuncList(f))->Simulate(*ds,theExclusions,catValues,catNames);
2992
2993 if (catValues) {
2994 catValVar->SetValue(catValues,false,true,NULL);
2995 }
2996 if (catNames) {
2997 catNameVar->SetValue(catNames,false,true,NULL);
2998 }
2999
3000 StoreADataSet (ds, resultingDSName);
3001 DeleteObject (resultingDSName);
3002 } else {
3003 _String newCorpus = chain.AddNameSpaceToID(*(_String*)parameters(0));
3004 CheckReceptacleAndStore (&newCorpus," SimulateDataSet (SCFG)", true, new _FString(((Scfg*)scfgList (s2))->SpawnRandomString()), false);
3005 }
3006 }
3007
3008 //____________________________________________________________________________________
3009
ExecuteCase47(_ExecutionList & chain)3010 void _ElementaryCommand::ExecuteCase47 (_ExecutionList& chain) {
3011 chain.currentCommand++;
3012
3013 _String *arg1 = GetIthParameter(0),
3014 *arg2 = GetIthParameter(1);
3015
3016 try {
3017 long type = HY_BL_LIKELIHOOD_FUNCTION;
3018 _LikelihoodFunction const * lf = (_LikelihoodFunction const *)_HYRetrieveBLObjectByName(AppendContainerName(*arg1, chain.nameSpacePrefix), type );
3019 if (lf) {
3020 type = HY_BL_HBL_FUNCTION;
3021 long function_index;
3022 if (_HYRetrieveBLObjectByName(ProcessLiteralArgument (arg2,chain.nameSpacePrefix), type, &function_index)) {
3023 if (GetBFFunctionArgumentCount(function_index)!=2L) {
3024 throw (arg2->Enquote() & " callback function must depend on 2 parameters ");
3025 } else {
3026 lf->StateCounter (function_index);
3027 }
3028 } else {
3029 throw (arg2->Enquote() & " is not a defined user batch language function ");
3030 }
3031
3032 } else {
3033 throw (arg1->Enquote() & " is not a defined likelihood function ID ");
3034 }
3035
3036 } catch (const _String & err) {
3037 HandleApplicationError (err);
3038 }
3039
3040 }
3041
3042
3043
3044 //____________________________________________________________________________________
3045
ExecuteCase52(_ExecutionList & chain)3046 void _ElementaryCommand::ExecuteCase52 (_ExecutionList& chain) {
3047 chain.currentCommand++;
3048
3049 long site_count = ProcessNumericArgument ((_String*)parameters (4),chain.nameSpacePrefix);
3050 _String given_state;
3051
3052 if (site_count < 1L) {
3053 given_state = ProcessLiteralArgument((_String*)parameters (4),chain.nameSpacePrefix);
3054 site_count = given_state.length();
3055 }
3056
3057 if (site_count < 1) {
3058 HandleApplicationError (*(_String*)parameters (4) & " must either evaluate to a positive integer or be a non-empty string of root states");
3059 return;
3060 }
3061
3062 _Variable * alphabet = FetchVar (LocateVarByName (AppendContainerName(*GetIthParameter(3),chain.nameSpacePrefix)), MATRIX),
3063 * tree_var = FetchVar (LocateVarByName (AppendContainerName(*GetIthParameter(1),chain.nameSpacePrefix)), TREE),
3064 * freq_var = FetchVar (LocateVarByName (AppendContainerName(*GetIthParameter(2),chain.nameSpacePrefix)), MATRIX);
3065
3066
3067 try {
3068 if (!alphabet) {
3069 throw (GetIthParameter(3)->Enquote() & " must be a defined matrix-valued variable");
3070 }
3071 if (!freq_var) {
3072 throw (GetIthParameter(2)->Enquote() & " must be a defined matrix-valued variable");
3073 }
3074 if (!tree_var) {
3075 throw (GetIthParameter(1)->Enquote() & " must be a defined tree-valued variable");
3076 }
3077
3078 _Matrix * alphabet_matrix = (_Matrix*)alphabet->GetValue();
3079
3080 if (!(alphabet_matrix->IsAStringMatrix() && alphabet_matrix->GetHDim() == 2 && alphabet_matrix->GetVDim () > 1)) {
3081 throw (_String("Alphabet specification variable ") & GetIthParameter(3)->Enquote() & " must be a string matrix with 2 rows and at least 2 columns");
3082 }
3083
3084 _String base_set;
3085
3086 for (unsigned long k=0UL; k < alphabet_matrix->GetVDim (); k++) {
3087 _FString * a_state = (_FString*)alphabet_matrix->GetFormula(0,k)->Compute();
3088 if (a_state) {
3089 if (a_state->get_str().length() == 1UL) {
3090 char c = a_state->get_str().char_at(0UL);
3091 if (base_set.Find(c) == -1) {
3092 base_set = base_set & c;
3093 } else {
3094 break;
3095 }
3096 } else {
3097 break;
3098 }
3099 } else {
3100 break;
3101 }
3102 }
3103
3104 if (base_set.length() < alphabet_matrix->GetVDim ()) {
3105 throw _String("The alphabet is mis-specified; it either has redundant characters or multi-character/non-string entries");
3106 }
3107
3108 long unit_size = ((_FString*)alphabet_matrix->GetFormula(1,0)->Compute())->get_str().to_long();
3109
3110 if (unit_size < 1L) {
3111 throw _String("The evolutionary unit size in the alphabet matrix is mis-specified");
3112 }
3113
3114 _Formula* exclusion_formula = alphabet_matrix->GetFormula(1,1);
3115 _String const* the_exclusions = &kEmptyString;
3116
3117 if (exclusion_formula) {
3118 the_exclusions = &((_FString*)exclusion_formula->Compute())->get_str();
3119 }
3120
3121 _TheTree * spawning_tree = (_TheTree*)tree_var;
3122
3123 if (parameters.lLength>6 && (spawning_tree->CountTreeCategories()>1)) {
3124 throw _String("Can't use spool to file option in Simulate when the tree depends on category variables.");
3125 }
3126
3127 if (given_state.length()>1) {
3128 // root state
3129 if ((given_state.length() >= unit_size)&&(given_state.length() % unit_size == 0)) {
3130 site_count = given_state.length()/unit_size;
3131 } else {
3132 throw _String("Root state string is either too short or has length which is not divisible by the unit size");
3133 }
3134 }
3135
3136 _TranslationTable newTT (base_set);
3137 _DataSet * ds = new _DataSet;
3138
3139 if (! newTT.IsStandardNucleotide() ) {
3140 ds->SetTranslationTable (&newTT); // mod 20060113 to properly deal with non-standard alphabets
3141 }
3142 // make a dummy
3143 spawning_tree->AddNodeNamesToDS (ds,true,false,1);
3144
3145 char c = base_set.char_at (0);
3146 long s = ds->GetNames().countitems();
3147
3148 if (s<2L) {
3149 ds->InsertName (_String ("Root"),0L);
3150 s ++;
3151 }
3152
3153
3154 ds->AddSite(c);
3155 for (long u = 1L; u < s; u++) {
3156 ds->Write2Site(0,c);
3157 }
3158 ds->Finalize();
3159 ds->SetNoSpecies (s);
3160
3161 unsigned long total_sites = site_count*unit_size;
3162
3163 _SimpleList * the_map = & ds->GetTheMap();
3164 the_map->RequestSpace (total_sites);
3165 InitializeArray (the_map->list_data, total_sites, 0L);
3166 the_map->lLength = total_sites;
3167
3168 _DataSetFilter* new_filter = new _DataSetFilter();
3169 _SimpleList h,v;
3170
3171 new_filter->SetFilter (ds,unit_size,h,v,false);
3172 new_filter->SetExclusions (*the_exclusions,true);
3173 new_filter->SetupConversion ();
3174
3175 _Matrix* root_states = nil;
3176 if (given_state.length()>=unit_size) {
3177 root_states = new _Matrix (1,site_count,false,true);
3178 hyFloat* holder = new hyFloat [new_filter->GetDimension(false)];
3179
3180 for (long cc = 0; cc < site_count; cc++) {
3181 unsigned long site_idex = cc*unit_size;
3182 _String root_char (given_state,site_idex,site_idex+unit_size-1L);
3183 long root_state = new_filter->Translate2Frequencies (root_char,holder,false);
3184 if (root_state<0) {
3185 throw (root_char & " found in the root state string at position " & _String ((long)site_idex) & " is an invalid/ambiguous state");
3186 } else {
3187 root_states->theData[cc] = root_state;
3188 }
3189 }
3190 delete [] holder;
3191 }
3192
3193
3194 long filter_id = StoreDataFilter (simulationFilter, new_filter);
3195
3196 spawning_tree->SetUp();
3197 spawning_tree->InitializeTreeFrequencies((_Matrix*)freq_var->Compute(),true);
3198
3199 _String filter_specification = *GetFilterName (filter_id) & spawning_tree->GetName()->Enquote(',') & *freq_var->GetName();
3200
3201
3202 bool do_internals = parameters.countitems() > 5 ? (ProcessNumericArgument ((_String*)parameters (5),chain.nameSpacePrefix)>0.5) : false;
3203
3204 _String spool_file;
3205
3206 FILE* main_file = nil;
3207
3208 if (parameters.countitems () > 6) {
3209 spool_file = ProcessLiteralArgument (GetIthParameter(6),chain.nameSpacePrefix);
3210 ProcessFileName(spool_file);
3211 main_file = doFileOpen (spool_file,"w");
3212 if (!main_file) {
3213 throw (_String("Failed to open ") & spool_file.Enquote() & " for writing");
3214 }
3215 if (do_internals) {
3216 spool_file = spool_file & ".anc";
3217 }
3218 }
3219
3220 _DataSet * sim_dataset;
3221
3222 if (main_file) {
3223 sim_dataset = new _DataSet (main_file);
3224 } else {
3225 sim_dataset = new _DataSet (site_count);
3226 }
3227
3228 _List exclusions;
3229
3230 _String *sim_name = new _String(AppendContainerName(*GetIthParameter(0),chain.nameSpacePrefix));
3231
3232
3233 _String rate_matrix_name = *sim_name & ".rates";
3234 _Variable *category_values_id = CheckReceptacle(&rate_matrix_name, __PRETTY_FUNCTION__);
3235 _Matrix* category_values = new _Matrix (1,1,false,true);
3236
3237 _String rate_variable_names = *sim_name & ".rateVars";
3238 _Variable * category_names_id = CheckReceptacle(&rate_variable_names, __PRETTY_FUNCTION__);
3239 _Matrix* category_names = new _Matrix (1,1,false,true);
3240
3241 SetStatusLine ("Simulating Data");
3242 { // lf must be deleted before the referenced datafilters
3243 _LikelihoodFunction lf (filter_specification, nil);
3244
3245 /*_SimpleList gl;
3246 lf.GetGlobalVars(gl);
3247 gl.Each ([] (long vi, unsigned long) -> void { StringToConsole(*LocateVar(vi)->GetName()); NLToConsole();});
3248 */
3249
3250 lf.Simulate (*sim_dataset, exclusions, category_values, category_names, root_states, do_internals?(main_file?&spool_file:&kEmptyString):nil);
3251 SetStatusLine ("Idle");
3252 }
3253
3254
3255 category_values_id->SetValue(category_values, false,true,NULL);
3256 category_names_id->SetValue(category_names, false,true,NULL);
3257
3258
3259 StoreADataSet (sim_dataset, sim_name);
3260 DeleteObject (sim_name);
3261 DeleteDataFilter (filter_id);
3262
3263 DeleteObject (ds);
3264 DeleteObject (root_states);
3265
3266 } catch (const _String & err) {
3267 HandleApplicationError(err & " in Simulate.");
3268 }
3269
3270 }
3271
3272
3273 //extern bool _debug_memory_leak;
3274
3275 //____________________________________________________________________________________
3276
Execute(_ExecutionList & chain)3277 bool _ElementaryCommand::Execute (_ExecutionList& chain) {
3278
3279 switch (code) {
3280
3281 case 0: // formula reparser
3282 ExecuteCase0 (chain);
3283 break;
3284
3285
3286 case 4:
3287 ExecuteCase4 (chain);
3288 break;
3289
3290
3291 case 5: // data set contruction
3292
3293 ExecuteCase5 (chain);
3294 break;
3295
3296 case 6: // data set filter construction
3297 case 27: // Permute
3298 case 28: // Bootstrap
3299 ExecuteDataFilterCases(chain);
3300 break;
3301
3302
3303 case 7: { // build a tree
3304 chain.currentCommand++;
3305
3306 _String treeIdent = chain.AddNameSpaceToID(*(_String*)parameters(0)),
3307 treeString = *(_String*)parameters(1);
3308
3309 SetStatusLine (_String("Constructing Tree ")&treeIdent);
3310 long varID = LocateVarByName (treeIdent);
3311
3312
3313 bool replace_tree_structure = hy_env::EnvVariableTrue(replaceTreeStructure);
3314
3315 _SimpleList leftOverVars; // mod 02/03/2003
3316 if (varID>=0)
3317 if (FetchVar(varID)->ObjectClass()==TREE) {
3318 if (replace_tree_structure) {
3319 DeleteVariable(*FetchVar(varID)->GetName()); // mod 11/19/2003
3320 } else {
3321 DeleteTreeVariable(varID,leftOverVars); // mod 02/03/2003
3322 }
3323 }
3324
3325
3326 _TheTree * tr = nil;
3327
3328 if (treeString.get_char(0)!='(') {
3329 _Formula nameForm (treeString,chain.nameSpacePrefix);
3330 HBLObjectRef formRes = nameForm.Compute();
3331 if (formRes) {
3332 if (formRes->ObjectClass () == STRING) {
3333 tr = new _TheTree (treeIdent,((_FString*)formRes)->get_str(),false);
3334 } else if (formRes->ObjectClass () == TOPOLOGY) {
3335 tr = new _TheTree (treeIdent,(_TreeTopology*)formRes);
3336 } else if (formRes->ObjectClass () == TREE) {
3337 for (unsigned long i = 0; i < leftOverVars.lLength; i++) {
3338 //rintf ("%s\n", LocateVar(leftOverVars.list_data[i])->GetName()->get_str());
3339 DeleteVariable(leftOverVars.list_data[i], true);
3340 }
3341 leftOverVars.Clear();
3342 tr = new _TheTree (treeIdent,(_TheTree*)formRes);
3343 }
3344 }
3345 } else {
3346 tr = new _TheTree (treeIdent,treeString,false);
3347 }
3348
3349 if (!tr) {
3350 DeleteObject (tr);
3351 HandleApplicationError("Illegal right hand side in call to Tree id = ...; it must be a string, a Newick tree spec or a topology");
3352 return false;
3353 }
3354
3355 if (leftOverVars.nonempty()) { // mod 02/03/2003 - the entire "if" block
3356 _SimpleList indep, dep, holder;
3357 {
3358 _AVLList indepA (&indep),
3359 depA (&dep);
3360
3361 tr->ScanContainerForVariables (indepA,depA);
3362 //tr.ScanForVariables (indepA,depA);
3363 indepA.ReorderList();
3364 depA.ReorderList();
3365 }
3366
3367
3368 //indep.Sort();
3369 //dep.Sort();
3370
3371 holder.Union (indep,dep);
3372 leftOverVars.Sort ();
3373 /*
3374 BufferToConsole("\nIndependents+nDependendts\n");
3375 ObjectToConsole(&holder); NLToConsole();
3376 BufferToConsole("\nLeftover\n");
3377 ObjectToConsole(&leftOverVars); NLToConsole();
3378 */
3379
3380 /*leftOverVars.Each ([](long v, unsigned long) -> void {
3381 StringToConsole(*LocateVar(v)->GetName()); NLToConsole();
3382 });*/
3383
3384 indep.Subtract (leftOverVars,holder);
3385
3386 /* the bit with freeSlots is here b/c
3387 some nodes variables may have been deleted during the unroot
3388 in the tree constructor and we don't want to delete them twice,
3389 do we? 08/22/2003 */
3390
3391 dep.Clear();
3392 dep.Duplicate (&freeSlots);
3393 dep.Sort ();
3394 holder.Subtract (indep,dep);
3395 for (varID = holder.lLength-1; varID >=0 ; varID--) {
3396 DeleteVariable (*LocateVar (holder.list_data[varID])->GetName());
3397 }
3398
3399 tr->Clear();
3400
3401 }
3402
3403 }
3404 break;
3405
3406 case HY_HBL_COMMAND_FPRINTF: { // print stuff to file (or stdout)
3407 return HandleFprintf(chain);
3408 }
3409
3410 case HY_HBL_COMMAND_HARVEST_FREQUENCIES: { // or HarvestFrequencies
3411 return HandleHarvestFrequencies(chain);
3412 }
3413
3414 case HY_HBL_COMMAND_OPTIMIZE: // optimize the likelihood function
3415 case HY_HBL_COMMAND_COVARIANCE_MATRIX: {
3416 return HandleOptimizeCovarianceMatrix (chain, code == HY_HBL_COMMAND_OPTIMIZE);
3417 }
3418
3419 case 11: // build the likelihood function
3420
3421 ExecuteCase11 (chain);
3422 break;
3423
3424 case 12: // data set contruction by simulation
3425
3426 ExecuteCase12 (chain);
3427 break;
3428
3429 case 14: {
3430 // a return statement
3431
3432 if (parameters.lLength) {
3433
3434 /*if (chain.is_compiled()) {
3435 chain.CopyCLIToVariables();
3436 }*/
3437
3438 _Formula * expression = nil;
3439 _String * errMsg = nil;
3440 try {
3441
3442
3443 if (simpleParameters.lLength < 2) {
3444
3445 expression = new _Formula;
3446 //printf ("Namespace: %x\nCode: %s\n", chain.nameSpacePrefix, ((_String*)parameters(0))->sData);
3447
3448 _FormulaParsingContext fpc (nil, chain.nameSpacePrefix);
3449 long status = Parse (expression, *(_String*)parameters(0), fpc, nil);
3450
3451 if (status== HY_FORMULA_EXPRESSION) {
3452 if (fpc.isVolatile() == false) {
3453 simpleParameters<<(long)expression;
3454 appendCompiledFormulae (expression);
3455 expression = nil;
3456 }
3457 } else {
3458 errMsg = new _String ("Invalid return statement");
3459 throw 0;
3460 }
3461 }
3462
3463 HBLObjectRef ret_val = nil;
3464 // important to store the return value in a local variable
3465 // because chain.result may be overwritten by recursive calls to
3466 // this function
3467
3468 if (expression) {
3469 //printf ("Return interpreted\n");
3470 ret_val = expression->Compute(0,nil,nil,nil,HY_ANY_OBJECT,false);
3471 }
3472 else{
3473 //printf ("Return compiled %d\n", ((_Formula*)simpleParameters(1))->GetList().lLength);
3474 //if (_debug_memory_leak) {
3475 // BufferToConsole("In return while parsing AssociateList string repr\n");
3476 //}
3477
3478 ret_val = ((_Formula*)simpleParameters(1))->Compute(0,nil,nil,nil,HY_ANY_OBJECT,false);
3479 }
3480
3481 DeleteObject (chain.result);
3482
3483 chain.result = ret_val;
3484 if (ret_val) {
3485 chain.result->AddAReference();
3486 }
3487
3488 if (expression) {
3489 delete (expression);
3490 }
3491 }
3492 catch (int e) {
3493 if (expression)
3494 delete expression;
3495 if (errMsg)
3496 HandleApplicationError (errMsg);
3497 return false;
3498 }
3499 }
3500
3501 chain.currentCommand = simpleParameters(0);
3502 if (chain.currentCommand<0) {
3503 chain.currentCommand = 0x7fffffff;
3504 }
3505 break;
3506 }
3507
3508
3509 case 16: { // data set merger operation
3510 chain.currentCommand++;
3511 SetStatusLine ("Merging Datasets");
3512 _SimpleList dsIndex;
3513 for (long di=1; di<parameters.lLength; di++) {
3514 _String dsname = chain.AddNameSpaceToID(*(_String*)parameters(di));
3515 long f = FindDataSetName (dsname);
3516 if (f==-1) {
3517 HandleApplicationError (((_String)("Identifier ")&dsname&_String(" doesn't correspond to a valid dataset.")));
3518 return false;
3519 } else {
3520 dsIndex<<f;
3521 }
3522 }
3523
3524 _DataSet* mergeResult = (simpleParameters(0)==1 || simpleParameters(0)==-1)?_DataSet::Concatenate(dsIndex):_DataSet::Combine(dsIndex);
3525 // xlc mod 03/08/2005
3526 _String * resultName = new _String (chain.AddNameSpaceToID(*(_String*)parameters(0)));
3527
3528 if (StoreADataSet (mergeResult, resultName) && simpleParameters(0)<0) {
3529 // purge all the datasets except the resulting one
3530 long newSetID = FindDataSetName (*resultName);
3531 for (long di=0; di<dsIndex.lLength; di++)
3532 if (dsIndex.list_data[di] != newSetID) {
3533 KillDataSetRecord(dsIndex.list_data[di]);
3534 }
3535 }
3536
3537 DeleteObject (resultName);
3538
3539 }
3540 break;
3541
3542 case HY_HBL_COMMAND_EXPORT: // matrix export operation
3543 HandleExport (chain);
3544 break;
3545
3546 case 18: // matrix import operation
3547
3548 {
3549 bool importResult = true;
3550 chain.currentCommand++;
3551 _String fName (*GetIthParameter(1));
3552 ProcessFileName(fName);
3553 if (terminate_execution) {
3554 return false;
3555 }
3556 FILE* theDump = doFileOpen (fName,"rb");
3557 if (!theDump) {
3558 HandleApplicationError (((_String)("File ")&fName&_String(" couldn't be open for reading.")));
3559 return false;
3560 }
3561
3562 fName = chain.AddNameSpaceToID(*(_String*)parameters(0));
3563 _Variable * result = CheckReceptacle(&fName,blImport.Cut(0,blImport.length()-2),true);
3564 if (result) {
3565 _Matrix * storage = new _Matrix (1,1,false,true);
3566 result->SetValue(storage,false,true,NULL);
3567 lastMatrixDeclared = result->get_index();
3568 if (!storage->ImportMatrixExp(theDump)) {
3569 HandleApplicationError("Matrix import failed - the file has an invalid format.");
3570 importResult = false;
3571 DeleteObject(storage);
3572 }
3573 } else {
3574 importResult = false;
3575 }
3576 fclose (theDump);
3577 return importResult;
3578 }
3579 break;
3580
3581 case HY_HBL_COMMAND_MOLECULAR_CLOCK: // molecular_clock constraint
3582 HandleMolecularClock(chain);
3583 break;
3584
3585 case 20: // category variable construction
3586
3587 {
3588 chain.currentCommand++;
3589 _String cName = chain.AddNameSpaceToID (*(_String*)parameters(0));
3590 _List parms (parameters);
3591 parms.Delete(0);
3592 _CategoryVariable newCat(cName,&parms,chain.nameSpacePrefix);
3593 ReplaceVar(&newCat);
3594 }
3595 break;
3596
3597 case HY_HBL_COMMAND_CONSTRUCT_CATEGORY_MATRIX: // construct the category matrix
3598 HandleConstructCategoryMatrix (chain);
3599 break;
3600
3601 case HY_HBL_COMMAND_CLEAR_CONSTRAINTS: // clear constraints
3602 HandleClearConstraints(chain);
3603 break;
3604
3605 case HY_HBL_COMMAND_SET_DIALOG_PROMPT: { // set dialog prompt
3606 chain.currentCommand++;
3607 dialogPrompt = ProcessLiteralArgument((_String*)parameters(0),chain.nameSpacePrefix);
3608 }
3609 break;
3610
3611 case HY_HBL_COMMAND_SELECT_TEMPLATE_MODEL: // prompt for a model file
3612 return HandleSelectTemplateModel(chain);
3613
3614 case HY_HBL_COMMAND_FSCANF: // fscanf
3615 HandleFscanf (chain,false);
3616 break;
3617
3618 case HY_HBL_COMMAND_SSCANF: // fscanf
3619 HandleFscanf (chain,true);
3620 break;
3621
3622
3623 case HY_HBL_COMMAND_USE_MODEL:
3624 return HandleUseModel(chain);
3625
3626 case 31:
3627 ExecuteCase31 (chain);
3628 break;
3629
3630 case HY_HBL_COMMAND_CHOICE_LIST:
3631 HandleChoiceList (chain);
3632 break;
3633
3634 case HY_HBL_COMMAND_GET_STRING:
3635 HandleGetString (chain);
3636 break;
3637
3638 case HY_HBL_COMMAND_KEYWORD_ARGUMENT:
3639 HandleKeywordArgument (chain);
3640 break;
3641
3642 case HY_HBL_COMMAND_SET_PARAMETER:
3643 return HandleSetParameter(chain);
3644
3645 case 38: // reconstruct ancestors
3646 ExecuteCase38 (chain, false);
3647 break;
3648
3649 case HY_HBL_COMMAND_EXECUTE_COMMANDS:
3650 HandleExecuteCommandsCases(chain, false, false);
3651 break;
3652
3653 case HY_HBL_COMMAND_EXECUTE_A_FILE:
3654 HandleExecuteCommandsCases(chain, true, false);
3655 break;
3656
3657 case HY_HBL_COMMAND_LOAD_FUNCTION_LIBRARY:
3658 HandleExecuteCommandsCases(chain, true, true);
3659 break;
3660
3661
3662 case HY_HBL_COMMAND_DIFFERENTIATE:
3663 return HandleDifferentiate (chain);
3664 break;
3665
3666 case HY_HBL_COMMAND_GET_INFORMATION:
3667 return HandleGetInformation(chain);
3668 break;
3669
3670 case HY_HBL_COMMAND_REPLICATE_CONSTRAINT:
3671 return HandleReplicateConstraint (chain);
3672 break;
3673
3674 case HY_HBL_COMMAND_INTEGRATE:
3675 case HY_HBL_COMMAND_FIND_ROOT:
3676 return HandleFindRootOrIntegrate(chain, code == HY_HBL_COMMAND_INTEGRATE);
3677 break;
3678
3679
3680 case HY_HBL_COMMAND_MPI_SEND:
3681 return HandleMPISend (chain);
3682 break;
3683
3684 case HY_HBL_COMMAND_MPI_RECEIVE:
3685 HandleMPIReceive(chain);
3686 break;
3687
3688
3689 case 47: // state counter; deprecate
3690 ExecuteCase47 (chain);
3691 break;
3692
3693 case HY_HBL_COMMAND_LFCOMPUTE:
3694 return HandleComputeLFFunction(chain);
3695
3696 case 50: // sample ancestors
3697 ExecuteCase38 (chain, true);
3698 break;
3699
3700 case HY_HBL_COMMAND_GET_URL:
3701 return HandleGetURL (chain);
3702
3703 case 52: // Simulate
3704 ExecuteCase52 (chain);
3705 break;
3706
3707 case HY_HBL_COMMAND_DO_SQL:
3708 return HandleDoSQL(chain);
3709
3710 case 54: // topology id =
3711 ExecuteCase54 (chain);
3712 break;
3713
3714 case HY_HBL_COMMAND_ALIGN_SEQUENCES:
3715 return HandleAlignSequences (chain);
3716 break;
3717
3718
3719 case 58: // #profile
3720 ExecuteCase58 (chain);
3721 break;
3722
3723 case HY_HBL_COMMAND_DELETE_OBJECT:
3724 return HandleDeleteObject (chain);
3725 break;
3726
3727 case HY_HBL_COMMAND_REQUIRE_VERSION:
3728 HandleRequireVersion(chain);
3729 break;
3730
3731 case 61: // SCFG =
3732 ExecuteCase61 (chain);
3733 break;
3734
3735 case 63: // dead?
3736 ExecuteCase63 (chain);
3737 break;
3738
3739 case 64: // BGM =
3740 ExecuteCase64 (chain);
3741 break;
3742
3743 case HY_HBL_COMMAND_ASSERT:
3744 HandleAssert (chain);
3745 break;
3746
3747 case HY_HBL_COMMAND_GET_DATA_INFO:
3748 HandleGetDataInfo(chain);
3749 break;
3750
3751 case HY_HBL_COMMAND_INIT_ITERATOR:
3752 HandleInitializeIterator(chain);
3753 break;
3754
3755 case HY_HBL_COMMAND_ADVANCE_ITERATOR:
3756 HandleAdvanceIterator(chain);
3757 break;
3758
3759 case HY_HBL_COMMAND_NESTED_LIST:
3760 chain.currentCommand++;
3761 {
3762 _FString *stash_current = (_FString*)hy_env::EnvVariableGet(kNamespaceName, STRING);
3763 if (stash_current) {
3764 stash_current->AddAReference();
3765 }
3766 hy_env::EnvVariableSet(kNamespaceName, new _FString ((_String*)GetIthParameter(1)->toStr(), false), false);
3767 ((_ExecutionList*)parameters.GetItem(0))->Execute(&chain);
3768 if (stash_current) {
3769 hy_env::EnvVariableSet(kNamespaceName, stash_current, false);
3770 } else {
3771 hy_env::EnvVariableSet(kNamespaceName, new _MathObject, false);
3772 }
3773 }
3774 break;
3775
3776
3777
3778 default:
3779 chain.currentCommand++;
3780 }
3781
3782 return true;
3783 }
3784
3785 //____________________________________________________________________________________
3786
3787
FindNextCommand(_StringBuffer & input,_StringBuffer & result)3788 void _ElementaryCommand::FindNextCommand (_StringBuffer& input, _StringBuffer &result) {
3789
3790 long index = input.length();
3791 result.Reset();
3792
3793 if (index == 0L) {
3794 return;
3795 }
3796
3797 bool skipping = false;
3798
3799 enum {
3800 normal_text = 0,
3801 double_quote = 1,
3802 single_quote = 2
3803
3804 } literal_state = normal_text;
3805
3806 enum {
3807 no_comment = 0,
3808 slash_star = 1,
3809 double_slash = 2
3810 } comment_state = no_comment;
3811
3812
3813 long scope_depth = 0L, // count how deep we are in {}
3814 matrix_depth = 0L, // count how deep we are in {} matrix definition (different from "scope")
3815 parentheses_depth = 0L, // count how deep we are in ()
3816 bracket_depth = 0L; // count how deep we are in []
3817
3818 _SimpleList is_DoWhileLoop;
3819
3820
3821 char last_char = '\0';
3822 // a look back character
3823
3824
3825 // non printable characters at the end ?
3826 while (index>0) {
3827 if (!isprint (input.char_at (index))) {
3828 index--;
3829 } else {
3830 break;
3831 }
3832 }
3833
3834 input.Trim (0,index);
3835
3836 for (index = 0L; index<input.length(); index++) {
3837 char c = input.char_at (index);
3838
3839 if (literal_state == normal_text && c=='\t') {
3840 c = ' ';
3841 }
3842
3843 // check for comments
3844 if (comment_state != no_comment) {
3845 if (comment_state == slash_star) {
3846 if (c=='/' && input.get_char(index-1)=='*') {
3847 comment_state = no_comment;
3848 }
3849 } else if (c == '\r' || c == '\n') {
3850 comment_state = no_comment;
3851 }
3852
3853 last_char = '\0';
3854 continue;
3855 } else {
3856 if (literal_state == normal_text && c=='/') {
3857 switch (input.get_char(index+1)) {
3858 case '*':
3859 comment_state = slash_star;
3860 break;
3861 case '/':
3862 comment_state = double_slash;
3863 }
3864
3865 if (comment_state != no_comment) {
3866 last_char = '\0';
3867 index++;
3868 continue;
3869 }
3870 }
3871 }
3872
3873
3874 // skip spaces, except for special cases, like return and data set filters
3875
3876 if (literal_state == normal_text && isspace(c)) {
3877
3878 // skip/compress spaces, unless we are in a higher level HBL statement
3879 // where spaces can't be compressed
3880 // examples include
3881 // DataSet|DataSetFilter|return|LikelihoodFunction (something)
3882 // need to maintain spaces for this to work appropriately
3883
3884
3885
3886 if (!skipping && index > 0L) {
3887
3888 long trie_match = _HY_HBL_KeywordsPreserveSpaces.FindKey(input.Cut (MAX (0, index - 20), index-1).Reverse(), nil, true);
3889 if (trie_match != kNotFound) {
3890 long matched_length = _HY_HBL_KeywordsPreserveSpaces.GetValue(trie_match);
3891 if (matched_length == index || !(isalnum(input.get_char(index-matched_length-1)) || input.get_char(index-matched_length-1) == '_' || input.get_char(index-matched_length-1) == '.')) {
3892 result << ' ';
3893 }
3894 }
3895 }
3896
3897
3898 skipping = true;
3899 continue;
3900 }
3901
3902 if (skipping && ( isalpha(c) || c=='_') && (isalnum(last_char) || last_char =='_')) {
3903 // SLKP 20170704: this seems incomplete : need to check more thorougly that this is an ident
3904 // this is meant to determine that we are at the beginning of a new ident-like
3905 // token and insert a space
3906 result<<' ';
3907 }
3908
3909 skipping = false;
3910
3911
3912 if (literal_state != normal_text && c == '\\') {
3913 // if the next character is 0, i.e. \0xx, treat as a terminal code
3914 char next_char = input.get_char(++index);
3915 if (next_char == '0') { // octal character; read all the subsequent numbers and convert
3916 _SimpleList digits;
3917 while (isdigit (next_char = input.get_char(++index)) && digits.countitems() <= 3) {
3918 digits << _String (next_char).to_long();
3919 }
3920 switch (digits.countitems()) {
3921 case 1:
3922 next_char = digits.get(0); break;
3923 case 2:
3924 next_char = digits.get(0) * 8 + digits.get(1) ; break;
3925 case 3:
3926 next_char = digits.get(0) * 64 + digits.get(1) * 8 + digits.get(2) ; break;
3927 default:
3928 next_char = '\0'; break;
3929 }
3930 index--;
3931 } else {
3932 result<<c;
3933 }
3934 // escape character \x
3935 result<< next_char;
3936 continue;
3937 }
3938
3939 result<<c;
3940
3941 // are we inside a string literal?
3942
3943 if (c=='"') {
3944 if (literal_state != single_quote) {
3945 literal_state = literal_state == normal_text ? double_quote : normal_text;
3946 last_char = '\0';
3947 continue;
3948 }
3949 } else {
3950 if (c == '\'') {
3951 if (literal_state != double_quote) {
3952 literal_state = literal_state == normal_text ? single_quote : normal_text;
3953 last_char = '\0';
3954 continue;
3955 }
3956 }
3957 }
3958
3959 if (literal_state != normal_text) {
3960 continue;
3961 }
3962
3963 // maybe we are done?
3964
3965 if (c==';' && scope_depth == 0L && matrix_depth == 0L && parentheses_depth == 0L && bracket_depth == 0L) {
3966 // SLKP 20170704 used to be parentheses_depth <= 0L && bracket_depth <= 0L
3967 break;
3968 }
3969
3970 // check to see whether we are defining a matrix
3971
3972 if (c=='(') {
3973 parentheses_depth ++;
3974 last_char = '\0';
3975 continue;
3976 }
3977
3978 if (c==')') {
3979 parentheses_depth --;
3980 if (parentheses_depth < 0L) {
3981 HandleApplicationError (_String("Too many closing ')' near '") & input.Cut (MAX(0,index-32),index) & "'.");
3982 input.Clear();
3983 result.Reset();
3984 return ;
3985 }
3986 last_char = '\0';
3987 continue;
3988 }
3989
3990 if (c=='[') {
3991 bracket_depth++;
3992 last_char = '\0';
3993 continue;
3994 }
3995
3996 if (c==']') {
3997 bracket_depth--;
3998 if (bracket_depth < 0L) {
3999 HandleApplicationError (_String("Too many closing ']' near '") & input.Cut (MAX(0,index-32),index) & "'.");
4000 input.Clear();
4001 result.Reset();
4002 return ;
4003 }
4004 last_char = '\0';
4005 continue;
4006 }
4007
4008
4009 if (c=='{') {
4010 if (matrix_depth) {
4011 matrix_depth++;
4012 } else if (last_char == '=') { // a matrix def
4013 matrix_depth++;
4014 } else {
4015 scope_depth++;
4016 if (index>=2L) {
4017 long t = input.FirstNonSpaceIndex (0, index-1, kStringDirectionBackward);
4018 if (t>=1) {
4019 if (input.get_char(t)=='o' && input.get_char(t-1)=='d') {
4020 is_DoWhileLoop << scope_depth-1L;
4021 //printf ("%d\n%s\n\n", isDoWhileLoop, input.Cut (t,-1).sData);
4022 }
4023 }
4024 }
4025 }
4026 last_char = '\0';
4027 continue;
4028 }
4029
4030 if (c=='}') {
4031 if (matrix_depth) {
4032 matrix_depth--;
4033 } else {
4034 scope_depth--;
4035 if (parentheses_depth == 0L && bracket_depth == 0L) {
4036 if (scope_depth >=0 && is_DoWhileLoop.lLength && is_DoWhileLoop.GetElement(-1L) == scope_depth) {
4037 is_DoWhileLoop.Pop();
4038 } else if (scope_depth == 0L) {
4039 break;
4040 }
4041 }
4042
4043 }
4044 last_char = '\0';
4045 continue;
4046 }
4047
4048 last_char = c;
4049 }
4050
4051
4052 if (scope_depth != 0L || comment_state == slash_star || literal_state != normal_text || matrix_depth != 0L || bracket_depth != 0L || parentheses_depth != 0L) {
4053 if (result!='}') {
4054 HandleApplicationError (_String("Expression appears to be incomplete/syntax error. {} scope: ") &scope_depth & ", () depth "
4055 & parentheses_depth & ", matrix scope: " & matrix_depth & '.' & (literal_state == double_quote ?" In a \"\" literal. ":kEmptyString)
4056 & (literal_state == single_quote?" In a '' literal. ":kEmptyString) &
4057 (comment_state == slash_star ? " In a /* */ comment ":kEmptyString) & '\n' & input);
4058 input.Clear();
4059 result.Reset();
4060 return ;
4061 } else {
4062 result.Reset();
4063 }
4064 }
4065
4066 long check_open = 0L;
4067 while (result.get_char(check_open)=='{') {
4068 check_open++;
4069 }
4070
4071 if (check_open) {
4072 long index2 = result.length() - 1L;
4073
4074 while (result[index2]=='}') {
4075 index2--;
4076 }
4077
4078 if (result.length () - index2 - 1 < check_open) {
4079 HandleApplicationError ((_String)("Expression appears to be incomplete/syntax error and will be ignored:")&input);
4080 result.Clear ();
4081 } else {
4082 result.Trim(check_open,result.length()-1-check_open);
4083 }
4084 }
4085
4086 if (index<input.length()-1) {
4087 input.Trim (index+1L, kStringEnd);
4088 } else {
4089 input.Clear();
4090 }
4091
4092 }
4093 //____________________________________________________________________________________
4094
ExtractConditions(_StringBuffer const & source,long start_at,_List & receptacle,char delimeter,bool include_empty_conditions)4095 long _ElementaryCommand::ExtractConditions (_StringBuffer const& source, long start_at, _List& receptacle, char delimeter, bool include_empty_conditions) {
4096
4097 long parentheses_depth = 1L,
4098 // this is because extaction will work from the first character following a '(', e.g. CreateFilter([start parsing here]....)
4099 last_delim = start_at,
4100 index = start_at,
4101 curly_depth = 0L,
4102 bracket_depth = 0L;
4103
4104
4105 enum {
4106 normal_text = 0,
4107 single_quote = 1,
4108 double_quote = 2
4109 } quote_type = normal_text;
4110
4111 auto strip_last_space = [] (_String const& source, long from, long to) -> _String* {
4112 if (to > from) {
4113 if (source.char_at(to-1L) == ' ') {
4114 return new _String (source, from, to-2L);
4115 }
4116 }
4117 return new _String (source, from, to-1L);
4118 };
4119
4120 for (; index<source.length(); index++) {
4121 char c = source.char_at (index);
4122 if (quote_type == normal_text) {
4123 if (c=='(') {
4124 parentheses_depth++;
4125 continue;
4126 }
4127 if (c=='{') {
4128 curly_depth++;
4129 continue;
4130 }
4131 if (c=='}') {
4132 curly_depth--;
4133 continue;
4134 }
4135 if (c=='[') {
4136 bracket_depth++;
4137 continue;
4138 }
4139 if (c==']') {
4140 bracket_depth--;
4141 continue;
4142 }
4143 if (c==')') {
4144 parentheses_depth --;
4145 if (parentheses_depth == 0L) {
4146 break;
4147 }
4148 continue;
4149 }
4150 }
4151 if (c=='"' && quote_type != single_quote) {
4152 if (index == start_at || source.char_at (index-1L) != '\\') {
4153 quote_type = quote_type == normal_text ? double_quote : normal_text;
4154 }
4155 continue;
4156 }
4157 if (c=='\'' && quote_type != double_quote) {
4158 if (index == start_at || source.char_at (index-1L) != '\\') {
4159 quote_type = quote_type == normal_text ? single_quote : normal_text;
4160 }
4161 continue;
4162 }
4163 if (c==delimeter) {
4164 if (parentheses_depth > 1 || quote_type != normal_text || curly_depth || bracket_depth) {
4165 continue;
4166 }
4167
4168 receptacle < strip_last_space (source,last_delim,index);
4169 //printf ("%s\n", ((_String*)receptacle.GetItem(receptacle.lLength-1))->get_str());
4170 last_delim = index+1;
4171 continue;
4172 }
4173 }
4174
4175 if (include_empty_conditions || last_delim <= index-1) {
4176 receptacle < strip_last_space (source,last_delim,index);
4177 //printf ("%s\n", ((_String*)receptacle.GetItem(receptacle.lLength-1))->get_str());
4178 }
4179 return index+1L;
4180 }
4181
4182 //____________________________________________________________________________________
4183
4184
MakeGeneralizedLoop(_String * p1,_String * p2,_String * p3,bool for_or_while,_StringBuffer & source,_ExecutionList & target)4185 bool _ElementaryCommand::MakeGeneralizedLoop (_String*p1, _String*p2, _String*p3 , bool for_or_while, _StringBuffer& source, _ExecutionList&target) {
4186
4187
4188 const _String kIterator ("in");
4189
4190 // extract the for enclosure
4191 long beginning = target.lLength,
4192 for_return = beginning;
4193
4194 bool success = true;
4195 bool has_increment = false;
4196 bool is_iterator = (p2 && *p2 == kIterator);
4197
4198 _SimpleList bc;
4199
4200 try {
4201 if (is_iterator) {
4202 if (! (p1 && p3)) {
4203 throw (kEmptyString);
4204 }
4205 for_return++;
4206 has_increment = true;
4207 _ElementaryCommand * init = new _ElementaryCommand (HY_HBL_COMMAND_INIT_ITERATOR);
4208
4209 long refer_to_init = target.countitems();
4210
4211 init->parameters << p3;
4212 target < init;
4213 _ElementaryCommand * advance = new _ElementaryCommand (HY_HBL_COMMAND_ADVANCE_ITERATOR);
4214
4215 ExtractConditions (*p1,0L,advance->parameters,',',false);
4216 if (advance->parameters.countitems() < 1 || advance->parameters.countitems() > 3) {
4217 throw (_String("Must have bewteen 1 and 3 arguments for the 'for' iterator loop"));
4218 }
4219
4220 advance->simpleParameters <<refer_to_init;
4221 advance->simpleParameters << advance->parameters.countitems();
4222 init->simpleParameters << target.countitems();
4223 init->simpleParameters << 0;
4224 target < advance;
4225
4226 for_return++;
4227 target < new _ElementaryCommand ();
4228
4229 if (source.get_char(0)=='{') {
4230 source.Trim(1,kStringEnd);
4231 }
4232
4233 if (target.BuildList (source, &bc) == false) { // construct the main body
4234 throw (kEmptyString);
4235 }
4236 target << advance;
4237
4238 _ElementaryCommand * loopback = new _ElementaryCommand;
4239 loopback->MakeJumpCommand (nil,for_return,0,target);
4240 target < loopback;
4241
4242 // automatically generate
4243 // None != iterator_value, but we don't know
4244
4245 _StringBuffer iterator_condition;
4246 iterator_condition << kEndIteration.Enquote() << "!=" << (_String*)advance->parameters.GetItem (advance->parameters.countitems() -1);
4247 //StringToConsole(iterator_condition);NLToConsole();
4248 target.GetIthCommand(for_return)->MakeJumpCommand (&iterator_condition, for_return+1, target.lLength,target);
4249
4250 } else {
4251
4252 if (p1 && p1->nonempty()) { // initialization stage
4253 for_return++;
4254 _StringBuffer code (*p1);
4255 success = success && target.BuildList (code, nil, true); // add init step
4256 }
4257
4258 // append condition now
4259
4260 if (!success) {
4261 throw (kEmptyString);
4262 }
4263
4264 if (for_or_while) {
4265 if (p2 && p2->nonempty()) { // condition stage
4266 target < new _ElementaryCommand (*p2);
4267 }
4268 }
4269
4270 if (source.get_char(0)=='{') {
4271 source.Trim(1,kStringEnd);
4272 }
4273
4274 if ((success = success && target.BuildList (source, &bc)) == false) { // construct the main body
4275 throw (kEmptyString);
4276 }
4277
4278 if (p3 && p3->nonempty ()) { // increment stage
4279 _StringBuffer code (*p3);
4280 success = success && target.BuildList (code, nil,true); // add increment step
4281 has_increment = true;
4282 }
4283
4284 if (!success) {
4285 throw (kEmptyString);
4286 }
4287
4288 if (for_or_while) {
4289 _ElementaryCommand * loopback = new _ElementaryCommand;
4290 loopback->MakeJumpCommand (nil,for_return,0,target);
4291 target < loopback;
4292 if (p2 && p2->nonempty()) {
4293 target.GetIthCommand(for_return)->MakeJumpCommand (p2, for_return+1, target.lLength,target);
4294 }
4295 } else {
4296 if (p2) {
4297 _ElementaryCommand* loopback = new _ElementaryCommand ;
4298 loopback->MakeJumpCommand (p2,for_return,target.lLength+1,target);
4299 target < loopback;
4300 }
4301 }
4302 }
4303 } catch (const _String& err) {
4304 for (long index = target.lLength - 1; index >= beginning; index--) {
4305 target.Delete (index);
4306 }
4307 if (err.nonempty()) {
4308 HandleApplicationError(err);
4309 }
4310 return false;
4311 }
4312
4313 bc.Each ([&target, has_increment] (long loc, unsigned long) -> void {
4314 if (loc>0) { // break
4315 (target.GetIthCommand(loc))->MakeJumpCommand (nil, target.lLength, 0,target);
4316 } else { // continue
4317 (target.GetIthCommand(-loc))->MakeJumpCommand (nil, target.lLength-(has_increment?2:1), 0,target);
4318 }
4319 });
4320
4321 return true;
4322 }
4323
4324 //____________________________________________________________________________________
4325
4326
BuildFor(_StringBuffer & source,_ExecutionList & target,_List * pieces)4327 bool _ElementaryCommand::BuildFor (_StringBuffer&source, _ExecutionList&target, _List * pieces)
4328
4329 /* the for loop becomes this:
4330
4331 1. initialize
4332 2. if (condition) then
4333 3. execute loop body
4334 4. else go to 7
4335 5. increment statement (if present)
4336 6. go to 2
4337 7. code following the loop
4338 */
4339
4340
4341 {
4342 if (pieces)
4343 return MakeGeneralizedLoop ((_String*)pieces->GetItem(0),(_String*)pieces->GetItem(1),(_String*)pieces->GetItem(2),true,source,target);
4344 else
4345 return MakeGeneralizedLoop (nil,nil,nil,true,source,target);
4346 }
4347
4348 //____________________________________________________________________________________
4349
BuildWhile(_StringBuffer & source,_ExecutionList & target,_List * pieces)4350 bool _ElementaryCommand::BuildWhile (_StringBuffer&source, _ExecutionList&target, _List * pieces)
4351 {
4352 if (pieces)
4353 return MakeGeneralizedLoop (nil,(_String*)pieces->GetItem(0),nil,true,source,target);
4354 else
4355 return MakeGeneralizedLoop (nil,nil,nil,true,source,target);
4356 }
4357
4358 //____________________________________________________________________________________
4359
BuildIfThenElse(_StringBuffer & source,_ExecutionList & target,_SimpleList * bc)4360 bool _ElementaryCommand::BuildIfThenElse (_StringBuffer &source, _ExecutionList&target, _SimpleList* bc)
4361 {
4362 _List pieces;
4363 long upto = ExtractConditions (source,3,pieces),
4364 beginning = target.lLength;
4365 target.lastif << target.lLength;
4366 int success = 1,
4367 intIfs = target.lastif.lLength;
4368
4369
4370 {
4371 if (pieces.lLength!=1) {
4372 HandleApplicationError ("'if' header makes no sense");
4373 }
4374
4375 source.Trim (upto,-1);
4376 target.AppendNewInstance (new _ElementaryCommand);
4377
4378 _StringBuffer nextCommand;
4379 _ElementaryCommand::FindNextCommand(source,nextCommand);
4380 success *= target.BuildList (nextCommand, bc, true);
4381
4382 }
4383
4384 if (!success) { // clean up
4385 for (unsigned long index = beginning; index<target.lLength; index++) {
4386 target.Delete (beginning);
4387 }
4388 return false;
4389 } else {
4390 _ElementaryCommand* ec=(_ElementaryCommand*)(target(beginning));
4391 ((_ElementaryCommand*)(target(beginning)))->MakeJumpCommand (((_String*)pieces(0)), beginning+1, (ec->simpleParameters.lLength<2)?target.lLength:ec->simpleParameters(1),target);
4392 }
4393
4394 while (target.lastif.lLength>intIfs) {
4395 target.lastif.Delete(target.lastif.lLength-1);
4396 }
4397
4398 return target.BuildList(source,bc,true);
4399 }
4400
4401
4402
4403 //____________________________________________________________________________________
BuildDoWhile(_StringBuffer & source,_ExecutionList & target)4404 bool _ElementaryCommand::BuildDoWhile (_StringBuffer&source, _ExecutionList&target)
4405 {
4406 long upto = source.FindBackwards(_String('}'), 0, -1);
4407 if (upto >= 0) {
4408 _String clipped (source, upto+1, -1);
4409 if (clipped.BeginsWith (blWhile)) {
4410 source.Trim (blDo.length(),upto);
4411 _List pieces;
4412 ExtractConditions (clipped,blWhile.length(),pieces);
4413 if (pieces.lLength != 1) {
4414 HandleApplicationError ("Malformed while clause in a do-while loop");
4415 return false;
4416 }
4417
4418 if (!MakeGeneralizedLoop (nil,(_String*)pieces(0),nil,false,source,target)) {
4419 return false;
4420 }
4421
4422 return true;
4423 }
4424 }
4425 HandleApplicationError ("Could not find a matching 'while' in the definition of a do-while loop");
4426
4427 return false;
4428 }
4429
4430 //____________________________________________________________________________________
4431
ProcessInclude(_StringBuffer & source,_ExecutionList & target)4432 bool _ElementaryCommand::ProcessInclude (_StringBuffer&source, _ExecutionList&target) {
4433 _String fileName (source, blInclude.length (),(long)source.length () - 2L);
4434 ProcessFileName(fileName, false,false,(hyPointer)target.nameSpacePrefix);
4435 if (terminate_execution) {
4436 return false;
4437 }
4438 PushFilePath (fileName);
4439 ReadBatchFile (fileName, target);
4440 PopFilePath ();
4441 return true;
4442 }
4443
4444 //____________________________________________________________________________________
4445
makeNewCommand(long ccode)4446 _ElementaryCommand* makeNewCommand (long ccode) {
4447 return new _ElementaryCommand (ccode);
4448 }
4449
4450 //____________________________________________________________________________________
4451
addAndClean(_ExecutionList & target,_List * parameter_list,long start_at)4452 void _ElementaryCommand::addAndClean (_ExecutionList&target,_List* parameter_list, long start_at) {
4453 if (parameter_list) {
4454 // TODO 20170913 SLKP : check that << works as well
4455 for (long i = start_at; i < parameter_list->countitems(); i++) {
4456 parameters << parameter_list->GetItem(i);
4457 }
4458 }
4459 target.AppendNewInstance(this);
4460 }
4461
4462 //____________________________________________________________________________________
4463
4464
ConstructDataSet(_StringBuffer & source,_ExecutionList & target)4465 bool _ElementaryCommand::ConstructDataSet (_StringBuffer&source, _ExecutionList&target)
4466 // DataSet dataSetid = ReadDataFile ("..");
4467 // or
4468 // DataSet dataSetid = SimulateDataSet (likeFunc);
4469 // or
4470 // DataSet dataSetid = Concatenate (<purge>,list of DataSets);
4471 // or
4472 // DataSet dataSetid = Combine (<purge>,list of DataSets);
4473 // or
4474 // DataSet dataSetid = ReconstructAncestors (lf)
4475 // or
4476 // DataSet dataSetid = SampleAncestors (lf)
4477 // or
4478 // DataSet dataSetid = Simulate (tree, freqs, alphabet, <store internal nodes, root vector>)
4479 // or
4480 // DataSet dataSetid = ReadFromString (string);
4481
4482
4483 {
4484 // first we must segment out the data set name
4485 // then the ReadDataFile command
4486 // then the data set file name
4487
4488 // look for the data set name first
4489
4490 const _String kConcat ("Concatenate"),
4491 kCombine ("Combine"),
4492 kReadDataFile ("ReadDataFile"),
4493 kReadFromString ("ReadFromString"),
4494 kPurge ("purge"),
4495 kReconstructAncestors ("ReconstructAncestors"),
4496 kSampleAncestors ("SampleAncestors"),
4497 kMarginalAncestors ("MARGINAL"),
4498 kDoLeavesAncestors ("DOLEAVES"),
4499 kSimulate ("Simulate");
4500
4501 try {
4502
4503 _String operation_type;
4504 _List arguments;
4505
4506 ProcessProcedureCall (source, operation_type, arguments);
4507
4508 if (operation_type == kReadDataFile || operation_type == kReadFromString) {
4509 if (arguments.countitems () != 2UL) {
4510 throw _String ("DataSet declaration missing a valid filename/string or has extra arguments");
4511 }
4512
4513 _ElementaryCommand * dsc = new _ElementaryCommand (5);
4514
4515 if (operation_type == kReadFromString) {
4516 dsc->simpleParameters << 1;
4517 }
4518 dsc->addAndClean (target, &arguments, 0L);
4519 } else if (operation_type == blSimulateDataSet) {
4520 if ( arguments.countitems()>5UL || arguments.countitems()==1UL ) {
4521 throw blSimulateDataSet.Enquote() & "expects 1-4 parameters: likelihood function ident (needed), a list of excluded states, a matrix to store random rates in, and a matrix to store the order of random rates in (last 3 - optional).";
4522 }
4523
4524 _ElementaryCommand * dsc = new _ElementaryCommand (12);
4525 dsc->addAndClean (target, &arguments, 0L);
4526 } else if ( operation_type == kConcat || operation_type == kCombine) {
4527 _ElementaryCommand * dsc = new _ElementaryCommand (16);
4528 dsc->simpleParameters<<((operation_type==kConcat)?1:2);
4529
4530 if ((*(_String*)arguments.GetItem(1)) == kPurge) {
4531 dsc->simpleParameters[0] = - dsc->simpleParameters[0];
4532 arguments.Delete (1);
4533 }
4534
4535 if (arguments.countitems() == 1UL) {
4536 delete (dsc);
4537 throw _String ("DataSet merging operation missing a valid list of arguments.");
4538 }
4539 dsc->addAndClean (target, &arguments, 0L);
4540 return true;
4541
4542 } else {
4543 if (operation_type == kReconstructAncestors || operation_type == kSampleAncestors) {
4544 if (arguments.countitems()>5UL || arguments.countitems()==1L) {
4545 throw operation_type.Enquote() & " expects 1-4 parameters: likelihood function ident (mandatory), an matrix expression to specify the list of partition(s) to reconstruct/sample from (optional), and, for ReconstructAncestors, an optional MARGINAL flag, plus an optional DOLEAVES flag.";
4546 }
4547 _ElementaryCommand * dsc = new _ElementaryCommand (operation_type == kReconstructAncestors ? 38 : 50);
4548 dsc->parameters << arguments (0) << arguments (1);
4549 for (long optP = 2L; optP < arguments.lLength; optP++) {
4550 _String * current_term = (_String*)arguments.GetItem(optP);
4551
4552 if (*current_term == kMarginalAncestors) {
4553 dsc->simpleParameters << -1;
4554 } else if (*current_term == kDoLeavesAncestors) {
4555 dsc->simpleParameters << -2;
4556 } else {
4557 dsc->parameters << current_term;
4558 }
4559 }
4560
4561 dsc->addAndClean (target);
4562 return true;
4563 } else if (operation_type == kSimulate) {
4564 if ((arguments.countitems()>8)||(arguments.countitems()<5UL)) {
4565 throw kSimulate.Enquote() & " expects 4-6 parameters: tree with attached models, equilibrium frequencies, character map, number of sites|root sequence, <save internal node sequences>, <file name for direct storage>";
4566
4567 }
4568
4569 _ElementaryCommand * dsc = new _ElementaryCommand (52);
4570 dsc->addAndClean (target, &arguments, 0);
4571 return true;
4572 } else {
4573 throw _String ("Expected DataSet ident = ReadDataFile(filename); or DataSet ident = SimulateDataSet (LikelihoodFunction); or DataSet ident = Combine (list of DataSets); or DataSet ident = Concatenate (list of DataSets); or DataSet ident = ReconstructAnscetors (likelihood function); or DataSet ident = SampleAnscetors (likelihood function) or DataSet dataSetid = ReadFromString (string);");
4574 }
4575 }
4576 } catch (const _String& err) {
4577 HandleErrorWhileParsing (err, source);
4578 return false;
4579 }
4580
4581 return false;
4582 }
4583 //____________________________________________________________________________________
4584
ConstructCategory(_StringBuffer & source,_ExecutionList & target)4585 bool _ElementaryCommand::ConstructCategory (_StringBuffer&source, _ExecutionList&target)
4586 // category <id> = (number of int, weights, method for representation, density, cumulative, left bound, right bound);
4587 {
4588
4589 long mark1 = source.FirstSpaceIndex (0,kStringEnd,kStringDirectionForward),
4590 mark2 = source.Find ('=', mark1, -1);
4591
4592 _String catID (source,mark1+1,mark2-1);
4593
4594 if (mark1==-1 || mark2==-1 || catID.empty () ) {
4595 HandleApplicationError("Category variable declaration missing a valid identifier");
4596 return false;
4597 }
4598
4599 // now look for the opening paren
4600
4601 mark1 = source.Find ('(',mark2,-1);
4602
4603 if (mark1!=-1) {
4604 mark2 = source.FindBackwards(')',mark1+1,-1);
4605 if (mark2!=-1) {
4606 _String definition (source,mark1+1,mark2-1);
4607 _List args;
4608 ExtractConditions (definition,0,args,',');
4609 if (args.countitems()>=7UL) {
4610 _ElementaryCommand * cv = new _ElementaryCommand (20);
4611 cv->parameters&&(&catID);
4612 cv->addAndClean(target,&args,0);
4613 return true;
4614 }
4615 }
4616 }
4617 HandleApplicationError ("Expected: category <id> = (number of intervals, weights, method for representation, density, cumulative, left bound, right bound,<optional mean cumulative function>,<optional hidden markov matrix>);");
4618 return false;
4619 }
4620
4621
4622 //____________________________________________________________________________________
ConstructChoiceList(_StringBuffer & source,_ExecutionList & target)4623 bool _ElementaryCommand::ConstructChoiceList(_StringBuffer&source, _ExecutionList&target) {
4624 _List args;
4625
4626
4627 ExtractConditions (source,blChoiceList.length(),args,',');
4628 if (args.lLength<5UL) {
4629 HandleApplicationError ("ChoiceList needs at least 5 arguments");
4630 return false;
4631 }
4632 _ElementaryCommand *cv = new _ElementaryCommand (32);
4633
4634 cv->parameters<<args(0);
4635 //((_String*)args.list_data[1])->StripQuotes();
4636 cv->parameters<<args(1)
4637 <<args(2)
4638 <<args(3);
4639
4640 if (args.lLength>5UL) {
4641 _List * choices = new _List;
4642 for (long k = 4L; k<args.lLength-1; k+=2) {
4643 ((_String*)args.list_data[k])->StripQuotes();
4644 ((_String*)args.list_data[k+1])->StripQuotes();
4645 _List * thisChoice = new _List;
4646 *thisChoice << args(k);
4647 *thisChoice << args(k+1);
4648 *choices < thisChoice;
4649 }
4650 cv->parameters < choices;
4651 cv->simpleParameters<<0;
4652 } else {
4653 cv->parameters<< args(4);
4654 cv->simpleParameters<<1;
4655 }
4656
4657
4658 cv->addAndClean(target,nil,0);
4659 return true;
4660 }
4661
4662
4663 //____________________________________________________________________________________
4664
ConstructTree(_StringBuffer & source,_ExecutionList & target)4665 bool _ElementaryCommand::ConstructTree (_StringBuffer&source, _ExecutionList&target)
4666 // Tree treeid = (...) or Topology = (...);
4667 {
4668 long mark1 = source.FirstSpaceIndex(0,-1,kStringDirectionForward);
4669 if (mark1 > 0) {
4670 mark1 = source.FirstNonSpaceIndex (mark1 + 1, -1);
4671 }
4672
4673
4674 long mark2 = source.FindTerminator(mark1, "=");
4675
4676
4677 if ( mark1 < 0 || mark2 < 0 || mark2 - mark1 < 1) {
4678 HandleApplicationError ("Tree declaration missing a valid identifier");
4679 return false;
4680 }
4681
4682 _String dsID = source.Cut (mark1,mark2-1);
4683 // now look for the opening paren
4684
4685 //(long& from, char open, char close, bool respectQuote, bool respectEscape)
4686 long mark3 = source.ExtractEnclosedExpression (mark1, '(', ')', fExtractRespectQuote | fExtractRespectEscape);
4687
4688
4689 if (mark1 < 0 || mark3 < 0 || mark3 <= mark1) {
4690 mark1 = mark2+1;
4691 mark3 = source.FindTerminator (mark1,";")-1;
4692 }
4693
4694 _ElementaryCommand * dsc = new _ElementaryCommand(source.BeginsWith(blTree)?7:54);
4695
4696 dsc->parameters&&(&dsID);
4697 dsc->parameters.AppendNewInstance(new _String(source,mark1,mark3));
4698
4699 dsc->addAndClean(target,nil,0);
4700 return true;
4701 }
4702
4703
4704
4705 //____________________________________________________________________________________
4706
ConstructDataSetFilter(_StringBuffer & source,_ExecutionList & target)4707 bool _ElementaryCommand::ConstructDataSetFilter (_StringBuffer&source, _ExecutionList&target) {
4708 // DataSetFilter dataSetFilterid = CreateFilter (datasetid;unit;vertical partition; horizontal partition; alphabet exclusions);
4709 // first we must segment out the data set name
4710
4711 const _String kCreateFilter ("CreateFilter"),
4712 kPermute ("Permute"),
4713 kBootstrap ("Bootstrap");
4714
4715 _ElementaryCommand * datafilter_command = nil;
4716
4717 try {
4718
4719 _String operation_type;
4720 _List arguments;
4721
4722 ProcessProcedureCall (source, operation_type, arguments);
4723
4724 if (operation_type == kCreateFilter) {
4725 datafilter_command = new _ElementaryCommand(6);
4726 } else if (operation_type == kPermute) {
4727 datafilter_command = new _ElementaryCommand(27);
4728 } else if (operation_type == kBootstrap) {
4729 datafilter_command = new _ElementaryCommand(28);
4730 } else {
4731 throw _String ("Expected: DataSetFilter dataSetFilterid = CreateFilter (datasetid,unit,vertical partition,horizontal partition,alphabet exclusions); or Permute/Bootstrap (dataset/filter,<atom>,<column partition>)");
4732 }
4733
4734 if (!(arguments.countitems()>=3UL || (arguments.countitems() == 2UL && datafilter_command->code == 6))) {
4735 throw _String ("Parameter(s) missing in DataSetFilter definition.");
4736 }
4737
4738 datafilter_command->addAndClean (target,&arguments);
4739 } catch (const _String& err) {
4740 HandleErrorWhileParsing (err, source);
4741 DeleteObject (datafilter_command);
4742 return false;
4743 }
4744
4745 return true;
4746 }
4747
4748 //____________________________________________________________________________________
4749
ConstructModel(_StringBuffer & source,_ExecutionList & target)4750 bool _ElementaryCommand::ConstructModel (_StringBuffer&source, _ExecutionList&target)
4751
4752 // Model ID = (inst transition matrix ident, equilibrium frequencies ident, <multiply by frequencies>);
4753 // if the third parameter is explicitFormMExp, then inst transition matrix ident is expected to be an explicit matrix exponential
4754 // EXPRESSION
4755
4756 {
4757 // first we must segment out the data set name
4758
4759 long mark1 = source.FirstSpaceIndex(0,-1,kStringDirectionForward),
4760 mark2 = source.Find ('=', mark1, -1);
4761
4762 _String modelID (source,mark1+1,mark2-1);
4763
4764 if (mark1==-1 || mark2==-1 || !modelID.IsValidIdentifier(fIDAllowCompound|fIDAllowFirstNumeric)) {
4765 HandleApplicationError ("Model declaration missing a valid identifier.");
4766 return false;
4767 }
4768
4769 // now look for the opening paren
4770 mark1 = source.Find ('(',mark2,-1);
4771 _List pieces;
4772 ExtractConditions (source,mark1+1,pieces,',');
4773
4774 if (pieces.lLength<2) {
4775 HandleApplicationError ("Parameter(s) missing in Model definition. Must have a matrix and a compatible eqiulibrium frequencies vector.");
4776 return false;
4777 } else {
4778 if (pieces.lLength>3) {
4779 HandleApplicationError ("Too many parameters (3 max) in Model definition");
4780 return false;
4781 }
4782 }
4783
4784 _ElementaryCommand * model = new _ElementaryCommand(31);
4785 model->parameters&&(&modelID);
4786 model->addAndClean (target,&pieces,0);
4787 return true;
4788
4789 }
4790
4791
4792 //____________________________________________________________________________________
4793
MakeJumpCommand(_String * source,long branch_true,long branch_false,_ExecutionList & parentList)4794 bool _ElementaryCommand::MakeJumpCommand (_String* source, long branch_true, long branch_false, _ExecutionList& parentList) {
4795 long existing_formula_handle = 0L;
4796 code = 4L;
4797
4798 if (simpleParameters.lLength==3) {
4799 if (source) {
4800 delete ((_Formula*)simpleParameters.get(2));
4801 } else {
4802 existing_formula_handle = simpleParameters.get(2);
4803 }
4804 }
4805
4806 if (branch_true==-1) {
4807 if (simpleParameters.empty()) {
4808 HandleApplicationError("An if-then-else scoping error. Check opening and closing brackets and double else's.");
4809 return false;
4810 }
4811 branch_true = simpleParameters.get (0);
4812 }
4813
4814 simpleParameters.Clear();
4815 simpleParameters<<branch_true<<branch_false;
4816 if (source) {
4817 parameters && source;
4818 } else if (existing_formula_handle) {
4819 simpleParameters<<existing_formula_handle;
4820 }
4821
4822 return true;
4823 }
4824
4825
4826
4827 //____________________________________________________________________________________
ConstructLF(_StringBuffer & source,_ExecutionList & target)4828 bool _ElementaryCommand::ConstructLF (_StringBuffer&source, _ExecutionList&target)
4829 // syntax: LikelihoodFunction id = (filter1, tree1, ..., filterN, treeN, optional compute template)
4830 // or LikelihoodFunction3 id = (filter1, tree1, freq1, ... filterN, treeN, freqN, optional compute template)
4831 {
4832 long mark1 = source.FirstNonSpaceFollowingSpace(),
4833 mark2 = mark1 > 0 ? source.FindTerminator (mark1 + 1, "=") : 0;
4834
4835 if ( mark1==-1 || mark2==-1 || mark1+1 > mark2 ) {
4836 HandleApplicationError ("Likelihood function declaration missing a valid identifier");
4837 return false;
4838 }
4839
4840 _String lfID (source,mark1,mark2-1);
4841 // now look for the opening paren
4842
4843 _List pieces;
4844 mark2 ++;
4845 mark1 = source.ExtractEnclosedExpression(mark2, '(', ')', fExtractRespectQuote | fExtractRespectEscape);
4846
4847 if ( mark1==-1 || mark2==-1 || mark1<mark2 ) {
4848 HandleApplicationError ("Expected: Likelihood Function ident = (tree1, datasetfilter1,...)");
4849 return false;
4850 }
4851
4852 ExtractConditions (source,mark2+1,pieces,',');
4853 _ElementaryCommand* dsc = new _ElementaryCommand (11);
4854 dsc->parameters&&(&lfID);
4855
4856 if (source.BeginsWith(blLF3)) {
4857 dsc->simpleParameters << 1;
4858 }
4859
4860 dsc->addAndClean(target,&pieces,0);
4861 return true;
4862 }
4863
4864
4865
4866 //____________________________________________________________________________________
ConstructFunction(_StringBuffer & source,_ExecutionList & chain)4867 bool _ElementaryCommand::ConstructFunction (_StringBuffer&source, _ExecutionList& chain) {
4868 // syntax: function <ident> (comma separated list of parameters) {body}
4869
4870 bool isFFunction = source.BeginsWith (blFFunction),
4871 isLFunction = ! isFFunction && source.BeginsWith (blLFunction),
4872 isCFunction = ! isFFunction && ! isLFunction && source.BeginsWith (blCFunction),
4873 isNameSpace = ! isFFunction && ! isLFunction && ! isCFunction && source.BeginsWith (blNameSpace);
4874
4875 _hy_nested_check save_state = isInFunction;
4876
4877
4878 if (!isNameSpace) {
4879 if (isInFunction == _HY_FUNCTION) {
4880 HandleApplicationError ("Nested function declarations are not allowed");
4881 return false;
4882 }
4883 }
4884
4885 long mark1, mark2;
4886
4887 _FString *save_nmspc = nil;
4888
4889 if (isNameSpace) {
4890 mark1 = source.FirstNonSpaceIndex(blNameSpace.length(), kStringEnd, kStringDirectionForward);
4891 mark2 = source.Find ('{', mark1, kStringEnd);
4892 } else {
4893 _String const * prefix = &blFunction;
4894
4895 if (isFFunction) prefix = &blFFunction; else
4896 if (isLFunction) prefix = &blLFunction; else
4897 if (isCFunction) prefix = &blCFunction;
4898
4899 mark1 = source.FirstNonSpaceIndex(prefix->length(),kStringEnd,kStringDirectionForward);
4900 mark2 = source.Find ('(', mark1, kStringEnd);
4901 }
4902
4903
4904 if ( mark1 == kNotFound || mark2 == kNotFound || mark1 > mark2-1) {
4905 HandleApplicationError (_String("Function declaration missing a valid function identifier or parameter list.\n-----------\n") & source & "\n-----------\n");
4906 return false;
4907 }
4908
4909 _String* funcID = new _String(source,mark1,mark2-1);
4910
4911 if (!funcID->IsValidIdentifier(fIDAllowCompound)) {
4912 HandleApplicationError (_String("Not a valid function/namespace identifier '") & _String(funcID) & "'");
4913 return false;
4914 }
4915
4916 *funcID = chain.AddNameSpaceToID (*funcID);
4917
4918 // now look for the opening paren
4919
4920 if (!isNameSpace) {
4921
4922
4923 _String extraNamespace;
4924
4925 if ((mark1=FindBFFunctionName(*funcID)) >= 0L) {
4926 ReportWarning (_String("Overwritten previously defined function:'") & *funcID & '\'');
4927 if (batchLanguageFunctionClassification.get (mark1) == kBLFunctionLocal) {
4928 // clean up previously used namespaces
4929 _ExecutionList * existing = (_ExecutionList *)batchLanguageFunctions.GetItem(mark1);
4930 _String * nm = existing->GetNameSpace();
4931 if (nm) {
4932 long best_index;
4933 char match = variableNames.FindBest(nm, best_index);
4934 //printf ("PREVIOUS LOCAL FUNCTION %s\n", nm->get_str());
4935 if (best_index >= 0) {
4936 _Variable * namespace_start = LocateVar (best_index);
4937 if (namespace_start->GetName()->BeginsWith(*nm)) {
4938 //printf ("Clearing namespace %s starting at %s for function %s \n", nm->get_str(), namespace_start->GetName()->get_str(), funcID->get_str());
4939 _SimpleList locals;
4940 DeleteTreeVariable (best_index, locals,nm, false);
4941 }
4942 }
4943 if (match == 0) {
4944 DeleteVariable(best_index, true, false);
4945 }
4946 }
4947 }
4948
4949 }
4950
4951 _List arguments;
4952 _SimpleList argument_types;
4953
4954 long upto = ExtractConditions (source,mark2+1,arguments,',',false);
4955
4956
4957 if (upto==source.length() || source[upto]!='{' || source (-1)!='}') {
4958 HandleApplicationError (_String("Function declaration is missing a valid function body."));
4959 isInFunction= save_state;
4960 return false;
4961 }
4962
4963 if (isLFunction && extraNamespace.empty())
4964 extraNamespace = _HYGenerateANameSpace();
4965
4966 for (long k = 0UL; k < arguments.lLength; k++) {
4967
4968 _String* namespaced = new _String(chain.AddNameSpaceToID (*(_String*)arguments(k), & extraNamespace));
4969 if ((*namespaced)(-1L) == '&') {
4970 namespaced->Trim(0,namespaced->length() -2);
4971 argument_types << kBLFunctionArgumentReference;
4972 } else {
4973 argument_types << kBLFunctionArgumentNormal;
4974 }
4975 arguments.Replace (k,namespaced,false);
4976 }
4977
4978
4979 _StringBuffer sfunctionBody (source, upto+1,source.length ()-2);
4980 _ExecutionList * functionBody;
4981
4982 isInFunction = _HY_FUNCTION;
4983 if (isLFunction) {
4984 _String * existing_namespace = chain.GetNameSpace();
4985 if (existing_namespace) {
4986 extraNamespace = *existing_namespace & '.' & extraNamespace;
4987 }
4988 functionBody = new _ExecutionList (sfunctionBody,&extraNamespace,true);
4989 if (existing_namespace) {
4990 functionBody->enclosingNamespace = *existing_namespace;
4991 }
4992 }
4993 else {
4994 functionBody = new _ExecutionList (sfunctionBody,chain.GetNameSpace(),true);
4995 }
4996
4997 if (isCFunction) {
4998 if (functionBody->TryToMakeSimple()) {
4999 ReportWarning(_String ("Successfully compiled code for function ") & funcID->Enquote());
5000 }
5001 }
5002
5003 // take care of all the return statements
5004 returnlist.Each ([functionBody] (long value, unsigned long) -> void {
5005 ((_ElementaryCommand*)functionBody->GetItem(value))->simpleParameters << functionBody->countitems();
5006 });
5007 returnlist.Clear();
5008
5009 if (mark1>=0) {
5010 batchLanguageFunctions.Replace (mark1, functionBody, false);
5011 batchLanguageFunctionNames.Replace (mark1, funcID, false);
5012 batchLanguageFunctionParameterLists.Replace (mark1, &arguments, true);
5013 batchLanguageFunctionParameterTypes.Replace (mark1, &argument_types, true);
5014 batchLanguageFunctionClassification.list_data[mark1] = isLFunction ? kBLFunctionLocal :( isFFunction? kBLFunctionSkipUpdate : kBLFunctionAlwaysUpdate);
5015 } else {
5016 batchLanguageFunctions.AppendNewInstance(functionBody);
5017 batchLanguageFunctionNamesIndexed.Insert (new _String (*funcID), batchLanguageFunctions.countitems() - 1, false, true);
5018 batchLanguageFunctionNames.AppendNewInstance(funcID);
5019 batchLanguageFunctionParameterLists &&(&arguments);
5020 batchLanguageFunctionParameterTypes &&(&argument_types);
5021 batchLanguageFunctionClassification <<(isLFunction ? kBLFunctionLocal :( isFFunction? kBLFunctionSkipUpdate : kBLFunctionAlwaysUpdate));
5022 }
5023 } else {
5024 if (mark2 == source.length () || source[mark2]!='{' || source (-1L) !='}') {
5025 HandleApplicationError (_String("Namespace declaration is missing a body."));
5026 return false;
5027 }
5028 _StringBuffer namespace_text (source, mark2+1,source.length()-2);
5029
5030 bool success = false;
5031
5032 isInFunction = _HY_NAMESPACE;
5033 _ExecutionList * namespace_payload = new _ExecutionList (namespace_text, funcID, false, &success);
5034
5035 // 20180713 SLKP -- this was marked as deleted in one of the v2.3 branches
5036 if (success) {
5037 _ElementaryCommand * nested_list = new _ElementaryCommand (HY_HBL_COMMAND_NESTED_LIST);
5038 nested_list->parameters.AppendNewInstance(namespace_payload);
5039 nested_list->parameters << funcID;
5040 chain.AppendNewInstance(nested_list);
5041 DeleteObject (funcID);
5042 } else {
5043 DeleteObject (namespace_payload);
5044 DeleteObject (funcID);
5045
5046 return false;
5047 }
5048
5049 }
5050 if (isNameSpace) {
5051
5052 }
5053
5054 isInFunction = save_state;
5055 return true;
5056 }
5057
5058 //____________________________________________________________________________________
ConstructReturn(_StringBuffer & source,_ExecutionList & target)5059 bool _ElementaryCommand::ConstructReturn (_StringBuffer&source, _ExecutionList&target) {
5060 // syntax: return <statement>
5061
5062 long mark1 = source.FirstNonSpaceIndex(blReturn.length(),kStringEnd,kStringDirectionForward);
5063
5064 _ElementaryCommand * return_statement = new _ElementaryCommand (14);
5065
5066 if (mark1 != kNotFound) { // not a trivial return statement;
5067 if (source (-1) ==';') {
5068 return_statement->parameters.AppendNewInstance (new _String (source, mark1, source.length () - 2L));
5069 } else {
5070 return_statement->parameters.AppendNewInstance (new _String (source, mark1, kStringEnd));
5071 }
5072 }
5073
5074 if (isInFunction == _HY_FUNCTION) {
5075 returnlist<<target.countitems();
5076 } else {
5077 if (isInFunction == _HY_NAMESPACE) {
5078 HandleApplicationError("return statements are not allowed in namespaces");
5079 }
5080 return_statement->simpleParameters << -1;
5081 }
5082
5083 return_statement->addAndClean (target);
5084 return true;
5085 }
5086
5087
5088 //____________________________________________________________________________________
5089
ReadBatchFile(_String & fName,_ExecutionList & target)5090 void ReadBatchFile (_String& fName, _ExecutionList& target) {
5091 // read/parse a file into an execution list
5092 // THE function!!!
5093
5094 ProcessFileName(fName, target.nameSpacePrefix);
5095
5096 if (terminate_execution) {
5097 return;
5098 }
5099 /*#else
5100 _Variable optprec (optimizationPrecision);
5101 _Constant precvalue (0.01);
5102 FetchVar(LocateVarByName (optimizationPrecision))->SetValue(&precvalue);
5103 #endif*/
5104
5105 hyFile *f = hyFile::openFile (fName.get_str (), "rb");
5106 SetStatusLine ("Parsing File");
5107 if (!f) {
5108 HandleApplicationError (_String("Could not read batch file '") & fName & "'.\nPath stack:\n\t" & GetPathStack("\n\t"));
5109 } else {
5110 _StringBuffer source_file (f);
5111
5112 if (source_file.BeginsWith ("#NEXUS",false)) {
5113 ReadDataSetFile (f,1,nil,&fName, nil, &hy_default_translation_table, &target);
5114 } else {
5115 target.BuildList (source_file);
5116 target.sourceFile = fName;
5117 }
5118 f->close();
5119 }
5120 if (f) delete f;
5121 }
5122
5123
5124
5125 //____________________________________________________________________________________
SerializeModel(_StringBuffer & rec,long theModel,_AVLList * alreadyDone,bool completeExport)5126 void SerializeModel (_StringBuffer & rec, long theModel, _AVLList* alreadyDone, bool completeExport)
5127 {
5128 bool mByF = true,
5129 do2 = false;
5130
5131 _Variable * tV = nil,
5132 * tV2 = nil;
5133
5134 _Formula * theExp = nil;
5135 _SimpleList matrices;
5136
5137 if (modelTypeList.list_data[theModel]) {
5138 theExp = (_Formula*)modelMatrixIndices.list_data[theModel];
5139 theExp->ScanFForType(matrices, MATRIX);
5140
5141 for (long mi = 0; mi < matrices.countitems(); mi++) {
5142 if (alreadyDone && alreadyDone->Insert ((BaseRef)matrices.list_data[mi]) < 0) {
5143 matrices.Delete(mi);
5144 mi--;
5145 }
5146 }
5147 } else {
5148 if (!alreadyDone || alreadyDone->Find ((BaseRef)modelMatrixIndices.list_data[theModel]) < 0) {
5149 if (alreadyDone) {
5150 alreadyDone->Insert ((BaseRef)modelMatrixIndices.list_data[theModel]);
5151 }
5152 matrices << modelMatrixIndices.list_data[theModel];
5153 }
5154 tV = LocateVar(modelMatrixIndices.list_data[theModel]);
5155 }
5156
5157 long freqID = modelFrequenciesIndices.list_data[theModel];
5158
5159 if (freqID>=0) {
5160 tV2 = LocateVar(freqID);
5161 } else {
5162 mByF = false;
5163 tV2 = LocateVar(-freqID-1);
5164 }
5165
5166 if (!alreadyDone || alreadyDone->Find ((BaseRef)tV2->get_index()) < 0) {
5167 if (alreadyDone) {
5168 alreadyDone->Insert ((BaseRef)tV2->get_index());
5169 }
5170 do2 = true;
5171 }
5172
5173 if (completeExport && (matrices.lLength || do2 || theExp)) {
5174 _SimpleList vl,
5175 ind,
5176 dep,
5177 cat;
5178
5179 _AVLList vlst (&vl);
5180
5181 if (theExp) {
5182 theExp->ScanFForVariables(vlst, true, false, true);
5183 }
5184
5185 for (long mi = 0; mi < matrices.lLength; mi++) {
5186 LocateVar(matrices.list_data[mi])->ScanForVariables (vlst,true);
5187 }
5188
5189 if (do2) {
5190 tV2->ScanForVariables (vlst,true);
5191 }
5192 vlst.ReorderList ();
5193 SplitVariablesIntoClasses (vl,ind,dep,cat);
5194
5195 _StringBuffer glVars (128L),
5196 locVars(128L);
5197
5198
5199 ExportIndVariables (glVars,locVars, &ind);
5200 ExportDepVariables (glVars,locVars, &dep);
5201 rec << "SetParameter (DEFER_CONSTRAINT_APPLICATION, 1, 0);\n" << glVars <<locVars << "SetParameter (DEFER_CONSTRAINT_APPLICATION, 0, 0);\n" ;
5202 ExportCatVariables (rec,&cat);
5203 }
5204
5205 if (matrices.lLength) {
5206 for (long k = 0; k < matrices.lLength; k++) {
5207 _Variable *tV = LocateVar (matrices.list_data[k]);
5208 ((_Matrix*) tV->GetValue())->Serialize (rec,*tV->GetName());
5209 rec << '\n';
5210 }
5211 }
5212
5213 if (do2) {
5214 ((_Matrix*) tV2->GetValue())->Serialize (rec,*tV2->GetName());
5215 }
5216
5217 rec << "\nModel "
5218 << *((_String*)modelNames (theModel))
5219 << "=(";
5220 if (theExp) {
5221 rec << _String((_String*)(theExp->toStr(kFormulaStringConversionNormal))).Enquote();
5222 } else {
5223 rec << *tV->GetName();
5224 }
5225 rec << ',' << *tV2->GetName();
5226 if (theExp) {
5227 rec << ',' << explicitFormMExp;
5228 } else if (!mByF) {
5229 rec << ",0";
5230 }
5231 rec << ");\n";
5232 }
5233