1#!/usr/local/bin/python3.8
2
3# general functions for use by uber_tool*.py
4
5import sys, os
6from time import asctime
7import glob
8
9from afnipy import afni_base as BASE
10from afnipy import afni_util as UTIL
11from afnipy import lib_subjects as SUBJ
12from afnipy import lib_vars_object as VO
13
14DEF_UBER_DIR = 'uber_results'        # top directory for output
15DEF_TOP_DIR  = 'tool_results'     # top subject dir under uber_results
16
17g_history = """
18  uber_skel.py history
19
20    0.0  12 Apr, 2011: initial revision (based on uber_align_test.py v. 0.2)
21"""
22
23g_version = '0.0 (May 12, 2011)'
24
25# ----------------------------------------------------------------------
26# global definition of default processing blocks
27g_tlrc_base_list  = ['TT_N27+tlrc', 'TT_avg152T1+tlrc', 'TT_icbm452+tlrc',
28                     'MNI_avg152T1+tlrc']
29g_def_tlrc_base   = 'TT_N27+tlrc'
30
31
32# ----------------------------------------------------------------------
33# global definitions of result, control and user defaults
34# (as well as string versions of control and user defaults)
35
36# ---- resulting values returned after class actions ----
37g_res_defs = VO.VarsObject("uber_skel result variables")
38g_res_defs.file_proc     = ''   # file name for process script
39g_res_defs.output_proc   = ''   # output from running proc script
40
41# ---- control variables: process control, not set by user in GUI
42
43g_ctrl_defs = VO.VarsObject("uber_skel control defaults")
44g_ctrl_defs.proc_dir     = '.'  # process dir: holds scripts and result dir
45
46
47# ---- user variables: process control, alignment inputs and options ----
48
49g_user_defs = VO.VarsObject("uber_skel user defaults")
50g_user_defs.verb           = 1          # verbose level
51g_user_defs.copy_scripts   = 'yes'      # do we make .orig copies of scripts?
52
53# required inputs
54g_user_defs.anat           = ''         # anatomical volume to align
55g_user_defs.epi            = ''         # EPI dataset
56g_user_defs.epi_base       = 0          # EPI alignment base index
57
58# options
59g_user_defs.results_dir    = 'align.results' # where script puts results
60g_user_defs.cost_list      = ['lpc', 'lpc+ZZ', 'lpc+', 'lpa', 'nmi', 'ls']
61g_user_defs.giant_move     = 'no'
62g_user_defs.align_centers  = 'no'
63g_user_defs.center_base    = 'TT_N27+tlrc'
64g_user_defs.aea_opts       = []         # other align_epi_anat.py options
65
66# todo...
67g_user_defs.add_edge       = 'no'
68g_user_defs.anat_has_skull = 'yes'
69g_user_defs.epi_strip_meth = '3dSkullStrip'
70
71
72# string versions of variables - used by GUI and main
73# (when creating AlignTest object, string versions of vars are passed)
74g_cdef_strs = g_ctrl_defs.copy(as_strings=1)
75g_udef_strs = g_user_defs.copy(as_strings=1)
76
77
78# main class definition
79class AlignTest(object):
80   """class for testing anat to EPI alignment
81
82        - cvars : control variables
83        - uvars : user variables
84        - rvars : return variables
85
86        ** input vars might be string types, convert on merge
87
88        variables:
89           LV            - local variables
90           cvars         - control variables
91           uvars         - user variables
92           cmd_text      - generated alignment script
93           errors            --> array of resulting error messages
94           warnings          --> array of resulting warning messages
95   """
96   def __init__(self, cvars=None, uvars=None):
97
98      # ------------------------------------------------------------
99      # variables
100
101      # LV: variables local to this interface, not passed
102      self.LV = VO.VarsObject("local AP_Subject vars")
103      self.LV.indent = 8                # default indent for main options
104      self.LV.istr   = ' '*self.LV.indent
105      self.LV.retdir = ''               # return directory (for jumping around)
106
107      # merge passed user variables with defaults
108      self.cvars = g_ctrl_defs.copy()
109      self.uvars = g_user_defs.copy()
110      self.cvars.merge(cvars, typedef=g_ctrl_defs)
111      self.uvars.merge(uvars, typedef=g_user_defs)
112
113      # output variables
114      self.rvars = g_res_defs.copy()    # init result vars
115      self.align_script = ''            # resulting script
116      self.errors = []                  # list of error strings
117      self.warnings = []                # list of warning strings
118      # ------------------------------------------------------------
119
120      # ------------------------------------------------------------
121      # preperatory settings
122
123      if self.check_inputs(): return    # require at least anat and epi
124
125      self.set_directories()            # data dirs: anat, epi
126
127      if self.uvars.verb > 3: self.LV.show('ready to start script')
128
129      # do the work
130      self.create_script()
131
132   def check_inputs(self):
133      """check for required inputs: anat, epi (check existence?)"""
134      if self.uvars.is_empty('anat'):
135         self.errors.append('** unspecified anatomical dataset')
136
137      if self.uvars.is_empty('epi'):
138         self.errors.append('** unspecified EPI dataset')
139
140      if len(self.uvars.cost_list) < 1:
141         self.errors.append('** unspecified cost function(s)')
142
143      if not UTIL.vals_are_unique(self.uvars.cost_list):
144         self.errors.append('** cost functions are not unique')
145
146      return len(self.errors)
147
148   def set_directories(self):
149      """decide on use of top_dir (use it or nothing - no anat_dir)
150
151         ==> if top_dir is set, use $top_dir/short_names
152             else just use anat and epi directly
153      """
154      top_dir, parent_dirs, short_dirs, short_names =    \
155                UTIL.common_parent_dirs([[self.uvars.anat, self.uvars.epi]])
156
157      self.LV.top_dir     = parent_dirs[0]  # common parent dir
158      self.LV.short_names = short_names # if top_dir is used, they are under it
159
160      if self.uvars.verb > 2:
161         print('-- set_dirs: top_dir    = %s\n' \
162               '             short_anat = %s\n' \
163               '             short_epi  = %s\n' \
164               % (self.LV.top_dir, short_names[0][0], short_names[0][1]))
165
166      # if top_dir isn't long enough, do not bother with it
167      if self.LV.top_dir.count('/') < 2:
168         self.LV.top_dir = ''
169         if self.uvars.verb > 2: print('   (top_dir not worth using...)')
170
171   def create_script(self):
172      """attempt to generate an alignment script
173            - write align_script
174            - keep a list of any warnings or errors
175            -
176            - if there are errors, align_script might not be filled
177      """
178
179      # script prep, headers and variable assignments
180      self.align_script  = self.script_init()
181      self.align_script += self.script_set_vars()
182
183      # do some actual work
184      self.align_script += self.script_results_dir()
185      self.align_script += self.script_copy_data()
186      if self.uvars.align_centers == 'yes':
187         self.align_script += self.script_align_centers()
188      self.align_script += self.script_align_datasets()
189
190      # add commands ...
191
192      if len(self.errors) > 0: return   # if any errors so far, give up
193
194      return
195
196   def script_align_datasets(self):
197      """actually run align_epi_anat.py
198
199         only current option is -mult_cost, everything else is via variables
200      """
201
202      if len(self.uvars.cost_list) > 1: mstr = ' -multi_cost $cost_list'
203      else:                             mstr = ''
204
205      cmd = SUBJ.comment_section_string('align data') + '\n'
206
207      cmd += \
208       '# test alignment, using variables set above\n'          \
209       'align_epi_anat.py -anat anat+orig -epi epi+orig '       \
210                         '-epi_base $in_ebase \\\n'             \
211       '                  -cost $cost_main%s $align_opts\n'     \
212       '\n' % mstr
213
214      return cmd
215
216   def script_align_centers(self):
217      """if align_centers should be run, deoblique both and align with
218         center_base (probably TT_N27+tlrc)
219
220         note: these commands should be fixed, since the dataset names are
221      """
222
223      if self.uvars.align_centers != 'yes': return ''
224
225      cmd = SUBJ.comment_section_string('align centers') + '\n'
226
227      cmd += '# since altering grid, remove any oblique transformation\n'    \
228             '3drefit -deoblique anat+orig epi+orig\n'                       \
229             '\n'                                                            \
230             '# align volume centers (we do not trust spatial locations)\n'  \
231             '@Align_Centers -no_cp -base TT_N27+tlrc -dset anat+orig\n'     \
232             '@Align_Centers -no_cp -base TT_N27+tlrc -dset epi+orig\n'      \
233             '\n'
234
235      return cmd
236
237   def script_copy_data(self):
238      """these commands only vary based on results_dir"""
239
240      cmd = SUBJ.comment_section_string('copy data') + '\n'
241
242      if self.uvars.is_trivial_dir('results_dir'): rdir = '.'
243      else:                                        rdir = '$results_dir'
244
245      cmd += '# copy dataset to processing directory\n'         \
246             '3dbucket -prefix %s/anat $in_anat\n'              \
247             '3dbucket -prefix %s/epi $in_epi"[$in_ebase]"\n'   \
248             '\n' % (rdir, rdir)
249
250      if rdir != '.':
251         cmd += '# enter the processing directory\n'    \
252                'cd %s\n\n' % rdir
253
254      return cmd
255
256   def script_init(self):
257      cmd = '#!/bin/tcsh -xef\n\n'                              \
258            '# created by uber_skel.py: version %s\n'     \
259            '# creation date: %s\n\n' % (g_version, asctime())
260
261      return cmd
262
263   def script_results_dir(self):
264
265      # if no results dir, just put everything here
266      if self.uvars.is_trivial_dir('results_dir'): return ''
267
268      cmd  = SUBJ.comment_section_string('test and create results dir') + '\n'
269
270      cmd += '# note directory for results\n'           \
271             'set results_dir = %s\n\n' % self.uvars.results_dir
272
273      cmd += '# make sure it does not yet exist\n'      \
274             'if ( -e $results_dir ) then\n'            \
275             '    echo "** results dir \'$results_dir\' already exists"\n'  \
276             '    exit\n'                               \
277             'endif\n\n'
278
279      cmd += '# create results directory, where the work will be done\n' \
280             'mkdir $results_dir\n\n'
281
282      return cmd
283
284   def script_set_vars(self):
285      """use variables for inputs (anat, epi, epi_base) and for
286         options (cost_main, cost_list, align_opts)
287      """
288
289      # init with a section comment
290      cmd = SUBJ.comment_section_string('set processing variables') + '\n'
291
292      # maybe init with top_dir
293      if not self.LV.is_trivial_dir('top_dir'):
294         cmd += '# top data directory\n' \
295                'set top_dir = %s\n\n' % self.LV.top_dir
296
297      # anat and epi might use top_dir
298      if self.LV.is_trivial_dir('top_dir'):
299         astr = self.uvars.anat
300         estr = self.uvars.epi
301      else:
302         astr = '$top_dir/%s' % self.LV.short_names[0][0]
303         estr = '$top_dir/%s' % self.LV.short_names[0][1]
304
305      cmd += '# input dataset options (ebase is EPI index)\n'           \
306             'set in_anat  = %s\n'                                      \
307             'set in_epi   = %s\n'                                      \
308             'set in_ebase = %d\n\n'                                    \
309             % (astr, estr, self.uvars.epi_base)
310
311      # note whether to use multi_cost
312      cmd += '# main options\n' \
313             'set cost_main = %s\n' % self.uvars.cost_list[0]
314      if len(self.uvars.cost_list) > 1:
315         cmd += 'set cost_list = ( %s )\n' % ' '.join(self.uvars.cost_list[1:])
316      cmd += '\n'
317
318      # possibly add align_opts list variable
319      cmd += self.make_align_opts_str()
320
321      return cmd
322
323   def make_align_opts_str(self):
324      """any align options, one per line"""
325
326      # keep comment separate to get indent length
327      cmnt = '# all other align_epi_anat.py options\n' \
328
329      cstr = 'set align_opts = ( '
330      clen = len(cstr)
331      istr = ' '*clen
332
333      # put one option on first line
334      cstr += '%s \\\n' % '-tshift off'
335
336      # ---------- here is the main option application ---------
337
338      # then add each option offset by initial indentation
339      cstr += '%s%s \\\n' % (istr, '-volreg off')
340
341      if self.uvars.giant_move == 'yes':
342         cstr += '%s%s \\\n' % (istr, '-giant_move')
343
344      if self.uvars.add_edge == 'yes':
345         if len(self.uvars.cost_list) > 1:
346            self.errors.append(
347               '** -AddEdge does not currently work with -multi_cost')
348         else:
349            cstr += '%s%s \\\n' % (istr, '-AddEdge')
350
351      if len(self.uvars.aea_opts) > 0:
352            cstr += '%s%s \\\n' % (istr, ' '.join(self.uvars.aea_opts))
353
354      # does the anatomy have a skull?
355      if self.uvars.anat_has_skull != 'yes':
356         cstr += '%s%s \\\n' % (istr, '-anat_has_skull no')
357
358      # -epi_strip method
359      if self.uvars.epi_strip_meth != '3dSkullStrip':
360         cstr += '%s%s \\\n' % (istr,'-epi_strip %s'%self.uvars.epi_strip_meth)
361
362      # want -save_all, -prep_off?
363
364      # last indent is left by 2 to align ()
365      cstr += '%*s)\n\n' % (clen-2, '')
366
367      # finally, align the line wrappers
368      cstr = UTIL.add_line_wrappers(cstr)
369
370      return cmnt + cstr
371
372   def get_script(self):
373      """return status, message
374
375                status = number of error messages
376                if 0: message = command
377                else: message = error string
378
379         Requests for warnings must be made separately, since they
380         are not fatal.
381      """
382
383      if len(self.errors) > 0:
384         return 1, SUBJ.make_message_list_string(self.errors, "errors")
385
386      return 0, self.align_script
387
388   def get_warnings(self):
389      """return the number of warnings and a warnings string"""
390
391      return len(self.warnings), \
392             SUBJ.make_message_list_string(self.warnings, "warnings")
393
394   def proc_dir_filename(self, vname):
395      """file is either fname or proc_dir/fname (if results is set)
396         vname : results file variable (must convert to fname)
397      """
398      fname = self.rvars.val(vname)
399      return self.cvars.file_under_dir('proc_dir', fname)
400
401   def nuke_old_results(self):
402      """if the results directory exists, remove it"""
403
404      if self.uvars.results_dir == '': return
405
406      # ------------------------- do the work -------------------------
407      self.LV.retdir = SUBJ.goto_proc_dir(self.cvars.proc_dir)
408
409      if os.path.isdir(self.uvars.results_dir):
410         print('-- nuking old results: %s' % self.uvars.results_dir)
411         os.system('rm -fr %s' % self.uvars.results_dir)
412
413      self.LV.retdir = SUBJ.ret_from_proc_dir(self.LV.retdir)
414      # ------------------------- done -------------------------
415
416
417   def copy_orig_proc(self):
418      """if the proc script exists, copy to .orig.SCRIPTNAME"""
419      if self.rvars.file_proc == '': return
420      pfile = self.rvars.file_proc
421
422      # ------------------------- do the work -------------------------
423      self.LV.retdir = SUBJ.goto_proc_dir(self.cvars.proc_dir)
424      if os.path.isfile(pfile):
425         cmd = 'cp -f %s .orig.%s' % (pfile, pfile)
426         if self.uvars.verb > 1: print('++ exec: %s' % cmd)
427         os.system(cmd)
428      elif self.uvars.verb > 1: print("** no proc '%s' to copy" % pfile)
429      self.LV.retdir = SUBJ.ret_from_proc_dir(self.LV.retdir)
430      # ------------------------- done -------------------------
431
432   def write_script(self, fname=''):
433      """write processing script to a file (in the proc_dir)
434         - if fname is set, use it, else generate
435         - set rvars.file_proc and output_proc
436      """
437
438      if not self.align_script:
439         print('** no alignment script to write out')
440         return 1
441      if fname: name = fname
442      else:
443         # if self.svars.sid: name = 'script.align.%s' % self.svars.sid
444         name = 'script.align'
445
446      # store (intended) names for calling tool to execute with
447      self.rvars.file_proc = name # store which file we have written to
448      self.rvars.output_proc = 'output.%s' % name # file for command output
449
450      if self.uvars.verb > 0: print('++ writing script to %s' % name)
451
452      # if requested, make an original copy
453      self.LV.retdir = SUBJ.goto_proc_dir(self.cvars.proc_dir)
454
455      if self.uvars.copy_scripts == 'yes': # make an orig copy
456         UTIL.write_text_to_file('.orig.%s'%name, self.align_script, exe=1)
457      rv = UTIL.write_text_to_file(name, self.align_script, exe=1)
458
459      self.LV.retdir = SUBJ.ret_from_proc_dir(self.LV.retdir)
460
461      return rv
462
463
464# ===========================================================================
465# help strings accessed both from command-line and GUI
466# ===========================================================================
467
468helpstr_todo = """
469---------------------------------------------------------------------------
470                        todo list:
471
472- create GUI
473- test center distance in GUI to suggest align centers
474- show corresponding afni_proc.py options
475- show corresponding uber_subjec.py options?
476- show afni command to look at results (basically point to directory)
477---------------------------------------------------------------------------
478"""
479
480helpstr_gui = """
481===========================================================================
482uber_skel.py (GUI)      - a graphical interface for testing alignment
483
484   Find good alignment options, possibly to add to afni_proc.py command
485   or uber_subject.py GUI options.
486
487   purposes:
488   required inputs:
489   optional inputs:
490   typical outputs:
491
492---------------------------------------------------------------------------
493Overview:
494
495- R Reynolds  Feb, 2011
496===========================================================================
497"""
498
499helpstr_create_program = """
500===========================================================================
501This is a brief overview about creating a new program/GUI for uber_proc.py.
502
503There are (currently) 3 basic files used, the main program, the library and
504the GUI.  The intention is that one can run the main program to generate or
505execute processing scripts (goal of the GUI) without actually using the GUI.
506Some common GUI routines/classes are in lib_qt_gui.py.
507
508So the purpose of the GUI is to set user variables to pass to the library.
509
510   1. uber_skel.py: main program
511      - handles just a few options
512         - options for help, version, etc.
513         - options to set user or other vars (for gui or library)
514         - options to execute main processing functions, akin to GUI
515      - should be able to create and execute scripts
516        (to be able to do the main operations of the GUI)
517      - by default, start GUI (unless -no_gui)
518      - give command help (this can be very simple, learning is via GUI)
519
520   2. lib_uber_skel.py: main library for program
521      - defines processing class that accepts user vars and generates
522        processing scripts
523      - main inputs:
524         - user variables *as strings*
525           These are converted to local user vars with types, e.g.
526           uvars.merge(new_uvars, typedef=user_vars_w_types).  This
527           allows higher-level interfaces to not worry about the types.
528      - return (internal) data:
529         - processing script
530         - error list (script is garbage if list is not empty)
531         - warning list (to be shown to user, but script may still be good)
532         - return vars struct
533            - suggested script file name and output file name
534      - script is currently created upon init
535      - library should be able to do the main work, so command line and GUI
536        programs do not repeat functionality
537      - examples of additional functions
538         - get_script() - return either error string or script text
539         - get_warnings() - return warnings string
540         - write_script: - go to proc dir, write script (and orig?), return
541
542   3. gui_uber_skel.py: graphical user interface
543      - defines main GUI class that accepts user vars
544      - init user vars from library defaults, then merge with any passed
545      - purpose is interface to display (string) options to user with useful
546        defaults, allowing them to create and execute processing scripts
547      - hopefully this can be integrated with uber_proc.py
548      - under the main 'uber_results' directory, each tool should write new
549        results under 'tool_results/tool.001.align_test', for example
550        (making 'tool' output, indexed, and with tool name)
551
552
553Writing the main program
554
555   This can generally be short.  Start with a few terminal options (e.g. help,
556   help_gui, hist, ver), add one for setting user or other options (e.g.
557   -uvar), and finally add ability to invoke GUI or create library script.
558
559
560Writing the library
561
562   Define a class that takes some user variables (VarsObject) as input
563   and attempts to create a processing script.  At first, a trivial "script"
564   could simply be returned.
565
566   Then have it merge passed vars with defaults and create the script (doing
567   error checking as it goes).  Any error messages should be added to the error
568   list, and any warnings to to the warnings list.  Errors can be terminal, so
569   returning early should be okay.
570
571   The script should be a simple string.  The error and warnings lists should
572   be lists of strings.  And the return vars struct is another VarsObject.
573
574
575Writing the GUI
576
577   Start by tracing the basic GUI to get a feel for what it is doing.  The
578   main work is just providing interfaces to control the main variables
579   passed to the library.  Smaller things like the menu bar, tool bar, status
580   bar, and menu functionality like creating and processing the script can be
581   traced separately.
582
583   One could start by having the library genrate a very simple script, and
584   then setting up the GUI to deal with it.  Then just add the interfaces for
585   all of the variables.  That will allow starting with a testable platform.
586
587   The current 'todo' list when adding a new variable interface to the GUI:
588
589        - init g_subj_defs in library (with default value or None)
590        - add GUI for it, initialized by uvar
591          (if table, consider 3 functions starting with group_box_gltsym)
592        - call-back update (check and set uvar)
593           - LineVAR in CB_line_text->update_textLine_check: set uvar
594           - if separate button list to update textLine, have callback to
595             both update the textLine and to set uvar
596           - if table, add to update_svars_from_tables()
597              - also, deal with table udpates (e.g. browse, clear, add, help)
598              - processed in CB_gbox_PushB?
599        - add var to apply_uvar_in_gui (for updating GUI from vars)
600        - add to restoration of defaults, if necessary
601          (i.e. button to clear all inputs and reset to defaults)
602        - add GUI help
603        - non-GUI: process in creation of script
604        - add regression testing case
605
606        * adding a table
607           - create table (gvars.Table_gltsym), e.g. write make_gltsym_table
608           - create group box for table (e.g. write group_box_gltsym)
609           - populate table (vars->table, e.g. write self.gltsym_list_to_table)
610           - table->vars noted above (updates_svars_from_tables())
611           - resize with resize_table_cols(table)
612           * when clearing or initializing the table (maybe with buttons),
613             update the vars and call the vars->table function
614           * when processing table edits (maybe only when writing script),
615             call table->vars function
616           * if adding/deleting rows, maybe resize_table_cols()
617===========================================================================
618"""
619