1#!/usr/local/bin/python3.8 2 3# general functions for use by uber_tool*.py 4 5import sys, os 6from time import asctime 7import glob 8 9from afnipy import afni_base as BASE 10from afnipy import afni_util as UTIL 11from afnipy import lib_subjects as SUBJ 12from afnipy import lib_vars_object as VO 13 14DEF_UBER_DIR = 'uber_results' # top directory for output 15DEF_TOP_DIR = 'tool_results' # top subject dir under uber_results 16 17g_history = """ 18 uber_skel.py history 19 20 0.0 12 Apr, 2011: initial revision (based on uber_align_test.py v. 0.2) 21""" 22 23g_version = '0.0 (May 12, 2011)' 24 25# ---------------------------------------------------------------------- 26# global definition of default processing blocks 27g_tlrc_base_list = ['TT_N27+tlrc', 'TT_avg152T1+tlrc', 'TT_icbm452+tlrc', 28 'MNI_avg152T1+tlrc'] 29g_def_tlrc_base = 'TT_N27+tlrc' 30 31 32# ---------------------------------------------------------------------- 33# global definitions of result, control and user defaults 34# (as well as string versions of control and user defaults) 35 36# ---- resulting values returned after class actions ---- 37g_res_defs = VO.VarsObject("uber_skel result variables") 38g_res_defs.file_proc = '' # file name for process script 39g_res_defs.output_proc = '' # output from running proc script 40 41# ---- control variables: process control, not set by user in GUI 42 43g_ctrl_defs = VO.VarsObject("uber_skel control defaults") 44g_ctrl_defs.proc_dir = '.' # process dir: holds scripts and result dir 45 46 47# ---- user variables: process control, alignment inputs and options ---- 48 49g_user_defs = VO.VarsObject("uber_skel user defaults") 50g_user_defs.verb = 1 # verbose level 51g_user_defs.copy_scripts = 'yes' # do we make .orig copies of scripts? 52 53# required inputs 54g_user_defs.anat = '' # anatomical volume to align 55g_user_defs.epi = '' # EPI dataset 56g_user_defs.epi_base = 0 # EPI alignment base index 57 58# options 59g_user_defs.results_dir = 'align.results' # where script puts results 60g_user_defs.cost_list = ['lpc', 'lpc+ZZ', 'lpc+', 'lpa', 'nmi', 'ls'] 61g_user_defs.giant_move = 'no' 62g_user_defs.align_centers = 'no' 63g_user_defs.center_base = 'TT_N27+tlrc' 64g_user_defs.aea_opts = [] # other align_epi_anat.py options 65 66# todo... 67g_user_defs.add_edge = 'no' 68g_user_defs.anat_has_skull = 'yes' 69g_user_defs.epi_strip_meth = '3dSkullStrip' 70 71 72# string versions of variables - used by GUI and main 73# (when creating AlignTest object, string versions of vars are passed) 74g_cdef_strs = g_ctrl_defs.copy(as_strings=1) 75g_udef_strs = g_user_defs.copy(as_strings=1) 76 77 78# main class definition 79class AlignTest(object): 80 """class for testing anat to EPI alignment 81 82 - cvars : control variables 83 - uvars : user variables 84 - rvars : return variables 85 86 ** input vars might be string types, convert on merge 87 88 variables: 89 LV - local variables 90 cvars - control variables 91 uvars - user variables 92 cmd_text - generated alignment script 93 errors --> array of resulting error messages 94 warnings --> array of resulting warning messages 95 """ 96 def __init__(self, cvars=None, uvars=None): 97 98 # ------------------------------------------------------------ 99 # variables 100 101 # LV: variables local to this interface, not passed 102 self.LV = VO.VarsObject("local AP_Subject vars") 103 self.LV.indent = 8 # default indent for main options 104 self.LV.istr = ' '*self.LV.indent 105 self.LV.retdir = '' # return directory (for jumping around) 106 107 # merge passed user variables with defaults 108 self.cvars = g_ctrl_defs.copy() 109 self.uvars = g_user_defs.copy() 110 self.cvars.merge(cvars, typedef=g_ctrl_defs) 111 self.uvars.merge(uvars, typedef=g_user_defs) 112 113 # output variables 114 self.rvars = g_res_defs.copy() # init result vars 115 self.align_script = '' # resulting script 116 self.errors = [] # list of error strings 117 self.warnings = [] # list of warning strings 118 # ------------------------------------------------------------ 119 120 # ------------------------------------------------------------ 121 # preperatory settings 122 123 if self.check_inputs(): return # require at least anat and epi 124 125 self.set_directories() # data dirs: anat, epi 126 127 if self.uvars.verb > 3: self.LV.show('ready to start script') 128 129 # do the work 130 self.create_script() 131 132 def check_inputs(self): 133 """check for required inputs: anat, epi (check existence?)""" 134 if self.uvars.is_empty('anat'): 135 self.errors.append('** unspecified anatomical dataset') 136 137 if self.uvars.is_empty('epi'): 138 self.errors.append('** unspecified EPI dataset') 139 140 if len(self.uvars.cost_list) < 1: 141 self.errors.append('** unspecified cost function(s)') 142 143 if not UTIL.vals_are_unique(self.uvars.cost_list): 144 self.errors.append('** cost functions are not unique') 145 146 return len(self.errors) 147 148 def set_directories(self): 149 """decide on use of top_dir (use it or nothing - no anat_dir) 150 151 ==> if top_dir is set, use $top_dir/short_names 152 else just use anat and epi directly 153 """ 154 top_dir, parent_dirs, short_dirs, short_names = \ 155 UTIL.common_parent_dirs([[self.uvars.anat, self.uvars.epi]]) 156 157 self.LV.top_dir = parent_dirs[0] # common parent dir 158 self.LV.short_names = short_names # if top_dir is used, they are under it 159 160 if self.uvars.verb > 2: 161 print('-- set_dirs: top_dir = %s\n' \ 162 ' short_anat = %s\n' \ 163 ' short_epi = %s\n' \ 164 % (self.LV.top_dir, short_names[0][0], short_names[0][1])) 165 166 # if top_dir isn't long enough, do not bother with it 167 if self.LV.top_dir.count('/') < 2: 168 self.LV.top_dir = '' 169 if self.uvars.verb > 2: print(' (top_dir not worth using...)') 170 171 def create_script(self): 172 """attempt to generate an alignment script 173 - write align_script 174 - keep a list of any warnings or errors 175 - 176 - if there are errors, align_script might not be filled 177 """ 178 179 # script prep, headers and variable assignments 180 self.align_script = self.script_init() 181 self.align_script += self.script_set_vars() 182 183 # do some actual work 184 self.align_script += self.script_results_dir() 185 self.align_script += self.script_copy_data() 186 if self.uvars.align_centers == 'yes': 187 self.align_script += self.script_align_centers() 188 self.align_script += self.script_align_datasets() 189 190 # add commands ... 191 192 if len(self.errors) > 0: return # if any errors so far, give up 193 194 return 195 196 def script_align_datasets(self): 197 """actually run align_epi_anat.py 198 199 only current option is -mult_cost, everything else is via variables 200 """ 201 202 if len(self.uvars.cost_list) > 1: mstr = ' -multi_cost $cost_list' 203 else: mstr = '' 204 205 cmd = SUBJ.comment_section_string('align data') + '\n' 206 207 cmd += \ 208 '# test alignment, using variables set above\n' \ 209 'align_epi_anat.py -anat anat+orig -epi epi+orig ' \ 210 '-epi_base $in_ebase \\\n' \ 211 ' -cost $cost_main%s $align_opts\n' \ 212 '\n' % mstr 213 214 return cmd 215 216 def script_align_centers(self): 217 """if align_centers should be run, deoblique both and align with 218 center_base (probably TT_N27+tlrc) 219 220 note: these commands should be fixed, since the dataset names are 221 """ 222 223 if self.uvars.align_centers != 'yes': return '' 224 225 cmd = SUBJ.comment_section_string('align centers') + '\n' 226 227 cmd += '# since altering grid, remove any oblique transformation\n' \ 228 '3drefit -deoblique anat+orig epi+orig\n' \ 229 '\n' \ 230 '# align volume centers (we do not trust spatial locations)\n' \ 231 '@Align_Centers -no_cp -base TT_N27+tlrc -dset anat+orig\n' \ 232 '@Align_Centers -no_cp -base TT_N27+tlrc -dset epi+orig\n' \ 233 '\n' 234 235 return cmd 236 237 def script_copy_data(self): 238 """these commands only vary based on results_dir""" 239 240 cmd = SUBJ.comment_section_string('copy data') + '\n' 241 242 if self.uvars.is_trivial_dir('results_dir'): rdir = '.' 243 else: rdir = '$results_dir' 244 245 cmd += '# copy dataset to processing directory\n' \ 246 '3dbucket -prefix %s/anat $in_anat\n' \ 247 '3dbucket -prefix %s/epi $in_epi"[$in_ebase]"\n' \ 248 '\n' % (rdir, rdir) 249 250 if rdir != '.': 251 cmd += '# enter the processing directory\n' \ 252 'cd %s\n\n' % rdir 253 254 return cmd 255 256 def script_init(self): 257 cmd = '#!/bin/tcsh -xef\n\n' \ 258 '# created by uber_skel.py: version %s\n' \ 259 '# creation date: %s\n\n' % (g_version, asctime()) 260 261 return cmd 262 263 def script_results_dir(self): 264 265 # if no results dir, just put everything here 266 if self.uvars.is_trivial_dir('results_dir'): return '' 267 268 cmd = SUBJ.comment_section_string('test and create results dir') + '\n' 269 270 cmd += '# note directory for results\n' \ 271 'set results_dir = %s\n\n' % self.uvars.results_dir 272 273 cmd += '# make sure it does not yet exist\n' \ 274 'if ( -e $results_dir ) then\n' \ 275 ' echo "** results dir \'$results_dir\' already exists"\n' \ 276 ' exit\n' \ 277 'endif\n\n' 278 279 cmd += '# create results directory, where the work will be done\n' \ 280 'mkdir $results_dir\n\n' 281 282 return cmd 283 284 def script_set_vars(self): 285 """use variables for inputs (anat, epi, epi_base) and for 286 options (cost_main, cost_list, align_opts) 287 """ 288 289 # init with a section comment 290 cmd = SUBJ.comment_section_string('set processing variables') + '\n' 291 292 # maybe init with top_dir 293 if not self.LV.is_trivial_dir('top_dir'): 294 cmd += '# top data directory\n' \ 295 'set top_dir = %s\n\n' % self.LV.top_dir 296 297 # anat and epi might use top_dir 298 if self.LV.is_trivial_dir('top_dir'): 299 astr = self.uvars.anat 300 estr = self.uvars.epi 301 else: 302 astr = '$top_dir/%s' % self.LV.short_names[0][0] 303 estr = '$top_dir/%s' % self.LV.short_names[0][1] 304 305 cmd += '# input dataset options (ebase is EPI index)\n' \ 306 'set in_anat = %s\n' \ 307 'set in_epi = %s\n' \ 308 'set in_ebase = %d\n\n' \ 309 % (astr, estr, self.uvars.epi_base) 310 311 # note whether to use multi_cost 312 cmd += '# main options\n' \ 313 'set cost_main = %s\n' % self.uvars.cost_list[0] 314 if len(self.uvars.cost_list) > 1: 315 cmd += 'set cost_list = ( %s )\n' % ' '.join(self.uvars.cost_list[1:]) 316 cmd += '\n' 317 318 # possibly add align_opts list variable 319 cmd += self.make_align_opts_str() 320 321 return cmd 322 323 def make_align_opts_str(self): 324 """any align options, one per line""" 325 326 # keep comment separate to get indent length 327 cmnt = '# all other align_epi_anat.py options\n' \ 328 329 cstr = 'set align_opts = ( ' 330 clen = len(cstr) 331 istr = ' '*clen 332 333 # put one option on first line 334 cstr += '%s \\\n' % '-tshift off' 335 336 # ---------- here is the main option application --------- 337 338 # then add each option offset by initial indentation 339 cstr += '%s%s \\\n' % (istr, '-volreg off') 340 341 if self.uvars.giant_move == 'yes': 342 cstr += '%s%s \\\n' % (istr, '-giant_move') 343 344 if self.uvars.add_edge == 'yes': 345 if len(self.uvars.cost_list) > 1: 346 self.errors.append( 347 '** -AddEdge does not currently work with -multi_cost') 348 else: 349 cstr += '%s%s \\\n' % (istr, '-AddEdge') 350 351 if len(self.uvars.aea_opts) > 0: 352 cstr += '%s%s \\\n' % (istr, ' '.join(self.uvars.aea_opts)) 353 354 # does the anatomy have a skull? 355 if self.uvars.anat_has_skull != 'yes': 356 cstr += '%s%s \\\n' % (istr, '-anat_has_skull no') 357 358 # -epi_strip method 359 if self.uvars.epi_strip_meth != '3dSkullStrip': 360 cstr += '%s%s \\\n' % (istr,'-epi_strip %s'%self.uvars.epi_strip_meth) 361 362 # want -save_all, -prep_off? 363 364 # last indent is left by 2 to align () 365 cstr += '%*s)\n\n' % (clen-2, '') 366 367 # finally, align the line wrappers 368 cstr = UTIL.add_line_wrappers(cstr) 369 370 return cmnt + cstr 371 372 def get_script(self): 373 """return status, message 374 375 status = number of error messages 376 if 0: message = command 377 else: message = error string 378 379 Requests for warnings must be made separately, since they 380 are not fatal. 381 """ 382 383 if len(self.errors) > 0: 384 return 1, SUBJ.make_message_list_string(self.errors, "errors") 385 386 return 0, self.align_script 387 388 def get_warnings(self): 389 """return the number of warnings and a warnings string""" 390 391 return len(self.warnings), \ 392 SUBJ.make_message_list_string(self.warnings, "warnings") 393 394 def proc_dir_filename(self, vname): 395 """file is either fname or proc_dir/fname (if results is set) 396 vname : results file variable (must convert to fname) 397 """ 398 fname = self.rvars.val(vname) 399 return self.cvars.file_under_dir('proc_dir', fname) 400 401 def nuke_old_results(self): 402 """if the results directory exists, remove it""" 403 404 if self.uvars.results_dir == '': return 405 406 # ------------------------- do the work ------------------------- 407 self.LV.retdir = SUBJ.goto_proc_dir(self.cvars.proc_dir) 408 409 if os.path.isdir(self.uvars.results_dir): 410 print('-- nuking old results: %s' % self.uvars.results_dir) 411 os.system('rm -fr %s' % self.uvars.results_dir) 412 413 self.LV.retdir = SUBJ.ret_from_proc_dir(self.LV.retdir) 414 # ------------------------- done ------------------------- 415 416 417 def copy_orig_proc(self): 418 """if the proc script exists, copy to .orig.SCRIPTNAME""" 419 if self.rvars.file_proc == '': return 420 pfile = self.rvars.file_proc 421 422 # ------------------------- do the work ------------------------- 423 self.LV.retdir = SUBJ.goto_proc_dir(self.cvars.proc_dir) 424 if os.path.isfile(pfile): 425 cmd = 'cp -f %s .orig.%s' % (pfile, pfile) 426 if self.uvars.verb > 1: print('++ exec: %s' % cmd) 427 os.system(cmd) 428 elif self.uvars.verb > 1: print("** no proc '%s' to copy" % pfile) 429 self.LV.retdir = SUBJ.ret_from_proc_dir(self.LV.retdir) 430 # ------------------------- done ------------------------- 431 432 def write_script(self, fname=''): 433 """write processing script to a file (in the proc_dir) 434 - if fname is set, use it, else generate 435 - set rvars.file_proc and output_proc 436 """ 437 438 if not self.align_script: 439 print('** no alignment script to write out') 440 return 1 441 if fname: name = fname 442 else: 443 # if self.svars.sid: name = 'script.align.%s' % self.svars.sid 444 name = 'script.align' 445 446 # store (intended) names for calling tool to execute with 447 self.rvars.file_proc = name # store which file we have written to 448 self.rvars.output_proc = 'output.%s' % name # file for command output 449 450 if self.uvars.verb > 0: print('++ writing script to %s' % name) 451 452 # if requested, make an original copy 453 self.LV.retdir = SUBJ.goto_proc_dir(self.cvars.proc_dir) 454 455 if self.uvars.copy_scripts == 'yes': # make an orig copy 456 UTIL.write_text_to_file('.orig.%s'%name, self.align_script, exe=1) 457 rv = UTIL.write_text_to_file(name, self.align_script, exe=1) 458 459 self.LV.retdir = SUBJ.ret_from_proc_dir(self.LV.retdir) 460 461 return rv 462 463 464# =========================================================================== 465# help strings accessed both from command-line and GUI 466# =========================================================================== 467 468helpstr_todo = """ 469--------------------------------------------------------------------------- 470 todo list: 471 472- create GUI 473- test center distance in GUI to suggest align centers 474- show corresponding afni_proc.py options 475- show corresponding uber_subjec.py options? 476- show afni command to look at results (basically point to directory) 477--------------------------------------------------------------------------- 478""" 479 480helpstr_gui = """ 481=========================================================================== 482uber_skel.py (GUI) - a graphical interface for testing alignment 483 484 Find good alignment options, possibly to add to afni_proc.py command 485 or uber_subject.py GUI options. 486 487 purposes: 488 required inputs: 489 optional inputs: 490 typical outputs: 491 492--------------------------------------------------------------------------- 493Overview: 494 495- R Reynolds Feb, 2011 496=========================================================================== 497""" 498 499helpstr_create_program = """ 500=========================================================================== 501This is a brief overview about creating a new program/GUI for uber_proc.py. 502 503There are (currently) 3 basic files used, the main program, the library and 504the GUI. The intention is that one can run the main program to generate or 505execute processing scripts (goal of the GUI) without actually using the GUI. 506Some common GUI routines/classes are in lib_qt_gui.py. 507 508So the purpose of the GUI is to set user variables to pass to the library. 509 510 1. uber_skel.py: main program 511 - handles just a few options 512 - options for help, version, etc. 513 - options to set user or other vars (for gui or library) 514 - options to execute main processing functions, akin to GUI 515 - should be able to create and execute scripts 516 (to be able to do the main operations of the GUI) 517 - by default, start GUI (unless -no_gui) 518 - give command help (this can be very simple, learning is via GUI) 519 520 2. lib_uber_skel.py: main library for program 521 - defines processing class that accepts user vars and generates 522 processing scripts 523 - main inputs: 524 - user variables *as strings* 525 These are converted to local user vars with types, e.g. 526 uvars.merge(new_uvars, typedef=user_vars_w_types). This 527 allows higher-level interfaces to not worry about the types. 528 - return (internal) data: 529 - processing script 530 - error list (script is garbage if list is not empty) 531 - warning list (to be shown to user, but script may still be good) 532 - return vars struct 533 - suggested script file name and output file name 534 - script is currently created upon init 535 - library should be able to do the main work, so command line and GUI 536 programs do not repeat functionality 537 - examples of additional functions 538 - get_script() - return either error string or script text 539 - get_warnings() - return warnings string 540 - write_script: - go to proc dir, write script (and orig?), return 541 542 3. gui_uber_skel.py: graphical user interface 543 - defines main GUI class that accepts user vars 544 - init user vars from library defaults, then merge with any passed 545 - purpose is interface to display (string) options to user with useful 546 defaults, allowing them to create and execute processing scripts 547 - hopefully this can be integrated with uber_proc.py 548 - under the main 'uber_results' directory, each tool should write new 549 results under 'tool_results/tool.001.align_test', for example 550 (making 'tool' output, indexed, and with tool name) 551 552 553Writing the main program 554 555 This can generally be short. Start with a few terminal options (e.g. help, 556 help_gui, hist, ver), add one for setting user or other options (e.g. 557 -uvar), and finally add ability to invoke GUI or create library script. 558 559 560Writing the library 561 562 Define a class that takes some user variables (VarsObject) as input 563 and attempts to create a processing script. At first, a trivial "script" 564 could simply be returned. 565 566 Then have it merge passed vars with defaults and create the script (doing 567 error checking as it goes). Any error messages should be added to the error 568 list, and any warnings to to the warnings list. Errors can be terminal, so 569 returning early should be okay. 570 571 The script should be a simple string. The error and warnings lists should 572 be lists of strings. And the return vars struct is another VarsObject. 573 574 575Writing the GUI 576 577 Start by tracing the basic GUI to get a feel for what it is doing. The 578 main work is just providing interfaces to control the main variables 579 passed to the library. Smaller things like the menu bar, tool bar, status 580 bar, and menu functionality like creating and processing the script can be 581 traced separately. 582 583 One could start by having the library genrate a very simple script, and 584 then setting up the GUI to deal with it. Then just add the interfaces for 585 all of the variables. That will allow starting with a testable platform. 586 587 The current 'todo' list when adding a new variable interface to the GUI: 588 589 - init g_subj_defs in library (with default value or None) 590 - add GUI for it, initialized by uvar 591 (if table, consider 3 functions starting with group_box_gltsym) 592 - call-back update (check and set uvar) 593 - LineVAR in CB_line_text->update_textLine_check: set uvar 594 - if separate button list to update textLine, have callback to 595 both update the textLine and to set uvar 596 - if table, add to update_svars_from_tables() 597 - also, deal with table udpates (e.g. browse, clear, add, help) 598 - processed in CB_gbox_PushB? 599 - add var to apply_uvar_in_gui (for updating GUI from vars) 600 - add to restoration of defaults, if necessary 601 (i.e. button to clear all inputs and reset to defaults) 602 - add GUI help 603 - non-GUI: process in creation of script 604 - add regression testing case 605 606 * adding a table 607 - create table (gvars.Table_gltsym), e.g. write make_gltsym_table 608 - create group box for table (e.g. write group_box_gltsym) 609 - populate table (vars->table, e.g. write self.gltsym_list_to_table) 610 - table->vars noted above (updates_svars_from_tables()) 611 - resize with resize_table_cols(table) 612 * when clearing or initializing the table (maybe with buttons), 613 update the vars and call the vars->table function 614 * when processing table edits (maybe only when writing script), 615 call table->vars function 616 * if adding/deleting rows, maybe resize_table_cols() 617=========================================================================== 618""" 619