1 /* Move registers around to reduce number of move instructions needed. 2 Copyright (C) 1987, 1988, 1989, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 3 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 4 Free Software Foundation, Inc. 5 6 This file is part of GCC. 7 8 GCC is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free 10 Software Foundation; either version 3, or (at your option) any later 11 version. 12 13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or 15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 for more details. 17 18 You should have received a copy of the GNU General Public License 19 along with GCC; see the file COPYING3. If not see 20 <http://www.gnu.org/licenses/>. */ 21 22 23 /* This module makes some simple RTL code transformations which 24 improve the subsequent register allocation. */ 25 26 #include "config.h" 27 #include "system.h" 28 #include "coretypes.h" 29 #include "tm.h" 30 #include "rtl.h" 31 #include "tm_p.h" 32 #include "insn-config.h" 33 #include "recog.h" 34 #include "target.h" 35 #include "output.h" 36 #include "regs.h" 37 #include "hard-reg-set.h" 38 #include "flags.h" 39 #include "function.h" 40 #include "expr.h" 41 #include "basic-block.h" 42 #include "except.h" 43 #include "diagnostic-core.h" 44 #include "reload.h" 45 #include "timevar.h" 46 #include "tree-pass.h" 47 #include "df.h" 48 #include "ira.h" 49 50 static int optimize_reg_copy_1 (rtx, rtx, rtx); 51 static void optimize_reg_copy_2 (rtx, rtx, rtx); 52 static void optimize_reg_copy_3 (rtx, rtx, rtx); 53 static void copy_src_to_dest (rtx, rtx, rtx); 54 55 enum match_use 56 { 57 READ, 58 WRITE, 59 READWRITE 60 }; 61 62 struct match { 63 int with[MAX_RECOG_OPERANDS]; 64 enum match_use use[MAX_RECOG_OPERANDS]; 65 int commutative[MAX_RECOG_OPERANDS]; 66 int early_clobber[MAX_RECOG_OPERANDS]; 67 }; 68 69 static int find_matches (rtx, struct match *); 70 static int fixup_match_2 (rtx, rtx, rtx, rtx); 71 72 /* Return nonzero if registers with CLASS1 and CLASS2 can be merged without 73 causing too much register allocation problems. */ 74 static int 75 regclass_compatible_p (reg_class_t class0, reg_class_t class1) 76 { 77 return (class0 == class1 78 || (reg_class_subset_p (class0, class1) 79 && ! targetm.class_likely_spilled_p (class0)) 80 || (reg_class_subset_p (class1, class0) 81 && ! targetm.class_likely_spilled_p (class1))); 82 } 83 84 85 #ifdef AUTO_INC_DEC 86 87 /* Find the place in the rtx X where REG is used as a memory address. 88 Return the MEM rtx that so uses it. 89 If PLUSCONST is nonzero, search instead for a memory address equivalent to 90 (plus REG (const_int PLUSCONST)). 91 92 If such an address does not appear, return 0. 93 If REG appears more than once, or is used other than in such an address, 94 return (rtx) 1. */ 95 96 static rtx 97 find_use_as_address (rtx x, rtx reg, HOST_WIDE_INT plusconst) 98 { 99 enum rtx_code code = GET_CODE (x); 100 const char * const fmt = GET_RTX_FORMAT (code); 101 int i; 102 rtx value = 0; 103 rtx tem; 104 105 if (code == MEM && XEXP (x, 0) == reg && plusconst == 0) 106 return x; 107 108 if (code == MEM && GET_CODE (XEXP (x, 0)) == PLUS 109 && XEXP (XEXP (x, 0), 0) == reg 110 && CONST_INT_P (XEXP (XEXP (x, 0), 1)) 111 && INTVAL (XEXP (XEXP (x, 0), 1)) == plusconst) 112 return x; 113 114 if (code == SIGN_EXTRACT || code == ZERO_EXTRACT) 115 { 116 /* If REG occurs inside a MEM used in a bit-field reference, 117 that is unacceptable. */ 118 if (find_use_as_address (XEXP (x, 0), reg, 0) != 0) 119 return (rtx) (size_t) 1; 120 } 121 122 if (x == reg) 123 return (rtx) (size_t) 1; 124 125 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--) 126 { 127 if (fmt[i] == 'e') 128 { 129 tem = find_use_as_address (XEXP (x, i), reg, plusconst); 130 if (value == 0) 131 value = tem; 132 else if (tem != 0) 133 return (rtx) (size_t) 1; 134 } 135 else if (fmt[i] == 'E') 136 { 137 int j; 138 for (j = XVECLEN (x, i) - 1; j >= 0; j--) 139 { 140 tem = find_use_as_address (XVECEXP (x, i, j), reg, plusconst); 141 if (value == 0) 142 value = tem; 143 else if (tem != 0) 144 return (rtx) (size_t) 1; 145 } 146 } 147 } 148 149 return value; 150 } 151 152 153 /* INC_INSN is an instruction that adds INCREMENT to REG. 154 Try to fold INC_INSN as a post/pre in/decrement into INSN. 155 Iff INC_INSN_SET is nonzero, inc_insn has a destination different from src. 156 Return nonzero for success. */ 157 static int 158 try_auto_increment (rtx insn, rtx inc_insn, rtx inc_insn_set, rtx reg, 159 HOST_WIDE_INT increment, int pre) 160 { 161 enum rtx_code inc_code; 162 163 rtx pset = single_set (insn); 164 if (pset) 165 { 166 /* Can't use the size of SET_SRC, we might have something like 167 (sign_extend:SI (mem:QI ... */ 168 rtx use = find_use_as_address (pset, reg, 0); 169 if (use != 0 && use != (rtx) (size_t) 1) 170 { 171 int size = GET_MODE_SIZE (GET_MODE (use)); 172 if (0 173 || (HAVE_POST_INCREMENT 174 && pre == 0 && (inc_code = POST_INC, increment == size)) 175 || (HAVE_PRE_INCREMENT 176 && pre == 1 && (inc_code = PRE_INC, increment == size)) 177 || (HAVE_POST_DECREMENT 178 && pre == 0 && (inc_code = POST_DEC, increment == -size)) 179 || (HAVE_PRE_DECREMENT 180 && pre == 1 && (inc_code = PRE_DEC, increment == -size)) 181 ) 182 { 183 if (inc_insn_set) 184 validate_change 185 (inc_insn, 186 &SET_SRC (inc_insn_set), 187 XEXP (SET_SRC (inc_insn_set), 0), 1); 188 validate_change (insn, &XEXP (use, 0), 189 gen_rtx_fmt_e (inc_code, 190 GET_MODE (XEXP (use, 0)), reg), 191 1); 192 if (apply_change_group ()) 193 { 194 /* If there is a REG_DEAD note on this insn, we must 195 change this not to REG_UNUSED meaning that the register 196 is set, but the value is dead. Failure to do so will 197 result in sched1 dying -- when it recomputes lifetime 198 information, the number of REG_DEAD notes will have 199 changed. */ 200 rtx note = find_reg_note (insn, REG_DEAD, reg); 201 if (note) 202 PUT_REG_NOTE_KIND (note, REG_UNUSED); 203 204 add_reg_note (insn, REG_INC, reg); 205 206 if (! inc_insn_set) 207 delete_insn (inc_insn); 208 return 1; 209 } 210 } 211 } 212 } 213 return 0; 214 } 215 #endif 216 217 218 static int *regno_src_regno; 219 220 /* INSN is a copy from SRC to DEST, both registers, and SRC does not die 221 in INSN. 222 223 Search forward to see if SRC dies before either it or DEST is modified, 224 but don't scan past the end of a basic block. If so, we can replace SRC 225 with DEST and let SRC die in INSN. 226 227 This will reduce the number of registers live in that range and may enable 228 DEST to be tied to SRC, thus often saving one register in addition to a 229 register-register copy. */ 230 231 static int 232 optimize_reg_copy_1 (rtx insn, rtx dest, rtx src) 233 { 234 rtx p, q; 235 rtx note; 236 rtx dest_death = 0; 237 int sregno = REGNO (src); 238 int dregno = REGNO (dest); 239 basic_block bb = BLOCK_FOR_INSN (insn); 240 241 /* We don't want to mess with hard regs if register classes are small. */ 242 if (sregno == dregno 243 || (targetm.small_register_classes_for_mode_p (GET_MODE (src)) 244 && (sregno < FIRST_PSEUDO_REGISTER 245 || dregno < FIRST_PSEUDO_REGISTER)) 246 /* We don't see all updates to SP if they are in an auto-inc memory 247 reference, so we must disallow this optimization on them. */ 248 || sregno == STACK_POINTER_REGNUM || dregno == STACK_POINTER_REGNUM) 249 return 0; 250 251 for (p = NEXT_INSN (insn); p; p = NEXT_INSN (p)) 252 { 253 if (! INSN_P (p)) 254 continue; 255 if (BLOCK_FOR_INSN (p) != bb) 256 break; 257 258 if (reg_set_p (src, p) || reg_set_p (dest, p) 259 /* If SRC is an asm-declared register, it must not be replaced 260 in any asm. Unfortunately, the REG_EXPR tree for the asm 261 variable may be absent in the SRC rtx, so we can't check the 262 actual register declaration easily (the asm operand will have 263 it, though). To avoid complicating the test for a rare case, 264 we just don't perform register replacement for a hard reg 265 mentioned in an asm. */ 266 || (sregno < FIRST_PSEUDO_REGISTER 267 && asm_noperands (PATTERN (p)) >= 0 268 && reg_overlap_mentioned_p (src, PATTERN (p))) 269 /* Don't change hard registers used by a call. */ 270 || (CALL_P (p) && sregno < FIRST_PSEUDO_REGISTER 271 && find_reg_fusage (p, USE, src)) 272 /* Don't change a USE of a register. */ 273 || (GET_CODE (PATTERN (p)) == USE 274 && reg_overlap_mentioned_p (src, XEXP (PATTERN (p), 0)))) 275 break; 276 277 /* See if all of SRC dies in P. This test is slightly more 278 conservative than it needs to be. */ 279 if ((note = find_regno_note (p, REG_DEAD, sregno)) != 0 280 && GET_MODE (XEXP (note, 0)) == GET_MODE (src)) 281 { 282 int failed = 0; 283 int d_length = 0; 284 int s_length = 0; 285 int d_n_calls = 0; 286 int s_n_calls = 0; 287 int s_freq_calls = 0; 288 int d_freq_calls = 0; 289 290 /* We can do the optimization. Scan forward from INSN again, 291 replacing regs as we go. Set FAILED if a replacement can't 292 be done. In that case, we can't move the death note for SRC. 293 This should be rare. */ 294 295 /* Set to stop at next insn. */ 296 for (q = next_real_insn (insn); 297 q != next_real_insn (p); 298 q = next_real_insn (q)) 299 { 300 if (reg_overlap_mentioned_p (src, PATTERN (q))) 301 { 302 /* If SRC is a hard register, we might miss some 303 overlapping registers with validate_replace_rtx, 304 so we would have to undo it. We can't if DEST is 305 present in the insn, so fail in that combination 306 of cases. */ 307 if (sregno < FIRST_PSEUDO_REGISTER 308 && reg_mentioned_p (dest, PATTERN (q))) 309 failed = 1; 310 311 /* Attempt to replace all uses. */ 312 else if (!validate_replace_rtx (src, dest, q)) 313 failed = 1; 314 315 /* If this succeeded, but some part of the register 316 is still present, undo the replacement. */ 317 else if (sregno < FIRST_PSEUDO_REGISTER 318 && reg_overlap_mentioned_p (src, PATTERN (q))) 319 { 320 validate_replace_rtx (dest, src, q); 321 failed = 1; 322 } 323 } 324 325 /* For SREGNO, count the total number of insns scanned. 326 For DREGNO, count the total number of insns scanned after 327 passing the death note for DREGNO. */ 328 if (!DEBUG_INSN_P (p)) 329 { 330 s_length++; 331 if (dest_death) 332 d_length++; 333 } 334 335 /* If the insn in which SRC dies is a CALL_INSN, don't count it 336 as a call that has been crossed. Otherwise, count it. */ 337 if (q != p && CALL_P (q)) 338 { 339 /* Similarly, total calls for SREGNO, total calls beyond 340 the death note for DREGNO. */ 341 s_n_calls++; 342 s_freq_calls += REG_FREQ_FROM_BB (BLOCK_FOR_INSN (q)); 343 if (dest_death) 344 { 345 d_n_calls++; 346 d_freq_calls += REG_FREQ_FROM_BB (BLOCK_FOR_INSN (q)); 347 } 348 } 349 350 /* If DEST dies here, remove the death note and save it for 351 later. Make sure ALL of DEST dies here; again, this is 352 overly conservative. */ 353 if (dest_death == 0 354 && (dest_death = find_regno_note (q, REG_DEAD, dregno)) != 0) 355 { 356 if (GET_MODE (XEXP (dest_death, 0)) != GET_MODE (dest)) 357 failed = 1, dest_death = 0; 358 else 359 remove_note (q, dest_death); 360 } 361 } 362 363 if (! failed) 364 { 365 /* These counters need to be updated if and only if we are 366 going to move the REG_DEAD note. */ 367 if (sregno >= FIRST_PSEUDO_REGISTER) 368 { 369 if (REG_LIVE_LENGTH (sregno) >= 0) 370 { 371 REG_LIVE_LENGTH (sregno) -= s_length; 372 /* REG_LIVE_LENGTH is only an approximation after 373 combine if sched is not run, so make sure that we 374 still have a reasonable value. */ 375 if (REG_LIVE_LENGTH (sregno) < 2) 376 REG_LIVE_LENGTH (sregno) = 2; 377 } 378 379 REG_N_CALLS_CROSSED (sregno) -= s_n_calls; 380 REG_FREQ_CALLS_CROSSED (sregno) -= s_freq_calls; 381 } 382 383 /* Move death note of SRC from P to INSN. */ 384 remove_note (p, note); 385 XEXP (note, 1) = REG_NOTES (insn); 386 REG_NOTES (insn) = note; 387 } 388 389 /* DEST is also dead if INSN has a REG_UNUSED note for DEST. */ 390 if (! dest_death 391 && (dest_death = find_regno_note (insn, REG_UNUSED, dregno))) 392 { 393 PUT_REG_NOTE_KIND (dest_death, REG_DEAD); 394 remove_note (insn, dest_death); 395 } 396 397 /* Put death note of DEST on P if we saw it die. */ 398 if (dest_death) 399 { 400 XEXP (dest_death, 1) = REG_NOTES (p); 401 REG_NOTES (p) = dest_death; 402 403 if (dregno >= FIRST_PSEUDO_REGISTER) 404 { 405 /* If and only if we are moving the death note for DREGNO, 406 then we need to update its counters. */ 407 if (REG_LIVE_LENGTH (dregno) >= 0) 408 REG_LIVE_LENGTH (dregno) += d_length; 409 REG_N_CALLS_CROSSED (dregno) += d_n_calls; 410 REG_FREQ_CALLS_CROSSED (dregno) += d_freq_calls; 411 } 412 } 413 414 return ! failed; 415 } 416 417 /* If SRC is a hard register which is set or killed in some other 418 way, we can't do this optimization. */ 419 else if (sregno < FIRST_PSEUDO_REGISTER 420 && dead_or_set_p (p, src)) 421 break; 422 } 423 return 0; 424 } 425 426 /* INSN is a copy of SRC to DEST, in which SRC dies. See if we now have 427 a sequence of insns that modify DEST followed by an insn that sets 428 SRC to DEST in which DEST dies, with no prior modification of DEST. 429 (There is no need to check if the insns in between actually modify 430 DEST. We should not have cases where DEST is not modified, but 431 the optimization is safe if no such modification is detected.) 432 In that case, we can replace all uses of DEST, starting with INSN and 433 ending with the set of SRC to DEST, with SRC. We do not do this 434 optimization if a CALL_INSN is crossed unless SRC already crosses a 435 call or if DEST dies before the copy back to SRC. 436 437 It is assumed that DEST and SRC are pseudos; it is too complicated to do 438 this for hard registers since the substitutions we may make might fail. */ 439 440 static void 441 optimize_reg_copy_2 (rtx insn, rtx dest, rtx src) 442 { 443 rtx p, q; 444 rtx set; 445 int sregno = REGNO (src); 446 int dregno = REGNO (dest); 447 basic_block bb = BLOCK_FOR_INSN (insn); 448 449 for (p = NEXT_INSN (insn); p; p = NEXT_INSN (p)) 450 { 451 if (! INSN_P (p)) 452 continue; 453 if (BLOCK_FOR_INSN (p) != bb) 454 break; 455 456 set = single_set (p); 457 if (set && SET_SRC (set) == dest && SET_DEST (set) == src 458 && find_reg_note (p, REG_DEAD, dest)) 459 { 460 /* We can do the optimization. Scan forward from INSN again, 461 replacing regs as we go. */ 462 463 /* Set to stop at next insn. */ 464 for (q = insn; q != NEXT_INSN (p); q = NEXT_INSN (q)) 465 if (INSN_P (q)) 466 { 467 if (reg_mentioned_p (dest, PATTERN (q))) 468 { 469 rtx note; 470 471 PATTERN (q) = replace_rtx (PATTERN (q), dest, src); 472 note = FIND_REG_INC_NOTE (q, dest); 473 if (note) 474 { 475 remove_note (q, note); 476 add_reg_note (q, REG_INC, src); 477 } 478 df_insn_rescan (q); 479 } 480 481 if (CALL_P (q)) 482 { 483 int freq = REG_FREQ_FROM_BB (BLOCK_FOR_INSN (q)); 484 REG_N_CALLS_CROSSED (dregno)--; 485 REG_N_CALLS_CROSSED (sregno)++; 486 REG_FREQ_CALLS_CROSSED (dregno) -= freq; 487 REG_FREQ_CALLS_CROSSED (sregno) += freq; 488 } 489 } 490 491 remove_note (p, find_reg_note (p, REG_DEAD, dest)); 492 REG_N_DEATHS (dregno)--; 493 remove_note (insn, find_reg_note (insn, REG_DEAD, src)); 494 REG_N_DEATHS (sregno)--; 495 return; 496 } 497 498 if (reg_set_p (src, p) 499 || find_reg_note (p, REG_DEAD, dest) 500 || (CALL_P (p) && REG_N_CALLS_CROSSED (sregno) == 0)) 501 break; 502 } 503 } 504 505 /* INSN is a ZERO_EXTEND or SIGN_EXTEND of SRC to DEST. 506 Look if SRC dies there, and if it is only set once, by loading 507 it from memory. If so, try to incorporate the zero/sign extension 508 into the memory read, change SRC to the mode of DEST, and alter 509 the remaining accesses to use the appropriate SUBREG. This allows 510 SRC and DEST to be tied later. */ 511 static void 512 optimize_reg_copy_3 (rtx insn, rtx dest, rtx src) 513 { 514 rtx src_reg = XEXP (src, 0); 515 int src_no = REGNO (src_reg); 516 int dst_no = REGNO (dest); 517 rtx p, set, set_insn; 518 enum machine_mode old_mode; 519 basic_block bb = BLOCK_FOR_INSN (insn); 520 521 if (src_no < FIRST_PSEUDO_REGISTER 522 || dst_no < FIRST_PSEUDO_REGISTER 523 || ! find_reg_note (insn, REG_DEAD, src_reg) 524 || REG_N_DEATHS (src_no) != 1 525 || REG_N_SETS (src_no) != 1) 526 return; 527 528 for (p = PREV_INSN (insn); p && ! reg_set_p (src_reg, p); p = PREV_INSN (p)) 529 if (INSN_P (p) && BLOCK_FOR_INSN (p) != bb) 530 break; 531 532 if (! p || BLOCK_FOR_INSN (p) != bb) 533 return; 534 535 if (! (set = single_set (p)) 536 || !MEM_P (SET_SRC (set)) 537 /* If there's a REG_EQUIV note, this must be an insn that loads an 538 argument. Prefer keeping the note over doing this optimization. */ 539 || find_reg_note (p, REG_EQUIV, NULL_RTX) 540 || SET_DEST (set) != src_reg) 541 return; 542 543 /* Be conservative: although this optimization is also valid for 544 volatile memory references, that could cause trouble in later passes. */ 545 if (MEM_VOLATILE_P (SET_SRC (set))) 546 return; 547 548 /* Do not use a SUBREG to truncate from one mode to another if truncation 549 is not a nop. */ 550 if (GET_MODE_BITSIZE (GET_MODE (src_reg)) <= GET_MODE_BITSIZE (GET_MODE (src)) 551 && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (src), GET_MODE (src_reg))) 552 return; 553 554 set_insn = p; 555 old_mode = GET_MODE (src_reg); 556 PUT_MODE (src_reg, GET_MODE (src)); 557 XEXP (src, 0) = SET_SRC (set); 558 559 /* Include this change in the group so that it's easily undone if 560 one of the changes in the group is invalid. */ 561 validate_change (p, &SET_SRC (set), src, 1); 562 563 /* Now walk forward making additional replacements. We want to be able 564 to undo all the changes if a later substitution fails. */ 565 while (p = NEXT_INSN (p), p != insn) 566 { 567 if (! INSN_P (p)) 568 continue; 569 570 /* Make a tentative change. */ 571 validate_replace_rtx_group (src_reg, 572 gen_lowpart_SUBREG (old_mode, src_reg), 573 p); 574 } 575 576 validate_replace_rtx_group (src, src_reg, insn); 577 578 /* Now see if all the changes are valid. */ 579 if (! apply_change_group ()) 580 { 581 /* One or more changes were no good. Back out everything. */ 582 PUT_MODE (src_reg, old_mode); 583 XEXP (src, 0) = src_reg; 584 } 585 else 586 { 587 rtx note = find_reg_note (set_insn, REG_EQUAL, NULL_RTX); 588 if (note) 589 { 590 if (rtx_equal_p (XEXP (note, 0), XEXP (src, 0))) 591 { 592 XEXP (note, 0) 593 = gen_rtx_fmt_e (GET_CODE (src), GET_MODE (src), 594 XEXP (note, 0)); 595 df_notes_rescan (set_insn); 596 } 597 else 598 remove_note (set_insn, note); 599 } 600 } 601 } 602 603 604 /* If we were not able to update the users of src to use dest directly, try 605 instead moving the value to dest directly before the operation. */ 606 607 static void 608 copy_src_to_dest (rtx insn, rtx src, rtx dest) 609 { 610 rtx seq; 611 rtx link; 612 rtx next; 613 rtx set; 614 rtx move_insn; 615 rtx *p_insn_notes; 616 rtx *p_move_notes; 617 int src_regno; 618 int dest_regno; 619 620 /* A REG_LIVE_LENGTH of -1 indicates the register is equivalent to a constant 621 or memory location and is used infrequently; a REG_LIVE_LENGTH of -2 is 622 parameter when there is no frame pointer that is not allocated a register. 623 For now, we just reject them, rather than incrementing the live length. */ 624 625 if (REG_P (src) 626 && REG_LIVE_LENGTH (REGNO (src)) > 0 627 && REG_P (dest) 628 && REG_LIVE_LENGTH (REGNO (dest)) > 0 629 && (set = single_set (insn)) != NULL_RTX 630 && !reg_mentioned_p (dest, SET_SRC (set)) 631 && GET_MODE (src) == GET_MODE (dest)) 632 { 633 int old_num_regs = reg_rtx_no; 634 635 /* Generate the src->dest move. */ 636 start_sequence (); 637 emit_move_insn (dest, src); 638 seq = get_insns (); 639 end_sequence (); 640 /* If this sequence uses new registers, we may not use it. */ 641 if (old_num_regs != reg_rtx_no 642 || ! validate_replace_rtx (src, dest, insn)) 643 { 644 /* We have to restore reg_rtx_no to its old value, lest 645 recompute_reg_usage will try to compute the usage of the 646 new regs, yet reg_n_info is not valid for them. */ 647 reg_rtx_no = old_num_regs; 648 return; 649 } 650 emit_insn_before (seq, insn); 651 move_insn = PREV_INSN (insn); 652 p_move_notes = ®_NOTES (move_insn); 653 p_insn_notes = ®_NOTES (insn); 654 655 /* Move any notes mentioning src to the move instruction. */ 656 for (link = REG_NOTES (insn); link != NULL_RTX; link = next) 657 { 658 next = XEXP (link, 1); 659 if (XEXP (link, 0) == src) 660 { 661 *p_move_notes = link; 662 p_move_notes = &XEXP (link, 1); 663 } 664 else 665 { 666 *p_insn_notes = link; 667 p_insn_notes = &XEXP (link, 1); 668 } 669 } 670 671 *p_move_notes = NULL_RTX; 672 *p_insn_notes = NULL_RTX; 673 674 /* Update the various register tables. */ 675 dest_regno = REGNO (dest); 676 INC_REG_N_SETS (dest_regno, 1); 677 REG_LIVE_LENGTH (dest_regno)++; 678 src_regno = REGNO (src); 679 if (! find_reg_note (move_insn, REG_DEAD, src)) 680 REG_LIVE_LENGTH (src_regno)++; 681 } 682 } 683 684 /* reg_set_in_bb[REGNO] points to basic block iff the register is set 685 only once in the given block and has REG_EQUAL note. */ 686 687 static basic_block *reg_set_in_bb; 688 689 /* Size of reg_set_in_bb array. */ 690 static unsigned int max_reg_computed; 691 692 693 /* Return whether REG is set in only one location, and is set to a 694 constant, but is set in a different basic block from INSN (an 695 instructions which uses REG). In this case REG is equivalent to a 696 constant, and we don't want to break that equivalence, because that 697 may increase register pressure and make reload harder. If REG is 698 set in the same basic block as INSN, we don't worry about it, 699 because we'll probably need a register anyhow (??? but what if REG 700 is used in a different basic block as well as this one?). */ 701 702 static bool 703 reg_is_remote_constant_p (rtx reg, rtx insn) 704 { 705 basic_block bb; 706 rtx p; 707 int max; 708 709 if (!reg_set_in_bb) 710 { 711 max_reg_computed = max = max_reg_num (); 712 reg_set_in_bb = XCNEWVEC (basic_block, max); 713 714 FOR_EACH_BB (bb) 715 FOR_BB_INSNS (bb, p) 716 { 717 rtx s; 718 719 if (!INSN_P (p)) 720 continue; 721 s = single_set (p); 722 /* This is the instruction which sets REG. If there is a 723 REG_EQUAL note, then REG is equivalent to a constant. */ 724 if (s != 0 725 && REG_P (SET_DEST (s)) 726 && REG_N_SETS (REGNO (SET_DEST (s))) == 1 727 && find_reg_note (p, REG_EQUAL, NULL_RTX)) 728 reg_set_in_bb[REGNO (SET_DEST (s))] = bb; 729 } 730 } 731 732 gcc_assert (REGNO (reg) < max_reg_computed); 733 if (reg_set_in_bb[REGNO (reg)] == NULL) 734 return false; 735 return (reg_set_in_bb[REGNO (reg)] != BLOCK_FOR_INSN (insn)); 736 } 737 738 /* INSN is adding a CONST_INT to a REG. We search backwards looking for 739 another add immediate instruction with the same source and dest registers, 740 and if we find one, we change INSN to an increment, and return 1. If 741 no changes are made, we return 0. 742 743 This changes 744 (set (reg100) (plus reg1 offset1)) 745 ... 746 (set (reg100) (plus reg1 offset2)) 747 to 748 (set (reg100) (plus reg1 offset1)) 749 ... 750 (set (reg100) (plus reg100 offset2-offset1)) */ 751 752 /* ??? What does this comment mean? */ 753 /* cse disrupts preincrement / postdecrement sequences when it finds a 754 hard register as ultimate source, like the frame pointer. */ 755 756 static int 757 fixup_match_2 (rtx insn, rtx dst, rtx src, rtx offset) 758 { 759 rtx p, dst_death = 0; 760 int length, num_calls = 0, freq_calls = 0; 761 basic_block bb = BLOCK_FOR_INSN (insn); 762 763 /* If SRC dies in INSN, we'd have to move the death note. This is 764 considered to be very unlikely, so we just skip the optimization 765 in this case. */ 766 if (find_regno_note (insn, REG_DEAD, REGNO (src))) 767 return 0; 768 769 /* Scan backward to find the first instruction that sets DST. */ 770 771 for (length = 0, p = PREV_INSN (insn); p; p = PREV_INSN (p)) 772 { 773 rtx pset; 774 775 if (! INSN_P (p)) 776 continue; 777 if (BLOCK_FOR_INSN (p) != bb) 778 break; 779 780 if (find_regno_note (p, REG_DEAD, REGNO (dst))) 781 dst_death = p; 782 if (! dst_death && !DEBUG_INSN_P (p)) 783 length++; 784 785 pset = single_set (p); 786 if (pset && SET_DEST (pset) == dst 787 && GET_CODE (SET_SRC (pset)) == PLUS 788 && XEXP (SET_SRC (pset), 0) == src 789 && CONST_INT_P (XEXP (SET_SRC (pset), 1))) 790 { 791 HOST_WIDE_INT newconst 792 = INTVAL (offset) - INTVAL (XEXP (SET_SRC (pset), 1)); 793 rtx add = gen_add3_insn (dst, dst, GEN_INT (newconst)); 794 795 if (add && validate_change (insn, &PATTERN (insn), add, 0)) 796 { 797 /* Remove the death note for DST from DST_DEATH. */ 798 if (dst_death) 799 { 800 remove_death (REGNO (dst), dst_death); 801 REG_LIVE_LENGTH (REGNO (dst)) += length; 802 REG_N_CALLS_CROSSED (REGNO (dst)) += num_calls; 803 REG_FREQ_CALLS_CROSSED (REGNO (dst)) += freq_calls; 804 } 805 806 if (dump_file) 807 fprintf (dump_file, 808 "Fixed operand of insn %d.\n", 809 INSN_UID (insn)); 810 811 #ifdef AUTO_INC_DEC 812 for (p = PREV_INSN (insn); p; p = PREV_INSN (p)) 813 { 814 if (! INSN_P (p)) 815 continue; 816 if (BLOCK_FOR_INSN (p) != bb) 817 break; 818 if (reg_overlap_mentioned_p (dst, PATTERN (p))) 819 { 820 if (try_auto_increment (p, insn, 0, dst, newconst, 0)) 821 return 1; 822 break; 823 } 824 } 825 for (p = NEXT_INSN (insn); p; p = NEXT_INSN (p)) 826 { 827 if (! INSN_P (p)) 828 continue; 829 if (BLOCK_FOR_INSN (p) != bb) 830 break; 831 if (reg_overlap_mentioned_p (dst, PATTERN (p))) 832 { 833 try_auto_increment (p, insn, 0, dst, newconst, 1); 834 break; 835 } 836 } 837 #endif 838 return 1; 839 } 840 } 841 842 if (reg_set_p (dst, PATTERN (p))) 843 break; 844 845 /* If we have passed a call instruction, and the 846 pseudo-reg SRC is not already live across a call, 847 then don't perform the optimization. */ 848 /* reg_set_p is overly conservative for CALL_INSNS, thinks that all 849 hard regs are clobbered. Thus, we only use it for src for 850 non-call insns. */ 851 if (CALL_P (p)) 852 { 853 if (! dst_death) 854 { 855 num_calls++; 856 freq_calls += REG_FREQ_FROM_BB (BLOCK_FOR_INSN (p)); 857 } 858 859 if (REG_N_CALLS_CROSSED (REGNO (src)) == 0) 860 break; 861 862 if ((HARD_REGISTER_P (dst) && call_used_regs [REGNO (dst)]) 863 || find_reg_fusage (p, CLOBBER, dst)) 864 break; 865 } 866 else if (reg_set_p (src, PATTERN (p))) 867 break; 868 } 869 870 return 0; 871 } 872 873 /* A forward pass. Replace output operands with input operands. */ 874 875 static void 876 regmove_forward_pass (void) 877 { 878 basic_block bb; 879 rtx insn; 880 881 if (! flag_expensive_optimizations) 882 return; 883 884 if (dump_file) 885 fprintf (dump_file, "Starting forward pass...\n"); 886 887 FOR_EACH_BB (bb) 888 { 889 FOR_BB_INSNS (bb, insn) 890 { 891 rtx set = single_set (insn); 892 if (! set) 893 continue; 894 895 if ((GET_CODE (SET_SRC (set)) == SIGN_EXTEND 896 || GET_CODE (SET_SRC (set)) == ZERO_EXTEND) 897 && REG_P (XEXP (SET_SRC (set), 0)) 898 && REG_P (SET_DEST (set))) 899 optimize_reg_copy_3 (insn, SET_DEST (set), SET_SRC (set)); 900 901 if (REG_P (SET_SRC (set)) 902 && REG_P (SET_DEST (set))) 903 { 904 /* If this is a register-register copy where SRC is not dead, 905 see if we can optimize it. If this optimization succeeds, 906 it will become a copy where SRC is dead. */ 907 if ((find_reg_note (insn, REG_DEAD, SET_SRC (set)) 908 || optimize_reg_copy_1 (insn, SET_DEST (set), SET_SRC (set))) 909 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER) 910 { 911 /* Similarly for a pseudo-pseudo copy when SRC is dead. */ 912 if (REGNO (SET_SRC (set)) >= FIRST_PSEUDO_REGISTER) 913 optimize_reg_copy_2 (insn, SET_DEST (set), SET_SRC (set)); 914 if (regno_src_regno[REGNO (SET_DEST (set))] < 0 915 && SET_SRC (set) != SET_DEST (set)) 916 { 917 int srcregno = REGNO (SET_SRC (set)); 918 if (regno_src_regno[srcregno] >= 0) 919 srcregno = regno_src_regno[srcregno]; 920 regno_src_regno[REGNO (SET_DEST (set))] = srcregno; 921 } 922 } 923 } 924 } 925 } 926 } 927 928 /* A backward pass. Replace input operands with output operands. */ 929 930 static void 931 regmove_backward_pass (void) 932 { 933 basic_block bb; 934 rtx insn, prev; 935 936 if (dump_file) 937 fprintf (dump_file, "Starting backward pass...\n"); 938 939 FOR_EACH_BB_REVERSE (bb) 940 { 941 /* ??? Use the safe iterator because fixup_match_2 can remove 942 insns via try_auto_increment. */ 943 FOR_BB_INSNS_REVERSE_SAFE (bb, insn, prev) 944 { 945 struct match match; 946 rtx copy_src, copy_dst; 947 int op_no, match_no; 948 int success = 0; 949 950 if (! INSN_P (insn)) 951 continue; 952 953 if (! find_matches (insn, &match)) 954 continue; 955 956 /* Now scan through the operands looking for a destination operand 957 which is supposed to match a source operand. 958 Then scan backward for an instruction which sets the source 959 operand. If safe, then replace the source operand with the 960 dest operand in both instructions. */ 961 962 copy_src = NULL_RTX; 963 copy_dst = NULL_RTX; 964 for (op_no = 0; op_no < recog_data.n_operands; op_no++) 965 { 966 rtx set, p, src, dst; 967 rtx src_note, dst_note; 968 int num_calls = 0, freq_calls = 0; 969 enum reg_class src_class, dst_class; 970 int length; 971 972 match_no = match.with[op_no]; 973 974 /* Nothing to do if the two operands aren't supposed to match. */ 975 if (match_no < 0) 976 continue; 977 978 dst = recog_data.operand[match_no]; 979 src = recog_data.operand[op_no]; 980 981 if (!REG_P (src)) 982 continue; 983 984 if (!REG_P (dst) 985 || REGNO (dst) < FIRST_PSEUDO_REGISTER 986 || REG_LIVE_LENGTH (REGNO (dst)) < 0 987 || GET_MODE (src) != GET_MODE (dst)) 988 continue; 989 990 /* If the operands already match, then there is nothing to do. */ 991 if (operands_match_p (src, dst)) 992 continue; 993 994 if (match.commutative[op_no] >= 0) 995 { 996 rtx comm = recog_data.operand[match.commutative[op_no]]; 997 if (operands_match_p (comm, dst)) 998 continue; 999 } 1000 1001 set = single_set (insn); 1002 if (! set) 1003 continue; 1004 1005 /* Note that single_set ignores parts of a parallel set for 1006 which one of the destinations is REG_UNUSED. We can't 1007 handle that here, since we can wind up rewriting things 1008 such that a single register is set twice within a single 1009 parallel. */ 1010 if (reg_set_p (src, insn)) 1011 continue; 1012 1013 /* match_no/dst must be a write-only operand, and 1014 operand_operand/src must be a read-only operand. */ 1015 if (match.use[op_no] != READ 1016 || match.use[match_no] != WRITE) 1017 continue; 1018 1019 if (match.early_clobber[match_no] 1020 && count_occurrences (PATTERN (insn), src, 0) > 1) 1021 continue; 1022 1023 /* Make sure match_no is the destination. */ 1024 if (recog_data.operand[match_no] != SET_DEST (set)) 1025 continue; 1026 1027 if (REGNO (src) < FIRST_PSEUDO_REGISTER) 1028 { 1029 if (GET_CODE (SET_SRC (set)) == PLUS 1030 && CONST_INT_P (XEXP (SET_SRC (set), 1)) 1031 && XEXP (SET_SRC (set), 0) == src 1032 && fixup_match_2 (insn, dst, src, 1033 XEXP (SET_SRC (set), 1))) 1034 break; 1035 continue; 1036 } 1037 src_class = reg_preferred_class (REGNO (src)); 1038 dst_class = reg_preferred_class (REGNO (dst)); 1039 1040 if (! (src_note = find_reg_note (insn, REG_DEAD, src))) 1041 { 1042 /* We used to force the copy here like in other cases, but 1043 it produces worse code, as it eliminates no copy 1044 instructions and the copy emitted will be produced by 1045 reload anyway. On patterns with multiple alternatives, 1046 there may be better solution available. 1047 1048 In particular this change produced slower code for numeric 1049 i387 programs. */ 1050 1051 continue; 1052 } 1053 1054 if (! regclass_compatible_p (src_class, dst_class)) 1055 { 1056 if (!copy_src) 1057 { 1058 copy_src = src; 1059 copy_dst = dst; 1060 } 1061 continue; 1062 } 1063 1064 /* Can not modify an earlier insn to set dst if this insn 1065 uses an old value in the source. */ 1066 if (reg_overlap_mentioned_p (dst, SET_SRC (set))) 1067 { 1068 if (!copy_src) 1069 { 1070 copy_src = src; 1071 copy_dst = dst; 1072 } 1073 continue; 1074 } 1075 1076 /* If src is set once in a different basic block, 1077 and is set equal to a constant, then do not use 1078 it for this optimization, as this would make it 1079 no longer equivalent to a constant. */ 1080 1081 if (reg_is_remote_constant_p (src, insn)) 1082 { 1083 if (!copy_src) 1084 { 1085 copy_src = src; 1086 copy_dst = dst; 1087 } 1088 continue; 1089 } 1090 1091 1092 if (dump_file) 1093 fprintf (dump_file, 1094 "Could fix operand %d of insn %d matching operand %d.\n", 1095 op_no, INSN_UID (insn), match_no); 1096 1097 /* Scan backward to find the first instruction that uses 1098 the input operand. If the operand is set here, then 1099 replace it in both instructions with match_no. */ 1100 1101 for (length = 0, p = PREV_INSN (insn); p; p = PREV_INSN (p)) 1102 { 1103 rtx pset; 1104 1105 if (! INSN_P (p)) 1106 continue; 1107 if (BLOCK_FOR_INSN (p) != bb) 1108 break; 1109 1110 if (!DEBUG_INSN_P (p)) 1111 length++; 1112 1113 /* ??? See if all of SRC is set in P. This test is much 1114 more conservative than it needs to be. */ 1115 pset = single_set (p); 1116 if (pset && SET_DEST (pset) == src) 1117 { 1118 /* We use validate_replace_rtx, in case there 1119 are multiple identical source operands. All 1120 of them have to be changed at the same time: 1121 when validate_replace_rtx() calls 1122 apply_change_group(). */ 1123 validate_change (p, &SET_DEST (pset), dst, 1); 1124 if (validate_replace_rtx (src, dst, insn)) 1125 success = 1; 1126 break; 1127 } 1128 1129 /* We can't make this change if DST is mentioned at 1130 all in P, since we are going to change its value. 1131 We can't make this change if SRC is read or 1132 partially written in P, since we are going to 1133 eliminate SRC. However, if it's a debug insn, we 1134 can't refrain from making the change, for this 1135 would cause codegen differences, so instead we 1136 invalidate debug expressions that reference DST, 1137 and adjust references to SRC in them so that they 1138 become references to DST. */ 1139 if (reg_mentioned_p (dst, PATTERN (p))) 1140 { 1141 if (DEBUG_INSN_P (p)) 1142 validate_change (p, &INSN_VAR_LOCATION_LOC (p), 1143 gen_rtx_UNKNOWN_VAR_LOC (), 1); 1144 else 1145 break; 1146 } 1147 if (reg_overlap_mentioned_p (src, PATTERN (p))) 1148 { 1149 if (DEBUG_INSN_P (p)) 1150 validate_replace_rtx_group (src, dst, p); 1151 else 1152 break; 1153 } 1154 1155 /* If we have passed a call instruction, and the 1156 pseudo-reg DST is not already live across a call, 1157 then don't perform the optimization. */ 1158 if (CALL_P (p)) 1159 { 1160 num_calls++; 1161 freq_calls += REG_FREQ_FROM_BB (BLOCK_FOR_INSN (p)); 1162 1163 if (REG_N_CALLS_CROSSED (REGNO (dst)) == 0) 1164 break; 1165 } 1166 } 1167 1168 if (success) 1169 { 1170 int dstno, srcno; 1171 1172 /* Remove the death note for SRC from INSN. */ 1173 remove_note (insn, src_note); 1174 /* Move the death note for SRC to P if it is used 1175 there. */ 1176 if (reg_overlap_mentioned_p (src, PATTERN (p))) 1177 { 1178 XEXP (src_note, 1) = REG_NOTES (p); 1179 REG_NOTES (p) = src_note; 1180 } 1181 /* If there is a REG_DEAD note for DST on P, then remove 1182 it, because DST is now set there. */ 1183 if ((dst_note = find_reg_note (p, REG_DEAD, dst))) 1184 remove_note (p, dst_note); 1185 1186 dstno = REGNO (dst); 1187 srcno = REGNO (src); 1188 1189 INC_REG_N_SETS (dstno, 1); 1190 INC_REG_N_SETS (srcno, -1); 1191 1192 REG_N_CALLS_CROSSED (dstno) += num_calls; 1193 REG_N_CALLS_CROSSED (srcno) -= num_calls; 1194 REG_FREQ_CALLS_CROSSED (dstno) += freq_calls; 1195 REG_FREQ_CALLS_CROSSED (srcno) -= freq_calls; 1196 1197 REG_LIVE_LENGTH (dstno) += length; 1198 if (REG_LIVE_LENGTH (srcno) >= 0) 1199 { 1200 REG_LIVE_LENGTH (srcno) -= length; 1201 /* REG_LIVE_LENGTH is only an approximation after 1202 combine if sched is not run, so make sure that we 1203 still have a reasonable value. */ 1204 if (REG_LIVE_LENGTH (srcno) < 2) 1205 REG_LIVE_LENGTH (srcno) = 2; 1206 } 1207 1208 if (dump_file) 1209 fprintf (dump_file, 1210 "Fixed operand %d of insn %d matching operand %d.\n", 1211 op_no, INSN_UID (insn), match_no); 1212 1213 break; 1214 } 1215 else if (num_changes_pending () > 0) 1216 cancel_changes (0); 1217 } 1218 1219 /* If we weren't able to replace any of the alternatives, try an 1220 alternative approach of copying the source to the destination. */ 1221 if (!success && copy_src != NULL_RTX) 1222 copy_src_to_dest (insn, copy_src, copy_dst); 1223 } 1224 } 1225 } 1226 1227 /* Main entry for the register move optimization. */ 1228 1229 static unsigned int 1230 regmove_optimize (void) 1231 { 1232 int i; 1233 int nregs = max_reg_num (); 1234 1235 df_note_add_problem (); 1236 df_analyze (); 1237 1238 regstat_init_n_sets_and_refs (); 1239 regstat_compute_ri (); 1240 1241 if (flag_ira_loop_pressure) 1242 ira_set_pseudo_classes (dump_file); 1243 1244 regno_src_regno = XNEWVEC (int, nregs); 1245 for (i = nregs; --i >= 0; ) 1246 regno_src_regno[i] = -1; 1247 1248 /* A forward pass. Replace output operands with input operands. */ 1249 regmove_forward_pass (); 1250 1251 /* A backward pass. Replace input operands with output operands. */ 1252 regmove_backward_pass (); 1253 1254 /* Clean up. */ 1255 free (regno_src_regno); 1256 if (reg_set_in_bb) 1257 { 1258 free (reg_set_in_bb); 1259 reg_set_in_bb = NULL; 1260 } 1261 regstat_free_n_sets_and_refs (); 1262 regstat_free_ri (); 1263 if (flag_ira_loop_pressure) 1264 free_reg_info (); 1265 return 0; 1266 } 1267 1268 /* Returns nonzero if INSN's pattern has matching constraints for any operand. 1269 Returns 0 if INSN can't be recognized, or if the alternative can't be 1270 determined. 1271 1272 Initialize the info in MATCHP based on the constraints. */ 1273 1274 static int 1275 find_matches (rtx insn, struct match *matchp) 1276 { 1277 int likely_spilled[MAX_RECOG_OPERANDS]; 1278 int op_no; 1279 int any_matches = 0; 1280 1281 extract_insn (insn); 1282 if (! constrain_operands (0)) 1283 return 0; 1284 1285 /* Must initialize this before main loop, because the code for 1286 the commutative case may set matches for operands other than 1287 the current one. */ 1288 for (op_no = recog_data.n_operands; --op_no >= 0; ) 1289 matchp->with[op_no] = matchp->commutative[op_no] = -1; 1290 1291 for (op_no = 0; op_no < recog_data.n_operands; op_no++) 1292 { 1293 const char *p; 1294 char c; 1295 int i = 0; 1296 1297 p = recog_data.constraints[op_no]; 1298 1299 likely_spilled[op_no] = 0; 1300 matchp->use[op_no] = READ; 1301 matchp->early_clobber[op_no] = 0; 1302 if (*p == '=') 1303 matchp->use[op_no] = WRITE; 1304 else if (*p == '+') 1305 matchp->use[op_no] = READWRITE; 1306 1307 for (;*p && i < which_alternative; p++) 1308 if (*p == ',') 1309 i++; 1310 1311 while ((c = *p) != '\0' && c != ',') 1312 { 1313 switch (c) 1314 { 1315 case '=': 1316 break; 1317 case '+': 1318 break; 1319 case '&': 1320 matchp->early_clobber[op_no] = 1; 1321 break; 1322 case '%': 1323 matchp->commutative[op_no] = op_no + 1; 1324 matchp->commutative[op_no + 1] = op_no; 1325 break; 1326 1327 case '0': case '1': case '2': case '3': case '4': 1328 case '5': case '6': case '7': case '8': case '9': 1329 { 1330 char *end; 1331 unsigned long match_ul = strtoul (p, &end, 10); 1332 int match = match_ul; 1333 1334 p = end; 1335 1336 if (match < op_no && likely_spilled[match]) 1337 continue; 1338 matchp->with[op_no] = match; 1339 any_matches = 1; 1340 if (matchp->commutative[op_no] >= 0) 1341 matchp->with[matchp->commutative[op_no]] = match; 1342 } 1343 continue; 1344 1345 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'h': 1346 case 'j': case 'k': case 'l': case 'p': case 'q': case 't': case 'u': 1347 case 'v': case 'w': case 'x': case 'y': case 'z': case 'A': case 'B': 1348 case 'C': case 'D': case 'W': case 'Y': case 'Z': 1349 if (targetm.class_likely_spilled_p (REG_CLASS_FROM_CONSTRAINT ((unsigned char) c, p))) 1350 likely_spilled[op_no] = 1; 1351 break; 1352 } 1353 p += CONSTRAINT_LEN (c, p); 1354 } 1355 } 1356 return any_matches; 1357 } 1358 1359 1360 1361 static bool 1362 gate_handle_regmove (void) 1363 { 1364 return (optimize > 0 && flag_regmove); 1365 } 1366 1367 1368 struct rtl_opt_pass pass_regmove = 1369 { 1370 { 1371 RTL_PASS, 1372 "regmove", /* name */ 1373 gate_handle_regmove, /* gate */ 1374 regmove_optimize, /* execute */ 1375 NULL, /* sub */ 1376 NULL, /* next */ 1377 0, /* static_pass_number */ 1378 TV_REGMOVE, /* tv_id */ 1379 0, /* properties_required */ 1380 0, /* properties_provided */ 1381 0, /* properties_destroyed */ 1382 0, /* todo_flags_start */ 1383 TODO_df_finish | TODO_verify_rtl_sharing | 1384 TODO_ggc_collect /* todo_flags_finish */ 1385 } 1386 }; 1387