1// Copyright 2020 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package ssa 6 7import ( 8 "cmd/compile/internal/abi" 9 "cmd/compile/internal/base" 10 "cmd/compile/internal/ir" 11 "cmd/compile/internal/types" 12 "cmd/internal/src" 13 "fmt" 14 "sort" 15) 16 17type selKey struct { 18 from *Value // what is selected from 19 offsetOrIndex int64 // whatever is appropriate for the selector 20 size int64 21 typ *types.Type 22} 23 24type Abi1RO uint8 // An offset within a parameter's slice of register indices, for abi1. 25 26func isBlockMultiValueExit(b *Block) bool { 27 return (b.Kind == BlockRet || b.Kind == BlockRetJmp) && b.Controls[0] != nil && b.Controls[0].Op == OpMakeResult 28} 29 30func badVal(s string, v *Value) error { 31 return fmt.Errorf("%s %s", s, v.LongString()) 32} 33 34// removeTrivialWrapperTypes unwraps layers of 35// struct { singleField SomeType } and [1]SomeType 36// until a non-wrapper type is reached. This is useful 37// for working with assignments to/from interface data 38// fields (either second operand to OpIMake or OpIData) 39// where the wrapping or type conversion can be elided 40// because of type conversions/assertions in source code 41// that do not appear in SSA. 42func removeTrivialWrapperTypes(t *types.Type) *types.Type { 43 for { 44 if t.IsStruct() && t.NumFields() == 1 { 45 t = t.Field(0).Type 46 continue 47 } 48 if t.IsArray() && t.NumElem() == 1 { 49 t = t.Elem() 50 continue 51 } 52 break 53 } 54 return t 55} 56 57// A registerCursor tracks which register is used for an Arg or regValues, or a piece of such. 58type registerCursor struct { 59 // TODO(register args) convert this to a generalized target cursor. 60 storeDest *Value // if there are no register targets, then this is the base of the store. 61 regsLen int // the number of registers available for this Arg/result (which is all in registers or not at all) 62 nextSlice Abi1RO // the next register/register-slice offset 63 config *abi.ABIConfig 64 regValues *[]*Value // values assigned to registers accumulate here 65} 66 67func (rc *registerCursor) String() string { 68 dest := "<none>" 69 if rc.storeDest != nil { 70 dest = rc.storeDest.String() 71 } 72 regs := "<none>" 73 if rc.regValues != nil { 74 regs = "" 75 for i, x := range *rc.regValues { 76 if i > 0 { 77 regs = regs + "; " 78 } 79 regs = regs + x.LongString() 80 } 81 } 82 // not printing the config because that has not been useful 83 return fmt.Sprintf("RCSR{storeDest=%v, regsLen=%d, nextSlice=%d, regValues=[%s]}", dest, rc.regsLen, rc.nextSlice, regs) 84} 85 86// next effectively post-increments the register cursor; the receiver is advanced, 87// the old value is returned. 88func (c *registerCursor) next(t *types.Type) registerCursor { 89 rc := *c 90 if int(c.nextSlice) < c.regsLen { 91 w := c.config.NumParamRegs(t) 92 c.nextSlice += Abi1RO(w) 93 } 94 return rc 95} 96 97// plus returns a register cursor offset from the original, without modifying the original. 98func (c *registerCursor) plus(regWidth Abi1RO) registerCursor { 99 rc := *c 100 rc.nextSlice += regWidth 101 return rc 102} 103 104const ( 105 // Register offsets for fields of built-in aggregate types; the ones not listed are zero. 106 RO_complex_imag = 1 107 RO_string_len = 1 108 RO_slice_len = 1 109 RO_slice_cap = 2 110 RO_iface_data = 1 111) 112 113func (x *expandState) regWidth(t *types.Type) Abi1RO { 114 return Abi1RO(x.abi1.NumParamRegs(t)) 115} 116 117// regOffset returns the register offset of the i'th element of type t 118func (x *expandState) regOffset(t *types.Type, i int) Abi1RO { 119 // TODO maybe cache this in a map if profiling recommends. 120 if i == 0 { 121 return 0 122 } 123 if t.IsArray() { 124 return Abi1RO(i) * x.regWidth(t.Elem()) 125 } 126 if t.IsStruct() { 127 k := Abi1RO(0) 128 for j := 0; j < i; j++ { 129 k += x.regWidth(t.FieldType(j)) 130 } 131 return k 132 } 133 panic("Haven't implemented this case yet, do I need to?") 134} 135 136// at returns the register cursor for component i of t, where the first 137// component is numbered 0. 138func (c *registerCursor) at(t *types.Type, i int) registerCursor { 139 rc := *c 140 if i == 0 || c.regsLen == 0 { 141 return rc 142 } 143 if t.IsArray() { 144 w := c.config.NumParamRegs(t.Elem()) 145 rc.nextSlice += Abi1RO(i * w) 146 return rc 147 } 148 if t.IsStruct() { 149 for j := 0; j < i; j++ { 150 rc.next(t.FieldType(j)) 151 } 152 return rc 153 } 154 panic("Haven't implemented this case yet, do I need to?") 155} 156 157func (c *registerCursor) init(regs []abi.RegIndex, info *abi.ABIParamResultInfo, result *[]*Value, storeDest *Value) { 158 c.regsLen = len(regs) 159 c.nextSlice = 0 160 if len(regs) == 0 { 161 c.storeDest = storeDest // only save this if there are no registers, will explode if misused. 162 return 163 } 164 c.config = info.Config() 165 c.regValues = result 166} 167 168func (c *registerCursor) addArg(v *Value) { 169 *c.regValues = append(*c.regValues, v) 170} 171 172func (c *registerCursor) hasRegs() bool { 173 return c.regsLen > 0 174} 175 176type expandState struct { 177 f *Func 178 abi1 *abi.ABIConfig 179 debug int // odd values log lost statement markers, so likely settings are 1 (stmts), 2 (expansion), and 3 (both) 180 canSSAType func(*types.Type) bool 181 regSize int64 182 sp *Value 183 typs *Types 184 ptrSize int64 185 hiOffset int64 186 lowOffset int64 187 hiRo Abi1RO 188 loRo Abi1RO 189 namedSelects map[*Value][]namedVal 190 sdom SparseTree 191 commonSelectors map[selKey]*Value // used to de-dupe selectors 192 commonArgs map[selKey]*Value // used to de-dupe OpArg/OpArgIntReg/OpArgFloatReg 193 memForCall map[ID]*Value // For a call, need to know the unique selector that gets the mem. 194 transformedSelects map[ID]bool // OpSelectN after rewriting, either created or renumbered. 195 indentLevel int // Indentation for debugging recursion 196} 197 198// intPairTypes returns the pair of 32-bit int types needed to encode a 64-bit integer type on a target 199// that has no 64-bit integer registers. 200func (x *expandState) intPairTypes(et types.Kind) (tHi, tLo *types.Type) { 201 tHi = x.typs.UInt32 202 if et == types.TINT64 { 203 tHi = x.typs.Int32 204 } 205 tLo = x.typs.UInt32 206 return 207} 208 209// isAlreadyExpandedAggregateType returns whether a type is an SSA-able "aggregate" (multiple register) type 210// that was expanded in an earlier phase (currently, expand_calls is intended to run after decomposeBuiltin, 211// so this is all aggregate types -- small struct and array, complex, interface, string, slice, and 64-bit 212// integer on 32-bit). 213func (x *expandState) isAlreadyExpandedAggregateType(t *types.Type) bool { 214 if !x.canSSAType(t) { 215 return false 216 } 217 return t.IsStruct() || t.IsArray() || t.IsComplex() || t.IsInterface() || t.IsString() || t.IsSlice() || 218 (t.Size() > x.regSize && (t.IsInteger() || (x.f.Config.SoftFloat && t.IsFloat()))) 219} 220 221// offsetFrom creates an offset from a pointer, simplifying chained offsets and offsets from SP 222// TODO should also optimize offsets from SB? 223func (x *expandState) offsetFrom(b *Block, from *Value, offset int64, pt *types.Type) *Value { 224 ft := from.Type 225 if offset == 0 { 226 if ft == pt { 227 return from 228 } 229 // This captures common, (apparently) safe cases. The unsafe cases involve ft == uintptr 230 if (ft.IsPtr() || ft.IsUnsafePtr()) && pt.IsPtr() { 231 return from 232 } 233 } 234 // Simplify, canonicalize 235 for from.Op == OpOffPtr { 236 offset += from.AuxInt 237 from = from.Args[0] 238 } 239 if from == x.sp { 240 return x.f.ConstOffPtrSP(pt, offset, x.sp) 241 } 242 return b.NewValue1I(from.Pos.WithNotStmt(), OpOffPtr, pt, offset, from) 243} 244 245// splitSlots splits one "field" (specified by sfx, offset, and ty) out of the LocalSlots in ls and returns the new LocalSlots this generates. 246func (x *expandState) splitSlots(ls []*LocalSlot, sfx string, offset int64, ty *types.Type) []*LocalSlot { 247 var locs []*LocalSlot 248 for i := range ls { 249 locs = append(locs, x.f.SplitSlot(ls[i], sfx, offset, ty)) 250 } 251 return locs 252} 253 254// prAssignForArg returns the ABIParamAssignment for v, assumed to be an OpArg. 255func (x *expandState) prAssignForArg(v *Value) *abi.ABIParamAssignment { 256 if v.Op != OpArg { 257 panic(badVal("Wanted OpArg, instead saw", v)) 258 } 259 return ParamAssignmentForArgName(x.f, v.Aux.(*ir.Name)) 260} 261 262// ParamAssignmentForArgName returns the ABIParamAssignment for f's arg with matching name. 263func ParamAssignmentForArgName(f *Func, name *ir.Name) *abi.ABIParamAssignment { 264 abiInfo := f.OwnAux.abiInfo 265 ip := abiInfo.InParams() 266 for i, a := range ip { 267 if a.Name == name { 268 return &ip[i] 269 } 270 } 271 panic(fmt.Errorf("Did not match param %v in prInfo %+v", name, abiInfo.InParams())) 272} 273 274// indent increments (or decrements) the indentation. 275func (x *expandState) indent(n int) { 276 x.indentLevel += n 277} 278 279// Printf does an indented fmt.Printf on te format and args. 280func (x *expandState) Printf(format string, a ...interface{}) (n int, err error) { 281 if x.indentLevel > 0 { 282 fmt.Printf("%[1]*s", x.indentLevel, "") 283 } 284 return fmt.Printf(format, a...) 285} 286 287// Calls that need lowering have some number of inputs, including a memory input, 288// and produce a tuple of (value1, value2, ..., mem) where valueK may or may not be SSA-able. 289 290// With the current ABI those inputs need to be converted into stores to memory, 291// rethreading the call's memory input to the first, and the new call now receiving the last. 292 293// With the current ABI, the outputs need to be converted to loads, which will all use the call's 294// memory output as their input. 295 296// rewriteSelect recursively walks from leaf selector to a root (OpSelectN, OpLoad, OpArg) 297// through a chain of Struct/Array/builtin Select operations. If the chain of selectors does not 298// end in an expected root, it does nothing (this can happen depending on compiler phase ordering). 299// The "leaf" provides the type, the root supplies the container, and the leaf-to-root path 300// accumulates the offset. 301// It emits the code necessary to implement the leaf select operation that leads to the root. 302// 303// TODO when registers really arrive, must also decompose anything split across two registers or registers and memory. 304func (x *expandState) rewriteSelect(leaf *Value, selector *Value, offset int64, regOffset Abi1RO) []*LocalSlot { 305 if x.debug > 1 { 306 x.indent(3) 307 defer x.indent(-3) 308 x.Printf("rewriteSelect(%s; %s; memOff=%d; regOff=%d)\n", leaf.LongString(), selector.LongString(), offset, regOffset) 309 } 310 var locs []*LocalSlot 311 leafType := leaf.Type 312 if len(selector.Args) > 0 { 313 w := selector.Args[0] 314 if w.Op == OpCopy { 315 for w.Op == OpCopy { 316 w = w.Args[0] 317 } 318 selector.SetArg(0, w) 319 } 320 } 321 switch selector.Op { 322 case OpArgIntReg, OpArgFloatReg: 323 if leafType == selector.Type { // OpIData leads us here, sometimes. 324 leaf.copyOf(selector) 325 } else { 326 x.f.Fatalf("Unexpected %s type, selector=%s, leaf=%s\n", selector.Op.String(), selector.LongString(), leaf.LongString()) 327 } 328 if x.debug > 1 { 329 x.Printf("---%s, break\n", selector.Op.String()) 330 } 331 case OpArg: 332 if !x.isAlreadyExpandedAggregateType(selector.Type) { 333 if leafType == selector.Type { // OpIData leads us here, sometimes. 334 x.newArgToMemOrRegs(selector, leaf, offset, regOffset, leafType, leaf.Pos) 335 } else { 336 x.f.Fatalf("Unexpected OpArg type, selector=%s, leaf=%s\n", selector.LongString(), leaf.LongString()) 337 } 338 if x.debug > 1 { 339 x.Printf("---OpArg, break\n") 340 } 341 break 342 } 343 switch leaf.Op { 344 case OpIData, OpStructSelect, OpArraySelect: 345 leafType = removeTrivialWrapperTypes(leaf.Type) 346 } 347 x.newArgToMemOrRegs(selector, leaf, offset, regOffset, leafType, leaf.Pos) 348 349 for _, s := range x.namedSelects[selector] { 350 locs = append(locs, x.f.Names[s.locIndex]) 351 } 352 353 case OpLoad: // We end up here because of IData of immediate structures. 354 // Failure case: 355 // (note the failure case is very rare; w/o this case, make.bash and run.bash both pass, as well as 356 // the hard cases of building {syscall,math,math/cmplx,math/bits,go/constant} on ppc64le and mips-softfloat). 357 // 358 // GOSSAFUNC='(*dumper).dump' go build -gcflags=-l -tags=math_big_pure_go cmd/compile/internal/gc 359 // cmd/compile/internal/gc/dump.go:136:14: internal compiler error: '(*dumper).dump': not lowered: v827, StructSelect PTR PTR 360 // b2: ← b1 361 // v20 (+142) = StaticLECall <interface {},mem> {AuxCall{reflect.Value.Interface([reflect.Value,0])[interface {},24]}} [40] v8 v1 362 // v21 (142) = SelectN <mem> [1] v20 363 // v22 (142) = SelectN <interface {}> [0] v20 364 // b15: ← b8 365 // v71 (+143) = IData <Nodes> v22 (v[Nodes]) 366 // v73 (+146) = StaticLECall <[]*Node,mem> {AuxCall{"".Nodes.Slice([Nodes,0])[[]*Node,8]}} [32] v71 v21 367 // 368 // translates (w/o the "case OpLoad:" above) to: 369 // 370 // b2: ← b1 371 // v20 (+142) = StaticCall <mem> {AuxCall{reflect.Value.Interface([reflect.Value,0])[interface {},24]}} [40] v715 372 // v23 (142) = Load <*uintptr> v19 v20 373 // v823 (142) = IsNonNil <bool> v23 374 // v67 (+143) = Load <*[]*Node> v880 v20 375 // b15: ← b8 376 // v827 (146) = StructSelect <*[]*Node> [0] v67 377 // v846 (146) = Store <mem> {*[]*Node} v769 v827 v20 378 // v73 (+146) = StaticCall <mem> {AuxCall{"".Nodes.Slice([Nodes,0])[[]*Node,8]}} [32] v846 379 // i.e., the struct select is generated and remains in because it is not applied to an actual structure. 380 // The OpLoad was created to load the single field of the IData 381 // This case removes that StructSelect. 382 if leafType != selector.Type { 383 if x.f.Config.SoftFloat && selector.Type.IsFloat() { 384 if x.debug > 1 { 385 x.Printf("---OpLoad, break\n") 386 } 387 break // softfloat pass will take care of that 388 } 389 x.f.Fatalf("Unexpected Load as selector, leaf=%s, selector=%s\n", leaf.LongString(), selector.LongString()) 390 } 391 leaf.copyOf(selector) 392 for _, s := range x.namedSelects[selector] { 393 locs = append(locs, x.f.Names[s.locIndex]) 394 } 395 396 case OpSelectN: 397 // TODO(register args) result case 398 // if applied to Op-mumble-call, the Aux tells us which result, regOffset specifies offset within result. If a register, should rewrite to OpSelectN for new call. 399 // TODO these may be duplicated. Should memoize. Intermediate selectors will go dead, no worries there. 400 call := selector.Args[0] 401 call0 := call 402 aux := call.Aux.(*AuxCall) 403 which := selector.AuxInt 404 if x.transformedSelects[selector.ID] { 405 // This is a minor hack. Either this select has had its operand adjusted (mem) or 406 // it is some other intermediate node that was rewritten to reference a register (not a generic arg). 407 // This can occur with chains of selection/indexing from single field/element aggregates. 408 leaf.copyOf(selector) 409 break 410 } 411 if which == aux.NResults() { // mem is after the results. 412 // rewrite v as a Copy of call -- the replacement call will produce a mem. 413 if leaf != selector { 414 panic(fmt.Errorf("Unexpected selector of memory, selector=%s, call=%s, leaf=%s", selector.LongString(), call.LongString(), leaf.LongString())) 415 } 416 if aux.abiInfo == nil { 417 panic(badVal("aux.abiInfo nil for call", call)) 418 } 419 if existing := x.memForCall[call.ID]; existing == nil { 420 selector.AuxInt = int64(aux.abiInfo.OutRegistersUsed()) 421 x.memForCall[call.ID] = selector 422 x.transformedSelects[selector.ID] = true // operand adjusted 423 } else { 424 selector.copyOf(existing) 425 } 426 427 } else { 428 leafType := removeTrivialWrapperTypes(leaf.Type) 429 if x.canSSAType(leafType) { 430 pt := types.NewPtr(leafType) 431 // Any selection right out of the arg area/registers has to be same Block as call, use call as mem input. 432 // Create a "mem" for any loads that need to occur. 433 if mem := x.memForCall[call.ID]; mem != nil { 434 if mem.Block != call.Block { 435 panic(fmt.Errorf("selector and call need to be in same block, selector=%s; call=%s", selector.LongString(), call.LongString())) 436 } 437 call = mem 438 } else { 439 mem = call.Block.NewValue1I(call.Pos.WithNotStmt(), OpSelectN, types.TypeMem, int64(aux.abiInfo.OutRegistersUsed()), call) 440 x.transformedSelects[mem.ID] = true // select uses post-expansion indexing 441 x.memForCall[call.ID] = mem 442 call = mem 443 } 444 outParam := aux.abiInfo.OutParam(int(which)) 445 if len(outParam.Registers) > 0 { 446 firstReg := uint32(0) 447 for i := 0; i < int(which); i++ { 448 firstReg += uint32(len(aux.abiInfo.OutParam(i).Registers)) 449 } 450 reg := int64(regOffset + Abi1RO(firstReg)) 451 if leaf.Block == call.Block { 452 leaf.reset(OpSelectN) 453 leaf.SetArgs1(call0) 454 leaf.Type = leafType 455 leaf.AuxInt = reg 456 x.transformedSelects[leaf.ID] = true // leaf, rewritten to use post-expansion indexing. 457 } else { 458 w := call.Block.NewValue1I(leaf.Pos, OpSelectN, leafType, reg, call0) 459 x.transformedSelects[w.ID] = true // select, using post-expansion indexing. 460 leaf.copyOf(w) 461 } 462 } else { 463 off := x.offsetFrom(x.f.Entry, x.sp, offset+aux.OffsetOfResult(which), pt) 464 if leaf.Block == call.Block { 465 leaf.reset(OpLoad) 466 leaf.SetArgs2(off, call) 467 leaf.Type = leafType 468 } else { 469 w := call.Block.NewValue2(leaf.Pos, OpLoad, leafType, off, call) 470 leaf.copyOf(w) 471 if x.debug > 1 { 472 x.Printf("---new %s\n", w.LongString()) 473 } 474 } 475 } 476 for _, s := range x.namedSelects[selector] { 477 locs = append(locs, x.f.Names[s.locIndex]) 478 } 479 } else { 480 x.f.Fatalf("Should not have non-SSA-able OpSelectN, selector=%s", selector.LongString()) 481 } 482 } 483 484 case OpStructSelect: 485 w := selector.Args[0] 486 var ls []*LocalSlot 487 if w.Type.Kind() != types.TSTRUCT { // IData artifact 488 ls = x.rewriteSelect(leaf, w, offset, regOffset) 489 } else { 490 fldi := int(selector.AuxInt) 491 ls = x.rewriteSelect(leaf, w, offset+w.Type.FieldOff(fldi), regOffset+x.regOffset(w.Type, fldi)) 492 if w.Op != OpIData { 493 for _, l := range ls { 494 locs = append(locs, x.f.SplitStruct(l, int(selector.AuxInt))) 495 } 496 } 497 } 498 499 case OpArraySelect: 500 w := selector.Args[0] 501 index := selector.AuxInt 502 x.rewriteSelect(leaf, w, offset+selector.Type.Size()*index, regOffset+x.regOffset(w.Type, int(index))) 503 504 case OpInt64Hi: 505 w := selector.Args[0] 506 ls := x.rewriteSelect(leaf, w, offset+x.hiOffset, regOffset+x.hiRo) 507 locs = x.splitSlots(ls, ".hi", x.hiOffset, leafType) 508 509 case OpInt64Lo: 510 w := selector.Args[0] 511 ls := x.rewriteSelect(leaf, w, offset+x.lowOffset, regOffset+x.loRo) 512 locs = x.splitSlots(ls, ".lo", x.lowOffset, leafType) 513 514 case OpStringPtr: 515 ls := x.rewriteSelect(leaf, selector.Args[0], offset, regOffset) 516 locs = x.splitSlots(ls, ".ptr", 0, x.typs.BytePtr) 517 518 case OpSlicePtr, OpSlicePtrUnchecked: 519 w := selector.Args[0] 520 ls := x.rewriteSelect(leaf, w, offset, regOffset) 521 locs = x.splitSlots(ls, ".ptr", 0, types.NewPtr(w.Type.Elem())) 522 523 case OpITab: 524 w := selector.Args[0] 525 ls := x.rewriteSelect(leaf, w, offset, regOffset) 526 sfx := ".itab" 527 if w.Type.IsEmptyInterface() { 528 sfx = ".type" 529 } 530 locs = x.splitSlots(ls, sfx, 0, x.typs.Uintptr) 531 532 case OpComplexReal: 533 ls := x.rewriteSelect(leaf, selector.Args[0], offset, regOffset) 534 locs = x.splitSlots(ls, ".real", 0, selector.Type) 535 536 case OpComplexImag: 537 ls := x.rewriteSelect(leaf, selector.Args[0], offset+selector.Type.Size(), regOffset+RO_complex_imag) // result is FloatNN, width of result is offset of imaginary part. 538 locs = x.splitSlots(ls, ".imag", selector.Type.Size(), selector.Type) 539 540 case OpStringLen, OpSliceLen: 541 ls := x.rewriteSelect(leaf, selector.Args[0], offset+x.ptrSize, regOffset+RO_slice_len) 542 locs = x.splitSlots(ls, ".len", x.ptrSize, leafType) 543 544 case OpIData: 545 ls := x.rewriteSelect(leaf, selector.Args[0], offset+x.ptrSize, regOffset+RO_iface_data) 546 locs = x.splitSlots(ls, ".data", x.ptrSize, leafType) 547 548 case OpSliceCap: 549 ls := x.rewriteSelect(leaf, selector.Args[0], offset+2*x.ptrSize, regOffset+RO_slice_cap) 550 locs = x.splitSlots(ls, ".cap", 2*x.ptrSize, leafType) 551 552 case OpCopy: // If it's an intermediate result, recurse 553 locs = x.rewriteSelect(leaf, selector.Args[0], offset, regOffset) 554 for _, s := range x.namedSelects[selector] { 555 // this copy may have had its own name, preserve that, too. 556 locs = append(locs, x.f.Names[s.locIndex]) 557 } 558 559 default: 560 // Ignore dead ends. These can occur if this phase is run before decompose builtin (which is not intended, but allowed). 561 } 562 563 return locs 564} 565 566func (x *expandState) rewriteDereference(b *Block, base, a, mem *Value, offset, size int64, typ *types.Type, pos src.XPos) *Value { 567 source := a.Args[0] 568 dst := x.offsetFrom(b, base, offset, source.Type) 569 if a.Uses == 1 && a.Block == b { 570 a.reset(OpMove) 571 a.Pos = pos 572 a.Type = types.TypeMem 573 a.Aux = typ 574 a.AuxInt = size 575 a.SetArgs3(dst, source, mem) 576 mem = a 577 } else { 578 mem = b.NewValue3A(pos, OpMove, types.TypeMem, typ, dst, source, mem) 579 mem.AuxInt = size 580 } 581 return mem 582} 583 584var indexNames [1]string = [1]string{"[0]"} 585 586// pathTo returns the selection path to the leaf type at offset within container. 587// e.g. len(thing.field[0]) => ".field[0].len" 588// this is for purposes of generating names ultimately fed to a debugger. 589func (x *expandState) pathTo(container, leaf *types.Type, offset int64) string { 590 if container == leaf || offset == 0 && container.Size() == leaf.Size() { 591 return "" 592 } 593 path := "" 594outer: 595 for { 596 switch container.Kind() { 597 case types.TARRAY: 598 container = container.Elem() 599 if container.Size() == 0 { 600 return path 601 } 602 i := offset / container.Size() 603 offset = offset % container.Size() 604 // If a future compiler/ABI supports larger SSA/Arg-able arrays, expand indexNames. 605 path = path + indexNames[i] 606 continue 607 case types.TSTRUCT: 608 for i := 0; i < container.NumFields(); i++ { 609 fld := container.Field(i) 610 if fld.Offset+fld.Type.Size() > offset { 611 offset -= fld.Offset 612 path += "." + fld.Sym.Name 613 container = fld.Type 614 continue outer 615 } 616 } 617 return path 618 case types.TINT64, types.TUINT64: 619 if container.Size() == x.regSize { 620 return path 621 } 622 if offset == x.hiOffset { 623 return path + ".hi" 624 } 625 return path + ".lo" 626 case types.TINTER: 627 if offset != 0 { 628 return path + ".data" 629 } 630 if container.IsEmptyInterface() { 631 return path + ".type" 632 } 633 return path + ".itab" 634 635 case types.TSLICE: 636 if offset == 2*x.regSize { 637 return path + ".cap" 638 } 639 fallthrough 640 case types.TSTRING: 641 if offset == 0 { 642 return path + ".ptr" 643 } 644 return path + ".len" 645 case types.TCOMPLEX64, types.TCOMPLEX128: 646 if offset == 0 { 647 return path + ".real" 648 } 649 return path + ".imag" 650 } 651 return path 652 } 653} 654 655// decomposeArg is a helper for storeArgOrLoad. 656// It decomposes a Load or an Arg into smaller parts and returns the new mem. 657// If the type does not match one of the expected aggregate types, it returns nil instead. 658// Parameters: 659// pos -- the location of any generated code. 660// b -- the block into which any generated code should normally be placed 661// source -- the value, possibly an aggregate, to be stored. 662// mem -- the mem flowing into this decomposition (loads depend on it, stores updated it) 663// t -- the type of the value to be stored 664// storeOffset -- if the value is stored in memory, it is stored at base (see storeRc) + storeOffset 665// loadRegOffset -- regarding source as a value in registers, the register offset in ABI1. Meaningful only if source is OpArg. 666// storeRc -- storeRC; if the value is stored in registers, this specifies the registers. 667// StoreRc also identifies whether the target is registers or memory, and has the base for the store operation. 668func (x *expandState) decomposeArg(pos src.XPos, b *Block, source, mem *Value, t *types.Type, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value { 669 670 pa := x.prAssignForArg(source) 671 var locs []*LocalSlot 672 for _, s := range x.namedSelects[source] { 673 locs = append(locs, x.f.Names[s.locIndex]) 674 } 675 676 if len(pa.Registers) > 0 { 677 // Handle the in-registers case directly 678 rts, offs := pa.RegisterTypesAndOffsets() 679 last := loadRegOffset + x.regWidth(t) 680 if offs[loadRegOffset] != 0 { 681 // Document the problem before panicking. 682 for i := 0; i < len(rts); i++ { 683 rt := rts[i] 684 off := offs[i] 685 fmt.Printf("rt=%s, off=%d, rt.Width=%d, rt.Align=%d\n", rt.String(), off, rt.Size(), uint8(rt.Alignment())) 686 } 687 panic(fmt.Errorf("offset %d of requested register %d should be zero, source=%s", offs[loadRegOffset], loadRegOffset, source.LongString())) 688 } 689 690 if x.debug > 1 { 691 x.Printf("decompose arg %s has %d locs\n", source.LongString(), len(locs)) 692 } 693 694 for i := loadRegOffset; i < last; i++ { 695 rt := rts[i] 696 off := offs[i] 697 w := x.commonArgs[selKey{source, off, rt.Size(), rt}] 698 if w == nil { 699 w = x.newArgToMemOrRegs(source, w, off, i, rt, pos) 700 suffix := x.pathTo(source.Type, rt, off) 701 if suffix != "" { 702 x.splitSlotsIntoNames(locs, suffix, off, rt, w) 703 } 704 } 705 if t.IsPtrShaped() { 706 // Preserve the original store type. This ensures pointer type 707 // properties aren't discarded (e.g, notinheap). 708 if rt.Size() != t.Size() || len(pa.Registers) != 1 || i != loadRegOffset { 709 b.Func.Fatalf("incompatible store type %v and %v, i=%d", t, rt, i) 710 } 711 rt = t 712 } 713 mem = x.storeArgOrLoad(pos, b, w, mem, rt, storeOffset+off, i, storeRc.next(rt)) 714 } 715 return mem 716 } 717 718 u := source.Type 719 switch u.Kind() { 720 case types.TARRAY: 721 elem := u.Elem() 722 elemRO := x.regWidth(elem) 723 for i := int64(0); i < u.NumElem(); i++ { 724 elemOff := i * elem.Size() 725 mem = storeOneArg(x, pos, b, locs, indexNames[i], source, mem, elem, elemOff, storeOffset+elemOff, loadRegOffset, storeRc.next(elem)) 726 loadRegOffset += elemRO 727 pos = pos.WithNotStmt() 728 } 729 return mem 730 case types.TSTRUCT: 731 for i := 0; i < u.NumFields(); i++ { 732 fld := u.Field(i) 733 mem = storeOneArg(x, pos, b, locs, "."+fld.Sym.Name, source, mem, fld.Type, fld.Offset, storeOffset+fld.Offset, loadRegOffset, storeRc.next(fld.Type)) 734 loadRegOffset += x.regWidth(fld.Type) 735 pos = pos.WithNotStmt() 736 } 737 return mem 738 case types.TINT64, types.TUINT64: 739 if t.Size() == x.regSize { 740 break 741 } 742 tHi, tLo := x.intPairTypes(t.Kind()) 743 mem = storeOneArg(x, pos, b, locs, ".hi", source, mem, tHi, x.hiOffset, storeOffset+x.hiOffset, loadRegOffset+x.hiRo, storeRc.plus(x.hiRo)) 744 pos = pos.WithNotStmt() 745 return storeOneArg(x, pos, b, locs, ".lo", source, mem, tLo, x.lowOffset, storeOffset+x.lowOffset, loadRegOffset+x.loRo, storeRc.plus(x.loRo)) 746 case types.TINTER: 747 sfx := ".itab" 748 if u.IsEmptyInterface() { 749 sfx = ".type" 750 } 751 return storeTwoArg(x, pos, b, locs, sfx, ".idata", source, mem, x.typs.Uintptr, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc) 752 case types.TSTRING: 753 return storeTwoArg(x, pos, b, locs, ".ptr", ".len", source, mem, x.typs.BytePtr, x.typs.Int, 0, storeOffset, loadRegOffset, storeRc) 754 case types.TCOMPLEX64: 755 return storeTwoArg(x, pos, b, locs, ".real", ".imag", source, mem, x.typs.Float32, x.typs.Float32, 0, storeOffset, loadRegOffset, storeRc) 756 case types.TCOMPLEX128: 757 return storeTwoArg(x, pos, b, locs, ".real", ".imag", source, mem, x.typs.Float64, x.typs.Float64, 0, storeOffset, loadRegOffset, storeRc) 758 case types.TSLICE: 759 mem = storeOneArg(x, pos, b, locs, ".ptr", source, mem, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr)) 760 return storeTwoArg(x, pos, b, locs, ".len", ".cap", source, mem, x.typs.Int, x.typs.Int, x.ptrSize, storeOffset+x.ptrSize, loadRegOffset+RO_slice_len, storeRc) 761 } 762 return nil 763} 764 765func (x *expandState) splitSlotsIntoNames(locs []*LocalSlot, suffix string, off int64, rt *types.Type, w *Value) { 766 wlocs := x.splitSlots(locs, suffix, off, rt) 767 for _, l := range wlocs { 768 old, ok := x.f.NamedValues[*l] 769 x.f.NamedValues[*l] = append(old, w) 770 if !ok { 771 x.f.Names = append(x.f.Names, l) 772 } 773 } 774} 775 776// decomposeLoad is a helper for storeArgOrLoad. 777// It decomposes a Load into smaller parts and returns the new mem. 778// If the type does not match one of the expected aggregate types, it returns nil instead. 779// Parameters: 780// pos -- the location of any generated code. 781// b -- the block into which any generated code should normally be placed 782// source -- the value, possibly an aggregate, to be stored. 783// mem -- the mem flowing into this decomposition (loads depend on it, stores updated it) 784// t -- the type of the value to be stored 785// storeOffset -- if the value is stored in memory, it is stored at base (see storeRc) + offset 786// loadRegOffset -- regarding source as a value in registers, the register offset in ABI1. Meaningful only if source is OpArg. 787// storeRc -- storeRC; if the value is stored in registers, this specifies the registers. 788// StoreRc also identifies whether the target is registers or memory, and has the base for the store operation. 789// 790// TODO -- this needs cleanup; it just works for SSA-able aggregates, and won't fully generalize to register-args aggregates. 791func (x *expandState) decomposeLoad(pos src.XPos, b *Block, source, mem *Value, t *types.Type, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value { 792 u := source.Type 793 switch u.Kind() { 794 case types.TARRAY: 795 elem := u.Elem() 796 elemRO := x.regWidth(elem) 797 for i := int64(0); i < u.NumElem(); i++ { 798 elemOff := i * elem.Size() 799 mem = storeOneLoad(x, pos, b, source, mem, elem, elemOff, storeOffset+elemOff, loadRegOffset, storeRc.next(elem)) 800 loadRegOffset += elemRO 801 pos = pos.WithNotStmt() 802 } 803 return mem 804 case types.TSTRUCT: 805 for i := 0; i < u.NumFields(); i++ { 806 fld := u.Field(i) 807 mem = storeOneLoad(x, pos, b, source, mem, fld.Type, fld.Offset, storeOffset+fld.Offset, loadRegOffset, storeRc.next(fld.Type)) 808 loadRegOffset += x.regWidth(fld.Type) 809 pos = pos.WithNotStmt() 810 } 811 return mem 812 case types.TINT64, types.TUINT64: 813 if t.Size() == x.regSize { 814 break 815 } 816 tHi, tLo := x.intPairTypes(t.Kind()) 817 mem = storeOneLoad(x, pos, b, source, mem, tHi, x.hiOffset, storeOffset+x.hiOffset, loadRegOffset+x.hiRo, storeRc.plus(x.hiRo)) 818 pos = pos.WithNotStmt() 819 return storeOneLoad(x, pos, b, source, mem, tLo, x.lowOffset, storeOffset+x.lowOffset, loadRegOffset+x.loRo, storeRc.plus(x.loRo)) 820 case types.TINTER: 821 return storeTwoLoad(x, pos, b, source, mem, x.typs.Uintptr, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc) 822 case types.TSTRING: 823 return storeTwoLoad(x, pos, b, source, mem, x.typs.BytePtr, x.typs.Int, 0, storeOffset, loadRegOffset, storeRc) 824 case types.TCOMPLEX64: 825 return storeTwoLoad(x, pos, b, source, mem, x.typs.Float32, x.typs.Float32, 0, storeOffset, loadRegOffset, storeRc) 826 case types.TCOMPLEX128: 827 return storeTwoLoad(x, pos, b, source, mem, x.typs.Float64, x.typs.Float64, 0, storeOffset, loadRegOffset, storeRc) 828 case types.TSLICE: 829 mem = storeOneLoad(x, pos, b, source, mem, x.typs.BytePtr, 0, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr)) 830 return storeTwoLoad(x, pos, b, source, mem, x.typs.Int, x.typs.Int, x.ptrSize, storeOffset+x.ptrSize, loadRegOffset+RO_slice_len, storeRc) 831 } 832 return nil 833} 834 835// storeOneArg creates a decomposed (one step) arg that is then stored. 836// pos and b locate the store instruction, source is the "base" of the value input, 837// mem is the input mem, t is the type in question, and offArg and offStore are the offsets from the respective bases. 838func storeOneArg(x *expandState, pos src.XPos, b *Block, locs []*LocalSlot, suffix string, source, mem *Value, t *types.Type, argOffset, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value { 839 if x.debug > 1 { 840 x.indent(3) 841 defer x.indent(-3) 842 x.Printf("storeOneArg(%s; %s; %s; aO=%d; sO=%d; lrO=%d; %s)\n", source.LongString(), mem.String(), t.String(), argOffset, storeOffset, loadRegOffset, storeRc.String()) 843 } 844 845 w := x.commonArgs[selKey{source, argOffset, t.Size(), t}] 846 if w == nil { 847 w = x.newArgToMemOrRegs(source, w, argOffset, loadRegOffset, t, pos) 848 x.splitSlotsIntoNames(locs, suffix, argOffset, t, w) 849 } 850 return x.storeArgOrLoad(pos, b, w, mem, t, storeOffset, loadRegOffset, storeRc) 851} 852 853// storeOneLoad creates a decomposed (one step) load that is then stored. 854func storeOneLoad(x *expandState, pos src.XPos, b *Block, source, mem *Value, t *types.Type, offArg, offStore int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value { 855 from := x.offsetFrom(source.Block, source.Args[0], offArg, types.NewPtr(t)) 856 w := source.Block.NewValue2(source.Pos, OpLoad, t, from, mem) 857 return x.storeArgOrLoad(pos, b, w, mem, t, offStore, loadRegOffset, storeRc) 858} 859 860func storeTwoArg(x *expandState, pos src.XPos, b *Block, locs []*LocalSlot, suffix1 string, suffix2 string, source, mem *Value, t1, t2 *types.Type, offArg, offStore int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value { 861 mem = storeOneArg(x, pos, b, locs, suffix1, source, mem, t1, offArg, offStore, loadRegOffset, storeRc.next(t1)) 862 pos = pos.WithNotStmt() 863 t1Size := t1.Size() 864 return storeOneArg(x, pos, b, locs, suffix2, source, mem, t2, offArg+t1Size, offStore+t1Size, loadRegOffset+1, storeRc) 865} 866 867// storeTwoLoad creates a pair of decomposed (one step) loads that are then stored. 868// the elements of the pair must not require any additional alignment. 869func storeTwoLoad(x *expandState, pos src.XPos, b *Block, source, mem *Value, t1, t2 *types.Type, offArg, offStore int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value { 870 mem = storeOneLoad(x, pos, b, source, mem, t1, offArg, offStore, loadRegOffset, storeRc.next(t1)) 871 pos = pos.WithNotStmt() 872 t1Size := t1.Size() 873 return storeOneLoad(x, pos, b, source, mem, t2, offArg+t1Size, offStore+t1Size, loadRegOffset+1, storeRc) 874} 875 876// storeArgOrLoad converts stores of SSA-able potentially aggregatable arguments (passed to a call) into a series of primitive-typed 877// stores of non-aggregate types. It recursively walks up a chain of selectors until it reaches a Load or an Arg. 878// If it does not reach a Load or an Arg, nothing happens; this allows a little freedom in phase ordering. 879func (x *expandState) storeArgOrLoad(pos src.XPos, b *Block, source, mem *Value, t *types.Type, storeOffset int64, loadRegOffset Abi1RO, storeRc registerCursor) *Value { 880 if x.debug > 1 { 881 x.indent(3) 882 defer x.indent(-3) 883 x.Printf("storeArgOrLoad(%s; %s; %s; %d; %s)\n", source.LongString(), mem.String(), t.String(), storeOffset, storeRc.String()) 884 } 885 886 // Start with Opcodes that can be disassembled 887 switch source.Op { 888 case OpCopy: 889 return x.storeArgOrLoad(pos, b, source.Args[0], mem, t, storeOffset, loadRegOffset, storeRc) 890 891 case OpLoad, OpDereference: 892 ret := x.decomposeLoad(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc) 893 if ret != nil { 894 return ret 895 } 896 897 case OpArg: 898 ret := x.decomposeArg(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc) 899 if ret != nil { 900 return ret 901 } 902 903 case OpArrayMake0, OpStructMake0: 904 // TODO(register args) is this correct for registers? 905 return mem 906 907 case OpStructMake1, OpStructMake2, OpStructMake3, OpStructMake4: 908 for i := 0; i < t.NumFields(); i++ { 909 fld := t.Field(i) 910 mem = x.storeArgOrLoad(pos, b, source.Args[i], mem, fld.Type, storeOffset+fld.Offset, 0, storeRc.next(fld.Type)) 911 pos = pos.WithNotStmt() 912 } 913 return mem 914 915 case OpArrayMake1: 916 return x.storeArgOrLoad(pos, b, source.Args[0], mem, t.Elem(), storeOffset, 0, storeRc.at(t, 0)) 917 918 case OpInt64Make: 919 tHi, tLo := x.intPairTypes(t.Kind()) 920 mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, tHi, storeOffset+x.hiOffset, 0, storeRc.next(tHi)) 921 pos = pos.WithNotStmt() 922 return x.storeArgOrLoad(pos, b, source.Args[1], mem, tLo, storeOffset+x.lowOffset, 0, storeRc) 923 924 case OpComplexMake: 925 tPart := x.typs.Float32 926 wPart := t.Size() / 2 927 if wPart == 8 { 928 tPart = x.typs.Float64 929 } 930 mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, tPart, storeOffset, 0, storeRc.next(tPart)) 931 pos = pos.WithNotStmt() 932 return x.storeArgOrLoad(pos, b, source.Args[1], mem, tPart, storeOffset+wPart, 0, storeRc) 933 934 case OpIMake: 935 mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, x.typs.Uintptr, storeOffset, 0, storeRc.next(x.typs.Uintptr)) 936 pos = pos.WithNotStmt() 937 return x.storeArgOrLoad(pos, b, source.Args[1], mem, x.typs.BytePtr, storeOffset+x.ptrSize, 0, storeRc) 938 939 case OpStringMake: 940 mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, x.typs.BytePtr, storeOffset, 0, storeRc.next(x.typs.BytePtr)) 941 pos = pos.WithNotStmt() 942 return x.storeArgOrLoad(pos, b, source.Args[1], mem, x.typs.Int, storeOffset+x.ptrSize, 0, storeRc) 943 944 case OpSliceMake: 945 mem = x.storeArgOrLoad(pos, b, source.Args[0], mem, x.typs.BytePtr, storeOffset, 0, storeRc.next(x.typs.BytePtr)) 946 pos = pos.WithNotStmt() 947 mem = x.storeArgOrLoad(pos, b, source.Args[1], mem, x.typs.Int, storeOffset+x.ptrSize, 0, storeRc.next(x.typs.Int)) 948 return x.storeArgOrLoad(pos, b, source.Args[2], mem, x.typs.Int, storeOffset+2*x.ptrSize, 0, storeRc) 949 } 950 951 // For nodes that cannot be taken apart -- OpSelectN, other structure selectors. 952 switch t.Kind() { 953 case types.TARRAY: 954 elt := t.Elem() 955 if source.Type != t && t.NumElem() == 1 && elt.Size() == t.Size() && t.Size() == x.regSize { 956 t = removeTrivialWrapperTypes(t) 957 // it could be a leaf type, but the "leaf" could be complex64 (for example) 958 return x.storeArgOrLoad(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc) 959 } 960 eltRO := x.regWidth(elt) 961 source.Type = t 962 for i := int64(0); i < t.NumElem(); i++ { 963 sel := source.Block.NewValue1I(pos, OpArraySelect, elt, i, source) 964 mem = x.storeArgOrLoad(pos, b, sel, mem, elt, storeOffset+i*elt.Size(), loadRegOffset, storeRc.at(t, 0)) 965 loadRegOffset += eltRO 966 pos = pos.WithNotStmt() 967 } 968 return mem 969 970 case types.TSTRUCT: 971 if source.Type != t && t.NumFields() == 1 && t.Field(0).Type.Size() == t.Size() && t.Size() == x.regSize { 972 // This peculiar test deals with accesses to immediate interface data. 973 // It works okay because everything is the same size. 974 // Example code that triggers this can be found in go/constant/value.go, function ToComplex 975 // v119 (+881) = IData <intVal> v6 976 // v121 (+882) = StaticLECall <floatVal,mem> {AuxCall{"".itof([intVal,0])[floatVal,8]}} [16] v119 v1 977 // This corresponds to the generic rewrite rule "(StructSelect [0] (IData x)) => (IData x)" 978 // Guard against "struct{struct{*foo}}" 979 // Other rewriting phases create minor glitches when they transform IData, for instance the 980 // interface-typed Arg "x" of ToFloat in go/constant/value.go 981 // v6 (858) = Arg <Value> {x} (x[Value], x[Value]) 982 // is rewritten by decomposeArgs into 983 // v141 (858) = Arg <uintptr> {x} 984 // v139 (858) = Arg <*uint8> {x} [8] 985 // because of a type case clause on line 862 of go/constant/value.go 986 // case intVal: 987 // return itof(x) 988 // v139 is later stored as an intVal == struct{val *big.Int} which naively requires the fields of 989 // of a *uint8, which does not succeed. 990 t = removeTrivialWrapperTypes(t) 991 // it could be a leaf type, but the "leaf" could be complex64 (for example) 992 return x.storeArgOrLoad(pos, b, source, mem, t, storeOffset, loadRegOffset, storeRc) 993 } 994 995 source.Type = t 996 for i := 0; i < t.NumFields(); i++ { 997 fld := t.Field(i) 998 sel := source.Block.NewValue1I(pos, OpStructSelect, fld.Type, int64(i), source) 999 mem = x.storeArgOrLoad(pos, b, sel, mem, fld.Type, storeOffset+fld.Offset, loadRegOffset, storeRc.next(fld.Type)) 1000 loadRegOffset += x.regWidth(fld.Type) 1001 pos = pos.WithNotStmt() 1002 } 1003 return mem 1004 1005 case types.TINT64, types.TUINT64: 1006 if t.Size() == x.regSize { 1007 break 1008 } 1009 tHi, tLo := x.intPairTypes(t.Kind()) 1010 sel := source.Block.NewValue1(pos, OpInt64Hi, tHi, source) 1011 mem = x.storeArgOrLoad(pos, b, sel, mem, tHi, storeOffset+x.hiOffset, loadRegOffset+x.hiRo, storeRc.plus(x.hiRo)) 1012 pos = pos.WithNotStmt() 1013 sel = source.Block.NewValue1(pos, OpInt64Lo, tLo, source) 1014 return x.storeArgOrLoad(pos, b, sel, mem, tLo, storeOffset+x.lowOffset, loadRegOffset+x.loRo, storeRc.plus(x.hiRo)) 1015 1016 case types.TINTER: 1017 sel := source.Block.NewValue1(pos, OpITab, x.typs.BytePtr, source) 1018 mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.BytePtr, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr)) 1019 pos = pos.WithNotStmt() 1020 sel = source.Block.NewValue1(pos, OpIData, x.typs.BytePtr, source) 1021 return x.storeArgOrLoad(pos, b, sel, mem, x.typs.BytePtr, storeOffset+x.ptrSize, loadRegOffset+RO_iface_data, storeRc) 1022 1023 case types.TSTRING: 1024 sel := source.Block.NewValue1(pos, OpStringPtr, x.typs.BytePtr, source) 1025 mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.BytePtr, storeOffset, loadRegOffset, storeRc.next(x.typs.BytePtr)) 1026 pos = pos.WithNotStmt() 1027 sel = source.Block.NewValue1(pos, OpStringLen, x.typs.Int, source) 1028 return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Int, storeOffset+x.ptrSize, loadRegOffset+RO_string_len, storeRc) 1029 1030 case types.TSLICE: 1031 et := types.NewPtr(t.Elem()) 1032 sel := source.Block.NewValue1(pos, OpSlicePtr, et, source) 1033 mem = x.storeArgOrLoad(pos, b, sel, mem, et, storeOffset, loadRegOffset, storeRc.next(et)) 1034 pos = pos.WithNotStmt() 1035 sel = source.Block.NewValue1(pos, OpSliceLen, x.typs.Int, source) 1036 mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.Int, storeOffset+x.ptrSize, loadRegOffset+RO_slice_len, storeRc.next(x.typs.Int)) 1037 sel = source.Block.NewValue1(pos, OpSliceCap, x.typs.Int, source) 1038 return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Int, storeOffset+2*x.ptrSize, loadRegOffset+RO_slice_cap, storeRc) 1039 1040 case types.TCOMPLEX64: 1041 sel := source.Block.NewValue1(pos, OpComplexReal, x.typs.Float32, source) 1042 mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float32, storeOffset, loadRegOffset, storeRc.next(x.typs.Float32)) 1043 pos = pos.WithNotStmt() 1044 sel = source.Block.NewValue1(pos, OpComplexImag, x.typs.Float32, source) 1045 return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float32, storeOffset+4, loadRegOffset+RO_complex_imag, storeRc) 1046 1047 case types.TCOMPLEX128: 1048 sel := source.Block.NewValue1(pos, OpComplexReal, x.typs.Float64, source) 1049 mem = x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float64, storeOffset, loadRegOffset, storeRc.next(x.typs.Float64)) 1050 pos = pos.WithNotStmt() 1051 sel = source.Block.NewValue1(pos, OpComplexImag, x.typs.Float64, source) 1052 return x.storeArgOrLoad(pos, b, sel, mem, x.typs.Float64, storeOffset+8, loadRegOffset+RO_complex_imag, storeRc) 1053 } 1054 1055 s := mem 1056 if source.Op == OpDereference { 1057 source.Op = OpLoad // For purposes of parameter passing expansion, a Dereference is a Load. 1058 } 1059 if storeRc.hasRegs() { 1060 storeRc.addArg(source) 1061 } else { 1062 dst := x.offsetFrom(b, storeRc.storeDest, storeOffset, types.NewPtr(t)) 1063 s = b.NewValue3A(pos, OpStore, types.TypeMem, t, dst, source, mem) 1064 } 1065 if x.debug > 1 { 1066 x.Printf("-->storeArg returns %s, storeRc=%s\n", s.LongString(), storeRc.String()) 1067 } 1068 return s 1069} 1070 1071// rewriteArgs replaces all the call-parameter Args to a call with their register translation (if any). 1072// Preceding parameters (code pointers, closure pointer) are preserved, and the memory input is modified 1073// to account for any parameter stores required. 1074// Any of the old Args that have their use count fall to zero are marked OpInvalid. 1075func (x *expandState) rewriteArgs(v *Value, firstArg int) { 1076 if x.debug > 1 { 1077 x.indent(3) 1078 defer x.indent(-3) 1079 x.Printf("rewriteArgs(%s; %d)\n", v.LongString(), firstArg) 1080 } 1081 // Thread the stores on the memory arg 1082 aux := v.Aux.(*AuxCall) 1083 m0 := v.MemoryArg() 1084 mem := m0 1085 newArgs := []*Value{} 1086 oldArgs := []*Value{} 1087 sp := x.sp 1088 if v.Op == OpTailLECall { 1089 // For tail call, we unwind the frame before the call so we'll use the caller's 1090 // SP. 1091 sp = x.f.Entry.NewValue0(src.NoXPos, OpGetCallerSP, x.typs.Uintptr) 1092 } 1093 for i, a := range v.Args[firstArg : len(v.Args)-1] { // skip leading non-parameter SSA Args and trailing mem SSA Arg. 1094 oldArgs = append(oldArgs, a) 1095 auxI := int64(i) 1096 aRegs := aux.RegsOfArg(auxI) 1097 aType := aux.TypeOfArg(auxI) 1098 if len(aRegs) == 0 && a.Op == OpDereference { 1099 aOffset := aux.OffsetOfArg(auxI) 1100 if a.MemoryArg() != m0 { 1101 x.f.Fatalf("Op...LECall and OpDereference have mismatched mem, %s and %s", v.LongString(), a.LongString()) 1102 } 1103 if v.Op == OpTailLECall { 1104 // It's common for a tail call passing the same arguments (e.g. method wrapper), 1105 // so this would be a self copy. Detect this and optimize it out. 1106 a0 := a.Args[0] 1107 if a0.Op == OpLocalAddr { 1108 n := a0.Aux.(*ir.Name) 1109 if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.ctxt.FixedFrameSize() == aOffset { 1110 continue 1111 } 1112 } 1113 } 1114 // "Dereference" of addressed (probably not-SSA-eligible) value becomes Move 1115 // TODO(register args) this will be more complicated with registers in the picture. 1116 mem = x.rewriteDereference(v.Block, sp, a, mem, aOffset, aux.SizeOfArg(auxI), aType, a.Pos) 1117 } else { 1118 var rc registerCursor 1119 var result *[]*Value 1120 var aOffset int64 1121 if len(aRegs) > 0 { 1122 result = &newArgs 1123 } else { 1124 aOffset = aux.OffsetOfArg(auxI) 1125 } 1126 if v.Op == OpTailLECall && a.Op == OpArg && a.AuxInt == 0 { 1127 // It's common for a tail call passing the same arguments (e.g. method wrapper), 1128 // so this would be a self copy. Detect this and optimize it out. 1129 n := a.Aux.(*ir.Name) 1130 if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.ctxt.FixedFrameSize() == aOffset { 1131 continue 1132 } 1133 } 1134 if x.debug > 1 { 1135 x.Printf("...storeArg %s, %v, %d\n", a.LongString(), aType, aOffset) 1136 } 1137 rc.init(aRegs, aux.abiInfo, result, sp) 1138 mem = x.storeArgOrLoad(a.Pos, v.Block, a, mem, aType, aOffset, 0, rc) 1139 } 1140 } 1141 var preArgStore [2]*Value 1142 preArgs := append(preArgStore[:0], v.Args[0:firstArg]...) 1143 v.resetArgs() 1144 v.AddArgs(preArgs...) 1145 v.AddArgs(newArgs...) 1146 v.AddArg(mem) 1147 for _, a := range oldArgs { 1148 if a.Uses == 0 { 1149 x.invalidateRecursively(a) 1150 } 1151 } 1152 1153 return 1154} 1155 1156func (x *expandState) invalidateRecursively(a *Value) { 1157 var s string 1158 if x.debug > 0 { 1159 plus := " " 1160 if a.Pos.IsStmt() == src.PosIsStmt { 1161 plus = " +" 1162 } 1163 s = a.String() + plus + a.Pos.LineNumber() + " " + a.LongString() 1164 if x.debug > 1 { 1165 x.Printf("...marking %v unused\n", s) 1166 } 1167 } 1168 lost := a.invalidateRecursively() 1169 if x.debug&1 != 0 && lost { // For odd values of x.debug, do this. 1170 x.Printf("Lost statement marker in %s on former %s\n", base.Ctxt.Pkgpath+"."+x.f.Name, s) 1171 } 1172} 1173 1174// expandCalls converts LE (Late Expansion) calls that act like they receive value args into a lower-level form 1175// that is more oriented to a platform's ABI. The SelectN operations that extract results are rewritten into 1176// more appropriate forms, and any StructMake or ArrayMake inputs are decomposed until non-struct values are 1177// reached. On the callee side, OpArg nodes are not decomposed until this phase is run. 1178// TODO results should not be lowered until this phase. 1179func expandCalls(f *Func) { 1180 // Calls that need lowering have some number of inputs, including a memory input, 1181 // and produce a tuple of (value1, value2, ..., mem) where valueK may or may not be SSA-able. 1182 1183 // With the current ABI those inputs need to be converted into stores to memory, 1184 // rethreading the call's memory input to the first, and the new call now receiving the last. 1185 1186 // With the current ABI, the outputs need to be converted to loads, which will all use the call's 1187 // memory output as their input. 1188 sp, _ := f.spSb() 1189 x := &expandState{ 1190 f: f, 1191 abi1: f.ABI1, 1192 debug: f.pass.debug, 1193 canSSAType: f.fe.CanSSA, 1194 regSize: f.Config.RegSize, 1195 sp: sp, 1196 typs: &f.Config.Types, 1197 ptrSize: f.Config.PtrSize, 1198 namedSelects: make(map[*Value][]namedVal), 1199 sdom: f.Sdom(), 1200 commonArgs: make(map[selKey]*Value), 1201 memForCall: make(map[ID]*Value), 1202 transformedSelects: make(map[ID]bool), 1203 } 1204 1205 // For 32-bit, need to deal with decomposition of 64-bit integers, which depends on endianness. 1206 if f.Config.BigEndian { 1207 x.lowOffset, x.hiOffset = 4, 0 1208 x.loRo, x.hiRo = 1, 0 1209 } else { 1210 x.lowOffset, x.hiOffset = 0, 4 1211 x.loRo, x.hiRo = 0, 1 1212 } 1213 1214 if x.debug > 1 { 1215 x.Printf("\nexpandsCalls(%s)\n", f.Name) 1216 } 1217 1218 for i, name := range f.Names { 1219 t := name.Type 1220 if x.isAlreadyExpandedAggregateType(t) { 1221 for j, v := range f.NamedValues[*name] { 1222 if v.Op == OpSelectN || v.Op == OpArg && x.isAlreadyExpandedAggregateType(v.Type) { 1223 ns := x.namedSelects[v] 1224 x.namedSelects[v] = append(ns, namedVal{locIndex: i, valIndex: j}) 1225 } 1226 } 1227 } 1228 } 1229 1230 // TODO if too slow, whole program iteration can be replaced w/ slices of appropriate values, accumulated in first loop here. 1231 1232 // Step 0: rewrite the calls to convert args to calls into stores/register movement. 1233 for _, b := range f.Blocks { 1234 for _, v := range b.Values { 1235 firstArg := 0 1236 switch v.Op { 1237 case OpStaticLECall, OpTailLECall: 1238 case OpInterLECall: 1239 firstArg = 1 1240 case OpClosureLECall: 1241 firstArg = 2 1242 default: 1243 continue 1244 } 1245 x.rewriteArgs(v, firstArg) 1246 } 1247 if isBlockMultiValueExit(b) { 1248 x.indent(3) 1249 // Very similar to code in rewriteArgs, but results instead of args. 1250 v := b.Controls[0] 1251 m0 := v.MemoryArg() 1252 mem := m0 1253 aux := f.OwnAux 1254 allResults := []*Value{} 1255 if x.debug > 1 { 1256 x.Printf("multiValueExit rewriting %s\n", v.LongString()) 1257 } 1258 var oldArgs []*Value 1259 for j, a := range v.Args[:len(v.Args)-1] { 1260 oldArgs = append(oldArgs, a) 1261 i := int64(j) 1262 auxType := aux.TypeOfResult(i) 1263 auxBase := b.NewValue2A(v.Pos, OpLocalAddr, types.NewPtr(auxType), aux.NameOfResult(i), x.sp, mem) 1264 auxOffset := int64(0) 1265 auxSize := aux.SizeOfResult(i) 1266 aRegs := aux.RegsOfResult(int64(j)) 1267 if len(aRegs) == 0 && a.Op == OpDereference { 1268 // Avoid a self-move, and if one is detected try to remove the already-inserted VarDef for the assignment that won't happen. 1269 if dAddr, dMem := a.Args[0], a.Args[1]; dAddr.Op == OpLocalAddr && dAddr.Args[0].Op == OpSP && 1270 dAddr.Args[1] == dMem && dAddr.Aux == aux.NameOfResult(i) { 1271 if dMem.Op == OpVarDef && dMem.Aux == dAddr.Aux { 1272 dMem.copyOf(dMem.MemoryArg()) // elide the VarDef 1273 } 1274 continue 1275 } 1276 mem = x.rewriteDereference(v.Block, auxBase, a, mem, auxOffset, auxSize, auxType, a.Pos) 1277 } else { 1278 if a.Op == OpLoad && a.Args[0].Op == OpLocalAddr { 1279 addr := a.Args[0] // This is a self-move. // TODO(register args) do what here for registers? 1280 if addr.MemoryArg() == a.MemoryArg() && addr.Aux == aux.NameOfResult(i) { 1281 continue 1282 } 1283 } 1284 var rc registerCursor 1285 var result *[]*Value 1286 if len(aRegs) > 0 { 1287 result = &allResults 1288 } 1289 rc.init(aRegs, aux.abiInfo, result, auxBase) 1290 mem = x.storeArgOrLoad(v.Pos, b, a, mem, aux.TypeOfResult(i), auxOffset, 0, rc) 1291 } 1292 } 1293 v.resetArgs() 1294 v.AddArgs(allResults...) 1295 v.AddArg(mem) 1296 v.Type = types.NewResults(append(abi.RegisterTypes(aux.abiInfo.OutParams()), types.TypeMem)) 1297 b.SetControl(v) 1298 for _, a := range oldArgs { 1299 if a.Uses == 0 { 1300 if x.debug > 1 { 1301 x.Printf("...marking %v unused\n", a.LongString()) 1302 } 1303 x.invalidateRecursively(a) 1304 } 1305 } 1306 if x.debug > 1 { 1307 x.Printf("...multiValueExit new result %s\n", v.LongString()) 1308 } 1309 x.indent(-3) 1310 } 1311 } 1312 1313 // Step 1: any stores of aggregates remaining are believed to be sourced from call results or args. 1314 // Decompose those stores into a series of smaller stores, adding selection ops as necessary. 1315 for _, b := range f.Blocks { 1316 for _, v := range b.Values { 1317 if v.Op == OpStore { 1318 t := v.Aux.(*types.Type) 1319 source := v.Args[1] 1320 tSrc := source.Type 1321 iAEATt := x.isAlreadyExpandedAggregateType(t) 1322 1323 if !iAEATt { 1324 // guarding against store immediate struct into interface data field -- store type is *uint8 1325 // TODO can this happen recursively? 1326 iAEATt = x.isAlreadyExpandedAggregateType(tSrc) 1327 if iAEATt { 1328 t = tSrc 1329 } 1330 } 1331 dst, mem := v.Args[0], v.Args[2] 1332 mem = x.storeArgOrLoad(v.Pos, b, source, mem, t, 0, 0, registerCursor{storeDest: dst}) 1333 v.copyOf(mem) 1334 } 1335 } 1336 } 1337 1338 val2Preds := make(map[*Value]int32) // Used to accumulate dependency graph of selection operations for topological ordering. 1339 1340 // Step 2: transform or accumulate selection operations for rewrite in topological order. 1341 // 1342 // Aggregate types that have already (in earlier phases) been transformed must be lowered comprehensively to finish 1343 // the transformation (user-defined structs and arrays, slices, strings, interfaces, complex, 64-bit on 32-bit architectures), 1344 // 1345 // Any select-for-addressing applied to call results can be transformed directly. 1346 for _, b := range f.Blocks { 1347 for _, v := range b.Values { 1348 // Accumulate chains of selectors for processing in topological order 1349 switch v.Op { 1350 case OpStructSelect, OpArraySelect, 1351 OpIData, OpITab, 1352 OpStringPtr, OpStringLen, 1353 OpSlicePtr, OpSliceLen, OpSliceCap, OpSlicePtrUnchecked, 1354 OpComplexReal, OpComplexImag, 1355 OpInt64Hi, OpInt64Lo: 1356 w := v.Args[0] 1357 switch w.Op { 1358 case OpStructSelect, OpArraySelect, OpSelectN, OpArg: 1359 val2Preds[w] += 1 1360 if x.debug > 1 { 1361 x.Printf("v2p[%s] = %d\n", w.LongString(), val2Preds[w]) 1362 } 1363 } 1364 fallthrough 1365 1366 case OpSelectN: 1367 if _, ok := val2Preds[v]; !ok { 1368 val2Preds[v] = 0 1369 if x.debug > 1 { 1370 x.Printf("v2p[%s] = %d\n", v.LongString(), val2Preds[v]) 1371 } 1372 } 1373 1374 case OpArg: 1375 if !x.isAlreadyExpandedAggregateType(v.Type) { 1376 continue 1377 } 1378 if _, ok := val2Preds[v]; !ok { 1379 val2Preds[v] = 0 1380 if x.debug > 1 { 1381 x.Printf("v2p[%s] = %d\n", v.LongString(), val2Preds[v]) 1382 } 1383 } 1384 1385 case OpSelectNAddr: 1386 // Do these directly, there are no chains of selectors. 1387 call := v.Args[0] 1388 which := v.AuxInt 1389 aux := call.Aux.(*AuxCall) 1390 pt := v.Type 1391 off := x.offsetFrom(x.f.Entry, x.sp, aux.OffsetOfResult(which), pt) 1392 v.copyOf(off) 1393 } 1394 } 1395 } 1396 1397 // Step 3: Compute topological order of selectors, 1398 // then process it in reverse to eliminate duplicates, 1399 // then forwards to rewrite selectors. 1400 // 1401 // All chains of selectors end up in same block as the call. 1402 1403 // Compilation must be deterministic, so sort after extracting first zeroes from map. 1404 // Sorting allows dominators-last order within each batch, 1405 // so that the backwards scan for duplicates will most often find copies from dominating blocks (it is best-effort). 1406 var toProcess []*Value 1407 less := func(i, j int) bool { 1408 vi, vj := toProcess[i], toProcess[j] 1409 bi, bj := vi.Block, vj.Block 1410 if bi == bj { 1411 return vi.ID < vj.ID 1412 } 1413 return x.sdom.domorder(bi) > x.sdom.domorder(bj) // reverse the order to put dominators last. 1414 } 1415 1416 // Accumulate order in allOrdered 1417 var allOrdered []*Value 1418 for v, n := range val2Preds { 1419 if n == 0 { 1420 allOrdered = append(allOrdered, v) 1421 } 1422 } 1423 last := 0 // allOrdered[0:last] has been top-sorted and processed 1424 for len(val2Preds) > 0 { 1425 toProcess = allOrdered[last:] 1426 last = len(allOrdered) 1427 sort.SliceStable(toProcess, less) 1428 for _, v := range toProcess { 1429 delete(val2Preds, v) 1430 if v.Op == OpArg { 1431 continue // no Args[0], hence done. 1432 } 1433 w := v.Args[0] 1434 n, ok := val2Preds[w] 1435 if !ok { 1436 continue 1437 } 1438 if n == 1 { 1439 allOrdered = append(allOrdered, w) 1440 delete(val2Preds, w) 1441 continue 1442 } 1443 val2Preds[w] = n - 1 1444 } 1445 } 1446 1447 x.commonSelectors = make(map[selKey]*Value) 1448 // Rewrite duplicate selectors as copies where possible. 1449 for i := len(allOrdered) - 1; i >= 0; i-- { 1450 v := allOrdered[i] 1451 if v.Op == OpArg { 1452 continue 1453 } 1454 w := v.Args[0] 1455 if w.Op == OpCopy { 1456 for w.Op == OpCopy { 1457 w = w.Args[0] 1458 } 1459 v.SetArg(0, w) 1460 } 1461 typ := v.Type 1462 if typ.IsMemory() { 1463 continue // handled elsewhere, not an indexable result 1464 } 1465 size := typ.Size() 1466 offset := int64(0) 1467 switch v.Op { 1468 case OpStructSelect: 1469 if w.Type.Kind() == types.TSTRUCT { 1470 offset = w.Type.FieldOff(int(v.AuxInt)) 1471 } else { // Immediate interface data artifact, offset is zero. 1472 f.Fatalf("Expand calls interface data problem, func %s, v=%s, w=%s\n", f.Name, v.LongString(), w.LongString()) 1473 } 1474 case OpArraySelect: 1475 offset = size * v.AuxInt 1476 case OpSelectN: 1477 offset = v.AuxInt // offset is just a key, really. 1478 case OpInt64Hi: 1479 offset = x.hiOffset 1480 case OpInt64Lo: 1481 offset = x.lowOffset 1482 case OpStringLen, OpSliceLen, OpIData: 1483 offset = x.ptrSize 1484 case OpSliceCap: 1485 offset = 2 * x.ptrSize 1486 case OpComplexImag: 1487 offset = size 1488 } 1489 sk := selKey{from: w, size: size, offsetOrIndex: offset, typ: typ} 1490 dupe := x.commonSelectors[sk] 1491 if dupe == nil { 1492 x.commonSelectors[sk] = v 1493 } else if x.sdom.IsAncestorEq(dupe.Block, v.Block) { 1494 if x.debug > 1 { 1495 x.Printf("Duplicate, make %s copy of %s\n", v, dupe) 1496 } 1497 v.copyOf(dupe) 1498 } else { 1499 // Because values are processed in dominator order, the old common[s] will never dominate after a miss is seen. 1500 // Installing the new value might match some future values. 1501 x.commonSelectors[sk] = v 1502 } 1503 } 1504 1505 // Indices of entries in f.Names that need to be deleted. 1506 var toDelete []namedVal 1507 1508 // Rewrite selectors. 1509 for i, v := range allOrdered { 1510 if x.debug > 1 { 1511 b := v.Block 1512 x.Printf("allOrdered[%d] = b%d, %s, uses=%d\n", i, b.ID, v.LongString(), v.Uses) 1513 } 1514 if v.Uses == 0 { 1515 x.invalidateRecursively(v) 1516 continue 1517 } 1518 if v.Op == OpCopy { 1519 continue 1520 } 1521 locs := x.rewriteSelect(v, v, 0, 0) 1522 // Install new names. 1523 if v.Type.IsMemory() { 1524 continue 1525 } 1526 // Leaf types may have debug locations 1527 if !x.isAlreadyExpandedAggregateType(v.Type) { 1528 for _, l := range locs { 1529 if _, ok := f.NamedValues[*l]; !ok { 1530 f.Names = append(f.Names, l) 1531 } 1532 f.NamedValues[*l] = append(f.NamedValues[*l], v) 1533 } 1534 continue 1535 } 1536 if ns, ok := x.namedSelects[v]; ok { 1537 // Not-leaf types that had debug locations need to lose them. 1538 1539 toDelete = append(toDelete, ns...) 1540 } 1541 } 1542 1543 deleteNamedVals(f, toDelete) 1544 1545 // Step 4: rewrite the calls themselves, correcting the type. 1546 for _, b := range f.Blocks { 1547 for _, v := range b.Values { 1548 switch v.Op { 1549 case OpArg: 1550 x.rewriteArgToMemOrRegs(v) 1551 case OpStaticLECall: 1552 v.Op = OpStaticCall 1553 rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams()) 1554 v.Type = types.NewResults(append(rts, types.TypeMem)) 1555 case OpTailLECall: 1556 v.Op = OpTailCall 1557 rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams()) 1558 v.Type = types.NewResults(append(rts, types.TypeMem)) 1559 case OpClosureLECall: 1560 v.Op = OpClosureCall 1561 rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams()) 1562 v.Type = types.NewResults(append(rts, types.TypeMem)) 1563 case OpInterLECall: 1564 v.Op = OpInterCall 1565 rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams()) 1566 v.Type = types.NewResults(append(rts, types.TypeMem)) 1567 } 1568 } 1569 } 1570 1571 // Step 5: dedup OpArgXXXReg values. Mostly it is already dedup'd by commonArgs, 1572 // but there are cases that we have same OpArgXXXReg values with different types. 1573 // E.g. string is sometimes decomposed as { *int8, int }, sometimes as { unsafe.Pointer, uintptr }. 1574 // (Can we avoid that?) 1575 var IArg, FArg [32]*Value 1576 for _, v := range f.Entry.Values { 1577 switch v.Op { 1578 case OpArgIntReg: 1579 i := v.AuxInt 1580 if w := IArg[i]; w != nil { 1581 if w.Type.Size() != v.Type.Size() { 1582 f.Fatalf("incompatible OpArgIntReg [%d]: %s and %s", i, v.LongString(), w.LongString()) 1583 } 1584 if w.Type.IsUnsafePtr() && !v.Type.IsUnsafePtr() { 1585 // Update unsafe.Pointer type if we know the actual pointer type. 1586 w.Type = v.Type 1587 } 1588 // TODO: don't dedup pointer and scalar? Rewrite to OpConvert? Can it happen? 1589 v.copyOf(w) 1590 } else { 1591 IArg[i] = v 1592 } 1593 case OpArgFloatReg: 1594 i := v.AuxInt 1595 if w := FArg[i]; w != nil { 1596 if w.Type.Size() != v.Type.Size() { 1597 f.Fatalf("incompatible OpArgFloatReg [%d]: %v and %v", i, v, w) 1598 } 1599 v.copyOf(w) 1600 } else { 1601 FArg[i] = v 1602 } 1603 } 1604 } 1605 1606 // Step 6: elide any copies introduced. 1607 // Update named values. 1608 for _, name := range f.Names { 1609 values := f.NamedValues[*name] 1610 for i, v := range values { 1611 if v.Op == OpCopy { 1612 a := v.Args[0] 1613 for a.Op == OpCopy { 1614 a = a.Args[0] 1615 } 1616 values[i] = a 1617 } 1618 } 1619 } 1620 for _, b := range f.Blocks { 1621 for _, v := range b.Values { 1622 for i, a := range v.Args { 1623 if a.Op != OpCopy { 1624 continue 1625 } 1626 aa := copySource(a) 1627 v.SetArg(i, aa) 1628 for a.Uses == 0 { 1629 b := a.Args[0] 1630 x.invalidateRecursively(a) 1631 a = b 1632 } 1633 } 1634 } 1635 } 1636 1637 // Rewriting can attach lines to values that are unlikely to survive code generation, so move them to a use. 1638 for _, b := range f.Blocks { 1639 for _, v := range b.Values { 1640 for _, a := range v.Args { 1641 if a.Pos.IsStmt() != src.PosIsStmt { 1642 continue 1643 } 1644 if a.Type.IsMemory() { 1645 continue 1646 } 1647 if a.Pos.Line() != v.Pos.Line() { 1648 continue 1649 } 1650 if !a.Pos.SameFile(v.Pos) { 1651 continue 1652 } 1653 switch a.Op { 1654 case OpArgIntReg, OpArgFloatReg, OpSelectN: 1655 v.Pos = v.Pos.WithIsStmt() 1656 a.Pos = a.Pos.WithDefaultStmt() 1657 } 1658 } 1659 } 1660 } 1661} 1662 1663// rewriteArgToMemOrRegs converts OpArg v in-place into the register version of v, 1664// if that is appropriate. 1665func (x *expandState) rewriteArgToMemOrRegs(v *Value) *Value { 1666 if x.debug > 1 { 1667 x.indent(3) 1668 defer x.indent(-3) 1669 x.Printf("rewriteArgToMemOrRegs(%s)\n", v.LongString()) 1670 } 1671 pa := x.prAssignForArg(v) 1672 switch len(pa.Registers) { 1673 case 0: 1674 frameOff := v.Aux.(*ir.Name).FrameOffset() 1675 if pa.Offset() != int32(frameOff+x.f.ABISelf.LocalsOffset()) { 1676 panic(fmt.Errorf("Parameter assignment %d and OpArg.Aux frameOffset %d disagree, op=%s", 1677 pa.Offset(), frameOff, v.LongString())) 1678 } 1679 case 1: 1680 t := v.Type 1681 key := selKey{v, 0, t.Size(), t} 1682 w := x.commonArgs[key] 1683 if w != nil && w.Uses != 0 { // do not reuse dead value 1684 v.copyOf(w) 1685 break 1686 } 1687 r := pa.Registers[0] 1688 var i int64 1689 v.Op, i = ArgOpAndRegisterFor(r, x.f.ABISelf) 1690 v.Aux = &AuxNameOffset{v.Aux.(*ir.Name), 0} 1691 v.AuxInt = i 1692 x.commonArgs[key] = v 1693 1694 default: 1695 panic(badVal("Saw unexpanded OpArg", v)) 1696 } 1697 if x.debug > 1 { 1698 x.Printf("-->%s\n", v.LongString()) 1699 } 1700 return v 1701} 1702 1703// newArgToMemOrRegs either rewrites toReplace into an OpArg referencing memory or into an OpArgXXXReg to a register, 1704// or rewrites it into a copy of the appropriate OpArgXXX. The actual OpArgXXX is determined by combining baseArg (an OpArg) 1705// with offset, regOffset, and t to determine which portion of it to reference (either all or a part, in memory or in registers). 1706func (x *expandState) newArgToMemOrRegs(baseArg, toReplace *Value, offset int64, regOffset Abi1RO, t *types.Type, pos src.XPos) *Value { 1707 if x.debug > 1 { 1708 x.indent(3) 1709 defer x.indent(-3) 1710 x.Printf("newArgToMemOrRegs(base=%s; toReplace=%s; t=%s; memOff=%d; regOff=%d)\n", baseArg.String(), toReplace.LongString(), t.String(), offset, regOffset) 1711 } 1712 key := selKey{baseArg, offset, t.Size(), t} 1713 w := x.commonArgs[key] 1714 if w != nil && w.Uses != 0 { // do not reuse dead value 1715 if toReplace != nil { 1716 toReplace.copyOf(w) 1717 if x.debug > 1 { 1718 x.Printf("...replace %s\n", toReplace.LongString()) 1719 } 1720 } 1721 if x.debug > 1 { 1722 x.Printf("-->%s\n", w.LongString()) 1723 } 1724 return w 1725 } 1726 1727 pa := x.prAssignForArg(baseArg) 1728 if len(pa.Registers) == 0 { // Arg is on stack 1729 frameOff := baseArg.Aux.(*ir.Name).FrameOffset() 1730 if pa.Offset() != int32(frameOff+x.f.ABISelf.LocalsOffset()) { 1731 panic(fmt.Errorf("Parameter assignment %d and OpArg.Aux frameOffset %d disagree, op=%s", 1732 pa.Offset(), frameOff, baseArg.LongString())) 1733 } 1734 aux := baseArg.Aux 1735 auxInt := baseArg.AuxInt + offset 1736 if toReplace != nil && toReplace.Block == baseArg.Block { 1737 toReplace.reset(OpArg) 1738 toReplace.Aux = aux 1739 toReplace.AuxInt = auxInt 1740 toReplace.Type = t 1741 w = toReplace 1742 } else { 1743 w = baseArg.Block.NewValue0IA(pos, OpArg, t, auxInt, aux) 1744 } 1745 x.commonArgs[key] = w 1746 if toReplace != nil { 1747 toReplace.copyOf(w) 1748 } 1749 if x.debug > 1 { 1750 x.Printf("-->%s\n", w.LongString()) 1751 } 1752 return w 1753 } 1754 // Arg is in registers 1755 r := pa.Registers[regOffset] 1756 op, auxInt := ArgOpAndRegisterFor(r, x.f.ABISelf) 1757 if op == OpArgIntReg && t.IsFloat() || op == OpArgFloatReg && t.IsInteger() { 1758 fmt.Printf("pa=%v\nx.f.OwnAux.abiInfo=%s\n", 1759 pa.ToString(x.f.ABISelf, true), 1760 x.f.OwnAux.abiInfo.String()) 1761 panic(fmt.Errorf("Op/Type mismatch, op=%s, type=%s", op.String(), t.String())) 1762 } 1763 if baseArg.AuxInt != 0 { 1764 base.Fatalf("BaseArg %s bound to registers has non-zero AuxInt", baseArg.LongString()) 1765 } 1766 aux := &AuxNameOffset{baseArg.Aux.(*ir.Name), offset} 1767 if toReplace != nil && toReplace.Block == baseArg.Block { 1768 toReplace.reset(op) 1769 toReplace.Aux = aux 1770 toReplace.AuxInt = auxInt 1771 toReplace.Type = t 1772 w = toReplace 1773 } else { 1774 w = baseArg.Block.NewValue0IA(pos, op, t, auxInt, aux) 1775 } 1776 x.commonArgs[key] = w 1777 if toReplace != nil { 1778 toReplace.copyOf(w) 1779 } 1780 if x.debug > 1 { 1781 x.Printf("-->%s\n", w.LongString()) 1782 } 1783 return w 1784 1785} 1786 1787// argOpAndRegisterFor converts an abi register index into an ssa Op and corresponding 1788// arg register index. 1789func ArgOpAndRegisterFor(r abi.RegIndex, abiConfig *abi.ABIConfig) (Op, int64) { 1790 i := abiConfig.FloatIndexFor(r) 1791 if i >= 0 { // float PR 1792 return OpArgFloatReg, i 1793 } 1794 return OpArgIntReg, int64(r) 1795} 1796