1 /* $Id: sparc-recode.c,v 1.6 2010/06/05 18:42:35 fredette Exp $ */
2
3 /* ic/sparc/sparc-recode.c - recodes SPARC instructions: */
4
5 /*
6 * Copyright (c) 2008 Matt Fredette
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by Matt Fredette.
20 * 4. The name of the author may not be used to endorse or promote products
21 * derived from this software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
27 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
32 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 * POSSIBILITY OF SUCH DAMAGE.
34 */
35
36 /* includes: */
37 #include "sparc-impl.h"
38
39 _TME_RCSID("$Id: sparc-recode.c,v 1.6 2010/06/05 18:42:35 fredette Exp $");
40
41 #if TME_HAVE_RECODE
42
43 /* macros: */
44
45 /* this gives the number of recode page bitmap bits needed to cover
46 the given number of bytes. this is always a multiple of eight: */
47 #define TME_SPARC_RECODE_PAGE_BITMAP_BITS(ic, size_bytes) \
48 ((((size_bytes) >> (ic)->tme_sparc_tlb_page_size_log2) \
49 + 7) \
50 & (0 - (unsigned long) 8))
51
52 /* this returns a read/write thunk index for an opcode: */
53 #define TME_SPARC_RECODE_RW_THUNK_INDEX(ic, address_size, endian, no_fault, opcode_0_3) \
54 ((opcode_0_3) \
55 + ((TME_SPARC_VERSION(ic) >= 9) \
56 * ((16 * ((no_fault) != 0)) \
57 + (32 * ((endian) == TME_ENDIAN_LITTLE)) \
58 + (64 * (TME_SPARC_RECODE_SIZE(ic) - (address_size))))))
59
60 /* how many source address hash positions are probed: */
61 #define TME_SPARC_RECODE_SRC_HASH_SIZE_PROBE (TME_SPARC_RECODE_SRC_HASH_SIZE_SET * 2)
62
63 /* the undefined source address hash key: */
64 #define TME_SPARC_RECODE_SRC_KEY_UNDEF ((tme_sparc_recode_src_key_t) (0 - (tme_sparc_recode_src_key_t) 1))
65
66 /* flags in a source address hash key: */
67 /* NB: an architecture can't have more than two flags, since those are
68 the only bits available in a source address hash key (because PCs
69 are 32-bit aligned): */
70 #define TME_SPARC64_RECODE_SRC_KEY_FLAG_AM (1 << 0)
71 #define TME_SPARC64_RECODE_SRC_KEY_FLAG_CLE (1 << 1)
72
73 /* the hit count threshold for recoding a source address: */
74 #define TME_SPARC_RECODE_HIT_COUNT_THRESHOLD (512)
75
76 /* this returns the recode size for the architecture size: */
77 #define TME_SPARC_RECODE_SIZE(ic) (TME_RECODE_SIZE_32 + (TME_SPARC_VERSION(ic) >= 9))
78
79 /* fix certain things based on the architecture size: */
80 #undef TME_SPARC_VERSION
81 #define TME_SPARC_VERSION(ic) (8 + (_TME_SPARC_RECODE_SIZE(/**/,/**/) == 64))
82 #define tme_sparc_ireg_t _TME_SPARC_RECODE_SIZE(tme_uint,_t)
83 #define tme_sparc_ireg _TME_SPARC_RECODE_SIZE(tme_sparc_ireg_uint,/**/)
84 #define TME_SPARC_IREG_MSBIT (((tme_sparc_ireg_t) 1) << (sizeof(tme_sparc_ireg_t) * 8 - 1))
85
86 /* the current verify state: */
87 #ifdef _TME_SPARC_RECODE_VERIFY
88 static int _tme_sparc_recode_off;
89 static int _tme_sparc_recode_verify_on;
90 #else /* !_TME_SPARC_RECODE_VERIFY */
91 #define _tme_sparc_recode_off (FALSE)
92 #define _tme_sparc_recode_verify_on (FALSE)
93 #endif /* !_TME_SPARC_RECODE_VERIFY */
94
95 /* make the sparc32 recode support: */
96 #define _TME_SPARC_RECODE_SIZE(x,y) _TME_CONCAT(_TME_CONCAT(x,32),y)
97 #include "sparc-rc-cc.c"
98 #include "sparc-rc-chain.c"
99 #include "sparc-rc-insns.c"
100 #include "sparc-rc-ls.c"
101 #undef _TME_SPARC_RECODE_SIZE
102
103 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
104
105 /* make the sparc64 recode support: */
106 #define _TME_SPARC_RECODE_SIZE(x,y) _TME_CONCAT(_TME_CONCAT(x,64),y)
107 #include "sparc-rc-cc.c"
108 #include "sparc-rc-chain.c"
109 #include "sparc-rc-insns.c"
110 #include "sparc-rc-ls.c"
111 #undef _TME_SPARC_RECODE_SIZE
112
113 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
114
115 /* unfix certain things: */
116 #undef TME_SPARC_VERSION
117 #define TME_SPARC_VERSION(ic) _TME_SPARC_VERSION(ic)
118 #undef tme_sparc_ireg_t
119 #undef tme_sparc_ireg
120 #undef TME_SPARC_IREG_MSBIT
121
122 /* this invalidates pages in the source address hash: */
123 static void
_tme_sparc_recode_src_hash_invalidate(struct tme_sparc * ic,tme_sparc_recode_src_key_t src_key_mask,tme_sparc_recode_src_key_t src_key_match)124 _tme_sparc_recode_src_hash_invalidate(struct tme_sparc *ic,
125 tme_sparc_recode_src_key_t src_key_mask,
126 tme_sparc_recode_src_key_t src_key_match)
127 {
128 signed long src_hash_i;
129 tme_sparc_recode_src_key_t src_key;
130 tme_recode_thunk_off_t insns_thunk;
131
132 /* walk all of the positions in the source address hash: */
133 src_hash_i
134 = ((TME_SPARC_RECODE_SRC_HASH_MODULUS
135 * TME_SPARC_RECODE_SRC_HASH_SIZE_SET)
136 - 1);
137 do {
138
139 /* if the source address key in this position matches: */
140 src_key
141 = (ic->tme_sparc_recode_src_hash
142 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
143 .tme_sparc_recode_src_hash_keys
144 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
145 if (((src_key ^ src_key_match) & src_key_mask) == 0) {
146
147 /* invalidate this source address key: */
148 (ic->tme_sparc_recode_src_hash
149 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
150 .tme_sparc_recode_src_hash_keys
151 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
152 = TME_SPARC_RECODE_SRC_KEY_UNDEF;
153
154 /* if this source address key has an instructions thunk: */
155 insns_thunk
156 = (ic->tme_sparc_recode_src_hash
157 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
158 .tme_sparc_recode_src_hash_values
159 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
160 if ((insns_thunk & TME_BIT(0)) == 0
161 && insns_thunk != 0) {
162
163 /* invalidate the instructions thunk: */
164 tme_recode_insns_thunk_invalidate(ic->tme_sparc_recode_ic,
165 insns_thunk);
166 }
167
168 /* reset the value for this position to a hit count of zero: */
169 (ic->tme_sparc_recode_src_hash
170 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
171 .tme_sparc_recode_src_hash_values
172 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
173 = TME_BIT(0);
174 }
175 } while (--src_hash_i >= 0);
176 }
177
178 /* this invalidates our entire source hash and all instructions
179 thunks: */
180 void
tme_sparc_recode_invalidate_all(struct tme_sparc * ic)181 tme_sparc_recode_invalidate_all(struct tme_sparc *ic)
182 {
183
184 /* invalidate our entire source hash: */
185 _tme_sparc_recode_src_hash_invalidate(ic, 0, 0);
186
187 /* invalidate all instructions thunks: */
188 tme_recode_insns_thunk_invalidate_all(ic->tme_sparc_recode_ic);
189
190 /* clear the return address stack: */
191 tme_recode_chain_ras_clear(ic->tme_sparc_recode_ic,
192 &ic->tme_sparc_ic);
193 }
194
195 /* include the verify support: */
196 #ifdef _TME_SPARC_RECODE_VERIFY
197 #include "sparc-rc-verify.c"
198 #endif /* _TME_SPARC_RECODE_VERIFY */
199
200 /* this adds a new cacheable: */
201 static struct tme_sparc_recode_cacheable *
_tme_sparc_recode_cacheable_new(struct tme_sparc * ic,const struct tme_sparc_tlb * itlb,const tme_shared tme_uint32_t * src)202 _tme_sparc_recode_cacheable_new(struct tme_sparc *ic,
203 const struct tme_sparc_tlb *itlb,
204 const tme_shared tme_uint32_t *src)
205 {
206 unsigned int page_size_log2;
207 tme_uint32_t page_size;
208 tme_uint32_t page_byte_offset;
209 struct tme_sparc_recode_cacheable *cacheable;
210 const struct tme_bus_cacheable *bus_cacheable;
211 unsigned long bitmap_bytes_old;
212 unsigned long bitmap_bytes;
213
214 /* get the page size: */
215 page_size_log2 = ic->tme_sparc_tlb_page_size_log2;
216 page_size = (((tme_uint32_t) 1) << page_size_log2);
217
218 /* get the offset of the current PC into this page: */
219 page_byte_offset
220 = (
221 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
222 TME_SPARC_VERSION(ic) >= 9
223 ? ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_PC)
224 :
225 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
226 ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_PC));
227 page_byte_offset %= page_size;
228
229 /* if we have run out of cacheables, return failure: */
230 cacheable = ic->tme_sparc_recode_cacheable_first;
231 if (cacheable == &ic->tme_sparc_recode_cacheables[TME_SPARC_RECODE_CACHEABLES_MAX - 1]) {
232 return (NULL);
233 }
234 cacheable += (ic->tme_sparc_recode_cacheables[0].tme_sparc_recode_cacheable_size != 0);
235
236 /* if this instruction TLB entry isn't for cacheable memory, return
237 failure: */
238 bus_cacheable = itlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cacheable;
239 if (bus_cacheable == NULL) {
240 return (NULL);
241 }
242
243 /* if the cacheable memory doesn't cover some whole number of
244 aligned pages, return failure: */
245 assert (bus_cacheable->tme_bus_cacheable_size > 0);
246 if ((((((tme_uint8_t *) src)
247 - bus_cacheable->tme_bus_cacheable_contents)
248 % page_size)
249 != page_byte_offset)
250 || ((bus_cacheable->tme_bus_cacheable_size
251 % page_size)
252 != 0)) {
253 return (NULL);
254 }
255
256 /* initialize our cacheable: */
257 /* NB: we round up the first source address key for this cacheable
258 so that its page index is a multiple of eight, which will be used
259 to index the least-significant bit in the first byte of the
260 valids bitmap we allocate below: */
261 cacheable->tme_sparc_recode_cacheable_contents = bus_cacheable->tme_bus_cacheable_contents;
262 cacheable->tme_sparc_recode_cacheable_size = bus_cacheable->tme_bus_cacheable_size;
263 cacheable->tme_sparc_recode_cacheable_src_key_first
264 = (cacheable == &ic->tme_sparc_recode_cacheables[0]
265 ? 0
266 : ((cacheable - 1)->tme_sparc_recode_cacheable_src_key_first
267
268 + (TME_SPARC_RECODE_PAGE_BITMAP_BITS(ic, (cacheable - 1)->tme_sparc_recode_cacheable_size)
269 * page_size)));
270
271 /* allocate the valids bitmap, and adjust the pointer to account
272 for the first source key in this cacheable: */
273 cacheable->tme_sparc_recode_cacheable_valids
274 = ((*bus_cacheable->tme_bus_cacheable_valids_new)
275 (bus_cacheable->tme_bus_cacheable_private,
276 page_size_log2)
277 - ((cacheable->tme_sparc_recode_cacheable_src_key_first
278 / page_size)
279 / 8));
280
281 /* allocate or reallocate the active recode pages bitmap, and zero
282 the new parts of the bitmap: */
283 bitmap_bytes_old
284 = (cacheable->tme_sparc_recode_cacheable_src_key_first
285 / (page_size * 8));
286 bitmap_bytes
287 = (bitmap_bytes_old
288 + (TME_SPARC_RECODE_PAGE_BITMAP_BITS(ic, cacheable->tme_sparc_recode_cacheable_size)
289 / 8));
290 ic->tme_sparc_recode_cacheable_actives
291 = (cacheable == &ic->tme_sparc_recode_cacheables[0]
292 ? tme_new(tme_uint8_t, bitmap_bytes)
293 : tme_renew(tme_uint8_t, ic->tme_sparc_recode_cacheable_actives, bitmap_bytes));
294 memset((ic->tme_sparc_recode_cacheable_actives + bitmap_bytes_old),
295 0,
296 (bitmap_bytes - bitmap_bytes_old));
297
298 /* finish allocating this cacheable: */
299 ic->tme_sparc_recode_cacheable_first = cacheable;
300
301 /* return success: */
302 return (cacheable);
303 }
304
305 tme_recode_thunk_off_t
tme_sparc_recode(struct tme_sparc * ic,const struct tme_sparc_tlb * itlb,const tme_shared tme_uint32_t * src)306 tme_sparc_recode(struct tme_sparc *ic,
307 const struct tme_sparc_tlb *itlb,
308 const tme_shared tme_uint32_t *src)
309 {
310
311 const struct tme_sparc_recode_cacheable *cacheable;
312 tme_sparc_recode_src_key_t src_key;
313 tme_uint32_t pstate;
314 unsigned long page_index;
315 tme_uint8_t page_invalid;
316 const tme_shared tme_uint8_t *cacheable_valid_byte;
317 unsigned int cacheable_valid_mask;
318 signed long src_hash_i;
319 signed int src_hash_probe;
320 tme_sparc_recode_src_key_t src_key_other;
321 tme_recode_thunk_off_t insns_thunk;
322 tme_recode_thunk_off_t hit_count;
323
324 TME_SPARC_STAT(ic, tme_sparc_stats_recode_calls);
325
326 /* if recoding is off: */
327 if (_tme_sparc_recode_off) {
328
329 /* we can't recode this source address: */
330 return (0);
331 }
332
333 /* if this instruction TLB entry doesn't extend to the end of the
334 page: */
335 /* XXX FIXME - this isn't right, since it will reject all itlb
336 entries that don't end right at a recode page boundary, instead
337 of only those that end before the end of the page containing the
338 current PC. maybe this check should move to sparc-rc-insns.c or
339 sparc-execute.c, where we have the current PC? */
340 if (__tme_predict_false((~itlb->tme_sparc_tlb_addr_last)
341 & ((1 << ic->tme_sparc_tlb_page_size_log2)
342 - sizeof(tme_uint32_t)))) {
343
344 /* we can't recode this source address: */
345 return (0);
346 }
347
348 /* assume that we will recode at this source address, and save it: */
349 ic->tme_sparc_recode_insns_group.tme_recode_insns_group_src
350 = (const tme_shared tme_uint8_t *) src;
351
352 /* search for a cacheable that contains this source address: */
353 cacheable = ic->tme_sparc_recode_cacheable_first;
354 for (;;) {
355
356 /* assume that this cacheable contains this source address, and get the
357 source address key relative to the start of the cacheable: */
358 src_key = ((tme_uint8_t *) src) - cacheable->tme_sparc_recode_cacheable_contents;
359
360 /* if this cacheable contains this source address: */
361 if (__tme_predict_true(src_key
362 < cacheable->tme_sparc_recode_cacheable_size)) {
363
364 /* stop now: */
365 break;
366 }
367
368 /* if we have not run out of cacheables: */
369 if (__tme_predict_true(--cacheable != (&ic->tme_sparc_recode_cacheables[0] - 1))) {
370 continue;
371 }
372
373 /* if this instruction TLB entry isn't for a cacheable, or if we
374 can't add this cacheable: */
375 cacheable = _tme_sparc_recode_cacheable_new(ic, itlb, src);
376 if (cacheable == NULL) {
377
378 /* we can't recode this source address: */
379 return (0);
380 }
381 }
382
383 /* make the absolute source address key by adding in the first
384 source address key in this cacheable: */
385 src_key += cacheable->tme_sparc_recode_cacheable_src_key_first;
386 assert ((src_key % sizeof(tme_uint32_t)) == 0);
387
388 /* if this is a v9 CPU: */
389 if (TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
390 && TME_SPARC_VERSION(ic) >= 9) {
391
392 /* add PSTATE.AM and PSTATE.CLE to the source address key: */
393 pstate = ic->tme_sparc64_ireg_pstate;
394 if (pstate & TME_SPARC64_PSTATE_AM) {
395 src_key += TME_SPARC64_RECODE_SRC_KEY_FLAG_AM;
396 }
397 if (pstate & TME_SPARC64_PSTATE_CLE) {
398 src_key += TME_SPARC64_RECODE_SRC_KEY_FLAG_CLE;
399 }
400 }
401
402 /* the source address key must be defined: */
403 assert (src_key != TME_SPARC_RECODE_SRC_KEY_UNDEF);
404
405 /* make the index for the page containing this source address: */
406 page_index = src_key >> ic->tme_sparc_tlb_page_size_log2;
407
408 /* assume that this recode page is still valid: */
409 page_invalid = FALSE;
410
411 /* get the cacheable valid byte and mask for this page: */
412 cacheable_valid_byte = &cacheable->tme_sparc_recode_cacheable_valids[page_index / 8];
413 cacheable_valid_mask = TME_BIT(page_index % 8);
414
415 /* assume that we will recode at this source address, and save the
416 cacheable valid byte and mask for this page: */
417 ic->tme_sparc_recode_insns_group.tme_recode_insns_group_valid_byte = cacheable_valid_byte;
418 ic->tme_sparc_recode_insns_group.tme_recode_insns_group_valid_mask = cacheable_valid_mask;
419
420 /* if the valid bit for this page is no longer set: */
421 if (__tme_predict_false(((*cacheable_valid_byte) & cacheable_valid_mask) == 0)) {
422
423 /* this page is invalid: */
424 page_invalid = TRUE;
425
426 /* if this page was active: */
427 if (__tme_predict_false(ic->tme_sparc_recode_cacheable_actives[page_index / 8]
428 & TME_BIT(page_index % 8))) {
429
430 TME_SPARC_STAT(ic, tme_sparc_stats_recode_page_invalids);
431
432 /* mark this page as inactive: */
433 ic->tme_sparc_recode_cacheable_actives[page_index / 8] &= ~TME_BIT(page_index % 8);
434
435 /* re-set the valid bit for the page containing this
436 source address: */
437 /* NB: we cheat and pass our
438 page-index-base-greater-than-zero-relative valids bitmap
439 pointer instead of the original pointer returned by the
440 allocator. this only works if the re-set function will work
441 given arbitrary valids bitmap pointers and page indices: */
442 (*itlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cacheable->tme_bus_cacheable_valids_set)
443 (itlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cacheable->tme_bus_cacheable_private,
444 cacheable->tme_sparc_recode_cacheable_valids,
445 page_index);
446
447 /* invalidate this page in the source address hash: */
448 _tme_sparc_recode_src_hash_invalidate(ic,
449 (0
450 - ((tme_sparc_recode_src_key_t)
451 (1 << ic->tme_sparc_tlb_page_size_log2))),
452 src_key);
453 }
454 }
455
456 /* hash the source address key: */
457 src_hash_i
458 = ((((src_key
459 / sizeof(tme_uint32_t))
460 % TME_SPARC_RECODE_SRC_HASH_MODULUS)
461 * TME_SPARC_RECODE_SRC_HASH_SIZE_SET)
462 + (TME_SPARC_RECODE_SRC_HASH_SIZE_SET
463 - 1));
464
465 /* search for the source address key in the hash: */
466 src_hash_probe = TME_SPARC_RECODE_SRC_HASH_SIZE_PROBE;
467 for (;;) {
468
469 TME_SPARC_STAT(ic, tme_sparc_stats_recode_source_hash_probes);
470
471 /* get the source address key at this position: */
472 src_key_other
473 = (ic->tme_sparc_recode_src_hash
474 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
475 .tme_sparc_recode_src_hash_keys
476 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
477
478 /* if we found the source address key, stop now: */
479 if (__tme_predict_true(src_key_other == src_key)) {
480 break;
481 }
482
483 /* if this position in the hash is free: */
484 if (src_key_other == TME_SPARC_RECODE_SRC_KEY_UNDEF) {
485
486 /* allocate this position in the hash: */
487 (ic->tme_sparc_recode_src_hash
488 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
489 .tme_sparc_recode_src_hash_keys
490 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
491 = src_key;
492
493 /* stop now: */
494 break;
495 }
496
497 /* if we have searched enough: */
498 if (__tme_predict_false(--src_hash_probe < 0)) {
499
500 TME_SPARC_STAT(ic, tme_sparc_stats_recode_source_hash_misses);
501
502 /* we won't recode this source address: */
503 return (0);
504 }
505
506 /* move to the next position to search: */
507 if (__tme_predict_false(--src_hash_i < 0)) {
508 src_hash_i
509 = ((TME_SPARC_RECODE_SRC_HASH_MODULUS
510 * TME_SPARC_RECODE_SRC_HASH_SIZE_SET)
511 - 1);
512 }
513 }
514
515 /* assume that this source address has been recoded, and get the
516 instructions thunk: */
517 insns_thunk
518 = (ic->tme_sparc_recode_src_hash
519 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
520 .tme_sparc_recode_src_hash_values
521 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
522
523 /* if the source address has not been recoded yet: */
524 if (__tme_predict_false(insns_thunk & TME_BIT(0))) {
525
526 /* update the hit count for this source address: */
527 hit_count = insns_thunk + 2;
528
529 /* if the hit count for this source address is still less than the
530 recode threshold: */
531 if (__tme_predict_true(hit_count
532 < ((TME_SPARC_RECODE_HIT_COUNT_THRESHOLD * 2)
533 + 1))) {
534
535 /* update the hit count for this source address: */
536 (ic->tme_sparc_recode_src_hash
537 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
538 .tme_sparc_recode_src_hash_values
539 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
540 = hit_count;
541
542 /* we won't recode this source address now: */
543 return (0);
544 }
545
546 /* mark this page as active: */
547 ic->tme_sparc_recode_cacheable_actives[page_index / 8] |= TME_BIT(page_index % 8);
548
549 /* if this page is invalid: */
550 if (page_invalid) {
551
552 /* return now and interpret instructions normally - the next
553 time we encounter this page again, we will mark this
554 page as inactive, re-set the page's valid bit and
555 invalidate the page in the source address hash.
556
557 this somewhat limits the work we do for pages that are
558 both executed from and written to frequently. we will only
559 recode an instructions thunk when its page was valid for the
560 entire time it took for its hit count to go from zero to the
561 threshold, and we will only invalidate an invalid page in the
562 source address hash when any hit count in the page reaches
563 the threshold: */
564 return (0);
565 }
566
567 /* recode starting from this source address: */
568 insns_thunk
569 = (
570 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
571 TME_SPARC_VERSION(ic) >= 9
572 ? _tme_sparc64_recode_recode(ic, itlb)
573 :
574 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
575 _tme_sparc32_recode_recode(ic, itlb));
576
577 /* if we couldn't recode at this source address: */
578 if (__tme_predict_false(insns_thunk < 0)) {
579
580 /* the only time we can't recode is when the host runs out of
581 memory for instructions thunks. when this happens, the host
582 invalidates all previous instructions thunks, so we have to
583 invalidate our entire source hash: */
584 tme_sparc_recode_invalidate_all(ic);
585 return (0);
586 }
587
588 /* set the instructions thunk for this source address: */
589 assert ((insns_thunk & TME_BIT(0)) == 0);
590 (ic->tme_sparc_recode_src_hash
591 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
592 .tme_sparc_recode_src_hash_values
593 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
594 = insns_thunk;
595
596 /* we can't run the new instructions thunk without re-checking the
597 valid bit for the page containing this source address
598 first - if it wasn't set for the entire time we were recoding,
599 the instructions thunk might not be consistent with the source
600 memory.
601
602 to keep things simple, we just return now and interpret
603 instructions normally - the next time we get to this source
604 address, we'll check the valid bit and either run the
605 instructions thunk for the first time, or we'll do the
606 invalidate: */
607 return (0);
608 }
609
610 /* return the instructions thunk offset: */
611 return (insns_thunk);
612 }
613
614 #ifdef TME_RECODE_DEBUG
615 #include <stdio.h>
616
617 void tme_sparc_recode_dump_ic _TME_P((const struct tme_sparc *));
618 void
tme_sparc_recode_dump_ic(const struct tme_sparc * ic)619 tme_sparc_recode_dump_ic(const struct tme_sparc *ic)
620 {
621 const char *conds_integer[] = {
622 "n", "e", "le", "l", "leu", "cs", "neg", "vs",
623 NULL
624 };
625
626 printf("sparc%u add flags thunk %p:\n",
627 (1 << TME_SPARC_RECODE_SIZE(ic)),
628 ic->tme_sparc_recode_flags_thunk_add);
629 tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
630 ic->tme_sparc_recode_flags_thunk_add);
631 printf("sparc%u sub flags thunk %p:\n",
632 (1 << TME_SPARC_RECODE_SIZE(ic)),
633 ic->tme_sparc_recode_flags_thunk_sub);
634 tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
635 ic->tme_sparc_recode_flags_thunk_sub);
636 printf("sparc%u logical flags thunk %p:\n",
637 (1 << TME_SPARC_RECODE_SIZE(ic)),
638 ic->tme_sparc_recode_flags_thunk_logical);
639 tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
640 ic->tme_sparc_recode_flags_thunk_logical);
641
642 if (TME_SPARC_VERSION(ic) >= 9) {
643 printf("sparc%u rcc flags thunk %p:\n",
644 (1 << TME_SPARC_RECODE_SIZE(ic)),
645 ic->tme_sparc_recode_flags_thunk_rcc);
646 tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
647 ic->tme_sparc_recode_flags_thunk_rcc);
648 }
649 printf("sparc%u %%icc conds thunk %p:\n",
650 (1 << TME_SPARC_RECODE_SIZE(ic)),
651 ic->tme_sparc_recode_conds_thunk_icc);
652 tme_recode_conds_thunk_dump(ic->tme_sparc_recode_ic,
653 ic->tme_sparc_recode_conds_thunk_icc,
654 conds_integer);
655 if (TME_SPARC_VERSION(ic) >= 9) {
656 printf("sparc%u %%xcc conds thunk %p:\n",
657 (1 << TME_SPARC_RECODE_SIZE(ic)),
658 ic->tme_sparc_recode_conds_thunk_xcc);
659 tme_recode_conds_thunk_dump(ic->tme_sparc_recode_ic,
660 ic->tme_sparc_recode_conds_thunk_xcc,
661 conds_integer);
662 printf("sparc%u %%rcc conds thunk %p:\n",
663 (1 << TME_SPARC_RECODE_SIZE(ic)),
664 ic->tme_sparc_recode_conds_thunk_rcc);
665 tme_recode_conds_thunk_dump(ic->tme_sparc_recode_ic,
666 ic->tme_sparc_recode_conds_thunk_rcc,
667 conds_integer);
668 }
669 }
670
671 void tme_sparc_recode_dump_insns _TME_P((const struct tme_sparc *));
672 void
tme_sparc_recode_dump_insns(const struct tme_sparc * ic)673 tme_sparc_recode_dump_insns(const struct tme_sparc *ic)
674 {
675 const struct tme_recode_insn *insns;
676 const struct tme_recode_insn *insns_end;
677 unsigned int insn_i;
678 const char *s;
679 unsigned int insn_size;
680 unsigned int operands_mask;
681 unsigned int thunk_i;
682 unsigned int operand_i;
683 int delim;
684 const char *regcs = "goli";
685 const char *conds_integer[] = {
686 "n", "e", "le", "l", "leu", "cs", "neg", "vs",
687 "a", "ne", "g", "ge", "gu", "cc", "pos", "vc",
688 };
689 tme_uint32_t chain_info;
690
691 printf("%%pc = ");
692 tme_recode_uguest_dump(
693 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
694 TME_SPARC_VERSION(ic) >= 9
695 ? ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_PC)
696 :
697 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
698 ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_PC));
699 printf("\n");
700
701 insns = ic->tme_sparc_recode_insns_group.tme_recode_insns_group_insns;
702 insns_end = ic->tme_sparc_recode_insns_group.tme_recode_insns_group_insns_end;
703 for (insn_i = 0; insns < insns_end; insns++, insn_i++) {
704
705 printf("%2u: ", insn_i);
706
707 s = tme_recode_opcode_dump(insns->tme_recode_insn_opcode);
708 insn_size = insns->tme_recode_insn_size;
709 switch (insns->tme_recode_insn_opcode) {
710 case TME_RECODE_OPCODE_EXTZ:
711 case TME_RECODE_OPCODE_EXTS:
712 printf("ext%u%c",
713 (1 << insns->tme_recode_insn_operand_src[1]),
714 (insns->tme_recode_insn_opcode == TME_RECODE_OPCODE_EXTZ ? 'z' : 's'));
715 s = "";
716 operands_mask = (1 << 0) | (1 << 2);
717 break;
718 case TME_RECODE_OPCODE_DEFC:
719 printf("defc\t %%%ccc:%s, %%c%u\n",
720 (insns->tme_recode_insn_conds_thunk == ic->tme_sparc_recode_conds_thunk_icc
721 ? 'i'
722 : insns->tme_recode_insn_conds_thunk == ic->tme_sparc_recode_conds_thunk_xcc
723 ? 'x'
724 : 'r'),
725 conds_integer
726 [insns->tme_recode_insn_operand_src[0]
727 + (TME_SPARC_COND_NOT
728 * (insns->tme_recode_insn_operand_src[1] != 0))],
729 insns->tme_recode_insn_operand_dst);
730 continue;
731 case TME_RECODE_OPCODE_IF:
732 printf("%s\t %%c%u\n", s, insns->tme_recode_insn_operand_src[0]);
733 continue;
734 case TME_RECODE_OPCODE_ELSE:
735 case TME_RECODE_OPCODE_ENDIF:
736 printf("%s\n", s);
737 continue;
738 case TME_RECODE_OPCODE_RW:
739 for (thunk_i = 0;; thunk_i++) {
740 assert (thunk_i < TME_ARRAY_ELS(ic->tme_sparc_recode_rw_thunks));
741 if (insns->tme_recode_insn_rw_thunk == ic->tme_sparc_recode_rw_thunks[thunk_i]) {
742 break;
743 }
744 }
745 s = "?rw?";
746 switch (thunk_i % 0x10) {
747 case 0x0: s = (TME_SPARC_VERSION(ic) >= 9 ? "lduw" : "ld"); break;
748 case 0x1: s = "ldub"; break;
749 case 0x2: s = "lduh"; break;
750 case 0x4: s = (TME_SPARC_VERSION(ic) >= 9 ? "stw" : "st"); break;
751 case 0x5: s = "stb"; break;
752 case 0x6: s = "sth"; break;
753 case 0x8: if (TME_SPARC_VERSION(ic) >= 9) { s = "ldsw"; } break;
754 case 0x9: s = "ldsb"; break;
755 case 0xa: s = "ldsh"; break;
756 case 0xb: if (TME_SPARC_VERSION(ic) >= 9) { s = "ldx"; } break;
757 case 0xe: if (TME_SPARC_VERSION(ic) >= 9) { s = "stx"; } break;
758 }
759 if (thunk_i & 16) {
760 printf("%s", s);
761 s = ".nf";
762 }
763 if (thunk_i & 32) {
764 printf("%s", s);
765 s = ".el";
766 }
767 if (thunk_i & 64) {
768 printf("%s", s);
769 s = ".am";
770 }
771 insn_size = 0;
772 operands_mask = (1 << 0);
773 if (insns->tme_recode_insn_rw_thunk->tme_recode_rw_thunk_write) {
774 operands_mask |= (1 << 1);
775 }
776 else {
777 operands_mask |= (1 << 2);
778 }
779 break;
780 default:
781 operands_mask = (1 << 0) | (1 << 1) | (1 << 2);
782 break;
783 }
784
785 printf("%s", s);
786 if (insn_size > 0) {
787 printf("%u", (1 << insns->tme_recode_insn_size));
788 }
789 delim = '\t';
790 for (operand_i = 0; operand_i <= 2; operand_i++) {
791 if (operands_mask & (1 << operand_i)) {
792 putchar(delim);
793 putchar(' ');
794 delim = ',';
795 switch (insns->tme_recode_insn_operands[operand_i]) {
796 case TME_RECODE_OPERAND_UNDEF:
797 s = "undef";
798 break;
799 case TME_RECODE_OPERAND_IMM:
800 tme_recode_uguest_dump(insns->tme_recode_insn_imm_uguest);
801 continue;
802 #if TME_RECODE_OPERAND_NULL != TME_RECODE_OPERAND_ZERO
803 #error "TME_RECODE_OPERAND_ values changed"
804 #endif
805 case TME_RECODE_OPERAND_ZERO:
806 s = "g0";
807 if (operand_i == 2
808 && insns->tme_recode_insn_opcode == TME_RECODE_OPCODE_GUEST) {
809 s = "null";
810 }
811 break;
812 case TME_SPARC_IREG_PC: s = "pc"; break;
813 case TME_SPARC_IREG_PC_NEXT: s = "pc_next"; break;
814 case TME_SPARC_IREG_PC_NEXT_NEXT: s = "pc_next_next"; break;
815 case TME_SPARC_IREG_INSN: s = "insn"; break;
816 case TME_SPARC_IREG_TMP(0): s = "tmp0"; break;
817 case TME_SPARC_IREG_TMP(1): s = "tmp1"; break;
818 case TME_SPARC_IREG_TMP(2): s = "tmp2"; break;
819 default:
820 if (insns->tme_recode_insn_operands[operand_i] < 32) {
821 printf("%%%c%u",
822 regcs[insns->tme_recode_insn_operands[operand_i] / 8],
823 (insns->tme_recode_insn_operands[operand_i] % 8));
824 continue;
825 }
826 s = "??";
827 break;
828 }
829 printf("%%%s", s);
830 }
831 }
832
833 if (insns->tme_recode_insn_opcode < TME_RECODE_OPCODES_INTEGER
834 && insns->tme_recode_insn_flags_thunk != NULL) {
835 printf(", %%%s",
836 (TME_SPARC_VERSION(ic) < 9
837 ? "icc"
838 : insns->tme_recode_insn_flags_thunk == ic->tme_sparc_recode_flags_thunk_rcc
839 ? "rcc"
840 : "ccr"));
841 }
842
843 putchar('\n');
844 }
845
846 chain_info = ic->tme_sparc_recode_insns_group.tme_recode_insns_group_chain_info;
847 printf("chain %s %s ",
848 ((chain_info & TME_RECODE_CHAIN_INFO_JUMP)
849 ? "jump"
850 : (chain_info & TME_RECODE_CHAIN_INFO_RETURN)
851 ? "return"
852 : "call"),
853 (((chain_info
854 & (TME_RECODE_CHAIN_INFO_NEAR
855 | TME_RECODE_CHAIN_INFO_FAR))
856 == TME_RECODE_CHAIN_INFO_NEAR)
857 ? "near"
858 : "far"));
859 if ((chain_info
860 & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
861 | TME_RECODE_CHAIN_INFO_CONDITIONAL))
862 == TME_RECODE_CHAIN_INFO_UNCONDITIONAL) {
863 printf("unconditional\n");
864 }
865 else {
866 printf("conditional alternate %s\n",
867 (((chain_info
868 & (TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR
869 | TME_RECODE_CHAIN_INFO_ALTERNATE_FAR))
870 == TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR)
871 ? "near"
872 : "far"));
873 }
874 }
875
876 #endif /* TME_RECODE_DEBUG */
877
878 #endif /* TME_HAVE_RECODE */
879
880 /* this initializes sparc recoding: */
881 void
tme_sparc_recode_init(struct tme_sparc * ic)882 tme_sparc_recode_init(struct tme_sparc *ic)
883 {
884 #if TME_HAVE_RECODE
885 unsigned long reg_guest_count;
886 unsigned long reg_guest;
887 struct tme_recode_ic *recode_ic;
888
889 /* get the number of guest registers: */
890 reg_guest_count = TME_SPARC_IREG_TMP(3);
891
892 /* allocate the recode ic: */
893 recode_ic
894 = tme_malloc0(sizeof(struct tme_recode_ic)
895 + (sizeof(union tme_recode_reginfo)
896 * reg_guest_count));
897 ic->tme_sparc_recode_ic = recode_ic;
898
899 /* set the register size: */
900 recode_ic->tme_recode_ic_reg_size = TME_SPARC_RECODE_SIZE(ic);
901
902 /* set the register count: */
903 recode_ic->tme_recode_ic_reg_count = reg_guest_count;
904
905 /* allocate some number of read-uses records: */
906 /* XXX FIXME - this can be anything, and doesn't affect correctness,
907 just performance. should it be tunable? */
908 recode_ic->tme_recode_ic_reg_guest_ruses_record_count = 512;
909 recode_ic->tme_recode_ic_reg_guest_ruses_records
910 = tme_new(tme_uint16_t,
911 recode_ic->tme_recode_ic_reg_guest_ruses_record_count + 1);
912 recode_ic->tme_recode_ic_reg_guest_ruses_records
913 [recode_ic->tme_recode_ic_reg_guest_ruses_record_count]
914 = TME_RECODE_REG_RUSES_RECORD_UNDEF;
915
916 /* assume that all registers use flat addressing: */
917 for (reg_guest = 0;
918 reg_guest < reg_guest_count;
919 reg_guest++) {
920 (recode_ic->tme_recode_ic_reginfo
921 + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_FLAT;
922 }
923
924 /* %g0 is fixed: */
925 (recode_ic->tme_recode_ic_reginfo
926 + TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0))->tme_recode_reginfo_all
927 |= TME_RECODE_REGINFO_TYPE_FIXED;
928
929 /* %r8 through %r23 are addressed through window zero, and %r24
930 through %r31 are addressed through window one: */
931 reg_guest = TME_RECODE_REG_GUEST(8);
932 do {
933 (recode_ic->tme_recode_ic_reginfo
934 + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_WINDOW(0);
935 } while (++reg_guest <= TME_RECODE_REG_GUEST(23));
936 do {
937 (recode_ic->tme_recode_ic_reginfo
938 + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_WINDOW(1);
939 } while (++reg_guest <= TME_RECODE_REG_GUEST(31));
940
941 /* on a v9 CPU, %r1 through %r7 are addressed through window two: */
942 if (TME_SPARC_VERSION(ic) >= 9) {
943 reg_guest = TME_RECODE_REG_GUEST(1);
944 do {
945 (recode_ic->tme_recode_ic_reginfo
946 + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_WINDOW(2);
947 } while (++reg_guest <= TME_RECODE_REG_GUEST(7));
948 }
949
950 /* the temporary registers are temporary: */
951 reg_guest = TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(0));
952 do {
953 (recode_ic->tme_recode_ic_reginfo
954 + reg_guest)->tme_recode_reginfo_all
955 |= TME_RECODE_REGINFO_TYPE_TEMPORARY;
956 } while (++reg_guest <= TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(2)));
957
958 /* set the window base offsets: */
959 recode_ic->tme_recode_ic_window_base_offsets[0]
960 = (((char *) &ic->tme_sparc_recode_window_base_offsets[0]) - ((char *) ic));
961 recode_ic->tme_recode_ic_window_base_offsets[1]
962 = (((char *) &ic->tme_sparc_recode_window_base_offsets[1]) - ((char *) ic));
963 recode_ic->tme_recode_ic_window_base_offsets[2]
964 = (((char *) &ic->tme_sparc_recode_window_base_offsets[2]) - ((char *) ic));
965
966 /* initialize the cacheables: */
967 ic->tme_sparc_recode_cacheables[0].tme_sparc_recode_cacheable_size = 0;
968 ic->tme_sparc_recode_cacheable_first = &ic->tme_sparc_recode_cacheables[0];
969
970 /* invalidate the entire source hash: */
971 _tme_sparc_recode_src_hash_invalidate(ic, 0, 0);
972
973 /* set the TLB page size: */
974 recode_ic->tme_recode_ic_tlb_page_size
975 = (((tme_uint32_t) 1) << ic->tme_sparc_tlb_page_size_log2);
976
977 /* set the pointer to the first instruction in a group: */
978 ic->tme_sparc_recode_insns_group.tme_recode_insns_group_insns
979 = &ic->tme_sparc_recode_insns[0];
980
981 /* initialize the recode ic: */
982 /* XXX FIXME - this run size should be passed in as an argument. it
983 needs to be big enough for the maximum number of non-variable
984 thunks (flags, conds, chain, etc.) that we will make, probably
985 TME_RECODE_HOST_THUNK_SIZE_MAX times some n? */
986 tme_recode_ic_new(recode_ic, 4 * 1024 * 1024);
987
988 /* call the architecture-specific init functions: */
989 if (TME_SPARC_VERSION(ic) >= 9) {
990 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
991 _tme_sparc64_recode_cc_init(ic);
992 _tme_sparc64_recode_chain_init(ic);
993 _tme_sparc64_recode_ls_init(ic);
994 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
995 }
996 else {
997 _tme_sparc32_recode_cc_init(ic);
998 _tme_sparc32_recode_chain_init(ic);
999 _tme_sparc32_recode_ls_init(ic);
1000 }
1001
1002 #ifdef _TME_SPARC_RECODE_VERIFY
1003
1004 /* initialize the verifier: */
1005 _tme_sparc_recode_verify_init(ic);
1006
1007 #endif /* _TME_SPARC_RECODE_VERIFY */
1008
1009 #else /* !TME_HAVE_RECODE */
1010 /* unused: */
1011 ic = 0;
1012 #endif /* !TME_HAVE_RECODE */
1013 }
1014