1 /* $Id: sparc-recode.c,v 1.6 2010/06/05 18:42:35 fredette Exp $ */
2 
3 /* ic/sparc/sparc-recode.c - recodes SPARC instructions: */
4 
5 /*
6  * Copyright (c) 2008 Matt Fredette
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  * 3. All advertising materials mentioning features or use of this software
18  *    must display the following acknowledgement:
19  *      This product includes software developed by Matt Fredette.
20  * 4. The name of the author may not be used to endorse or promote products
21  *    derived from this software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
27  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
32  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33  * POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 /* includes: */
37 #include "sparc-impl.h"
38 
39 _TME_RCSID("$Id: sparc-recode.c,v 1.6 2010/06/05 18:42:35 fredette Exp $");
40 
41 #if TME_HAVE_RECODE
42 
43 /* macros: */
44 
45 /* this gives the number of recode page bitmap bits needed to cover
46    the given number of bytes.  this is always a multiple of eight: */
47 #define TME_SPARC_RECODE_PAGE_BITMAP_BITS(ic, size_bytes)	\
48   ((((size_bytes) >> (ic)->tme_sparc_tlb_page_size_log2)	\
49     + 7)							\
50    & (0 - (unsigned long) 8))
51 
52 /* this returns a read/write thunk index for an opcode: */
53 #define TME_SPARC_RECODE_RW_THUNK_INDEX(ic, address_size, endian, no_fault, opcode_0_3) \
54   ((opcode_0_3)							\
55    + ((TME_SPARC_VERSION(ic) >= 9)				\
56       * ((16 * ((no_fault) != 0))				\
57 	 + (32 * ((endian) == TME_ENDIAN_LITTLE))		\
58 	 + (64 * (TME_SPARC_RECODE_SIZE(ic) - (address_size))))))
59 
60 /* how many source address hash positions are probed: */
61 #define TME_SPARC_RECODE_SRC_HASH_SIZE_PROBE	(TME_SPARC_RECODE_SRC_HASH_SIZE_SET * 2)
62 
63 /* the undefined source address hash key: */
64 #define TME_SPARC_RECODE_SRC_KEY_UNDEF		((tme_sparc_recode_src_key_t) (0 - (tme_sparc_recode_src_key_t) 1))
65 
66 /* flags in a source address hash key: */
67 /* NB: an architecture can't have more than two flags, since those are
68    the only bits available in a source address hash key (because PCs
69    are 32-bit aligned): */
70 #define TME_SPARC64_RECODE_SRC_KEY_FLAG_AM	(1 << 0)
71 #define TME_SPARC64_RECODE_SRC_KEY_FLAG_CLE	(1 << 1)
72 
73 /* the hit count threshold for recoding a source address: */
74 #define TME_SPARC_RECODE_HIT_COUNT_THRESHOLD	(512)
75 
76 /* this returns the recode size for the architecture size: */
77 #define TME_SPARC_RECODE_SIZE(ic) (TME_RECODE_SIZE_32 + (TME_SPARC_VERSION(ic) >= 9))
78 
79 /* fix certain things based on the architecture size: */
80 #undef TME_SPARC_VERSION
81 #define TME_SPARC_VERSION(ic) (8 + (_TME_SPARC_RECODE_SIZE(/**/,/**/) == 64))
82 #define tme_sparc_ireg_t _TME_SPARC_RECODE_SIZE(tme_uint,_t)
83 #define tme_sparc_ireg _TME_SPARC_RECODE_SIZE(tme_sparc_ireg_uint,/**/)
84 #define TME_SPARC_IREG_MSBIT (((tme_sparc_ireg_t) 1) << (sizeof(tme_sparc_ireg_t) * 8 - 1))
85 
86 /* the current verify state: */
87 #ifdef _TME_SPARC_RECODE_VERIFY
88 static int _tme_sparc_recode_off;
89 static int _tme_sparc_recode_verify_on;
90 #else  /* !_TME_SPARC_RECODE_VERIFY */
91 #define _tme_sparc_recode_off (FALSE)
92 #define _tme_sparc_recode_verify_on (FALSE)
93 #endif /* !_TME_SPARC_RECODE_VERIFY */
94 
95 /* make the sparc32 recode support: */
96 #define _TME_SPARC_RECODE_SIZE(x,y) _TME_CONCAT(_TME_CONCAT(x,32),y)
97 #include "sparc-rc-cc.c"
98 #include "sparc-rc-chain.c"
99 #include "sparc-rc-insns.c"
100 #include "sparc-rc-ls.c"
101 #undef _TME_SPARC_RECODE_SIZE
102 
103 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
104 
105 /* make the sparc64 recode support: */
106 #define _TME_SPARC_RECODE_SIZE(x,y) _TME_CONCAT(_TME_CONCAT(x,64),y)
107 #include "sparc-rc-cc.c"
108 #include "sparc-rc-chain.c"
109 #include "sparc-rc-insns.c"
110 #include "sparc-rc-ls.c"
111 #undef _TME_SPARC_RECODE_SIZE
112 
113 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
114 
115 /* unfix certain things: */
116 #undef  TME_SPARC_VERSION
117 #define TME_SPARC_VERSION(ic) _TME_SPARC_VERSION(ic)
118 #undef tme_sparc_ireg_t
119 #undef tme_sparc_ireg
120 #undef TME_SPARC_IREG_MSBIT
121 
122 /* this invalidates pages in the source address hash: */
123 static void
_tme_sparc_recode_src_hash_invalidate(struct tme_sparc * ic,tme_sparc_recode_src_key_t src_key_mask,tme_sparc_recode_src_key_t src_key_match)124 _tme_sparc_recode_src_hash_invalidate(struct tme_sparc *ic,
125 				      tme_sparc_recode_src_key_t src_key_mask,
126 				      tme_sparc_recode_src_key_t src_key_match)
127 {
128   signed long src_hash_i;
129   tme_sparc_recode_src_key_t src_key;
130   tme_recode_thunk_off_t insns_thunk;
131 
132   /* walk all of the positions in the source address hash: */
133   src_hash_i
134     = ((TME_SPARC_RECODE_SRC_HASH_MODULUS
135 	* TME_SPARC_RECODE_SRC_HASH_SIZE_SET)
136        - 1);
137   do {
138 
139     /* if the source address key in this position matches: */
140     src_key
141       = (ic->tme_sparc_recode_src_hash
142 	 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
143 	 .tme_sparc_recode_src_hash_keys
144 	 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
145     if (((src_key ^ src_key_match) & src_key_mask) == 0) {
146 
147       /* invalidate this source address key: */
148       (ic->tme_sparc_recode_src_hash
149        [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
150        .tme_sparc_recode_src_hash_keys
151        [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
152 	= TME_SPARC_RECODE_SRC_KEY_UNDEF;
153 
154       /* if this source address key has an instructions thunk: */
155       insns_thunk
156 	= (ic->tme_sparc_recode_src_hash
157 	   [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
158 	   .tme_sparc_recode_src_hash_values
159 	   [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
160       if ((insns_thunk & TME_BIT(0)) == 0
161 	  && insns_thunk != 0) {
162 
163 	/* invalidate the instructions thunk: */
164 	tme_recode_insns_thunk_invalidate(ic->tme_sparc_recode_ic,
165 					  insns_thunk);
166       }
167 
168       /* reset the value for this position to a hit count of zero: */
169       (ic->tme_sparc_recode_src_hash
170        [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
171        .tme_sparc_recode_src_hash_values
172        [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
173 	= TME_BIT(0);
174     }
175   } while (--src_hash_i >= 0);
176 }
177 
178 /* this invalidates our entire source hash and all instructions
179    thunks: */
180 void
tme_sparc_recode_invalidate_all(struct tme_sparc * ic)181 tme_sparc_recode_invalidate_all(struct tme_sparc *ic)
182 {
183 
184   /* invalidate our entire source hash: */
185   _tme_sparc_recode_src_hash_invalidate(ic, 0, 0);
186 
187   /* invalidate all instructions thunks: */
188   tme_recode_insns_thunk_invalidate_all(ic->tme_sparc_recode_ic);
189 
190   /* clear the return address stack: */
191   tme_recode_chain_ras_clear(ic->tme_sparc_recode_ic,
192 			     &ic->tme_sparc_ic);
193 }
194 
195 /* include the verify support: */
196 #ifdef _TME_SPARC_RECODE_VERIFY
197 #include "sparc-rc-verify.c"
198 #endif /* _TME_SPARC_RECODE_VERIFY */
199 
200 /* this adds a new cacheable: */
201 static struct tme_sparc_recode_cacheable *
_tme_sparc_recode_cacheable_new(struct tme_sparc * ic,const struct tme_sparc_tlb * itlb,const tme_shared tme_uint32_t * src)202 _tme_sparc_recode_cacheable_new(struct tme_sparc *ic,
203 				const struct tme_sparc_tlb *itlb,
204 				const tme_shared tme_uint32_t *src)
205 {
206   unsigned int page_size_log2;
207   tme_uint32_t page_size;
208   tme_uint32_t page_byte_offset;
209   struct tme_sparc_recode_cacheable *cacheable;
210   const struct tme_bus_cacheable *bus_cacheable;
211   unsigned long bitmap_bytes_old;
212   unsigned long bitmap_bytes;
213 
214   /* get the page size: */
215   page_size_log2 = ic->tme_sparc_tlb_page_size_log2;
216   page_size = (((tme_uint32_t) 1) << page_size_log2);
217 
218   /* get the offset of the current PC into this page: */
219   page_byte_offset
220     = (
221 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
222        TME_SPARC_VERSION(ic) >= 9
223        ? ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_PC)
224        :
225 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
226        ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_PC));
227   page_byte_offset %= page_size;
228 
229   /* if we have run out of cacheables, return failure: */
230   cacheable = ic->tme_sparc_recode_cacheable_first;
231   if (cacheable == &ic->tme_sparc_recode_cacheables[TME_SPARC_RECODE_CACHEABLES_MAX - 1]) {
232     return (NULL);
233   }
234   cacheable += (ic->tme_sparc_recode_cacheables[0].tme_sparc_recode_cacheable_size != 0);
235 
236   /* if this instruction TLB entry isn't for cacheable memory, return
237      failure: */
238   bus_cacheable = itlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cacheable;
239   if (bus_cacheable == NULL) {
240     return (NULL);
241   }
242 
243   /* if the cacheable memory doesn't cover some whole number of
244      aligned pages, return failure: */
245   assert (bus_cacheable->tme_bus_cacheable_size > 0);
246   if ((((((tme_uint8_t *) src)
247 	 - bus_cacheable->tme_bus_cacheable_contents)
248 	% page_size)
249        != page_byte_offset)
250       || ((bus_cacheable->tme_bus_cacheable_size
251 	   % page_size)
252 	  != 0)) {
253     return (NULL);
254   }
255 
256   /* initialize our cacheable: */
257   /* NB: we round up the first source address key for this cacheable
258      so that its page index is a multiple of eight, which will be used
259      to index the least-significant bit in the first byte of the
260      valids bitmap we allocate below: */
261   cacheable->tme_sparc_recode_cacheable_contents = bus_cacheable->tme_bus_cacheable_contents;
262   cacheable->tme_sparc_recode_cacheable_size = bus_cacheable->tme_bus_cacheable_size;
263   cacheable->tme_sparc_recode_cacheable_src_key_first
264     = (cacheable == &ic->tme_sparc_recode_cacheables[0]
265        ? 0
266        : ((cacheable - 1)->tme_sparc_recode_cacheable_src_key_first
267 
268 	  + (TME_SPARC_RECODE_PAGE_BITMAP_BITS(ic, (cacheable - 1)->tme_sparc_recode_cacheable_size)
269 	     * page_size)));
270 
271   /* allocate the valids bitmap, and adjust the pointer to account
272      for the first source key in this cacheable: */
273   cacheable->tme_sparc_recode_cacheable_valids
274     = ((*bus_cacheable->tme_bus_cacheable_valids_new)
275        (bus_cacheable->tme_bus_cacheable_private,
276 	page_size_log2)
277        - ((cacheable->tme_sparc_recode_cacheable_src_key_first
278 	   / page_size)
279 	  / 8));
280 
281   /* allocate or reallocate the active recode pages bitmap, and zero
282      the new parts of the bitmap: */
283   bitmap_bytes_old
284     = (cacheable->tme_sparc_recode_cacheable_src_key_first
285        / (page_size * 8));
286   bitmap_bytes
287     = (bitmap_bytes_old
288        + (TME_SPARC_RECODE_PAGE_BITMAP_BITS(ic, cacheable->tme_sparc_recode_cacheable_size)
289 	  / 8));
290   ic->tme_sparc_recode_cacheable_actives
291     = (cacheable == &ic->tme_sparc_recode_cacheables[0]
292        ? tme_new(tme_uint8_t, bitmap_bytes)
293        : tme_renew(tme_uint8_t, ic->tme_sparc_recode_cacheable_actives, bitmap_bytes));
294   memset((ic->tme_sparc_recode_cacheable_actives + bitmap_bytes_old),
295 	 0,
296 	 (bitmap_bytes - bitmap_bytes_old));
297 
298   /* finish allocating this cacheable: */
299   ic->tme_sparc_recode_cacheable_first = cacheable;
300 
301   /* return success: */
302   return (cacheable);
303 }
304 
305 tme_recode_thunk_off_t
tme_sparc_recode(struct tme_sparc * ic,const struct tme_sparc_tlb * itlb,const tme_shared tme_uint32_t * src)306 tme_sparc_recode(struct tme_sparc *ic,
307 		 const struct tme_sparc_tlb *itlb,
308 		 const tme_shared tme_uint32_t *src)
309 {
310 
311   const struct tme_sparc_recode_cacheable *cacheable;
312   tme_sparc_recode_src_key_t src_key;
313   tme_uint32_t pstate;
314   unsigned long page_index;
315   tme_uint8_t page_invalid;
316   const tme_shared tme_uint8_t *cacheable_valid_byte;
317   unsigned int cacheable_valid_mask;
318   signed long src_hash_i;
319   signed int src_hash_probe;
320   tme_sparc_recode_src_key_t src_key_other;
321   tme_recode_thunk_off_t insns_thunk;
322   tme_recode_thunk_off_t hit_count;
323 
324   TME_SPARC_STAT(ic, tme_sparc_stats_recode_calls);
325 
326   /* if recoding is off: */
327   if (_tme_sparc_recode_off) {
328 
329     /* we can't recode this source address: */
330     return (0);
331   }
332 
333   /* if this instruction TLB entry doesn't extend to the end of the
334      page: */
335   /* XXX FIXME - this isn't right, since it will reject all itlb
336      entries that don't end right at a recode page boundary, instead
337      of only those that end before the end of the page containing the
338      current PC.  maybe this check should move to sparc-rc-insns.c or
339      sparc-execute.c, where we have the current PC? */
340   if (__tme_predict_false((~itlb->tme_sparc_tlb_addr_last)
341 			  & ((1 << ic->tme_sparc_tlb_page_size_log2)
342 			     - sizeof(tme_uint32_t)))) {
343 
344     /* we can't recode this source address: */
345     return (0);
346   }
347 
348   /* assume that we will recode at this source address, and save it: */
349   ic->tme_sparc_recode_insns_group.tme_recode_insns_group_src
350     = (const tme_shared tme_uint8_t *) src;
351 
352   /* search for a cacheable that contains this source address: */
353   cacheable = ic->tme_sparc_recode_cacheable_first;
354   for (;;) {
355 
356     /* assume that this cacheable contains this source address, and get the
357        source address key relative to the start of the cacheable: */
358     src_key = ((tme_uint8_t *) src) - cacheable->tme_sparc_recode_cacheable_contents;
359 
360     /* if this cacheable contains this source address: */
361     if (__tme_predict_true(src_key
362 			   < cacheable->tme_sparc_recode_cacheable_size)) {
363 
364       /* stop now: */
365       break;
366     }
367 
368     /* if we have not run out of cacheables: */
369     if (__tme_predict_true(--cacheable != (&ic->tme_sparc_recode_cacheables[0] - 1))) {
370       continue;
371     }
372 
373     /* if this instruction TLB entry isn't for a cacheable, or if we
374        can't add this cacheable: */
375     cacheable = _tme_sparc_recode_cacheable_new(ic, itlb, src);
376     if (cacheable == NULL) {
377 
378       /* we can't recode this source address: */
379       return (0);
380     }
381   }
382 
383   /* make the absolute source address key by adding in the first
384      source address key in this cacheable: */
385   src_key += cacheable->tme_sparc_recode_cacheable_src_key_first;
386   assert ((src_key % sizeof(tme_uint32_t)) == 0);
387 
388   /* if this is a v9 CPU: */
389   if (TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
390       && TME_SPARC_VERSION(ic) >= 9) {
391 
392     /* add PSTATE.AM and PSTATE.CLE to the source address key: */
393     pstate = ic->tme_sparc64_ireg_pstate;
394     if (pstate & TME_SPARC64_PSTATE_AM) {
395       src_key += TME_SPARC64_RECODE_SRC_KEY_FLAG_AM;
396     }
397     if (pstate & TME_SPARC64_PSTATE_CLE) {
398       src_key += TME_SPARC64_RECODE_SRC_KEY_FLAG_CLE;
399     }
400   }
401 
402   /* the source address key must be defined: */
403   assert (src_key != TME_SPARC_RECODE_SRC_KEY_UNDEF);
404 
405   /* make the index for the page containing this source address: */
406   page_index = src_key >> ic->tme_sparc_tlb_page_size_log2;
407 
408   /* assume that this recode page is still valid: */
409   page_invalid = FALSE;
410 
411   /* get the cacheable valid byte and mask for this page: */
412   cacheable_valid_byte = &cacheable->tme_sparc_recode_cacheable_valids[page_index / 8];
413   cacheable_valid_mask = TME_BIT(page_index % 8);
414 
415   /* assume that we will recode at this source address, and save the
416      cacheable valid byte and mask for this page: */
417   ic->tme_sparc_recode_insns_group.tme_recode_insns_group_valid_byte = cacheable_valid_byte;
418   ic->tme_sparc_recode_insns_group.tme_recode_insns_group_valid_mask = cacheable_valid_mask;
419 
420   /* if the valid bit for this page is no longer set: */
421   if (__tme_predict_false(((*cacheable_valid_byte) & cacheable_valid_mask) == 0)) {
422 
423     /* this page is invalid: */
424     page_invalid = TRUE;
425 
426     /* if this page was active: */
427     if (__tme_predict_false(ic->tme_sparc_recode_cacheable_actives[page_index / 8]
428 			    & TME_BIT(page_index % 8))) {
429 
430       TME_SPARC_STAT(ic, tme_sparc_stats_recode_page_invalids);
431 
432       /* mark this page as inactive: */
433       ic->tme_sparc_recode_cacheable_actives[page_index / 8] &= ~TME_BIT(page_index % 8);
434 
435       /* re-set the valid bit for the page containing this
436 	 source address: */
437       /* NB: we cheat and pass our
438 	 page-index-base-greater-than-zero-relative valids bitmap
439 	 pointer instead of the original pointer returned by the
440 	 allocator.  this only works if the re-set function will work
441 	 given arbitrary valids bitmap pointers and page indices: */
442       (*itlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cacheable->tme_bus_cacheable_valids_set)
443 	(itlb->tme_sparc_tlb_bus_tlb.tme_bus_tlb_cacheable->tme_bus_cacheable_private,
444 	 cacheable->tme_sparc_recode_cacheable_valids,
445 	 page_index);
446 
447       /* invalidate this page in the source address hash: */
448       _tme_sparc_recode_src_hash_invalidate(ic,
449 					    (0
450 					     - ((tme_sparc_recode_src_key_t)
451 						(1 << ic->tme_sparc_tlb_page_size_log2))),
452 					    src_key);
453     }
454   }
455 
456   /* hash the source address key: */
457   src_hash_i
458     = ((((src_key
459 	  / sizeof(tme_uint32_t))
460 	 % TME_SPARC_RECODE_SRC_HASH_MODULUS)
461 	* TME_SPARC_RECODE_SRC_HASH_SIZE_SET)
462        + (TME_SPARC_RECODE_SRC_HASH_SIZE_SET
463 	  - 1));
464 
465   /* search for the source address key in the hash: */
466   src_hash_probe = TME_SPARC_RECODE_SRC_HASH_SIZE_PROBE;
467   for (;;) {
468 
469     TME_SPARC_STAT(ic, tme_sparc_stats_recode_source_hash_probes);
470 
471     /* get the source address key at this position: */
472     src_key_other
473       = (ic->tme_sparc_recode_src_hash
474 	 [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
475 	 .tme_sparc_recode_src_hash_keys
476 	 [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
477 
478     /* if we found the source address key, stop now: */
479     if (__tme_predict_true(src_key_other == src_key)) {
480       break;
481     }
482 
483     /* if this position in the hash is free: */
484     if (src_key_other == TME_SPARC_RECODE_SRC_KEY_UNDEF) {
485 
486       /* allocate this position in the hash: */
487       (ic->tme_sparc_recode_src_hash
488        [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
489        .tme_sparc_recode_src_hash_keys
490        [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
491 	= src_key;
492 
493       /* stop now: */
494       break;
495     }
496 
497     /* if we have searched enough: */
498     if (__tme_predict_false(--src_hash_probe < 0)) {
499 
500       TME_SPARC_STAT(ic, tme_sparc_stats_recode_source_hash_misses);
501 
502       /* we won't recode this source address: */
503       return (0);
504     }
505 
506     /* move to the next position to search: */
507     if (__tme_predict_false(--src_hash_i < 0)) {
508       src_hash_i
509 	= ((TME_SPARC_RECODE_SRC_HASH_MODULUS
510 	    * TME_SPARC_RECODE_SRC_HASH_SIZE_SET)
511 	   - 1);
512     }
513   }
514 
515   /* assume that this source address has been recoded, and get the
516      instructions thunk: */
517   insns_thunk
518     = (ic->tme_sparc_recode_src_hash
519        [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
520        .tme_sparc_recode_src_hash_values
521        [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]);
522 
523   /* if the source address has not been recoded yet: */
524   if (__tme_predict_false(insns_thunk & TME_BIT(0))) {
525 
526     /* update the hit count for this source address: */
527     hit_count = insns_thunk + 2;
528 
529     /* if the hit count for this source address is still less than the
530        recode threshold: */
531     if (__tme_predict_true(hit_count
532 			   < ((TME_SPARC_RECODE_HIT_COUNT_THRESHOLD * 2)
533 			      + 1))) {
534 
535       /* update the hit count for this source address: */
536       (ic->tme_sparc_recode_src_hash
537        [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
538        .tme_sparc_recode_src_hash_values
539        [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
540 	= hit_count;
541 
542       /* we won't recode this source address now: */
543       return (0);
544     }
545 
546     /* mark this page as active: */
547     ic->tme_sparc_recode_cacheable_actives[page_index / 8] |= TME_BIT(page_index % 8);
548 
549     /* if this page is invalid: */
550     if (page_invalid) {
551 
552       /* return now and interpret instructions normally - the next
553 	 time we encounter this page again, we will mark this
554 	 page as inactive, re-set the page's valid bit and
555 	 invalidate the page in the source address hash.
556 
557 	 this somewhat limits the work we do for pages that are
558 	 both executed from and written to frequently.  we will only
559 	 recode an instructions thunk when its page was valid for the
560 	 entire time it took for its hit count to go from zero to the
561 	 threshold, and we will only invalidate an invalid page in the
562 	 source address hash when any hit count in the page reaches
563 	 the threshold: */
564       return (0);
565     }
566 
567     /* recode starting from this source address: */
568     insns_thunk
569       = (
570 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
571 	 TME_SPARC_VERSION(ic) >= 9
572 	 ? _tme_sparc64_recode_recode(ic, itlb)
573 	 :
574 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
575 	 _tme_sparc32_recode_recode(ic, itlb));
576 
577     /* if we couldn't recode at this source address: */
578     if (__tme_predict_false(insns_thunk < 0)) {
579 
580       /* the only time we can't recode is when the host runs out of
581 	 memory for instructions thunks.  when this happens, the host
582 	 invalidates all previous instructions thunks, so we have to
583 	 invalidate our entire source hash: */
584       tme_sparc_recode_invalidate_all(ic);
585       return (0);
586     }
587 
588     /* set the instructions thunk for this source address: */
589     assert ((insns_thunk & TME_BIT(0)) == 0);
590     (ic->tme_sparc_recode_src_hash
591      [src_hash_i / TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT]
592      .tme_sparc_recode_src_hash_values
593      [src_hash_i % TME_SPARC_RECODE_SRC_HASH_SIZE_ELEMENT])
594       = insns_thunk;
595 
596     /* we can't run the new instructions thunk without re-checking the
597        valid bit for the page containing this source address
598        first - if it wasn't set for the entire time we were recoding,
599        the instructions thunk might not be consistent with the source
600        memory.
601 
602        to keep things simple, we just return now and interpret
603        instructions normally - the next time we get to this source
604        address, we'll check the valid bit and either run the
605        instructions thunk for the first time, or we'll do the
606        invalidate: */
607     return (0);
608   }
609 
610   /* return the instructions thunk offset: */
611   return (insns_thunk);
612 }
613 
614 #ifdef TME_RECODE_DEBUG
615 #include <stdio.h>
616 
617 void tme_sparc_recode_dump_ic _TME_P((const struct tme_sparc *));
618 void
tme_sparc_recode_dump_ic(const struct tme_sparc * ic)619 tme_sparc_recode_dump_ic(const struct tme_sparc *ic)
620 {
621   const char *conds_integer[] = {
622     "n", "e", "le", "l", "leu", "cs", "neg", "vs",
623     NULL
624   };
625 
626   printf("sparc%u add flags thunk %p:\n",
627 	 (1 << TME_SPARC_RECODE_SIZE(ic)),
628 	 ic->tme_sparc_recode_flags_thunk_add);
629   tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
630 			      ic->tme_sparc_recode_flags_thunk_add);
631   printf("sparc%u sub flags thunk %p:\n",
632 	 (1 << TME_SPARC_RECODE_SIZE(ic)),
633 	 ic->tme_sparc_recode_flags_thunk_sub);
634   tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
635 			      ic->tme_sparc_recode_flags_thunk_sub);
636   printf("sparc%u logical flags thunk %p:\n",
637 	 (1 << TME_SPARC_RECODE_SIZE(ic)),
638 	 ic->tme_sparc_recode_flags_thunk_logical);
639   tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
640 			      ic->tme_sparc_recode_flags_thunk_logical);
641 
642   if (TME_SPARC_VERSION(ic) >= 9) {
643     printf("sparc%u rcc flags thunk %p:\n",
644 	   (1 << TME_SPARC_RECODE_SIZE(ic)),
645 	   ic->tme_sparc_recode_flags_thunk_rcc);
646     tme_recode_flags_thunk_dump(ic->tme_sparc_recode_ic,
647 				ic->tme_sparc_recode_flags_thunk_rcc);
648   }
649   printf("sparc%u %%icc conds thunk %p:\n",
650 	 (1 << TME_SPARC_RECODE_SIZE(ic)),
651 	 ic->tme_sparc_recode_conds_thunk_icc);
652   tme_recode_conds_thunk_dump(ic->tme_sparc_recode_ic,
653 			      ic->tme_sparc_recode_conds_thunk_icc,
654 			      conds_integer);
655   if (TME_SPARC_VERSION(ic) >= 9) {
656     printf("sparc%u %%xcc conds thunk %p:\n",
657 	   (1 << TME_SPARC_RECODE_SIZE(ic)),
658 	   ic->tme_sparc_recode_conds_thunk_xcc);
659     tme_recode_conds_thunk_dump(ic->tme_sparc_recode_ic,
660 				ic->tme_sparc_recode_conds_thunk_xcc,
661 				conds_integer);
662     printf("sparc%u %%rcc conds thunk %p:\n",
663 	   (1 << TME_SPARC_RECODE_SIZE(ic)),
664 	   ic->tme_sparc_recode_conds_thunk_rcc);
665     tme_recode_conds_thunk_dump(ic->tme_sparc_recode_ic,
666 				ic->tme_sparc_recode_conds_thunk_rcc,
667 				conds_integer);
668   }
669 }
670 
671 void tme_sparc_recode_dump_insns _TME_P((const struct tme_sparc *));
672 void
tme_sparc_recode_dump_insns(const struct tme_sparc * ic)673 tme_sparc_recode_dump_insns(const struct tme_sparc *ic)
674 {
675   const struct tme_recode_insn *insns;
676   const struct tme_recode_insn *insns_end;
677   unsigned int insn_i;
678   const char *s;
679   unsigned int insn_size;
680   unsigned int operands_mask;
681   unsigned int thunk_i;
682   unsigned int operand_i;
683   int delim;
684   const char *regcs = "goli";
685   const char *conds_integer[] = {
686     "n", "e", "le", "l", "leu", "cs", "neg", "vs",
687     "a", "ne", "g", "ge", "gu", "cc", "pos", "vc",
688   };
689   tme_uint32_t chain_info;
690 
691   printf("%%pc = ");
692   tme_recode_uguest_dump(
693 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
694 			 TME_SPARC_VERSION(ic) >= 9
695 			 ? ic->tme_sparc_ireg_uint64(TME_SPARC_IREG_PC)
696 			 :
697 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
698 			 ic->tme_sparc_ireg_uint32(TME_SPARC_IREG_PC));
699   printf("\n");
700 
701   insns = ic->tme_sparc_recode_insns_group.tme_recode_insns_group_insns;
702   insns_end = ic->tme_sparc_recode_insns_group.tme_recode_insns_group_insns_end;
703   for (insn_i = 0; insns < insns_end; insns++, insn_i++) {
704 
705     printf("%2u: ", insn_i);
706 
707     s = tme_recode_opcode_dump(insns->tme_recode_insn_opcode);
708     insn_size = insns->tme_recode_insn_size;
709     switch (insns->tme_recode_insn_opcode) {
710     case TME_RECODE_OPCODE_EXTZ:
711     case TME_RECODE_OPCODE_EXTS:
712       printf("ext%u%c",
713 	     (1 << insns->tme_recode_insn_operand_src[1]),
714 	     (insns->tme_recode_insn_opcode == TME_RECODE_OPCODE_EXTZ ? 'z' : 's'));
715       s = "";
716       operands_mask = (1 << 0) | (1 << 2);
717       break;
718     case TME_RECODE_OPCODE_DEFC:
719       printf("defc\t %%%ccc:%s, %%c%u\n",
720 	     (insns->tme_recode_insn_conds_thunk == ic->tme_sparc_recode_conds_thunk_icc
721 	      ? 'i'
722 	      : insns->tme_recode_insn_conds_thunk == ic->tme_sparc_recode_conds_thunk_xcc
723 	      ? 'x'
724 	      : 'r'),
725 	     conds_integer
726 	     [insns->tme_recode_insn_operand_src[0]
727 	      + (TME_SPARC_COND_NOT
728 		 * (insns->tme_recode_insn_operand_src[1] != 0))],
729 	     insns->tme_recode_insn_operand_dst);
730       continue;
731     case TME_RECODE_OPCODE_IF:
732       printf("%s\t %%c%u\n", s, insns->tme_recode_insn_operand_src[0]);
733       continue;
734     case TME_RECODE_OPCODE_ELSE:
735     case TME_RECODE_OPCODE_ENDIF:
736       printf("%s\n", s);
737       continue;
738     case TME_RECODE_OPCODE_RW:
739       for (thunk_i = 0;; thunk_i++) {
740 	assert (thunk_i < TME_ARRAY_ELS(ic->tme_sparc_recode_rw_thunks));
741 	if (insns->tme_recode_insn_rw_thunk == ic->tme_sparc_recode_rw_thunks[thunk_i]) {
742 	  break;
743 	}
744       }
745       s = "?rw?";
746       switch (thunk_i % 0x10) {
747       case 0x0: s = (TME_SPARC_VERSION(ic) >= 9 ? "lduw" : "ld"); break;
748       case 0x1: s = "ldub"; break;
749       case 0x2: s = "lduh"; break;
750       case 0x4: s = (TME_SPARC_VERSION(ic) >= 9 ? "stw" : "st"); break;
751       case 0x5: s = "stb"; break;
752       case 0x6: s = "sth"; break;
753       case 0x8: if (TME_SPARC_VERSION(ic) >= 9) { s = "ldsw"; } break;
754       case 0x9: s = "ldsb"; break;
755       case 0xa: s = "ldsh"; break;
756       case 0xb: if (TME_SPARC_VERSION(ic) >= 9) { s = "ldx"; } break;
757       case 0xe: if (TME_SPARC_VERSION(ic) >= 9) { s = "stx"; } break;
758       }
759       if (thunk_i & 16) {
760 	printf("%s", s);
761 	s = ".nf";
762       }
763       if (thunk_i & 32) {
764 	printf("%s", s);
765 	s = ".el";
766       }
767       if (thunk_i & 64) {
768 	printf("%s", s);
769 	s = ".am";
770       }
771       insn_size = 0;
772       operands_mask = (1 << 0);
773       if (insns->tme_recode_insn_rw_thunk->tme_recode_rw_thunk_write) {
774 	operands_mask |= (1 << 1);
775       }
776       else {
777 	operands_mask |= (1 << 2);
778       }
779       break;
780     default:
781       operands_mask = (1 << 0) | (1 << 1) | (1 << 2);
782       break;
783     }
784 
785     printf("%s", s);
786     if (insn_size > 0) {
787       printf("%u", (1 << insns->tme_recode_insn_size));
788     }
789     delim = '\t';
790     for (operand_i = 0; operand_i <= 2; operand_i++) {
791       if (operands_mask & (1 << operand_i)) {
792 	putchar(delim);
793 	putchar(' ');
794 	delim = ',';
795 	switch (insns->tme_recode_insn_operands[operand_i]) {
796 	case TME_RECODE_OPERAND_UNDEF:
797 	  s = "undef";
798 	  break;
799 	case TME_RECODE_OPERAND_IMM:
800 	  tme_recode_uguest_dump(insns->tme_recode_insn_imm_uguest);
801 	  continue;
802 #if TME_RECODE_OPERAND_NULL != TME_RECODE_OPERAND_ZERO
803 #error "TME_RECODE_OPERAND_ values changed"
804 #endif
805 	case TME_RECODE_OPERAND_ZERO:
806 	  s = "g0";
807 	  if (operand_i == 2
808 	      && insns->tme_recode_insn_opcode == TME_RECODE_OPCODE_GUEST) {
809 	    s = "null";
810 	  }
811 	  break;
812 	case TME_SPARC_IREG_PC: s = "pc"; break;
813 	case TME_SPARC_IREG_PC_NEXT: s = "pc_next"; break;
814 	case TME_SPARC_IREG_PC_NEXT_NEXT: s = "pc_next_next"; break;
815 	case TME_SPARC_IREG_INSN: s = "insn"; break;
816 	case TME_SPARC_IREG_TMP(0): s = "tmp0"; break;
817 	case TME_SPARC_IREG_TMP(1): s = "tmp1"; break;
818 	case TME_SPARC_IREG_TMP(2): s = "tmp2"; break;
819 	default:
820 	  if (insns->tme_recode_insn_operands[operand_i] < 32) {
821 	    printf("%%%c%u",
822 		   regcs[insns->tme_recode_insn_operands[operand_i] / 8],
823 		   (insns->tme_recode_insn_operands[operand_i] % 8));
824 	    continue;
825 	  }
826 	  s = "??";
827 	  break;
828 	}
829 	printf("%%%s", s);
830       }
831     }
832 
833     if (insns->tme_recode_insn_opcode < TME_RECODE_OPCODES_INTEGER
834 	&& insns->tme_recode_insn_flags_thunk != NULL) {
835       printf(", %%%s",
836 	     (TME_SPARC_VERSION(ic) < 9
837 	      ? "icc"
838 	      : insns->tme_recode_insn_flags_thunk == ic->tme_sparc_recode_flags_thunk_rcc
839 	      ? "rcc"
840 	      : "ccr"));
841     }
842 
843     putchar('\n');
844   }
845 
846   chain_info = ic->tme_sparc_recode_insns_group.tme_recode_insns_group_chain_info;
847   printf("chain %s %s ",
848 	 ((chain_info & TME_RECODE_CHAIN_INFO_JUMP)
849 	  ? "jump"
850 	  : (chain_info & TME_RECODE_CHAIN_INFO_RETURN)
851 	  ? "return"
852 	  : "call"),
853 	 (((chain_info
854 	    & (TME_RECODE_CHAIN_INFO_NEAR
855 	       | TME_RECODE_CHAIN_INFO_FAR))
856 	   == TME_RECODE_CHAIN_INFO_NEAR)
857 	  ? "near"
858 	  : "far"));
859   if ((chain_info
860        & (TME_RECODE_CHAIN_INFO_UNCONDITIONAL
861 	  | TME_RECODE_CHAIN_INFO_CONDITIONAL))
862       == TME_RECODE_CHAIN_INFO_UNCONDITIONAL) {
863     printf("unconditional\n");
864   }
865   else {
866     printf("conditional alternate %s\n",
867 	   (((chain_info
868 	      & (TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR
869 		 | TME_RECODE_CHAIN_INFO_ALTERNATE_FAR))
870 	     == TME_RECODE_CHAIN_INFO_ALTERNATE_NEAR)
871 	    ? "near"
872 	    : "far"));
873   }
874 }
875 
876 #endif /* TME_RECODE_DEBUG */
877 
878 #endif /* TME_HAVE_RECODE */
879 
880 /* this initializes sparc recoding: */
881 void
tme_sparc_recode_init(struct tme_sparc * ic)882 tme_sparc_recode_init(struct tme_sparc *ic)
883 {
884 #if TME_HAVE_RECODE
885   unsigned long reg_guest_count;
886   unsigned long reg_guest;
887   struct tme_recode_ic *recode_ic;
888 
889   /* get the number of guest registers: */
890   reg_guest_count = TME_SPARC_IREG_TMP(3);
891 
892   /* allocate the recode ic: */
893   recode_ic
894     = tme_malloc0(sizeof(struct tme_recode_ic)
895 		  + (sizeof(union tme_recode_reginfo)
896 		     * reg_guest_count));
897   ic->tme_sparc_recode_ic = recode_ic;
898 
899   /* set the register size: */
900   recode_ic->tme_recode_ic_reg_size = TME_SPARC_RECODE_SIZE(ic);
901 
902   /* set the register count: */
903   recode_ic->tme_recode_ic_reg_count = reg_guest_count;
904 
905   /* allocate some number of read-uses records: */
906   /* XXX FIXME - this can be anything, and doesn't affect correctness,
907      just performance.  should it be tunable? */
908   recode_ic->tme_recode_ic_reg_guest_ruses_record_count = 512;
909   recode_ic->tme_recode_ic_reg_guest_ruses_records
910     = tme_new(tme_uint16_t,
911 	      recode_ic->tme_recode_ic_reg_guest_ruses_record_count + 1);
912   recode_ic->tme_recode_ic_reg_guest_ruses_records
913     [recode_ic->tme_recode_ic_reg_guest_ruses_record_count]
914     = TME_RECODE_REG_RUSES_RECORD_UNDEF;
915 
916   /* assume that all registers use flat addressing: */
917   for (reg_guest = 0;
918        reg_guest < reg_guest_count;
919        reg_guest++) {
920     (recode_ic->tme_recode_ic_reginfo
921      + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_FLAT;
922   }
923 
924   /* %g0 is fixed: */
925   (recode_ic->tme_recode_ic_reginfo
926    + TME_RECODE_REG_GUEST(TME_SPARC_IREG_G0))->tme_recode_reginfo_all
927     |= TME_RECODE_REGINFO_TYPE_FIXED;
928 
929   /* %r8 through %r23 are addressed through window zero, and %r24
930      through %r31 are addressed through window one: */
931   reg_guest = TME_RECODE_REG_GUEST(8);
932   do {
933     (recode_ic->tme_recode_ic_reginfo
934      + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_WINDOW(0);
935   } while (++reg_guest <= TME_RECODE_REG_GUEST(23));
936   do {
937     (recode_ic->tme_recode_ic_reginfo
938      + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_WINDOW(1);
939   } while (++reg_guest <= TME_RECODE_REG_GUEST(31));
940 
941   /* on a v9 CPU, %r1 through %r7 are addressed through window two: */
942   if (TME_SPARC_VERSION(ic) >= 9) {
943     reg_guest = TME_RECODE_REG_GUEST(1);
944     do {
945       (recode_ic->tme_recode_ic_reginfo
946        + reg_guest)->tme_recode_reginfo_all = TME_RECODE_REGINFO_TYPE_WINDOW(2);
947     } while (++reg_guest <= TME_RECODE_REG_GUEST(7));
948   }
949 
950   /* the temporary registers are temporary: */
951   reg_guest = TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(0));
952   do {
953     (recode_ic->tme_recode_ic_reginfo
954      + reg_guest)->tme_recode_reginfo_all
955       |= TME_RECODE_REGINFO_TYPE_TEMPORARY;
956   } while (++reg_guest <= TME_RECODE_REG_GUEST(TME_SPARC_IREG_TMP(2)));
957 
958   /* set the window base offsets: */
959   recode_ic->tme_recode_ic_window_base_offsets[0]
960     = (((char *) &ic->tme_sparc_recode_window_base_offsets[0]) - ((char *) ic));
961   recode_ic->tme_recode_ic_window_base_offsets[1]
962     = (((char *) &ic->tme_sparc_recode_window_base_offsets[1]) - ((char *) ic));
963   recode_ic->tme_recode_ic_window_base_offsets[2]
964     = (((char *) &ic->tme_sparc_recode_window_base_offsets[2]) - ((char *) ic));
965 
966   /* initialize the cacheables: */
967   ic->tme_sparc_recode_cacheables[0].tme_sparc_recode_cacheable_size = 0;
968   ic->tme_sparc_recode_cacheable_first = &ic->tme_sparc_recode_cacheables[0];
969 
970   /* invalidate the entire source hash: */
971   _tme_sparc_recode_src_hash_invalidate(ic, 0, 0);
972 
973   /* set the TLB page size: */
974   recode_ic->tme_recode_ic_tlb_page_size
975     = (((tme_uint32_t) 1) << ic->tme_sparc_tlb_page_size_log2);
976 
977   /* set the pointer to the first instruction in a group: */
978   ic->tme_sparc_recode_insns_group.tme_recode_insns_group_insns
979     = &ic->tme_sparc_recode_insns[0];
980 
981   /* initialize the recode ic: */
982   /* XXX FIXME - this run size should be passed in as an argument.  it
983      needs to be big enough for the maximum number of non-variable
984      thunks (flags, conds, chain, etc.) that we will make, probably
985      TME_RECODE_HOST_THUNK_SIZE_MAX times some n? */
986   tme_recode_ic_new(recode_ic, 4 * 1024 * 1024);
987 
988   /* call the architecture-specific init functions: */
989   if (TME_SPARC_VERSION(ic) >= 9) {
990 #if TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32
991     _tme_sparc64_recode_cc_init(ic);
992     _tme_sparc64_recode_chain_init(ic);
993     _tme_sparc64_recode_ls_init(ic);
994 #endif /* TME_RECODE_SIZE_GUEST_MAX > TME_RECODE_SIZE_32 */
995   }
996   else {
997     _tme_sparc32_recode_cc_init(ic);
998     _tme_sparc32_recode_chain_init(ic);
999     _tme_sparc32_recode_ls_init(ic);
1000   }
1001 
1002 #ifdef _TME_SPARC_RECODE_VERIFY
1003 
1004   /* initialize the verifier: */
1005   _tme_sparc_recode_verify_init(ic);
1006 
1007 #endif /* _TME_SPARC_RECODE_VERIFY */
1008 
1009 #else  /* !TME_HAVE_RECODE */
1010   /* unused: */
1011   ic = 0;
1012 #endif /* !TME_HAVE_RECODE */
1013 }
1014