1 /* longlong.h -- definitions for mixed size 32/64 bit arithmetic.
2    Copyright (C) 1991-2016 Free Software Foundation, Inc.
3 
4    This file is part of the GNU C Library.
5 
6    The GNU C Library is free software; you can redistribute it and/or
7    modify it under the terms of the GNU Lesser General Public
8    License as published by the Free Software Foundation; either
9    version 2.1 of the License, or (at your option) any later version.
10 
11    In addition to the permissions in the GNU Lesser General Public
12    License, the Free Software Foundation gives you unlimited
13    permission to link the compiled version of this file into
14    combinations with other programs, and to distribute those
15    combinations without any restriction coming from the use of this
16    file.  (The Lesser General Public License restrictions do apply in
17    other respects; for example, they cover modification of the file,
18    and distribution when not linked into a combine executable.)
19 
20    The GNU C Library is distributed in the hope that it will be useful,
21    but WITHOUT ANY WARRANTY; without even the implied warranty of
22    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
23    Lesser General Public License for more details.
24 
25    You should have received a copy of the GNU Lesser General Public
26    License along with the GNU C Library; if not, see
27    <http://www.gnu.org/licenses/>.  */
28 
29 /* You have to define the following before including this file:
30 
31    UWtype -- An unsigned type, default type for operations (typically a "word")
32    UHWtype -- An unsigned type, at least half the size of UWtype.
33    UDWtype -- An unsigned type, at least twice as large a UWtype
34    W_TYPE_SIZE -- size in bits of UWtype
35 
36    UQItype -- Unsigned 8 bit type.
37    SItype, USItype -- Signed and unsigned 32 bit types.
38    DItype, UDItype -- Signed and unsigned 64 bit types.
39 
40    On a 32 bit machine UWtype should typically be USItype;
41    on a 64 bit machine, UWtype should typically be UDItype.  */
42 
43 #define __BITS4 (W_TYPE_SIZE / 4)
44 #define __ll_B ((UWtype) 1 << (W_TYPE_SIZE / 2))
45 #define __ll_lowpart(t) ((UWtype) (t) & (__ll_B - 1))
46 #define __ll_highpart(t) ((UWtype) (t) >> (W_TYPE_SIZE / 2))
47 
48 #ifndef W_TYPE_SIZE
49 #define W_TYPE_SIZE	32
50 #define UWtype		USItype
51 #define UHWtype		USItype
52 #define UDWtype		UDItype
53 #endif
54 
55 /* Used in glibc only.  */
56 #ifndef attribute_hidden
57 #define attribute_hidden
58 #endif
59 
60 extern const UQItype __clz_tab[256] attribute_hidden;
61 
62 /* Define auxiliary asm macros.
63 
64    1) umul_ppmm(high_prod, low_prod, multiplier, multiplicand) multiplies two
65    UWtype integers MULTIPLIER and MULTIPLICAND, and generates a two UWtype
66    word product in HIGH_PROD and LOW_PROD.
67 
68    2) __umulsidi3(a,b) multiplies two UWtype integers A and B, and returns a
69    UDWtype product.  This is just a variant of umul_ppmm.
70 
71    3) udiv_qrnnd(quotient, remainder, high_numerator, low_numerator,
72    denominator) divides a UDWtype, composed by the UWtype integers
73    HIGH_NUMERATOR and LOW_NUMERATOR, by DENOMINATOR and places the quotient
74    in QUOTIENT and the remainder in REMAINDER.  HIGH_NUMERATOR must be less
75    than DENOMINATOR for correct operation.  If, in addition, the most
76    significant bit of DENOMINATOR must be 1, then the pre-processor symbol
77    UDIV_NEEDS_NORMALIZATION is defined to 1.
78 
79    4) sdiv_qrnnd(quotient, remainder, high_numerator, low_numerator,
80    denominator).  Like udiv_qrnnd but the numbers are signed.  The quotient
81    is rounded towards 0.
82 
83    5) count_leading_zeros(count, x) counts the number of zero-bits from the
84    msb to the first nonzero bit in the UWtype X.  This is the number of
85    steps X needs to be shifted left to set the msb.  Undefined for X == 0,
86    unless the symbol COUNT_LEADING_ZEROS_0 is defined to some value.
87 
88    6) count_trailing_zeros(count, x) like count_leading_zeros, but counts
89    from the least significant end.
90 
91    7) add_ssaaaa(high_sum, low_sum, high_addend_1, low_addend_1,
92    high_addend_2, low_addend_2) adds two UWtype integers, composed by
93    HIGH_ADDEND_1 and LOW_ADDEND_1, and HIGH_ADDEND_2 and LOW_ADDEND_2
94    respectively.  The result is placed in HIGH_SUM and LOW_SUM.  Overflow
95    (i.e. carry out) is not stored anywhere, and is lost.
96 
97    8) sub_ddmmss(high_difference, low_difference, high_minuend, low_minuend,
98    high_subtrahend, low_subtrahend) subtracts two two-word UWtype integers,
99    composed by HIGH_MINUEND_1 and LOW_MINUEND_1, and HIGH_SUBTRAHEND_2 and
100    LOW_SUBTRAHEND_2 respectively.  The result is placed in HIGH_DIFFERENCE
101    and LOW_DIFFERENCE.  Overflow (i.e. carry out) is not stored anywhere,
102    and is lost.
103 
104    If any of these macros are left undefined for a particular CPU,
105    C macros are used.  */
106 
107 /* The CPUs come in alphabetical order below.
108 
109    Please add support for more CPUs here, or improve the current support
110    for the CPUs below!
111    (E.g. WE32100, IBM360.)  */
112 
113 #if defined (__GNUC__) && !defined (NO_ASM)
114 
115 /* We sometimes need to clobber "cc" with gcc2, but that would not be
116    understood by gcc1.  Use cpp to avoid major code duplication.  */
117 #if __GNUC__ < 2
118 #define __CLOBBER_CC
119 #define __AND_CLOBBER_CC
120 #else /* __GNUC__ >= 2 */
121 #define __CLOBBER_CC : "cc"
122 #define __AND_CLOBBER_CC , "cc"
123 #endif /* __GNUC__ < 2 */
124 
125 #if defined (__aarch64__)
126 
127 #if W_TYPE_SIZE == 32
128 #define count_leading_zeros(COUNT, X)	((COUNT) = __builtin_clz (X))
129 #define count_trailing_zeros(COUNT, X)   ((COUNT) = __builtin_ctz (X))
130 #define COUNT_LEADING_ZEROS_0 32
131 #endif /* W_TYPE_SIZE == 32 */
132 
133 #if W_TYPE_SIZE == 64
134 #define count_leading_zeros(COUNT, X)	((COUNT) = __builtin_clzll (X))
135 #define count_trailing_zeros(COUNT, X)   ((COUNT) = __builtin_ctzll (X))
136 #define COUNT_LEADING_ZEROS_0 64
137 #endif /* W_TYPE_SIZE == 64 */
138 
139 #endif /* __aarch64__ */
140 
141 #if defined (__alpha) && W_TYPE_SIZE == 64
142 /* There is a bug in g++ before version 5 that
143    errors on __builtin_alpha_umulh.  */
144 #if !defined(__cplusplus) || __GNUC__ >= 5
145 #define umul_ppmm(ph, pl, m0, m1) \
146   do {									\
147     UDItype __m0 = (m0), __m1 = (m1);					\
148     (ph) = __builtin_alpha_umulh (__m0, __m1);				\
149     (pl) = __m0 * __m1;							\
150   } while (0)
151 #define UMUL_TIME 46
152 #endif /* !c++ */
153 #ifndef LONGLONG_STANDALONE
154 #define udiv_qrnnd(q, r, n1, n0, d) \
155   do { UDItype __r;							\
156     (q) = __udiv_qrnnd (&__r, (n1), (n0), (d));				\
157     (r) = __r;								\
158   } while (0)
159 extern UDItype __udiv_qrnnd (UDItype *, UDItype, UDItype, UDItype);
160 #define UDIV_TIME 220
161 #endif /* LONGLONG_STANDALONE */
162 #ifdef __alpha_cix__
163 #define count_leading_zeros(COUNT,X)	((COUNT) = __builtin_clzl (X))
164 #define count_trailing_zeros(COUNT,X)	((COUNT) = __builtin_ctzl (X))
165 #define COUNT_LEADING_ZEROS_0 64
166 #else
167 #define count_leading_zeros(COUNT,X) \
168   do {									\
169     UDItype __xr = (X), __t, __a;					\
170     __t = __builtin_alpha_cmpbge (0, __xr);				\
171     __a = __clz_tab[__t ^ 0xff] - 1;					\
172     __t = __builtin_alpha_extbl (__xr, __a);				\
173     (COUNT) = 64 - (__clz_tab[__t] + __a*8);				\
174   } while (0)
175 #define count_trailing_zeros(COUNT,X) \
176   do {									\
177     UDItype __xr = (X), __t, __a;					\
178     __t = __builtin_alpha_cmpbge (0, __xr);				\
179     __t = ~__t & -~__t;							\
180     __a = ((__t & 0xCC) != 0) * 2;					\
181     __a += ((__t & 0xF0) != 0) * 4;					\
182     __a += ((__t & 0xAA) != 0);						\
183     __t = __builtin_alpha_extbl (__xr, __a);				\
184     __a <<= 3;								\
185     __t &= -__t;							\
186     __a += ((__t & 0xCC) != 0) * 2;					\
187     __a += ((__t & 0xF0) != 0) * 4;					\
188     __a += ((__t & 0xAA) != 0);						\
189     (COUNT) = __a;							\
190   } while (0)
191 #endif /* __alpha_cix__ */
192 #endif /* __alpha */
193 
194 #if defined (__arc__) && W_TYPE_SIZE == 32
195 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
196   __asm__ ("add.f	%1, %4, %5\n\tadc	%0, %2, %3"		\
197 	   : "=r" ((USItype) (sh)),					\
198 	     "=&r" ((USItype) (sl))					\
199 	   : "%r" ((USItype) (ah)),					\
200 	     "rIJ" ((USItype) (bh)),					\
201 	     "%r" ((USItype) (al)),					\
202 	     "rIJ" ((USItype) (bl)))
203 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
204   __asm__ ("sub.f	%1, %4, %5\n\tsbc	%0, %2, %3"		\
205 	   : "=r" ((USItype) (sh)),					\
206 	     "=&r" ((USItype) (sl))					\
207 	   : "r" ((USItype) (ah)),					\
208 	     "rIJ" ((USItype) (bh)),					\
209 	     "r" ((USItype) (al)),					\
210 	     "rIJ" ((USItype) (bl)))
211 
212 #define __umulsidi3(u,v) ((UDItype)(USItype)u*(USItype)v)
213 #ifdef __ARC_NORM__
214 #define count_leading_zeros(count, x) \
215   do									\
216     {									\
217       SItype c_;							\
218 									\
219       __asm__ ("norm.f\t%0,%1\n\tmov.mi\t%0,-1" : "=r" (c_) : "r" (x) : "cc");\
220       (count) = c_ + 1;							\
221     }									\
222   while (0)
223 #define COUNT_LEADING_ZEROS_0 32
224 #endif
225 #endif
226 
227 #if defined (__arm__) && (defined (__thumb2__) || !defined (__thumb__)) \
228  && W_TYPE_SIZE == 32
229 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
230   __asm__ ("adds	%1, %4, %5\n\tadc	%0, %2, %3"		\
231 	   : "=r" ((USItype) (sh)),					\
232 	     "=&r" ((USItype) (sl))					\
233 	   : "%r" ((USItype) (ah)),					\
234 	     "rI" ((USItype) (bh)),					\
235 	     "%r" ((USItype) (al)),					\
236 	     "rI" ((USItype) (bl)) __CLOBBER_CC)
237 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
238   __asm__ ("subs	%1, %4, %5\n\tsbc	%0, %2, %3"		\
239 	   : "=r" ((USItype) (sh)),					\
240 	     "=&r" ((USItype) (sl))					\
241 	   : "r" ((USItype) (ah)),					\
242 	     "rI" ((USItype) (bh)),					\
243 	     "r" ((USItype) (al)),					\
244 	     "rI" ((USItype) (bl)) __CLOBBER_CC)
245 # if defined(__ARM_ARCH_2__) || defined(__ARM_ARCH_2A__) \
246      || defined(__ARM_ARCH_3__)
247 #  define umul_ppmm(xh, xl, a, b)					\
248   do {									\
249     register USItype __t0, __t1, __t2;					\
250     __asm__ ("%@ Inlined umul_ppmm\n"					\
251 	   "	mov	%2, %5, lsr #16\n"				\
252 	   "	mov	%0, %6, lsr #16\n"				\
253 	   "	bic	%3, %5, %2, lsl #16\n"				\
254 	   "	bic	%4, %6, %0, lsl #16\n"				\
255 	   "	mul	%1, %3, %4\n"					\
256 	   "	mul	%4, %2, %4\n"					\
257 	   "	mul	%3, %0, %3\n"					\
258 	   "	mul	%0, %2, %0\n"					\
259 	   "	adds	%3, %4, %3\n"					\
260 	   "	addcs	%0, %0, #65536\n"				\
261 	   "	adds	%1, %1, %3, lsl #16\n"				\
262 	   "	adc	%0, %0, %3, lsr #16"				\
263 	   : "=&r" ((USItype) (xh)),					\
264 	     "=r" ((USItype) (xl)),					\
265 	     "=&r" (__t0), "=&r" (__t1), "=r" (__t2)			\
266 	   : "r" ((USItype) (a)),					\
267 	     "r" ((USItype) (b)) __CLOBBER_CC );			\
268   } while (0)
269 #  define UMUL_TIME 20
270 # else
271 #  define umul_ppmm(xh, xl, a, b)					\
272   do {									\
273     /* Generate umull, under compiler control.  */			\
274     register UDItype __t0 = (UDItype)(USItype)(a) * (USItype)(b);	\
275     (xl) = (USItype)__t0;						\
276     (xh) = (USItype)(__t0 >> 32);					\
277   } while (0)
278 #  define UMUL_TIME 3
279 # endif
280 # define UDIV_TIME 100
281 #endif /* __arm__ */
282 
283 #if defined(__arm__)
284 /* Let gcc decide how best to implement count_leading_zeros.  */
285 #define count_leading_zeros(COUNT,X)	((COUNT) = __builtin_clz (X))
286 #define count_trailing_zeros(COUNT,X)   ((COUNT) = __builtin_ctz (X))
287 #define COUNT_LEADING_ZEROS_0 32
288 #endif
289 
290 #if defined (__AVR__)
291 
292 #if W_TYPE_SIZE == 16
293 #define count_leading_zeros(COUNT,X)  ((COUNT) = __builtin_clz (X))
294 #define count_trailing_zeros(COUNT,X) ((COUNT) = __builtin_ctz (X))
295 #define COUNT_LEADING_ZEROS_0 16
296 #endif /* W_TYPE_SIZE == 16 */
297 
298 #if W_TYPE_SIZE == 32
299 #define count_leading_zeros(COUNT,X)  ((COUNT) = __builtin_clzl (X))
300 #define count_trailing_zeros(COUNT,X) ((COUNT) = __builtin_ctzl (X))
301 #define COUNT_LEADING_ZEROS_0 32
302 #endif /* W_TYPE_SIZE == 32 */
303 
304 #if W_TYPE_SIZE == 64
305 #define count_leading_zeros(COUNT,X)  ((COUNT) = __builtin_clzll (X))
306 #define count_trailing_zeros(COUNT,X) ((COUNT) = __builtin_ctzll (X))
307 #define COUNT_LEADING_ZEROS_0 64
308 #endif /* W_TYPE_SIZE == 64 */
309 
310 #endif /* defined (__AVR__) */
311 
312 #if defined (__CRIS__)
313 
314 #if __CRIS_arch_version >= 3
315 #define count_leading_zeros(COUNT, X) ((COUNT) = __builtin_clz (X))
316 #define COUNT_LEADING_ZEROS_0 32
317 #endif /* __CRIS_arch_version >= 3 */
318 
319 #if __CRIS_arch_version >= 8
320 #define count_trailing_zeros(COUNT, X) ((COUNT) = __builtin_ctz (X))
321 #endif /* __CRIS_arch_version >= 8 */
322 
323 #if __CRIS_arch_version >= 10
324 #define __umulsidi3(u,v) ((UDItype)(USItype) (u) * (UDItype)(USItype) (v))
325 #else
326 #define __umulsidi3 __umulsidi3
327 extern UDItype __umulsidi3 (USItype, USItype);
328 #endif /* __CRIS_arch_version >= 10 */
329 
330 #define umul_ppmm(w1, w0, u, v)		\
331   do {					\
332     UDItype __x = __umulsidi3 (u, v);	\
333     (w0) = (USItype) (__x);		\
334     (w1) = (USItype) (__x >> 32);	\
335   } while (0)
336 
337 /* FIXME: defining add_ssaaaa and sub_ddmmss should be advantageous for
338    DFmode ("double" intrinsics, avoiding two of the three insns handling
339    carry), but defining them as open-code C composing and doing the
340    operation in DImode (UDImode) shows that the DImode needs work:
341    register pressure from requiring neighboring registers and the
342    traffic to and from them come to dominate, in the 4.7 series.  */
343 
344 #endif /* defined (__CRIS__) */
345 
346 #if defined (__hppa) && W_TYPE_SIZE == 32
347 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
348   __asm__ ("add %4,%5,%1\n\taddc %2,%3,%0"				\
349 	   : "=r" ((USItype) (sh)),					\
350 	     "=&r" ((USItype) (sl))					\
351 	   : "%rM" ((USItype) (ah)),					\
352 	     "rM" ((USItype) (bh)),					\
353 	     "%rM" ((USItype) (al)),					\
354 	     "rM" ((USItype) (bl)))
355 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
356   __asm__ ("sub %4,%5,%1\n\tsubb %2,%3,%0"				\
357 	   : "=r" ((USItype) (sh)),					\
358 	     "=&r" ((USItype) (sl))					\
359 	   : "rM" ((USItype) (ah)),					\
360 	     "rM" ((USItype) (bh)),					\
361 	     "rM" ((USItype) (al)),					\
362 	     "rM" ((USItype) (bl)))
363 #if defined (_PA_RISC1_1)
364 #define umul_ppmm(w1, w0, u, v) \
365   do {									\
366     union								\
367       {									\
368 	UDItype __f;							\
369 	struct {USItype __w1, __w0;} __w1w0;				\
370       } __t;								\
371     __asm__ ("xmpyu %1,%2,%0"						\
372 	     : "=x" (__t.__f)						\
373 	     : "x" ((USItype) (u)),					\
374 	       "x" ((USItype) (v)));					\
375     (w1) = __t.__w1w0.__w1;						\
376     (w0) = __t.__w1w0.__w0;						\
377      } while (0)
378 #define UMUL_TIME 8
379 #else
380 #define UMUL_TIME 30
381 #endif
382 #define UDIV_TIME 40
383 #define count_leading_zeros(count, x) \
384   do {									\
385     USItype __tmp;							\
386     __asm__ (								\
387        "ldi		1,%0\n"						\
388 "	extru,=		%1,15,16,%%r0		; Bits 31..16 zero?\n"	\
389 "	extru,tr	%1,15,16,%1		; No.  Shift down, skip add.\n"\
390 "	ldo		16(%0),%0		; Yes.  Perform add.\n"	\
391 "	extru,=		%1,23,8,%%r0		; Bits 15..8 zero?\n"	\
392 "	extru,tr	%1,23,8,%1		; No.  Shift down, skip add.\n"\
393 "	ldo		8(%0),%0		; Yes.  Perform add.\n"	\
394 "	extru,=		%1,27,4,%%r0		; Bits 7..4 zero?\n"	\
395 "	extru,tr	%1,27,4,%1		; No.  Shift down, skip add.\n"\
396 "	ldo		4(%0),%0		; Yes.  Perform add.\n"	\
397 "	extru,=		%1,29,2,%%r0		; Bits 3..2 zero?\n"	\
398 "	extru,tr	%1,29,2,%1		; No.  Shift down, skip add.\n"\
399 "	ldo		2(%0),%0		; Yes.  Perform add.\n"	\
400 "	extru		%1,30,1,%1		; Extract bit 1.\n"	\
401 "	sub		%0,%1,%0		; Subtract it.\n"	\
402 	: "=r" (count), "=r" (__tmp) : "1" (x));			\
403   } while (0)
404 #endif
405 
406 #if (defined (__i370__) || defined (__s390__) || defined (__mvs__)) && W_TYPE_SIZE == 32
407 #if !defined (__zarch__)
408 #define smul_ppmm(xh, xl, m0, m1) \
409   do {									\
410     union {DItype __ll;							\
411 	   struct {USItype __h, __l;} __i;				\
412 	  } __x;							\
413     __asm__ ("lr %N0,%1\n\tmr %0,%2"					\
414 	     : "=&r" (__x.__ll)						\
415 	     : "r" (m0), "r" (m1));					\
416     (xh) = __x.__i.__h; (xl) = __x.__i.__l;				\
417   } while (0)
418 #define sdiv_qrnnd(q, r, n1, n0, d) \
419   do {									\
420     union {DItype __ll;							\
421 	   struct {USItype __h, __l;} __i;				\
422 	  } __x;							\
423     __x.__i.__h = n1; __x.__i.__l = n0;					\
424     __asm__ ("dr %0,%2"							\
425 	     : "=r" (__x.__ll)						\
426 	     : "0" (__x.__ll), "r" (d));				\
427     (q) = __x.__i.__l; (r) = __x.__i.__h;				\
428   } while (0)
429 #else
430 #define smul_ppmm(xh, xl, m0, m1) \
431   do {                                                                  \
432     register SItype __r0 __asm__ ("0");					\
433     register SItype __r1 __asm__ ("1") = (m0);				\
434 									\
435     __asm__ ("mr\t%%r0,%3"                                              \
436 	     : "=r" (__r0), "=r" (__r1)					\
437 	     : "r"  (__r1),  "r" (m1));					\
438     (xh) = __r0; (xl) = __r1;						\
439   } while (0)
440 
441 #define sdiv_qrnnd(q, r, n1, n0, d) \
442   do {									\
443     register SItype __r0 __asm__ ("0") = (n1);				\
444     register SItype __r1 __asm__ ("1") = (n0);				\
445 									\
446     __asm__ ("dr\t%%r0,%4"                                              \
447 	     : "=r" (__r0), "=r" (__r1)					\
448 	     : "r" (__r0), "r" (__r1), "r" (d));			\
449     (q) = __r1; (r) = __r0;						\
450   } while (0)
451 #endif /* __zarch__ */
452 #endif
453 
454 #if (defined (__i386__) || defined (__i486__)) && W_TYPE_SIZE == 32
455 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
456   __asm__ ("add{l} {%5,%1|%1,%5}\n\tadc{l} {%3,%0|%0,%3}"		\
457 	   : "=r" ((USItype) (sh)),					\
458 	     "=&r" ((USItype) (sl))					\
459 	   : "%0" ((USItype) (ah)),					\
460 	     "g" ((USItype) (bh)),					\
461 	     "%1" ((USItype) (al)),					\
462 	     "g" ((USItype) (bl)))
463 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
464   __asm__ ("sub{l} {%5,%1|%1,%5}\n\tsbb{l} {%3,%0|%0,%3}"		\
465 	   : "=r" ((USItype) (sh)),					\
466 	     "=&r" ((USItype) (sl))					\
467 	   : "0" ((USItype) (ah)),					\
468 	     "g" ((USItype) (bh)),					\
469 	     "1" ((USItype) (al)),					\
470 	     "g" ((USItype) (bl)))
471 #define umul_ppmm(w1, w0, u, v) \
472   __asm__ ("mul{l} %3"							\
473 	   : "=a" ((USItype) (w0)),					\
474 	     "=d" ((USItype) (w1))					\
475 	   : "%0" ((USItype) (u)),					\
476 	     "rm" ((USItype) (v)))
477 #define udiv_qrnnd(q, r, n1, n0, dv) \
478   __asm__ ("div{l} %4"							\
479 	   : "=a" ((USItype) (q)),					\
480 	     "=d" ((USItype) (r))					\
481 	   : "0" ((USItype) (n0)),					\
482 	     "1" ((USItype) (n1)),					\
483 	     "rm" ((USItype) (dv)))
484 #define count_leading_zeros(count, x)	((count) = __builtin_clz (x))
485 #define count_trailing_zeros(count, x)	((count) = __builtin_ctz (x))
486 #define UMUL_TIME 40
487 #define UDIV_TIME 40
488 #endif /* 80x86 */
489 
490 #if defined (__x86_64__) && W_TYPE_SIZE == 64
491 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
492   __asm__ ("add{q} {%5,%1|%1,%5}\n\tadc{q} {%3,%0|%0,%3}"		\
493 	   : "=r" ((UDItype) (sh)),					\
494 	     "=&r" ((UDItype) (sl))					\
495 	   : "%0" ((UDItype) (ah)),					\
496 	     "rme" ((UDItype) (bh)),					\
497 	     "%1" ((UDItype) (al)),					\
498 	     "rme" ((UDItype) (bl)))
499 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
500   __asm__ ("sub{q} {%5,%1|%1,%5}\n\tsbb{q} {%3,%0|%0,%3}"		\
501 	   : "=r" ((UDItype) (sh)),					\
502 	     "=&r" ((UDItype) (sl))					\
503 	   : "0" ((UDItype) (ah)),					\
504 	     "rme" ((UDItype) (bh)),					\
505 	     "1" ((UDItype) (al)),					\
506 	     "rme" ((UDItype) (bl)))
507 #define umul_ppmm(w1, w0, u, v) \
508   __asm__ ("mul{q} %3"							\
509 	   : "=a" ((UDItype) (w0)),					\
510 	     "=d" ((UDItype) (w1))					\
511 	   : "%0" ((UDItype) (u)),					\
512 	     "rm" ((UDItype) (v)))
513 #define udiv_qrnnd(q, r, n1, n0, dv) \
514   __asm__ ("div{q} %4"							\
515 	   : "=a" ((UDItype) (q)),					\
516 	     "=d" ((UDItype) (r))					\
517 	   : "0" ((UDItype) (n0)),					\
518 	     "1" ((UDItype) (n1)),					\
519 	     "rm" ((UDItype) (dv)))
520 #define count_leading_zeros(count, x)	((count) = __builtin_clzll (x))
521 #define count_trailing_zeros(count, x)	((count) = __builtin_ctzll (x))
522 #define UMUL_TIME 40
523 #define UDIV_TIME 40
524 #endif /* x86_64 */
525 
526 #if defined (__i960__) && W_TYPE_SIZE == 32
527 #define umul_ppmm(w1, w0, u, v) \
528   ({union {UDItype __ll;						\
529 	   struct {USItype __l, __h;} __i;				\
530 	  } __xx;							\
531   __asm__ ("emul	%2,%1,%0"					\
532 	   : "=d" (__xx.__ll)						\
533 	   : "%dI" ((USItype) (u)),					\
534 	     "dI" ((USItype) (v)));					\
535   (w1) = __xx.__i.__h; (w0) = __xx.__i.__l;})
536 #define __umulsidi3(u, v) \
537   ({UDItype __w;							\
538     __asm__ ("emul	%2,%1,%0"					\
539 	     : "=d" (__w)						\
540 	     : "%dI" ((USItype) (u)),					\
541 	       "dI" ((USItype) (v)));					\
542     __w; })
543 #endif /* __i960__ */
544 
545 #if defined (__ia64) && W_TYPE_SIZE == 64
546 /* This form encourages gcc (pre-release 3.4 at least) to emit predicated
547    "sub r=r,r" and "sub r=r,r,1", giving a 2 cycle latency.  The generic
548    code using "al<bl" arithmetically comes out making an actual 0 or 1 in a
549    register, which takes an extra cycle.  */
550 #define sub_ddmmss(sh, sl, ah, al, bh, bl)				\
551   do {									\
552     UWtype __x;								\
553     __x = (al) - (bl);							\
554     if ((al) < (bl))							\
555       (sh) = (ah) - (bh) - 1;						\
556     else								\
557       (sh) = (ah) - (bh);						\
558     (sl) = __x;								\
559   } while (0)
560 
561 /* Do both product parts in assembly, since that gives better code with
562    all gcc versions.  Some callers will just use the upper part, and in
563    that situation we waste an instruction, but not any cycles.  */
564 #define umul_ppmm(ph, pl, m0, m1)					\
565   __asm__ ("xma.hu %0 = %2, %3, f0\n\txma.l %1 = %2, %3, f0"		\
566 	   : "=&f" (ph), "=f" (pl)					\
567 	   : "f" (m0), "f" (m1))
568 #define count_leading_zeros(count, x)					\
569   do {									\
570     UWtype _x = (x), _y, _a, _c;					\
571     __asm__ ("mux1 %0 = %1, @rev" : "=r" (_y) : "r" (_x));		\
572     __asm__ ("czx1.l %0 = %1" : "=r" (_a) : "r" (-_y | _y));		\
573     _c = (_a - 1) << 3;							\
574     _x >>= _c;								\
575     if (_x >= 1 << 4)							\
576       _x >>= 4, _c += 4;						\
577     if (_x >= 1 << 2)							\
578       _x >>= 2, _c += 2;						\
579     _c += _x >> 1;							\
580     (count) =  W_TYPE_SIZE - 1 - _c;					\
581   } while (0)
582 /* similar to what gcc does for __builtin_ffs, but 0 based rather than 1
583    based, and we don't need a special case for x==0 here */
584 #define count_trailing_zeros(count, x)					\
585   do {									\
586     UWtype __ctz_x = (x);						\
587     __asm__ ("popcnt %0 = %1"						\
588 	     : "=r" (count)						\
589 	     : "r" ((__ctz_x-1) & ~__ctz_x));				\
590   } while (0)
591 #define UMUL_TIME 14
592 #endif
593 
594 #if defined (__M32R__) && W_TYPE_SIZE == 32
595 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
596   /* The cmp clears the condition bit.  */ \
597   __asm__ ("cmp %0,%0\n\taddx %1,%5\n\taddx %0,%3"			\
598 	   : "=r" ((USItype) (sh)),					\
599 	     "=&r" ((USItype) (sl))					\
600 	   : "0" ((USItype) (ah)),					\
601 	     "r" ((USItype) (bh)),					\
602 	     "1" ((USItype) (al)),					\
603 	     "r" ((USItype) (bl))					\
604 	   : "cbit")
605 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
606   /* The cmp clears the condition bit.  */ \
607   __asm__ ("cmp %0,%0\n\tsubx %1,%5\n\tsubx %0,%3"			\
608 	   : "=r" ((USItype) (sh)),					\
609 	     "=&r" ((USItype) (sl))					\
610 	   : "0" ((USItype) (ah)),					\
611 	     "r" ((USItype) (bh)),					\
612 	     "1" ((USItype) (al)),					\
613 	     "r" ((USItype) (bl))					\
614 	   : "cbit")
615 #endif /* __M32R__ */
616 
617 #if defined (__mc68000__) && W_TYPE_SIZE == 32
618 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
619   __asm__ ("add%.l %5,%1\n\taddx%.l %3,%0"				\
620 	   : "=d" ((USItype) (sh)),					\
621 	     "=&d" ((USItype) (sl))					\
622 	   : "%0" ((USItype) (ah)),					\
623 	     "d" ((USItype) (bh)),					\
624 	     "%1" ((USItype) (al)),					\
625 	     "g" ((USItype) (bl)))
626 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
627   __asm__ ("sub%.l %5,%1\n\tsubx%.l %3,%0"				\
628 	   : "=d" ((USItype) (sh)),					\
629 	     "=&d" ((USItype) (sl))					\
630 	   : "0" ((USItype) (ah)),					\
631 	     "d" ((USItype) (bh)),					\
632 	     "1" ((USItype) (al)),					\
633 	     "g" ((USItype) (bl)))
634 
635 /* The '020, '030, '040, '060 and CPU32 have 32x32->64 and 64/32->32q-32r.  */
636 #if (defined (__mc68020__) && !defined (__mc68060__))
637 #define umul_ppmm(w1, w0, u, v) \
638   __asm__ ("mulu%.l %3,%1:%0"						\
639 	   : "=d" ((USItype) (w0)),					\
640 	     "=d" ((USItype) (w1))					\
641 	   : "%0" ((USItype) (u)),					\
642 	     "dmi" ((USItype) (v)))
643 #define UMUL_TIME 45
644 #define udiv_qrnnd(q, r, n1, n0, d) \
645   __asm__ ("divu%.l %4,%1:%0"						\
646 	   : "=d" ((USItype) (q)),					\
647 	     "=d" ((USItype) (r))					\
648 	   : "0" ((USItype) (n0)),					\
649 	     "1" ((USItype) (n1)),					\
650 	     "dmi" ((USItype) (d)))
651 #define UDIV_TIME 90
652 #define sdiv_qrnnd(q, r, n1, n0, d) \
653   __asm__ ("divs%.l %4,%1:%0"						\
654 	   : "=d" ((USItype) (q)),					\
655 	     "=d" ((USItype) (r))					\
656 	   : "0" ((USItype) (n0)),					\
657 	     "1" ((USItype) (n1)),					\
658 	     "dmi" ((USItype) (d)))
659 
660 #elif defined (__mcoldfire__) /* not mc68020 */
661 
662 #define umul_ppmm(xh, xl, a, b) \
663   __asm__ ("| Inlined umul_ppmm\n"					\
664 	   "	move%.l	%2,%/d0\n"					\
665 	   "	move%.l	%3,%/d1\n"					\
666 	   "	move%.l	%/d0,%/d2\n"					\
667 	   "	swap	%/d0\n"						\
668 	   "	move%.l	%/d1,%/d3\n"					\
669 	   "	swap	%/d1\n"						\
670 	   "	move%.w	%/d2,%/d4\n"					\
671 	   "	mulu	%/d3,%/d4\n"					\
672 	   "	mulu	%/d1,%/d2\n"					\
673 	   "	mulu	%/d0,%/d3\n"					\
674 	   "	mulu	%/d0,%/d1\n"					\
675 	   "	move%.l	%/d4,%/d0\n"					\
676 	   "	clr%.w	%/d0\n"						\
677 	   "	swap	%/d0\n"						\
678 	   "	add%.l	%/d0,%/d2\n"					\
679 	   "	add%.l	%/d3,%/d2\n"					\
680 	   "	jcc	1f\n"						\
681 	   "	add%.l	%#65536,%/d1\n"					\
682 	   "1:	swap	%/d2\n"						\
683 	   "	moveq	%#0,%/d0\n"					\
684 	   "	move%.w	%/d2,%/d0\n"					\
685 	   "	move%.w	%/d4,%/d2\n"					\
686 	   "	move%.l	%/d2,%1\n"					\
687 	   "	add%.l	%/d1,%/d0\n"					\
688 	   "	move%.l	%/d0,%0"					\
689 	   : "=g" ((USItype) (xh)),					\
690 	     "=g" ((USItype) (xl))					\
691 	   : "g" ((USItype) (a)),					\
692 	     "g" ((USItype) (b))					\
693 	   : "d0", "d1", "d2", "d3", "d4")
694 #define UMUL_TIME 100
695 #define UDIV_TIME 400
696 #else /* not ColdFire */
697 /* %/ inserts REGISTER_PREFIX, %# inserts IMMEDIATE_PREFIX.  */
698 #define umul_ppmm(xh, xl, a, b) \
699   __asm__ ("| Inlined umul_ppmm\n"					\
700 	   "	move%.l	%2,%/d0\n"					\
701 	   "	move%.l	%3,%/d1\n"					\
702 	   "	move%.l	%/d0,%/d2\n"					\
703 	   "	swap	%/d0\n"						\
704 	   "	move%.l	%/d1,%/d3\n"					\
705 	   "	swap	%/d1\n"						\
706 	   "	move%.w	%/d2,%/d4\n"					\
707 	   "	mulu	%/d3,%/d4\n"					\
708 	   "	mulu	%/d1,%/d2\n"					\
709 	   "	mulu	%/d0,%/d3\n"					\
710 	   "	mulu	%/d0,%/d1\n"					\
711 	   "	move%.l	%/d4,%/d0\n"					\
712 	   "	eor%.w	%/d0,%/d0\n"					\
713 	   "	swap	%/d0\n"						\
714 	   "	add%.l	%/d0,%/d2\n"					\
715 	   "	add%.l	%/d3,%/d2\n"					\
716 	   "	jcc	1f\n"						\
717 	   "	add%.l	%#65536,%/d1\n"					\
718 	   "1:	swap	%/d2\n"						\
719 	   "	moveq	%#0,%/d0\n"					\
720 	   "	move%.w	%/d2,%/d0\n"					\
721 	   "	move%.w	%/d4,%/d2\n"					\
722 	   "	move%.l	%/d2,%1\n"					\
723 	   "	add%.l	%/d1,%/d0\n"					\
724 	   "	move%.l	%/d0,%0"					\
725 	   : "=g" ((USItype) (xh)),					\
726 	     "=g" ((USItype) (xl))					\
727 	   : "g" ((USItype) (a)),					\
728 	     "g" ((USItype) (b))					\
729 	   : "d0", "d1", "d2", "d3", "d4")
730 #define UMUL_TIME 100
731 #define UDIV_TIME 400
732 
733 #endif /* not mc68020 */
734 
735 /* The '020, '030, '040 and '060 have bitfield insns.
736    cpu32 disguises as a 68020, but lacks them.  */
737 #if defined (__mc68020__) && !defined (__mcpu32__)
738 #define count_leading_zeros(count, x) \
739   __asm__ ("bfffo %1{%b2:%b2},%0"					\
740 	   : "=d" ((USItype) (count))					\
741 	   : "od" ((USItype) (x)), "n" (0))
742 /* Some ColdFire architectures have a ff1 instruction supported via
743    __builtin_clz. */
744 #elif defined (__mcfisaaplus__) || defined (__mcfisac__)
745 #define count_leading_zeros(count,x) ((count) = __builtin_clz (x))
746 #define COUNT_LEADING_ZEROS_0 32
747 #endif
748 #endif /* mc68000 */
749 
750 #if defined (__m88000__) && W_TYPE_SIZE == 32
751 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
752   __asm__ ("addu.co %1,%r4,%r5\n\taddu.ci %0,%r2,%r3"			\
753 	   : "=r" ((USItype) (sh)),					\
754 	     "=&r" ((USItype) (sl))					\
755 	   : "%rJ" ((USItype) (ah)),					\
756 	     "rJ" ((USItype) (bh)),					\
757 	     "%rJ" ((USItype) (al)),					\
758 	     "rJ" ((USItype) (bl)))
759 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
760   __asm__ ("subu.co %1,%r4,%r5\n\tsubu.ci %0,%r2,%r3"			\
761 	   : "=r" ((USItype) (sh)),					\
762 	     "=&r" ((USItype) (sl))					\
763 	   : "rJ" ((USItype) (ah)),					\
764 	     "rJ" ((USItype) (bh)),					\
765 	     "rJ" ((USItype) (al)),					\
766 	     "rJ" ((USItype) (bl)))
767 #define count_leading_zeros(count, x) \
768   do {									\
769     USItype __cbtmp;							\
770     __asm__ ("ff1 %0,%1"						\
771 	     : "=r" (__cbtmp)						\
772 	     : "r" ((USItype) (x)));					\
773     (count) = __cbtmp ^ 31;						\
774   } while (0)
775 #define COUNT_LEADING_ZEROS_0 63 /* sic */
776 #if defined (__mc88110__)
777 #define umul_ppmm(wh, wl, u, v) \
778   do {									\
779     union {UDItype __ll;						\
780 	   struct {USItype __h, __l;} __i;				\
781 	  } __xx;							\
782     __asm__ ("mulu.d	%0,%1,%2"					\
783 	     : "=r" (__xx.__ll)						\
784 	     : "r" ((USItype) (u)),					\
785 	       "r" ((USItype) (v)));					\
786     (wh) = __xx.__i.__h;						\
787     (wl) = __xx.__i.__l;						\
788   } while (0)
789 #define udiv_qrnnd(q, r, n1, n0, d) \
790   ({union {UDItype __ll;						\
791 	   struct {USItype __h, __l;} __i;				\
792 	  } __xx;							\
793   USItype __q;								\
794   __xx.__i.__h = (n1); __xx.__i.__l = (n0);				\
795   __asm__ ("divu.d %0,%1,%2"						\
796 	   : "=r" (__q)							\
797 	   : "r" (__xx.__ll),						\
798 	     "r" ((USItype) (d)));					\
799   (r) = (n0) - __q * (d); (q) = __q; })
800 #define UMUL_TIME 5
801 #define UDIV_TIME 25
802 #else
803 #define UMUL_TIME 17
804 #define UDIV_TIME 150
805 #endif /* __mc88110__ */
806 #endif /* __m88000__ */
807 
808 #if defined (__mn10300__)
809 # if defined (__AM33__)
810 #  define count_leading_zeros(COUNT,X)	((COUNT) = __builtin_clz (X))
811 #  define umul_ppmm(w1, w0, u, v)		\
812     asm("mulu %3,%2,%1,%0" : "=r"(w0), "=r"(w1) : "r"(u), "r"(v))
813 #  define smul_ppmm(w1, w0, u, v)		\
814     asm("mul %3,%2,%1,%0" : "=r"(w0), "=r"(w1) : "r"(u), "r"(v))
815 # else
816 #  define umul_ppmm(w1, w0, u, v)		\
817     asm("nop; nop; mulu %3,%0" : "=d"(w0), "=z"(w1) : "%0"(u), "d"(v))
818 #  define smul_ppmm(w1, w0, u, v)		\
819     asm("nop; nop; mul %3,%0" : "=d"(w0), "=z"(w1) : "%0"(u), "d"(v))
820 # endif
821 # define add_ssaaaa(sh, sl, ah, al, bh, bl)	\
822   do {						\
823     DWunion __s, __a, __b;			\
824     __a.s.low = (al); __a.s.high = (ah);	\
825     __b.s.low = (bl); __b.s.high = (bh);	\
826     __s.ll = __a.ll + __b.ll;			\
827     (sl) = __s.s.low; (sh) = __s.s.high;	\
828   } while (0)
829 # define sub_ddmmss(sh, sl, ah, al, bh, bl)	\
830   do {						\
831     DWunion __s, __a, __b;			\
832     __a.s.low = (al); __a.s.high = (ah);	\
833     __b.s.low = (bl); __b.s.high = (bh);	\
834     __s.ll = __a.ll - __b.ll;			\
835     (sl) = __s.s.low; (sh) = __s.s.high;	\
836   } while (0)
837 # define udiv_qrnnd(q, r, nh, nl, d)		\
838   asm("divu %2,%0" : "=D"(q), "=z"(r) : "D"(d), "0"(nl), "1"(nh))
839 # define sdiv_qrnnd(q, r, nh, nl, d)		\
840   asm("div %2,%0" : "=D"(q), "=z"(r) : "D"(d), "0"(nl), "1"(nh))
841 # define UMUL_TIME 3
842 # define UDIV_TIME 38
843 #endif
844 
845 #if defined (__mips__) && W_TYPE_SIZE == 32
846 #define umul_ppmm(w1, w0, u, v)						\
847   do {									\
848     UDItype __x = (UDItype) (USItype) (u) * (USItype) (v);		\
849     (w1) = (USItype) (__x >> 32);					\
850     (w0) = (USItype) (__x);						\
851   } while (0)
852 #define UMUL_TIME 10
853 #define UDIV_TIME 100
854 
855 #if (__mips == 32 || __mips == 64) && ! defined (__mips16)
856 #define count_leading_zeros(COUNT,X)	((COUNT) = __builtin_clz (X))
857 #define COUNT_LEADING_ZEROS_0 32
858 #endif
859 #endif /* __mips__ */
860 
861 #if defined (__ns32000__) && W_TYPE_SIZE == 32
862 #define umul_ppmm(w1, w0, u, v) \
863   ({union {UDItype __ll;						\
864 	   struct {USItype __l, __h;} __i;				\
865 	  } __xx;							\
866   __asm__ ("meid %2,%0"							\
867 	   : "=g" (__xx.__ll)						\
868 	   : "%0" ((USItype) (u)),					\
869 	     "g" ((USItype) (v)));					\
870   (w1) = __xx.__i.__h; (w0) = __xx.__i.__l;})
871 #define __umulsidi3(u, v) \
872   ({UDItype __w;							\
873     __asm__ ("meid %2,%0"						\
874 	     : "=g" (__w)						\
875 	     : "%0" ((USItype) (u)),					\
876 	       "g" ((USItype) (v)));					\
877     __w; })
878 #define udiv_qrnnd(q, r, n1, n0, d) \
879   ({union {UDItype __ll;						\
880 	   struct {USItype __l, __h;} __i;				\
881 	  } __xx;							\
882   __xx.__i.__h = (n1); __xx.__i.__l = (n0);				\
883   __asm__ ("deid %2,%0"							\
884 	   : "=g" (__xx.__ll)						\
885 	   : "0" (__xx.__ll),						\
886 	     "g" ((USItype) (d)));					\
887   (r) = __xx.__i.__l; (q) = __xx.__i.__h; })
888 #define count_trailing_zeros(count,x) \
889   do {									\
890     __asm__ ("ffsd     %2,%0"						\
891 	    : "=r" ((USItype) (count))					\
892 	    : "0" ((USItype) 0),					\
893 	      "r" ((USItype) (x)));					\
894   } while (0)
895 #endif /* __ns32000__ */
896 
897 /* FIXME: We should test _IBMR2 here when we add assembly support for the
898    system vendor compilers.
899    FIXME: What's needed for gcc PowerPC VxWorks?  __vxworks__ is not good
900    enough, since that hits ARM and m68k too.  */
901 #if (defined (_ARCH_PPC)	/* AIX */				\
902      || defined (__powerpc__)	/* gcc */				\
903      || defined (__POWERPC__)	/* BEOS */				\
904      || defined (__ppc__)	/* Darwin */				\
905      || (defined (PPC) && ! defined (CPU_FAMILY)) /* gcc 2.7.x GNU&SysV */    \
906      || (defined (PPC) && defined (CPU_FAMILY)    /* VxWorks */               \
907 	 && CPU_FAMILY == PPC)                                                \
908      ) && W_TYPE_SIZE == 32
909 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
910   do {									\
911     if (__builtin_constant_p (bh) && (bh) == 0)				\
912       __asm__ ("add%I4c %1,%3,%4\n\taddze %0,%2"		\
913 	     : "=r" (sh), "=&r" (sl) : "r" (ah), "%r" (al), "rI" (bl));\
914     else if (__builtin_constant_p (bh) && (bh) == ~(USItype) 0)		\
915       __asm__ ("add%I4c %1,%3,%4\n\taddme %0,%2"		\
916 	     : "=r" (sh), "=&r" (sl) : "r" (ah), "%r" (al), "rI" (bl));\
917     else								\
918       __asm__ ("add%I5c %1,%4,%5\n\tadde %0,%2,%3"		\
919 	     : "=r" (sh), "=&r" (sl)					\
920 	     : "%r" (ah), "r" (bh), "%r" (al), "rI" (bl));		\
921   } while (0)
922 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
923   do {									\
924     if (__builtin_constant_p (ah) && (ah) == 0)				\
925       __asm__ ("subf%I3c %1,%4,%3\n\tsubfze %0,%2"	\
926 	       : "=r" (sh), "=&r" (sl) : "r" (bh), "rI" (al), "r" (bl));\
927     else if (__builtin_constant_p (ah) && (ah) == ~(USItype) 0)		\
928       __asm__ ("subf%I3c %1,%4,%3\n\tsubfme %0,%2"	\
929 	       : "=r" (sh), "=&r" (sl) : "r" (bh), "rI" (al), "r" (bl));\
930     else if (__builtin_constant_p (bh) && (bh) == 0)			\
931       __asm__ ("subf%I3c %1,%4,%3\n\taddme %0,%2"		\
932 	       : "=r" (sh), "=&r" (sl) : "r" (ah), "rI" (al), "r" (bl));\
933     else if (__builtin_constant_p (bh) && (bh) == ~(USItype) 0)		\
934       __asm__ ("subf%I3c %1,%4,%3\n\taddze %0,%2"		\
935 	       : "=r" (sh), "=&r" (sl) : "r" (ah), "rI" (al), "r" (bl));\
936     else								\
937       __asm__ ("subf%I4c %1,%5,%4\n\tsubfe %0,%3,%2"	\
938 	       : "=r" (sh), "=&r" (sl)					\
939 	       : "r" (ah), "r" (bh), "rI" (al), "r" (bl));		\
940   } while (0)
941 #define count_leading_zeros(count, x) \
942   __asm__ ("cntlzw %0,%1" : "=r" (count) : "r" (x))
943 #define COUNT_LEADING_ZEROS_0 32
944 #if defined (_ARCH_PPC) || defined (__powerpc__) || defined (__POWERPC__) \
945   || defined (__ppc__)                                                    \
946   || (defined (PPC) && ! defined (CPU_FAMILY)) /* gcc 2.7.x GNU&SysV */       \
947   || (defined (PPC) && defined (CPU_FAMILY)    /* VxWorks */                  \
948 	 && CPU_FAMILY == PPC)
949 #define umul_ppmm(ph, pl, m0, m1) \
950   do {									\
951     USItype __m0 = (m0), __m1 = (m1);					\
952     __asm__ ("mulhwu %0,%1,%2" : "=r" (ph) : "%r" (m0), "r" (m1));	\
953     (pl) = __m0 * __m1;							\
954   } while (0)
955 #define UMUL_TIME 15
956 #define smul_ppmm(ph, pl, m0, m1) \
957   do {									\
958     SItype __m0 = (m0), __m1 = (m1);					\
959     __asm__ ("mulhw %0,%1,%2" : "=r" (ph) : "%r" (m0), "r" (m1));	\
960     (pl) = __m0 * __m1;							\
961   } while (0)
962 #define SMUL_TIME 14
963 #define UDIV_TIME 120
964 #endif
965 #endif /* 32-bit POWER architecture variants.  */
966 
967 /* We should test _IBMR2 here when we add assembly support for the system
968    vendor compilers.  */
969 #if (defined (_ARCH_PPC64) || defined (__powerpc64__)) && W_TYPE_SIZE == 64
970 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
971   do {									\
972     if (__builtin_constant_p (bh) && (bh) == 0)				\
973       __asm__ ("add%I4c %1,%3,%4\n\taddze %0,%2"		\
974 	     : "=r" (sh), "=&r" (sl) : "r" (ah), "%r" (al), "rI" (bl));\
975     else if (__builtin_constant_p (bh) && (bh) == ~(UDItype) 0)		\
976       __asm__ ("add%I4c %1,%3,%4\n\taddme %0,%2"		\
977 	     : "=r" (sh), "=&r" (sl) : "r" (ah), "%r" (al), "rI" (bl));\
978     else								\
979       __asm__ ("add%I5c %1,%4,%5\n\tadde %0,%2,%3"		\
980 	     : "=r" (sh), "=&r" (sl)					\
981 	     : "%r" (ah), "r" (bh), "%r" (al), "rI" (bl));		\
982   } while (0)
983 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
984   do {									\
985     if (__builtin_constant_p (ah) && (ah) == 0)				\
986       __asm__ ("subf%I3c %1,%4,%3\n\tsubfze %0,%2"	\
987 	       : "=r" (sh), "=&r" (sl) : "r" (bh), "rI" (al), "r" (bl));\
988     else if (__builtin_constant_p (ah) && (ah) == ~(UDItype) 0)		\
989       __asm__ ("subf%I3c %1,%4,%3\n\tsubfme %0,%2"	\
990 	       : "=r" (sh), "=&r" (sl) : "r" (bh), "rI" (al), "r" (bl));\
991     else if (__builtin_constant_p (bh) && (bh) == 0)			\
992       __asm__ ("subf%I3c %1,%4,%3\n\taddme %0,%2"		\
993 	       : "=r" (sh), "=&r" (sl) : "r" (ah), "rI" (al), "r" (bl));\
994     else if (__builtin_constant_p (bh) && (bh) == ~(UDItype) 0)		\
995       __asm__ ("subf%I3c %1,%4,%3\n\taddze %0,%2"		\
996 	       : "=r" (sh), "=&r" (sl) : "r" (ah), "rI" (al), "r" (bl));\
997     else								\
998       __asm__ ("subf%I4c %1,%5,%4\n\tsubfe %0,%3,%2"	\
999 	       : "=r" (sh), "=&r" (sl)					\
1000 	       : "r" (ah), "r" (bh), "rI" (al), "r" (bl));		\
1001   } while (0)
1002 #define count_leading_zeros(count, x) \
1003   __asm__ ("cntlzd %0,%1" : "=r" (count) : "r" (x))
1004 #define COUNT_LEADING_ZEROS_0 64
1005 #define umul_ppmm(ph, pl, m0, m1) \
1006   do {									\
1007     UDItype __m0 = (m0), __m1 = (m1);					\
1008     __asm__ ("mulhdu %0,%1,%2" : "=r" (ph) : "%r" (m0), "r" (m1));	\
1009     (pl) = __m0 * __m1;							\
1010   } while (0)
1011 #define UMUL_TIME 15
1012 #define smul_ppmm(ph, pl, m0, m1) \
1013   do {									\
1014     DItype __m0 = (m0), __m1 = (m1);					\
1015     __asm__ ("mulhd %0,%1,%2" : "=r" (ph) : "%r" (m0), "r" (m1));	\
1016     (pl) = __m0 * __m1;							\
1017   } while (0)
1018 #define SMUL_TIME 14  /* ??? */
1019 #define UDIV_TIME 120 /* ??? */
1020 #endif /* 64-bit PowerPC.  */
1021 
1022 #if defined (__ibm032__) /* RT/ROMP */ && W_TYPE_SIZE == 32
1023 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
1024   __asm__ ("a %1,%5\n\tae %0,%3"					\
1025 	   : "=r" ((USItype) (sh)),					\
1026 	     "=&r" ((USItype) (sl))					\
1027 	   : "%0" ((USItype) (ah)),					\
1028 	     "r" ((USItype) (bh)),					\
1029 	     "%1" ((USItype) (al)),					\
1030 	     "r" ((USItype) (bl)))
1031 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
1032   __asm__ ("s %1,%5\n\tse %0,%3"					\
1033 	   : "=r" ((USItype) (sh)),					\
1034 	     "=&r" ((USItype) (sl))					\
1035 	   : "0" ((USItype) (ah)),					\
1036 	     "r" ((USItype) (bh)),					\
1037 	     "1" ((USItype) (al)),					\
1038 	     "r" ((USItype) (bl)))
1039 #define umul_ppmm(ph, pl, m0, m1) \
1040   do {									\
1041     USItype __m0 = (m0), __m1 = (m1);					\
1042     __asm__ (								\
1043        "s	r2,r2\n"						\
1044 "	mts	r10,%2\n"						\
1045 "	m	r2,%3\n"						\
1046 "	m	r2,%3\n"						\
1047 "	m	r2,%3\n"						\
1048 "	m	r2,%3\n"						\
1049 "	m	r2,%3\n"						\
1050 "	m	r2,%3\n"						\
1051 "	m	r2,%3\n"						\
1052 "	m	r2,%3\n"						\
1053 "	m	r2,%3\n"						\
1054 "	m	r2,%3\n"						\
1055 "	m	r2,%3\n"						\
1056 "	m	r2,%3\n"						\
1057 "	m	r2,%3\n"						\
1058 "	m	r2,%3\n"						\
1059 "	m	r2,%3\n"						\
1060 "	m	r2,%3\n"						\
1061 "	cas	%0,r2,r0\n"						\
1062 "	mfs	r10,%1"							\
1063 	     : "=r" ((USItype) (ph)),					\
1064 	       "=r" ((USItype) (pl))					\
1065 	     : "%r" (__m0),						\
1066 		"r" (__m1)						\
1067 	     : "r2");							\
1068     (ph) += ((((SItype) __m0 >> 31) & __m1)				\
1069 	     + (((SItype) __m1 >> 31) & __m0));				\
1070   } while (0)
1071 #define UMUL_TIME 20
1072 #define UDIV_TIME 200
1073 #define count_leading_zeros(count, x) \
1074   do {									\
1075     if ((x) >= 0x10000)							\
1076       __asm__ ("clz	%0,%1"						\
1077 	       : "=r" ((USItype) (count))				\
1078 	       : "r" ((USItype) (x) >> 16));				\
1079     else								\
1080       {									\
1081 	__asm__ ("clz	%0,%1"						\
1082 		 : "=r" ((USItype) (count))				\
1083 		 : "r" ((USItype) (x)));					\
1084 	(count) += 16;							\
1085       }									\
1086   } while (0)
1087 #endif
1088 
1089 #if defined(__sh__) && (!defined (__SHMEDIA__) || !__SHMEDIA__) && W_TYPE_SIZE == 32
1090 #ifndef __sh1__
1091 #define umul_ppmm(w1, w0, u, v) \
1092   __asm__ (								\
1093        "dmulu.l	%2,%3\n\tsts%M1	macl,%1\n\tsts%M0	mach,%0"	\
1094 	   : "=r<" ((USItype)(w1)),					\
1095 	     "=r<" ((USItype)(w0))					\
1096 	   : "r" ((USItype)(u)),					\
1097 	     "r" ((USItype)(v))						\
1098 	   : "macl", "mach")
1099 #define UMUL_TIME 5
1100 #endif
1101 
1102 /* This is the same algorithm as __udiv_qrnnd_c.  */
1103 #define UDIV_NEEDS_NORMALIZATION 1
1104 
1105 #ifdef __FDPIC__
1106 /* FDPIC needs a special version of the asm fragment to extract the
1107    code address from the function descriptor. __udiv_qrnnd_16 is
1108    assumed to be local and not to use the GOT, so loading r12 is
1109    not needed. */
1110 #define udiv_qrnnd(q, r, n1, n0, d) \
1111   do {									\
1112     extern UWtype __udiv_qrnnd_16 (UWtype, UWtype)			\
1113 			__attribute__ ((visibility ("hidden")));	\
1114     /* r0: rn r1: qn */ /* r0: n1 r4: n0 r5: d r6: d1 */ /* r2: __m */	\
1115     __asm__ (								\
1116 	"mov%M4	%4,r5\n"						\
1117 "	swap.w	%3,r4\n"						\
1118 "	swap.w	r5,r6\n"						\
1119 "	mov.l	@%5,r2\n"						\
1120 "	jsr	@r2\n"							\
1121 "	shll16	r6\n"							\
1122 "	swap.w	r4,r4\n"						\
1123 "	mov.l	@%5,r2\n"						\
1124 "	jsr	@r2\n"							\
1125 "	swap.w	r1,%0\n"						\
1126 "	or	r1,%0"							\
1127 	: "=r" (q), "=&z" (r)						\
1128 	: "1" (n1), "r" (n0), "rm" (d), "r" (&__udiv_qrnnd_16)		\
1129 	: "r1", "r2", "r4", "r5", "r6", "pr", "t");			\
1130   } while (0)
1131 #else
1132 #define udiv_qrnnd(q, r, n1, n0, d) \
1133   do {									\
1134     extern UWtype __udiv_qrnnd_16 (UWtype, UWtype)			\
1135 			__attribute__ ((visibility ("hidden")));	\
1136     /* r0: rn r1: qn */ /* r0: n1 r4: n0 r5: d r6: d1 */ /* r2: __m */	\
1137     __asm__ (								\
1138 	"mov%M4 %4,r5\n"						\
1139 "	swap.w %3,r4\n"							\
1140 "	swap.w r5,r6\n"							\
1141 "	jsr @%5\n"							\
1142 "	shll16 r6\n"							\
1143 "	swap.w r4,r4\n"							\
1144 "	jsr @%5\n"							\
1145 "	swap.w r1,%0\n"							\
1146 "	or r1,%0"							\
1147 	: "=r" (q), "=&z" (r)						\
1148 	: "1" (n1), "r" (n0), "rm" (d), "r" (&__udiv_qrnnd_16)		\
1149 	: "r1", "r2", "r4", "r5", "r6", "pr", "t");			\
1150   } while (0)
1151 #endif /* __FDPIC__  */
1152 
1153 #define UDIV_TIME 80
1154 
1155 #define sub_ddmmss(sh, sl, ah, al, bh, bl)				\
1156   __asm__ ("clrt;subc %5,%1; subc %4,%0"				\
1157 	   : "=r" (sh), "=r" (sl)					\
1158 	   : "0" (ah), "1" (al), "r" (bh), "r" (bl) : "t")
1159 
1160 #endif /* __sh__ */
1161 
1162 #if defined (__SH5__) && defined (__SHMEDIA__) && __SHMEDIA__ && W_TYPE_SIZE == 32
1163 #define __umulsidi3(u,v) ((UDItype)(USItype)u*(USItype)v)
1164 #define count_leading_zeros(count, x) \
1165   do									\
1166     {									\
1167       UDItype x_ = (USItype)(x);					\
1168       SItype c_;							\
1169 									\
1170       __asm__ ("nsb %1, %0" : "=r" (c_) : "r" (x_));			\
1171       (count) = c_ - 31;						\
1172     }									\
1173   while (0)
1174 #define COUNT_LEADING_ZEROS_0 32
1175 #endif
1176 
1177 #if defined (__sparc__) && !defined (__arch64__) && !defined (__sparcv9) \
1178     && W_TYPE_SIZE == 32
1179 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
1180   __asm__ ("addcc %r4,%5,%1\n\taddx %r2,%3,%0"				\
1181 	   : "=r" ((USItype) (sh)),					\
1182 	     "=&r" ((USItype) (sl))					\
1183 	   : "%rJ" ((USItype) (ah)),					\
1184 	     "rI" ((USItype) (bh)),					\
1185 	     "%rJ" ((USItype) (al)),					\
1186 	     "rI" ((USItype) (bl))					\
1187 	   __CLOBBER_CC)
1188 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
1189   __asm__ ("subcc %r4,%5,%1\n\tsubx %r2,%3,%0"				\
1190 	   : "=r" ((USItype) (sh)),					\
1191 	     "=&r" ((USItype) (sl))					\
1192 	   : "rJ" ((USItype) (ah)),					\
1193 	     "rI" ((USItype) (bh)),					\
1194 	     "rJ" ((USItype) (al)),					\
1195 	     "rI" ((USItype) (bl))					\
1196 	   __CLOBBER_CC)
1197 #if defined (__sparc_v9__)
1198 #define umul_ppmm(w1, w0, u, v) \
1199   do {									\
1200     register USItype __g1 asm ("g1");					\
1201     __asm__ ("umul\t%2,%3,%1\n\t"					\
1202 	     "srlx\t%1, 32, %0"						\
1203 	     : "=r" ((USItype) (w1)),					\
1204 	       "=r" (__g1)						\
1205 	     : "r" ((USItype) (u)),					\
1206 	       "r" ((USItype) (v)));					\
1207     (w0) = __g1;							\
1208   } while (0)
1209 #define udiv_qrnnd(__q, __r, __n1, __n0, __d) \
1210   __asm__ ("mov\t%2,%%y\n\t"						\
1211 	   "udiv\t%3,%4,%0\n\t"						\
1212 	   "umul\t%0,%4,%1\n\t"						\
1213 	   "sub\t%3,%1,%1"						\
1214 	   : "=&r" ((USItype) (__q)),					\
1215 	     "=&r" ((USItype) (__r))					\
1216 	   : "r" ((USItype) (__n1)),					\
1217 	     "r" ((USItype) (__n0)),					\
1218 	     "r" ((USItype) (__d)))
1219 #else
1220 #if defined (__sparc_v8__)
1221 #define umul_ppmm(w1, w0, u, v) \
1222   __asm__ ("umul %2,%3,%1;rd %%y,%0"					\
1223 	   : "=r" ((USItype) (w1)),					\
1224 	     "=r" ((USItype) (w0))					\
1225 	   : "r" ((USItype) (u)),					\
1226 	     "r" ((USItype) (v)))
1227 #define udiv_qrnnd(__q, __r, __n1, __n0, __d) \
1228   __asm__ ("mov %2,%%y;nop;nop;nop;udiv %3,%4,%0;umul %0,%4,%1;sub %3,%1,%1"\
1229 	   : "=&r" ((USItype) (__q)),					\
1230 	     "=&r" ((USItype) (__r))					\
1231 	   : "r" ((USItype) (__n1)),					\
1232 	     "r" ((USItype) (__n0)),					\
1233 	     "r" ((USItype) (__d)))
1234 #else
1235 #if defined (__sparclite__)
1236 /* This has hardware multiply but not divide.  It also has two additional
1237    instructions scan (ffs from high bit) and divscc.  */
1238 #define umul_ppmm(w1, w0, u, v) \
1239   __asm__ ("umul %2,%3,%1;rd %%y,%0"					\
1240 	   : "=r" ((USItype) (w1)),					\
1241 	     "=r" ((USItype) (w0))					\
1242 	   : "r" ((USItype) (u)),					\
1243 	     "r" ((USItype) (v)))
1244 #define udiv_qrnnd(q, r, n1, n0, d) \
1245   __asm__ ("! Inlined udiv_qrnnd\n"					\
1246 "	wr	%%g0,%2,%%y	! Not a delayed write for sparclite\n"	\
1247 "	tst	%%g0\n"							\
1248 "	divscc	%3,%4,%%g1\n"						\
1249 "	divscc	%%g1,%4,%%g1\n"						\
1250 "	divscc	%%g1,%4,%%g1\n"						\
1251 "	divscc	%%g1,%4,%%g1\n"						\
1252 "	divscc	%%g1,%4,%%g1\n"						\
1253 "	divscc	%%g1,%4,%%g1\n"						\
1254 "	divscc	%%g1,%4,%%g1\n"						\
1255 "	divscc	%%g1,%4,%%g1\n"						\
1256 "	divscc	%%g1,%4,%%g1\n"						\
1257 "	divscc	%%g1,%4,%%g1\n"						\
1258 "	divscc	%%g1,%4,%%g1\n"						\
1259 "	divscc	%%g1,%4,%%g1\n"						\
1260 "	divscc	%%g1,%4,%%g1\n"						\
1261 "	divscc	%%g1,%4,%%g1\n"						\
1262 "	divscc	%%g1,%4,%%g1\n"						\
1263 "	divscc	%%g1,%4,%%g1\n"						\
1264 "	divscc	%%g1,%4,%%g1\n"						\
1265 "	divscc	%%g1,%4,%%g1\n"						\
1266 "	divscc	%%g1,%4,%%g1\n"						\
1267 "	divscc	%%g1,%4,%%g1\n"						\
1268 "	divscc	%%g1,%4,%%g1\n"						\
1269 "	divscc	%%g1,%4,%%g1\n"						\
1270 "	divscc	%%g1,%4,%%g1\n"						\
1271 "	divscc	%%g1,%4,%%g1\n"						\
1272 "	divscc	%%g1,%4,%%g1\n"						\
1273 "	divscc	%%g1,%4,%%g1\n"						\
1274 "	divscc	%%g1,%4,%%g1\n"						\
1275 "	divscc	%%g1,%4,%%g1\n"						\
1276 "	divscc	%%g1,%4,%%g1\n"						\
1277 "	divscc	%%g1,%4,%%g1\n"						\
1278 "	divscc	%%g1,%4,%%g1\n"						\
1279 "	divscc	%%g1,%4,%0\n"						\
1280 "	rd	%%y,%1\n"						\
1281 "	bl,a 1f\n"							\
1282 "	add	%1,%4,%1\n"						\
1283 "1:	! End of inline udiv_qrnnd"					\
1284 	   : "=r" ((USItype) (q)),					\
1285 	     "=r" ((USItype) (r))					\
1286 	   : "r" ((USItype) (n1)),					\
1287 	     "r" ((USItype) (n0)),					\
1288 	     "rI" ((USItype) (d))					\
1289 	   : "g1" __AND_CLOBBER_CC)
1290 #define UDIV_TIME 37
1291 #define count_leading_zeros(count, x) \
1292   do {                                                                  \
1293   __asm__ ("scan %1,1,%0"                                               \
1294 	   : "=r" ((USItype) (count))                                   \
1295 	   : "r" ((USItype) (x)));					\
1296   } while (0)
1297 /* Early sparclites return 63 for an argument of 0, but they warn that future
1298    implementations might change this.  Therefore, leave COUNT_LEADING_ZEROS_0
1299    undefined.  */
1300 #else
1301 /* SPARC without integer multiplication and divide instructions.
1302    (i.e. at least Sun4/20,40,60,65,75,110,260,280,330,360,380,470,490) */
1303 #define umul_ppmm(w1, w0, u, v) \
1304   __asm__ ("! Inlined umul_ppmm\n"					\
1305 "	wr	%%g0,%2,%%y	! SPARC has 0-3 delay insn after a wr\n"\
1306 "	sra	%3,31,%%o5	! Don't move this insn\n"		\
1307 "	and	%2,%%o5,%%o5	! Don't move this insn\n"		\
1308 "	andcc	%%g0,0,%%g1	! Don't move this insn\n"		\
1309 "	mulscc	%%g1,%3,%%g1\n"						\
1310 "	mulscc	%%g1,%3,%%g1\n"						\
1311 "	mulscc	%%g1,%3,%%g1\n"						\
1312 "	mulscc	%%g1,%3,%%g1\n"						\
1313 "	mulscc	%%g1,%3,%%g1\n"						\
1314 "	mulscc	%%g1,%3,%%g1\n"						\
1315 "	mulscc	%%g1,%3,%%g1\n"						\
1316 "	mulscc	%%g1,%3,%%g1\n"						\
1317 "	mulscc	%%g1,%3,%%g1\n"						\
1318 "	mulscc	%%g1,%3,%%g1\n"						\
1319 "	mulscc	%%g1,%3,%%g1\n"						\
1320 "	mulscc	%%g1,%3,%%g1\n"						\
1321 "	mulscc	%%g1,%3,%%g1\n"						\
1322 "	mulscc	%%g1,%3,%%g1\n"						\
1323 "	mulscc	%%g1,%3,%%g1\n"						\
1324 "	mulscc	%%g1,%3,%%g1\n"						\
1325 "	mulscc	%%g1,%3,%%g1\n"						\
1326 "	mulscc	%%g1,%3,%%g1\n"						\
1327 "	mulscc	%%g1,%3,%%g1\n"						\
1328 "	mulscc	%%g1,%3,%%g1\n"						\
1329 "	mulscc	%%g1,%3,%%g1\n"						\
1330 "	mulscc	%%g1,%3,%%g1\n"						\
1331 "	mulscc	%%g1,%3,%%g1\n"						\
1332 "	mulscc	%%g1,%3,%%g1\n"						\
1333 "	mulscc	%%g1,%3,%%g1\n"						\
1334 "	mulscc	%%g1,%3,%%g1\n"						\
1335 "	mulscc	%%g1,%3,%%g1\n"						\
1336 "	mulscc	%%g1,%3,%%g1\n"						\
1337 "	mulscc	%%g1,%3,%%g1\n"						\
1338 "	mulscc	%%g1,%3,%%g1\n"						\
1339 "	mulscc	%%g1,%3,%%g1\n"						\
1340 "	mulscc	%%g1,%3,%%g1\n"						\
1341 "	mulscc	%%g1,0,%%g1\n"						\
1342 "	add	%%g1,%%o5,%0\n"						\
1343 "	rd	%%y,%1"							\
1344 	   : "=r" ((USItype) (w1)),					\
1345 	     "=r" ((USItype) (w0))					\
1346 	   : "%rI" ((USItype) (u)),					\
1347 	     "r" ((USItype) (v))						\
1348 	   : "g1", "o5" __AND_CLOBBER_CC)
1349 #define UMUL_TIME 39		/* 39 instructions */
1350 /* It's quite necessary to add this much assembler for the sparc.
1351    The default udiv_qrnnd (in C) is more than 10 times slower!  */
1352 #define udiv_qrnnd(__q, __r, __n1, __n0, __d) \
1353   __asm__ ("! Inlined udiv_qrnnd\n"					\
1354 "	mov	32,%%g1\n"						\
1355 "	subcc	%1,%2,%%g0\n"						\
1356 "1:	bcs	5f\n"							\
1357 "	 addxcc %0,%0,%0	! shift n1n0 and a q-bit in lsb\n"	\
1358 "	sub	%1,%2,%1	! this kills msb of n\n"		\
1359 "	addx	%1,%1,%1	! so this can't give carry\n"		\
1360 "	subcc	%%g1,1,%%g1\n"						\
1361 "2:	bne	1b\n"							\
1362 "	 subcc	%1,%2,%%g0\n"						\
1363 "	bcs	3f\n"							\
1364 "	 addxcc %0,%0,%0	! shift n1n0 and a q-bit in lsb\n"	\
1365 "	b	3f\n"							\
1366 "	 sub	%1,%2,%1	! this kills msb of n\n"		\
1367 "4:	sub	%1,%2,%1\n"						\
1368 "5:	addxcc	%1,%1,%1\n"						\
1369 "	bcc	2b\n"							\
1370 "	 subcc	%%g1,1,%%g1\n"						\
1371 "! Got carry from n.  Subtract next step to cancel this carry.\n"	\
1372 "	bne	4b\n"							\
1373 "	 addcc	%0,%0,%0	! shift n1n0 and a 0-bit in lsb\n"	\
1374 "	sub	%1,%2,%1\n"						\
1375 "3:	xnor	%0,0,%0\n"						\
1376 "	! End of inline udiv_qrnnd"					\
1377 	   : "=&r" ((USItype) (__q)),					\
1378 	     "=&r" ((USItype) (__r))					\
1379 	   : "r" ((USItype) (__d)),					\
1380 	     "1" ((USItype) (__n1)),					\
1381 	     "0" ((USItype) (__n0)) : "g1" __AND_CLOBBER_CC)
1382 #define UDIV_TIME (3+7*32)	/* 7 instructions/iteration. 32 iterations.  */
1383 #endif /* __sparclite__ */
1384 #endif /* __sparc_v8__ */
1385 #endif /* __sparc_v9__ */
1386 #endif /* sparc32 */
1387 
1388 #if ((defined (__sparc__) && defined (__arch64__)) || defined (__sparcv9)) \
1389     && W_TYPE_SIZE == 64
1390 #define add_ssaaaa(sh, sl, ah, al, bh, bl)				\
1391   do {									\
1392     UDItype __carry = 0;						\
1393     __asm__ ("addcc\t%r5,%6,%1\n\t"					\
1394 	     "add\t%r3,%4,%0\n\t"					\
1395 	     "movcs\t%%xcc, 1, %2\n\t"					\
1396 	     "add\t%0, %2, %0"						\
1397 	     : "=r" ((UDItype)(sh)),				      	\
1398 	       "=&r" ((UDItype)(sl)),				      	\
1399 	       "+r" (__carry)				      		\
1400 	     : "%rJ" ((UDItype)(ah)),				     	\
1401 	       "rI" ((UDItype)(bh)),				      	\
1402 	       "%rJ" ((UDItype)(al)),				     	\
1403 	       "rI" ((UDItype)(bl))				       	\
1404 	     __CLOBBER_CC);						\
1405   } while (0)
1406 
1407 #define sub_ddmmss(sh, sl, ah, al, bh, bl)				\
1408   do {									\
1409     UDItype __carry = 0;						\
1410     __asm__ ("subcc\t%r5,%6,%1\n\t"					\
1411 	     "sub\t%r3,%4,%0\n\t"					\
1412 	     "movcs\t%%xcc, 1, %2\n\t"					\
1413 	     "sub\t%0, %2, %0"						\
1414 	     : "=r" ((UDItype)(sh)),				      	\
1415 	       "=&r" ((UDItype)(sl)),				      	\
1416 	       "+r" (__carry)				      		\
1417 	     : "%rJ" ((UDItype)(ah)),				     	\
1418 	       "rI" ((UDItype)(bh)),				      	\
1419 	       "%rJ" ((UDItype)(al)),				     	\
1420 	       "rI" ((UDItype)(bl))				       	\
1421 	     __CLOBBER_CC);						\
1422   } while (0)
1423 
1424 #define umul_ppmm(wh, wl, u, v)						\
1425   do {									\
1426 	  UDItype tmp1, tmp2, tmp3, tmp4;				\
1427 	  __asm__ __volatile__ (					\
1428 		   "srl %7,0,%3\n\t"					\
1429 		   "mulx %3,%6,%1\n\t"					\
1430 		   "srlx %6,32,%2\n\t"					\
1431 		   "mulx %2,%3,%4\n\t"					\
1432 		   "sllx %4,32,%5\n\t"					\
1433 		   "srl %6,0,%3\n\t"					\
1434 		   "sub %1,%5,%5\n\t"					\
1435 		   "srlx %5,32,%5\n\t"					\
1436 		   "addcc %4,%5,%4\n\t"					\
1437 		   "srlx %7,32,%5\n\t"					\
1438 		   "mulx %3,%5,%3\n\t"					\
1439 		   "mulx %2,%5,%5\n\t"					\
1440 		   "sethi %%hi(0x80000000),%2\n\t"			\
1441 		   "addcc %4,%3,%4\n\t"					\
1442 		   "srlx %4,32,%4\n\t"					\
1443 		   "add %2,%2,%2\n\t"					\
1444 		   "movcc %%xcc,%%g0,%2\n\t"				\
1445 		   "addcc %5,%4,%5\n\t"					\
1446 		   "sllx %3,32,%3\n\t"					\
1447 		   "add %1,%3,%1\n\t"					\
1448 		   "add %5,%2,%0"					\
1449 	   : "=r" ((UDItype)(wh)),					\
1450 	     "=&r" ((UDItype)(wl)),					\
1451 	     "=&r" (tmp1), "=&r" (tmp2), "=&r" (tmp3), "=&r" (tmp4)	\
1452 	   : "r" ((UDItype)(u)),					\
1453 	     "r" ((UDItype)(v))						\
1454 	   __CLOBBER_CC);						\
1455   } while (0)
1456 #define UMUL_TIME 96
1457 #define UDIV_TIME 230
1458 #endif /* sparc64 */
1459 
1460 #if defined (__vax__) && W_TYPE_SIZE == 32
1461 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
1462   __asm__ ("addl2 %5,%1\n\tadwc %3,%0"					\
1463 	   : "=g" ((USItype) (sh)),					\
1464 	     "=&g" ((USItype) (sl))					\
1465 	   : "%0" ((USItype) (ah)),					\
1466 	     "g" ((USItype) (bh)),					\
1467 	     "%1" ((USItype) (al)),					\
1468 	     "g" ((USItype) (bl)))
1469 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
1470   __asm__ ("subl2 %5,%1\n\tsbwc %3,%0"					\
1471 	   : "=g" ((USItype) (sh)),					\
1472 	     "=&g" ((USItype) (sl))					\
1473 	   : "0" ((USItype) (ah)),					\
1474 	     "g" ((USItype) (bh)),					\
1475 	     "1" ((USItype) (al)),					\
1476 	     "g" ((USItype) (bl)))
1477 #define umul_ppmm(xh, xl, m0, m1) \
1478   do {									\
1479     union {								\
1480 	UDItype __ll;							\
1481 	struct {USItype __l, __h;} __i;					\
1482       } __xx;								\
1483     USItype __m0 = (m0), __m1 = (m1);					\
1484     __asm__ ("emul %1,%2,$0,%0"						\
1485 	     : "=r" (__xx.__ll)						\
1486 	     : "g" (__m0),						\
1487 	       "g" (__m1));						\
1488     (xh) = __xx.__i.__h;						\
1489     (xl) = __xx.__i.__l;						\
1490     (xh) += ((((SItype) __m0 >> 31) & __m1)				\
1491 	     + (((SItype) __m1 >> 31) & __m0));				\
1492   } while (0)
1493 #define sdiv_qrnnd(q, r, n1, n0, d) \
1494   do {									\
1495     union {DItype __ll;							\
1496 	   struct {SItype __l, __h;} __i;				\
1497 	  } __xx;							\
1498     __xx.__i.__h = n1; __xx.__i.__l = n0;				\
1499     __asm__ ("ediv %3,%2,%0,%1"						\
1500 	     : "=g" (q), "=g" (r)					\
1501 	     : "g" (__xx.__ll), "g" (d));				\
1502   } while (0)
1503 #endif /* __vax__ */
1504 
1505 #ifdef _TMS320C6X
1506 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
1507   do									\
1508     {									\
1509       UDItype __ll;							\
1510       __asm__ ("addu .l1 %1, %2, %0"					\
1511 	       : "=a" (__ll) : "a" (al), "a" (bl));			\
1512       (sl) = (USItype)__ll;						\
1513       (sh) = ((USItype)(__ll >> 32)) + (ah) + (bh);			\
1514     }									\
1515   while (0)
1516 
1517 #ifdef _TMS320C6400_PLUS
1518 #define __umulsidi3(u,v) ((UDItype)(USItype)u*(USItype)v)
1519 #define umul_ppmm(w1, w0, u, v)						\
1520   do {									\
1521     UDItype __x = (UDItype) (USItype) (u) * (USItype) (v);		\
1522     (w1) = (USItype) (__x >> 32);					\
1523     (w0) = (USItype) (__x);						\
1524   } while (0)
1525 #endif  /* _TMS320C6400_PLUS */
1526 
1527 #define count_leading_zeros(count, x)	((count) = __builtin_clz (x))
1528 #ifdef _TMS320C6400
1529 #define count_trailing_zeros(count, x)	((count) = __builtin_ctz (x))
1530 #endif
1531 #define UMUL_TIME 4
1532 #define UDIV_TIME 40
1533 #endif /* _TMS320C6X */
1534 
1535 #if defined (__xtensa__) && W_TYPE_SIZE == 32
1536 /* This code is not Xtensa-configuration-specific, so rely on the compiler
1537    to expand builtin functions depending on what configuration features
1538    are available.  This avoids library calls when the operation can be
1539    performed in-line.  */
1540 #define umul_ppmm(w1, w0, u, v)						\
1541   do {									\
1542     DWunion __w;							\
1543     __w.ll = __builtin_umulsidi3 (u, v);				\
1544     w1 = __w.s.high;							\
1545     w0 = __w.s.low;							\
1546   } while (0)
1547 #define __umulsidi3(u, v)		__builtin_umulsidi3 (u, v)
1548 #define count_leading_zeros(COUNT, X)	((COUNT) = __builtin_clz (X))
1549 #define count_trailing_zeros(COUNT, X)	((COUNT) = __builtin_ctz (X))
1550 #endif /* __xtensa__ */
1551 
1552 #if defined xstormy16
1553 extern UHItype __stormy16_count_leading_zeros (UHItype);
1554 #define count_leading_zeros(count, x)					\
1555   do									\
1556     {									\
1557       UHItype size;							\
1558 									\
1559       /* We assume that W_TYPE_SIZE is a multiple of 16...  */		\
1560       for ((count) = 0, size = W_TYPE_SIZE; size; size -= 16)		\
1561 	{								\
1562 	  UHItype c;							\
1563 									\
1564 	  c = __clzhi2 ((x) >> (size - 16));				\
1565 	  (count) += c;							\
1566 	  if (c != 16)							\
1567 	    break;							\
1568 	}								\
1569     }									\
1570   while (0)
1571 #define COUNT_LEADING_ZEROS_0 W_TYPE_SIZE
1572 #endif
1573 
1574 #if defined (__z8000__) && W_TYPE_SIZE == 16
1575 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
1576   __asm__ ("add	%H1,%H5\n\tadc	%H0,%H3"				\
1577 	   : "=r" ((unsigned int)(sh)),					\
1578 	     "=&r" ((unsigned int)(sl))					\
1579 	   : "%0" ((unsigned int)(ah)),					\
1580 	     "r" ((unsigned int)(bh)),					\
1581 	     "%1" ((unsigned int)(al)),					\
1582 	     "rQR" ((unsigned int)(bl)))
1583 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
1584   __asm__ ("sub	%H1,%H5\n\tsbc	%H0,%H3"				\
1585 	   : "=r" ((unsigned int)(sh)),					\
1586 	     "=&r" ((unsigned int)(sl))					\
1587 	   : "0" ((unsigned int)(ah)),					\
1588 	     "r" ((unsigned int)(bh)),					\
1589 	     "1" ((unsigned int)(al)),					\
1590 	     "rQR" ((unsigned int)(bl)))
1591 #define umul_ppmm(xh, xl, m0, m1) \
1592   do {									\
1593     union {long int __ll;						\
1594 	   struct {unsigned int __h, __l;} __i;				\
1595 	  } __xx;							\
1596     unsigned int __m0 = (m0), __m1 = (m1);				\
1597     __asm__ ("mult	%S0,%H3"					\
1598 	     : "=r" (__xx.__i.__h),					\
1599 	       "=r" (__xx.__i.__l)					\
1600 	     : "%1" (__m0),						\
1601 	       "rQR" (__m1));						\
1602     (xh) = __xx.__i.__h; (xl) = __xx.__i.__l;				\
1603     (xh) += ((((signed int) __m0 >> 15) & __m1)				\
1604 	     + (((signed int) __m1 >> 15) & __m0));			\
1605   } while (0)
1606 #endif /* __z8000__ */
1607 
1608 #endif /* __GNUC__ */
1609 
1610 /* If this machine has no inline assembler, use C macros.  */
1611 
1612 #if !defined (add_ssaaaa)
1613 #define add_ssaaaa(sh, sl, ah, al, bh, bl) \
1614   do {									\
1615     UWtype __x;								\
1616     __x = (al) + (bl);							\
1617     (sh) = (ah) + (bh) + (__x < (al));					\
1618     (sl) = __x;								\
1619   } while (0)
1620 #endif
1621 
1622 #if !defined (sub_ddmmss)
1623 #define sub_ddmmss(sh, sl, ah, al, bh, bl) \
1624   do {									\
1625     UWtype __x;								\
1626     __x = (al) - (bl);							\
1627     (sh) = (ah) - (bh) - (__x > (al));					\
1628     (sl) = __x;								\
1629   } while (0)
1630 #endif
1631 
1632 /* If we lack umul_ppmm but have smul_ppmm, define umul_ppmm in terms of
1633    smul_ppmm.  */
1634 #if !defined (umul_ppmm) && defined (smul_ppmm)
1635 #define umul_ppmm(w1, w0, u, v)						\
1636   do {									\
1637     UWtype __w1;							\
1638     UWtype __xm0 = (u), __xm1 = (v);					\
1639     smul_ppmm (__w1, w0, __xm0, __xm1);					\
1640     (w1) = __w1 + (-(__xm0 >> (W_TYPE_SIZE - 1)) & __xm1)		\
1641 		+ (-(__xm1 >> (W_TYPE_SIZE - 1)) & __xm0);		\
1642   } while (0)
1643 #endif
1644 
1645 /* If we still don't have umul_ppmm, define it using plain C.  */
1646 #if !defined (umul_ppmm)
1647 #define umul_ppmm(w1, w0, u, v)						\
1648   do {									\
1649     UWtype __x0, __x1, __x2, __x3;					\
1650     UHWtype __ul, __vl, __uh, __vh;					\
1651 									\
1652     __ul = __ll_lowpart (u);						\
1653     __uh = __ll_highpart (u);						\
1654     __vl = __ll_lowpart (v);						\
1655     __vh = __ll_highpart (v);						\
1656 									\
1657     __x0 = (UWtype) __ul * __vl;					\
1658     __x1 = (UWtype) __ul * __vh;					\
1659     __x2 = (UWtype) __uh * __vl;					\
1660     __x3 = (UWtype) __uh * __vh;					\
1661 									\
1662     __x1 += __ll_highpart (__x0);/* this can't give carry */		\
1663     __x1 += __x2;		/* but this indeed can */		\
1664     if (__x1 < __x2)		/* did we get it? */			\
1665       __x3 += __ll_B;		/* yes, add it in the proper pos.  */	\
1666 									\
1667     (w1) = __x3 + __ll_highpart (__x1);					\
1668     (w0) = __ll_lowpart (__x1) * __ll_B + __ll_lowpart (__x0);		\
1669   } while (0)
1670 #endif
1671 
1672 #if !defined (__umulsidi3)
1673 #define __umulsidi3(u, v) \
1674   ({DWunion __w;							\
1675     umul_ppmm (__w.s.high, __w.s.low, u, v);				\
1676     __w.ll; })
1677 #endif
1678 
1679 /* Define this unconditionally, so it can be used for debugging.  */
1680 #define __udiv_qrnnd_c(q, r, n1, n0, d) \
1681   do {									\
1682     UWtype __d1, __d0, __q1, __q0;					\
1683     UWtype __r1, __r0, __m;						\
1684     __d1 = __ll_highpart (d);						\
1685     __d0 = __ll_lowpart (d);						\
1686 									\
1687     __r1 = (n1) % __d1;							\
1688     __q1 = (n1) / __d1;							\
1689     __m = (UWtype) __q1 * __d0;						\
1690     __r1 = __r1 * __ll_B | __ll_highpart (n0);				\
1691     if (__r1 < __m)							\
1692       {									\
1693 	__q1--, __r1 += (d);						\
1694 	if (__r1 >= (d)) /* i.e. we didn't get carry when adding to __r1 */\
1695 	  if (__r1 < __m)						\
1696 	    __q1--, __r1 += (d);					\
1697       }									\
1698     __r1 -= __m;							\
1699 									\
1700     __r0 = __r1 % __d1;							\
1701     __q0 = __r1 / __d1;							\
1702     __m = (UWtype) __q0 * __d0;						\
1703     __r0 = __r0 * __ll_B | __ll_lowpart (n0);				\
1704     if (__r0 < __m)							\
1705       {									\
1706 	__q0--, __r0 += (d);						\
1707 	if (__r0 >= (d))						\
1708 	  if (__r0 < __m)						\
1709 	    __q0--, __r0 += (d);					\
1710       }									\
1711     __r0 -= __m;							\
1712 									\
1713     (q) = (UWtype) __q1 * __ll_B | __q0;				\
1714     (r) = __r0;								\
1715   } while (0)
1716 
1717 /* If the processor has no udiv_qrnnd but sdiv_qrnnd, go through
1718    __udiv_w_sdiv (defined in libgcc or elsewhere).  */
1719 #if !defined (udiv_qrnnd) && defined (sdiv_qrnnd)
1720 #define udiv_qrnnd(q, r, nh, nl, d) \
1721   do {									\
1722     extern UWtype __udiv_w_sdiv (UWtype *, UWtype, UWtype, UWtype);	\
1723     UWtype __r;								\
1724     (q) = __udiv_w_sdiv (&__r, nh, nl, d);				\
1725     (r) = __r;								\
1726   } while (0)
1727 #endif
1728 
1729 /* If udiv_qrnnd was not defined for this processor, use __udiv_qrnnd_c.  */
1730 #if !defined (udiv_qrnnd)
1731 #define UDIV_NEEDS_NORMALIZATION 1
1732 #define udiv_qrnnd __udiv_qrnnd_c
1733 #endif
1734 
1735 #if !defined (count_leading_zeros)
1736 #define count_leading_zeros(count, x) \
1737   do {									\
1738     UWtype __xr = (x);							\
1739     UWtype __a;								\
1740 									\
1741     if (W_TYPE_SIZE <= 32)						\
1742       {									\
1743 	__a = __xr < ((UWtype)1<<2*__BITS4)				\
1744 	  ? (__xr < ((UWtype)1<<__BITS4) ? 0 : __BITS4)			\
1745 	  : (__xr < ((UWtype)1<<3*__BITS4) ?  2*__BITS4 : 3*__BITS4);	\
1746       }									\
1747     else								\
1748       {									\
1749 	for (__a = W_TYPE_SIZE - 8; __a > 0; __a -= 8)			\
1750 	  if (((__xr >> __a) & 0xff) != 0)				\
1751 	    break;							\
1752       }									\
1753 									\
1754     (count) = W_TYPE_SIZE - (__clz_tab[__xr >> __a] + __a);		\
1755   } while (0)
1756 #define COUNT_LEADING_ZEROS_0 W_TYPE_SIZE
1757 #endif
1758 
1759 #if !defined (count_trailing_zeros)
1760 /* Define count_trailing_zeros using count_leading_zeros.  The latter might be
1761    defined in asm, but if it is not, the C version above is good enough.  */
1762 #define count_trailing_zeros(count, x) \
1763   do {									\
1764     UWtype __ctz_x = (x);						\
1765     UWtype __ctz_c;							\
1766     count_leading_zeros (__ctz_c, __ctz_x & -__ctz_x);			\
1767     (count) = W_TYPE_SIZE - 1 - __ctz_c;				\
1768   } while (0)
1769 #endif
1770 
1771 #ifndef UDIV_NEEDS_NORMALIZATION
1772 #define UDIV_NEEDS_NORMALIZATION 0
1773 #endif
1774