1*86d7f5d3SJohn Marino /* Implementation of the multiplication algorithm for Toom-Cook 8.5-way.
2*86d7f5d3SJohn Marino
3*86d7f5d3SJohn Marino Contributed to the GNU project by Marco Bodrato.
4*86d7f5d3SJohn Marino
5*86d7f5d3SJohn Marino THE FUNCTION IN THIS FILE IS INTERNAL WITH A MUTABLE INTERFACE. IT IS ONLY
6*86d7f5d3SJohn Marino SAFE TO REACH IT THROUGH DOCUMENTED INTERFACES. IN FACT, IT IS ALMOST
7*86d7f5d3SJohn Marino GUARANTEED THAT IT WILL CHANGE OR DISAPPEAR IN A FUTURE GNU MP RELEASE.
8*86d7f5d3SJohn Marino
9*86d7f5d3SJohn Marino Copyright 2009, 2010 Free Software Foundation, Inc.
10*86d7f5d3SJohn Marino
11*86d7f5d3SJohn Marino This file is part of the GNU MP Library.
12*86d7f5d3SJohn Marino
13*86d7f5d3SJohn Marino The GNU MP Library is free software; you can redistribute it and/or modify
14*86d7f5d3SJohn Marino it under the terms of the GNU Lesser General Public License as published by
15*86d7f5d3SJohn Marino the Free Software Foundation; either version 3 of the License, or (at your
16*86d7f5d3SJohn Marino option) any later version.
17*86d7f5d3SJohn Marino
18*86d7f5d3SJohn Marino The GNU MP Library is distributed in the hope that it will be useful, but
19*86d7f5d3SJohn Marino WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
20*86d7f5d3SJohn Marino or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
21*86d7f5d3SJohn Marino License for more details.
22*86d7f5d3SJohn Marino
23*86d7f5d3SJohn Marino You should have received a copy of the GNU Lesser General Public License
24*86d7f5d3SJohn Marino along with the GNU MP Library. If not, see http://www.gnu.org/licenses/. */
25*86d7f5d3SJohn Marino
26*86d7f5d3SJohn Marino
27*86d7f5d3SJohn Marino #include "gmp.h"
28*86d7f5d3SJohn Marino #include "gmp-impl.h"
29*86d7f5d3SJohn Marino
30*86d7f5d3SJohn Marino
31*86d7f5d3SJohn Marino #if GMP_NUMB_BITS < 29
32*86d7f5d3SJohn Marino #error Not implemented.
33*86d7f5d3SJohn Marino #endif
34*86d7f5d3SJohn Marino
35*86d7f5d3SJohn Marino #if GMP_NUMB_BITS < 43
36*86d7f5d3SJohn Marino #define BIT_CORRECTION 1
37*86d7f5d3SJohn Marino #define CORRECTION_BITS GMP_NUMB_BITS
38*86d7f5d3SJohn Marino #else
39*86d7f5d3SJohn Marino #define BIT_CORRECTION 0
40*86d7f5d3SJohn Marino #define CORRECTION_BITS 0
41*86d7f5d3SJohn Marino #endif
42*86d7f5d3SJohn Marino
43*86d7f5d3SJohn Marino
44*86d7f5d3SJohn Marino #if TUNE_PROGRAM_BUILD
45*86d7f5d3SJohn Marino #define MAYBE_mul_basecase 1
46*86d7f5d3SJohn Marino #define MAYBE_mul_toom22 1
47*86d7f5d3SJohn Marino #define MAYBE_mul_toom33 1
48*86d7f5d3SJohn Marino #define MAYBE_mul_toom44 1
49*86d7f5d3SJohn Marino #define MAYBE_mul_toom8h 1
50*86d7f5d3SJohn Marino #else
51*86d7f5d3SJohn Marino #define MAYBE_mul_basecase \
52*86d7f5d3SJohn Marino (MUL_TOOM8H_THRESHOLD < 8 * MUL_TOOM22_THRESHOLD)
53*86d7f5d3SJohn Marino #define MAYBE_mul_toom22 \
54*86d7f5d3SJohn Marino (MUL_TOOM8H_THRESHOLD < 8 * MUL_TOOM33_THRESHOLD)
55*86d7f5d3SJohn Marino #define MAYBE_mul_toom33 \
56*86d7f5d3SJohn Marino (MUL_TOOM8H_THRESHOLD < 8 * MUL_TOOM44_THRESHOLD)
57*86d7f5d3SJohn Marino #define MAYBE_mul_toom44 \
58*86d7f5d3SJohn Marino (MUL_TOOM8H_THRESHOLD < 8 * MUL_TOOM6H_THRESHOLD)
59*86d7f5d3SJohn Marino #define MAYBE_mul_toom8h \
60*86d7f5d3SJohn Marino (MUL_FFT_THRESHOLD >= 8 * MUL_TOOM8H_THRESHOLD)
61*86d7f5d3SJohn Marino #endif
62*86d7f5d3SJohn Marino
63*86d7f5d3SJohn Marino #define TOOM8H_MUL_N_REC(p, a, b, n, ws) \
64*86d7f5d3SJohn Marino do { \
65*86d7f5d3SJohn Marino if (MAYBE_mul_basecase \
66*86d7f5d3SJohn Marino && BELOW_THRESHOLD (n, MUL_TOOM22_THRESHOLD)) \
67*86d7f5d3SJohn Marino mpn_mul_basecase (p, a, n, b, n); \
68*86d7f5d3SJohn Marino else if (MAYBE_mul_toom22 \
69*86d7f5d3SJohn Marino && BELOW_THRESHOLD (n, MUL_TOOM33_THRESHOLD)) \
70*86d7f5d3SJohn Marino mpn_toom22_mul (p, a, n, b, n, ws); \
71*86d7f5d3SJohn Marino else if (MAYBE_mul_toom33 \
72*86d7f5d3SJohn Marino && BELOW_THRESHOLD (n, MUL_TOOM44_THRESHOLD)) \
73*86d7f5d3SJohn Marino mpn_toom33_mul (p, a, n, b, n, ws); \
74*86d7f5d3SJohn Marino else if (MAYBE_mul_toom44 \
75*86d7f5d3SJohn Marino && BELOW_THRESHOLD (n, MUL_TOOM6H_THRESHOLD)) \
76*86d7f5d3SJohn Marino mpn_toom44_mul (p, a, n, b, n, ws); \
77*86d7f5d3SJohn Marino else if (! MAYBE_mul_toom8h \
78*86d7f5d3SJohn Marino || BELOW_THRESHOLD (n, MUL_TOOM8H_THRESHOLD)) \
79*86d7f5d3SJohn Marino mpn_toom6h_mul (p, a, n, b, n, ws); \
80*86d7f5d3SJohn Marino else \
81*86d7f5d3SJohn Marino mpn_toom8h_mul (p, a, n, b, n, ws); \
82*86d7f5d3SJohn Marino } while (0)
83*86d7f5d3SJohn Marino
84*86d7f5d3SJohn Marino #define TOOM8H_MUL_REC(p, a, na, b, nb, ws) \
85*86d7f5d3SJohn Marino do { mpn_mul (p, a, na, b, nb); \
86*86d7f5d3SJohn Marino } while (0)
87*86d7f5d3SJohn Marino
88*86d7f5d3SJohn Marino /* Toom-8.5 , compute the product {pp,an+bn} <- {ap,an} * {bp,bn}
89*86d7f5d3SJohn Marino With: an >= bn >= 86, an*5 < bn * 11.
90*86d7f5d3SJohn Marino It _may_ work with bn<=?? and bn*?? < an*? < bn*??
91*86d7f5d3SJohn Marino
92*86d7f5d3SJohn Marino Evaluate in: infinity, +8,-8,+4,-4,+2,-2,+1,-1,+1/2,-1/2,+1/4,-1/4,+1/8,-1/8,0.
93*86d7f5d3SJohn Marino */
94*86d7f5d3SJohn Marino /* Estimate on needed scratch:
95*86d7f5d3SJohn Marino S(n) <= (n+7)\8*13+5+MAX(S((n+7)\8),1+2*(n+7)\8),
96*86d7f5d3SJohn Marino since n>80; S(n) <= ceil(log(n/10)/log(8))*(13+5)+n*15\8 < n*15\8 + lg2(n)*6
97*86d7f5d3SJohn Marino */
98*86d7f5d3SJohn Marino
99*86d7f5d3SJohn Marino void
mpn_toom8h_mul(mp_ptr pp,mp_srcptr ap,mp_size_t an,mp_srcptr bp,mp_size_t bn,mp_ptr scratch)100*86d7f5d3SJohn Marino mpn_toom8h_mul (mp_ptr pp,
101*86d7f5d3SJohn Marino mp_srcptr ap, mp_size_t an,
102*86d7f5d3SJohn Marino mp_srcptr bp, mp_size_t bn, mp_ptr scratch)
103*86d7f5d3SJohn Marino {
104*86d7f5d3SJohn Marino mp_size_t n, s, t;
105*86d7f5d3SJohn Marino int p, q, half;
106*86d7f5d3SJohn Marino int sign;
107*86d7f5d3SJohn Marino
108*86d7f5d3SJohn Marino /***************************** decomposition *******************************/
109*86d7f5d3SJohn Marino
110*86d7f5d3SJohn Marino ASSERT (an >= bn);
111*86d7f5d3SJohn Marino /* Can not handle too small operands */
112*86d7f5d3SJohn Marino ASSERT (bn >= 86);
113*86d7f5d3SJohn Marino /* Can not handle too much unbalancement */
114*86d7f5d3SJohn Marino ASSERT (an*4 <= bn*13);
115*86d7f5d3SJohn Marino ASSERT (GMP_NUMB_BITS > 12*3 || an*4 <= bn*12);
116*86d7f5d3SJohn Marino ASSERT (GMP_NUMB_BITS > 11*3 || an*5 <= bn*11);
117*86d7f5d3SJohn Marino ASSERT (GMP_NUMB_BITS > 10*3 || an*6 <= bn*10);
118*86d7f5d3SJohn Marino ASSERT (GMP_NUMB_BITS > 9*3 || an*7 <= bn* 9);
119*86d7f5d3SJohn Marino
120*86d7f5d3SJohn Marino /* Limit num/den is a rational number between
121*86d7f5d3SJohn Marino (16/15)^(log(6)/log(2*6-1)) and (16/15)^(log(8)/log(2*8-1)) */
122*86d7f5d3SJohn Marino #define LIMIT_numerator (21)
123*86d7f5d3SJohn Marino #define LIMIT_denominat (20)
124*86d7f5d3SJohn Marino
125*86d7f5d3SJohn Marino if (LIKELY (an == bn) || an * (LIMIT_denominat>>1) < LIMIT_numerator * (bn>>1) ) /* is 8*... < 8*... */
126*86d7f5d3SJohn Marino {
127*86d7f5d3SJohn Marino half = 0;
128*86d7f5d3SJohn Marino n = 1 + ((an - 1)>>3);
129*86d7f5d3SJohn Marino p = q = 7;
130*86d7f5d3SJohn Marino s = an - p * n;
131*86d7f5d3SJohn Marino t = bn - q * n;
132*86d7f5d3SJohn Marino }
133*86d7f5d3SJohn Marino else
134*86d7f5d3SJohn Marino {
135*86d7f5d3SJohn Marino if (an * 13 < 16 * bn) /* (an*7*LIMIT_numerator<LIMIT_denominat*9*bn) */
136*86d7f5d3SJohn Marino { p = 9; q = 8; }
137*86d7f5d3SJohn Marino else if (GMP_NUMB_BITS <= 9*3 ||
138*86d7f5d3SJohn Marino an *(LIMIT_denominat>>1) < (LIMIT_numerator/7*9) * (bn>>1))
139*86d7f5d3SJohn Marino { p = 9; q = 7; }
140*86d7f5d3SJohn Marino else if (an * 10 < 33 * (bn>>1)) /* (an*3*LIMIT_numerator<LIMIT_denominat*5*bn) */
141*86d7f5d3SJohn Marino { p =10; q = 7; }
142*86d7f5d3SJohn Marino else if (GMP_NUMB_BITS <= 10*3 ||
143*86d7f5d3SJohn Marino an * (LIMIT_denominat/5) < (LIMIT_numerator/3) * bn)
144*86d7f5d3SJohn Marino { p =10; q = 6; }
145*86d7f5d3SJohn Marino else if (an * 6 < 13 * bn) /*(an * 5 * LIMIT_numerator < LIMIT_denominat *11 * bn)*/
146*86d7f5d3SJohn Marino { p =11; q = 6; }
147*86d7f5d3SJohn Marino else if (GMP_NUMB_BITS <= 11*3 ||
148*86d7f5d3SJohn Marino an * 4 < 9 * bn)
149*86d7f5d3SJohn Marino { p =11; q = 5; }
150*86d7f5d3SJohn Marino else if (an *(LIMIT_numerator/3) < LIMIT_denominat * bn ) /* is 4*... <12*... */
151*86d7f5d3SJohn Marino { p =12; q = 5; }
152*86d7f5d3SJohn Marino else if (GMP_NUMB_BITS <= 12*3 ||
153*86d7f5d3SJohn Marino an * 9 < 28 * bn ) /* is 4*... <12*... */
154*86d7f5d3SJohn Marino { p =12; q = 4; }
155*86d7f5d3SJohn Marino else
156*86d7f5d3SJohn Marino { p =13; q = 4; }
157*86d7f5d3SJohn Marino
158*86d7f5d3SJohn Marino half = (p+q)&1;
159*86d7f5d3SJohn Marino n = 1 + (q * an >= p * bn ? (an - 1) / (size_t) p : (bn - 1) / (size_t) q);
160*86d7f5d3SJohn Marino p--; q--;
161*86d7f5d3SJohn Marino
162*86d7f5d3SJohn Marino s = an - p * n;
163*86d7f5d3SJohn Marino t = bn - q * n;
164*86d7f5d3SJohn Marino
165*86d7f5d3SJohn Marino if(half) { /* Recover from badly chosen splitting */
166*86d7f5d3SJohn Marino if (s<1) {p--; s+=n; half=0;}
167*86d7f5d3SJohn Marino else if (t<1) {q--; t+=n; half=0;}
168*86d7f5d3SJohn Marino }
169*86d7f5d3SJohn Marino }
170*86d7f5d3SJohn Marino #undef LIMIT_numerator
171*86d7f5d3SJohn Marino #undef LIMIT_denominat
172*86d7f5d3SJohn Marino
173*86d7f5d3SJohn Marino ASSERT (0 < s && s <= n);
174*86d7f5d3SJohn Marino ASSERT (0 < t && t <= n);
175*86d7f5d3SJohn Marino ASSERT (half || s + t > 3);
176*86d7f5d3SJohn Marino ASSERT (n > 2);
177*86d7f5d3SJohn Marino
178*86d7f5d3SJohn Marino #define r6 (pp + 3 * n) /* 3n+1 */
179*86d7f5d3SJohn Marino #define r4 (pp + 7 * n) /* 3n+1 */
180*86d7f5d3SJohn Marino #define r2 (pp +11 * n) /* 3n+1 */
181*86d7f5d3SJohn Marino #define r0 (pp +15 * n) /* s+t <= 2*n */
182*86d7f5d3SJohn Marino #define r7 (scratch) /* 3n+1 */
183*86d7f5d3SJohn Marino #define r5 (scratch + 3 * n + 1) /* 3n+1 */
184*86d7f5d3SJohn Marino #define r3 (scratch + 6 * n + 2) /* 3n+1 */
185*86d7f5d3SJohn Marino #define r1 (scratch + 9 * n + 3) /* 3n+1 */
186*86d7f5d3SJohn Marino #define v0 (pp +11 * n) /* n+1 */
187*86d7f5d3SJohn Marino #define v1 (pp +12 * n+1) /* n+1 */
188*86d7f5d3SJohn Marino #define v2 (pp +13 * n+2) /* n+1 */
189*86d7f5d3SJohn Marino #define v3 (scratch +12 * n + 4) /* n+1 */
190*86d7f5d3SJohn Marino #define wsi (scratch +12 * n + 4) /* 3n+1 */
191*86d7f5d3SJohn Marino #define wse (scratch +13 * n + 5) /* 2n+1 */
192*86d7f5d3SJohn Marino
193*86d7f5d3SJohn Marino /* Alloc also 3n+1 limbs for wsi... toom_interpolate_16pts may
194*86d7f5d3SJohn Marino need all of them */
195*86d7f5d3SJohn Marino /* if (scratch == NULL) */
196*86d7f5d3SJohn Marino /* scratch = TMP_SALLOC_LIMBS(mpn_toom8_sqr_itch(n * 8)); */
197*86d7f5d3SJohn Marino ASSERT (15 * n + 6 <= mpn_toom8h_mul_itch (an, bn));
198*86d7f5d3SJohn Marino ASSERT (15 * n + 6 <= mpn_toom8_sqr_itch (n * 8));
199*86d7f5d3SJohn Marino
200*86d7f5d3SJohn Marino /********************** evaluation and recursive calls *********************/
201*86d7f5d3SJohn Marino
202*86d7f5d3SJohn Marino /* $\pm1/8$ */
203*86d7f5d3SJohn Marino sign = mpn_toom_eval_pm2rexp (v2, v0, p, ap, n, s, 3, pp) ^
204*86d7f5d3SJohn Marino mpn_toom_eval_pm2rexp (v3, v1, q, bp, n, t, 3, pp);
205*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, v0, v1, n + 1, wse); /* A(-1/8)*B(-1/8)*8^. */
206*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(r7, v2, v3, n + 1, wse); /* A(+1/8)*B(+1/8)*8^. */
207*86d7f5d3SJohn Marino mpn_toom_couple_handling (r7, 2 * n + 1 + BIT_CORRECTION, pp, sign, n, 3*(1+half), 3*(half));
208*86d7f5d3SJohn Marino
209*86d7f5d3SJohn Marino /* $\pm1/4$ */
210*86d7f5d3SJohn Marino sign = mpn_toom_eval_pm2rexp (v2, v0, p, ap, n, s, 2, pp) ^
211*86d7f5d3SJohn Marino mpn_toom_eval_pm2rexp (v3, v1, q, bp, n, t, 2, pp);
212*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, v0, v1, n + 1, wse); /* A(-1/4)*B(-1/4)*4^. */
213*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(r5, v2, v3, n + 1, wse); /* A(+1/4)*B(+1/4)*4^. */
214*86d7f5d3SJohn Marino mpn_toom_couple_handling (r5, 2 * n + 1, pp, sign, n, 2*(1+half), 2*(half));
215*86d7f5d3SJohn Marino
216*86d7f5d3SJohn Marino /* $\pm2$ */
217*86d7f5d3SJohn Marino sign = mpn_toom_eval_pm2 (v2, v0, p, ap, n, s, pp) ^
218*86d7f5d3SJohn Marino mpn_toom_eval_pm2 (v3, v1, q, bp, n, t, pp);
219*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, v0, v1, n + 1, wse); /* A(-2)*B(-2) */
220*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(r3, v2, v3, n + 1, wse); /* A(+2)*B(+2) */
221*86d7f5d3SJohn Marino mpn_toom_couple_handling (r3, 2 * n + 1, pp, sign, n, 1, 2);
222*86d7f5d3SJohn Marino
223*86d7f5d3SJohn Marino /* $\pm8$ */
224*86d7f5d3SJohn Marino sign = mpn_toom_eval_pm2exp (v2, v0, p, ap, n, s, 3, pp) ^
225*86d7f5d3SJohn Marino mpn_toom_eval_pm2exp (v3, v1, q, bp, n, t, 3, pp);
226*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, v0, v1, n + 1, wse); /* A(-8)*B(-8) */
227*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(r1, v2, v3, n + 1, wse); /* A(+8)*B(+8) */
228*86d7f5d3SJohn Marino mpn_toom_couple_handling (r1, 2 * n + 1 + BIT_CORRECTION, pp, sign, n, 3, 6);
229*86d7f5d3SJohn Marino
230*86d7f5d3SJohn Marino /* $\pm1/2$ */
231*86d7f5d3SJohn Marino sign = mpn_toom_eval_pm2rexp (v2, v0, p, ap, n, s, 1, pp) ^
232*86d7f5d3SJohn Marino mpn_toom_eval_pm2rexp (v3, v1, q, bp, n, t, 1, pp);
233*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, v0, v1, n + 1, wse); /* A(-1/2)*B(-1/2)*2^. */
234*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(r6, v2, v3, n + 1, wse); /* A(+1/2)*B(+1/2)*2^. */
235*86d7f5d3SJohn Marino mpn_toom_couple_handling (r6, 2 * n + 1, pp, sign, n, 1+half, half);
236*86d7f5d3SJohn Marino
237*86d7f5d3SJohn Marino /* $\pm1$ */
238*86d7f5d3SJohn Marino sign = mpn_toom_eval_pm1 (v2, v0, p, ap, n, s, pp);
239*86d7f5d3SJohn Marino if (q == 3)
240*86d7f5d3SJohn Marino sign ^= mpn_toom_eval_dgr3_pm1 (v3, v1, bp, n, t, pp);
241*86d7f5d3SJohn Marino else
242*86d7f5d3SJohn Marino sign ^= mpn_toom_eval_pm1 (v3, v1, q, bp, n, t, pp);
243*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, v0, v1, n + 1, wse); /* A(-1)*B(-1) */
244*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(r4, v2, v3, n + 1, wse); /* A(1)*B(1) */
245*86d7f5d3SJohn Marino mpn_toom_couple_handling (r4, 2 * n + 1, pp, sign, n, 0, 0);
246*86d7f5d3SJohn Marino
247*86d7f5d3SJohn Marino /* $\pm4$ */
248*86d7f5d3SJohn Marino sign = mpn_toom_eval_pm2exp (v2, v0, p, ap, n, s, 2, pp) ^
249*86d7f5d3SJohn Marino mpn_toom_eval_pm2exp (v3, v1, q, bp, n, t, 2, pp);
250*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, v0, v1, n + 1, wse); /* A(-4)*B(-4) */
251*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(r2, v2, v3, n + 1, wse); /* A(+4)*B(+4) */
252*86d7f5d3SJohn Marino mpn_toom_couple_handling (r2, 2 * n + 1, pp, sign, n, 2, 4);
253*86d7f5d3SJohn Marino
254*86d7f5d3SJohn Marino #undef v0
255*86d7f5d3SJohn Marino #undef v1
256*86d7f5d3SJohn Marino #undef v2
257*86d7f5d3SJohn Marino #undef v3
258*86d7f5d3SJohn Marino #undef wse
259*86d7f5d3SJohn Marino
260*86d7f5d3SJohn Marino /* A(0)*B(0) */
261*86d7f5d3SJohn Marino TOOM8H_MUL_N_REC(pp, ap, bp, n, wsi);
262*86d7f5d3SJohn Marino
263*86d7f5d3SJohn Marino /* Infinity */
264*86d7f5d3SJohn Marino if( half != 0) {
265*86d7f5d3SJohn Marino if(s>t) {
266*86d7f5d3SJohn Marino TOOM8H_MUL_REC(r0, ap + p * n, s, bp + q * n, t, wsi);
267*86d7f5d3SJohn Marino } else {
268*86d7f5d3SJohn Marino TOOM8H_MUL_REC(r0, bp + q * n, t, ap + p * n, s, wsi);
269*86d7f5d3SJohn Marino };
270*86d7f5d3SJohn Marino };
271*86d7f5d3SJohn Marino
272*86d7f5d3SJohn Marino mpn_toom_interpolate_16pts (pp, r1, r3, r5, r7, n, s+t, half, wsi);
273*86d7f5d3SJohn Marino
274*86d7f5d3SJohn Marino #undef r0
275*86d7f5d3SJohn Marino #undef r1
276*86d7f5d3SJohn Marino #undef r2
277*86d7f5d3SJohn Marino #undef r3
278*86d7f5d3SJohn Marino #undef r4
279*86d7f5d3SJohn Marino #undef r5
280*86d7f5d3SJohn Marino #undef r6
281*86d7f5d3SJohn Marino #undef wsi
282*86d7f5d3SJohn Marino }
283*86d7f5d3SJohn Marino
284*86d7f5d3SJohn Marino #undef TOOM8H_MUL_N_REC
285*86d7f5d3SJohn Marino #undef TOOM8H_MUL_REC
286*86d7f5d3SJohn Marino #undef MAYBE_mul_basecase
287*86d7f5d3SJohn Marino #undef MAYBE_mul_toom22
288*86d7f5d3SJohn Marino #undef MAYBE_mul_toom33
289*86d7f5d3SJohn Marino #undef MAYBE_mul_toom44
290*86d7f5d3SJohn Marino #undef MAYBE_mul_toom8h
291