xref: /freebsd/sys/dev/sound/pcm/feeder_rate.c (revision e17f5b1d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2005-2009 Ariff Abdullah <ariff@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 /*
30  * feeder_rate: (Codename: Z Resampler), which means any effort to create
31  *              future replacement for this resampler are simply absurd unless
32  *              the world decide to add new alphabet after Z.
33  *
34  * FreeBSD bandlimited sinc interpolator, technically based on
35  * "Digital Audio Resampling" by Julius O. Smith III
36  *  - http://ccrma.stanford.edu/~jos/resample/
37  *
38  * The Good:
39  * + all out fixed point integer operations, no soft-float or anything like
40  *   that.
41  * + classic polyphase converters with high quality coefficient's polynomial
42  *   interpolators.
43  * + fast, faster, or the fastest of its kind.
44  * + compile time configurable.
45  * + etc etc..
46  *
47  * The Bad:
48  * - The z, z_, and Z_ . Due to mental block (or maybe just 0x7a69), I
49  *   couldn't think of anything simpler than that (feeder_rate_xxx is just
50  *   too long). Expect possible clashes with other zitizens (any?).
51  */
52 
53 #ifdef _KERNEL
54 #ifdef HAVE_KERNEL_OPTION_HEADERS
55 #include "opt_snd.h"
56 #endif
57 #include <dev/sound/pcm/sound.h>
58 #include <dev/sound/pcm/pcm.h>
59 #include "feeder_if.h"
60 
61 #define SND_USE_FXDIV
62 #include "snd_fxdiv_gen.h"
63 
64 SND_DECLARE_FILE("$FreeBSD$");
65 #endif
66 
67 #include "feeder_rate_gen.h"
68 
69 #if !defined(_KERNEL) && defined(SND_DIAGNOSTIC)
70 #undef Z_DIAGNOSTIC
71 #define Z_DIAGNOSTIC		1
72 #elif defined(_KERNEL)
73 #undef Z_DIAGNOSTIC
74 #endif
75 
76 #ifndef Z_QUALITY_DEFAULT
77 #define Z_QUALITY_DEFAULT	Z_QUALITY_LINEAR
78 #endif
79 
80 #define Z_RESERVOIR		2048
81 #define Z_RESERVOIR_MAX		131072
82 
83 #define Z_SINC_MAX		0x3fffff
84 #define Z_SINC_DOWNMAX		48		/* 384000 / 8000 */
85 
86 #ifdef _KERNEL
87 #define Z_POLYPHASE_MAX		183040		/* 286 taps, 640 phases */
88 #else
89 #define Z_POLYPHASE_MAX		1464320		/* 286 taps, 5120 phases */
90 #endif
91 
92 #define Z_RATE_DEFAULT		48000
93 
94 #define Z_RATE_MIN		FEEDRATE_RATEMIN
95 #define Z_RATE_MAX		FEEDRATE_RATEMAX
96 #define Z_ROUNDHZ		FEEDRATE_ROUNDHZ
97 #define Z_ROUNDHZ_MIN		FEEDRATE_ROUNDHZ_MIN
98 #define Z_ROUNDHZ_MAX		FEEDRATE_ROUNDHZ_MAX
99 
100 #define Z_RATE_SRC		FEEDRATE_SRC
101 #define Z_RATE_DST		FEEDRATE_DST
102 #define Z_RATE_QUALITY		FEEDRATE_QUALITY
103 #define Z_RATE_CHANNELS		FEEDRATE_CHANNELS
104 
105 #define Z_PARANOID		1
106 
107 #define Z_MULTIFORMAT		1
108 
109 #ifdef _KERNEL
110 #undef Z_USE_ALPHADRIFT
111 #define Z_USE_ALPHADRIFT	1
112 #endif
113 
114 #define Z_FACTOR_MIN		1
115 #define Z_FACTOR_MAX		Z_MASK
116 #define Z_FACTOR_SAFE(v)	(!((v) < Z_FACTOR_MIN || (v) > Z_FACTOR_MAX))
117 
118 struct z_info;
119 
120 typedef void (*z_resampler_t)(struct z_info *, uint8_t *);
121 
122 struct z_info {
123 	int32_t rsrc, rdst;	/* original source / destination rates */
124 	int32_t src, dst;	/* rounded source / destination rates */
125 	int32_t channels;	/* total channels */
126 	int32_t bps;		/* bytes-per-sample */
127 	int32_t quality;	/* resampling quality */
128 
129 	int32_t z_gx, z_gy;	/* interpolation / decimation ratio */
130 	int32_t z_alpha;	/* output sample time phase / drift */
131 	uint8_t *z_delay;	/* FIR delay line / linear buffer */
132 	int32_t *z_coeff;	/* FIR coefficients */
133 	int32_t *z_dcoeff;	/* FIR coefficients differences */
134 	int32_t *z_pcoeff;	/* FIR polyphase coefficients */
135 	int32_t z_scale;	/* output scaling */
136 	int32_t z_dx;		/* input sample drift increment */
137 	int32_t z_dy;		/* output sample drift increment */
138 #ifdef Z_USE_ALPHADRIFT
139 	int32_t z_alphadrift;	/* alpha drift rate */
140 	int32_t z_startdrift;	/* buffer start position drift rate */
141 #endif
142 	int32_t z_mask;		/* delay line full length mask */
143 	int32_t z_size;		/* half width of FIR taps */
144 	int32_t z_full;		/* full size of delay line */
145 	int32_t z_alloc;	/* largest allocated full size of delay line */
146 	int32_t z_start;	/* buffer processing start position */
147 	int32_t z_pos;		/* current position for the next feed */
148 #ifdef Z_DIAGNOSTIC
149 	uint32_t z_cycle;	/* output cycle, purely for statistical */
150 #endif
151 	int32_t z_maxfeed;	/* maximum feed to avoid 32bit overflow */
152 
153 	z_resampler_t z_resample;
154 };
155 
156 int feeder_rate_min = Z_RATE_MIN;
157 int feeder_rate_max = Z_RATE_MAX;
158 int feeder_rate_round = Z_ROUNDHZ;
159 int feeder_rate_quality = Z_QUALITY_DEFAULT;
160 
161 static int feeder_rate_polyphase_max = Z_POLYPHASE_MAX;
162 
163 #ifdef _KERNEL
164 static char feeder_rate_presets[] = FEEDER_RATE_PRESETS;
165 SYSCTL_STRING(_hw_snd, OID_AUTO, feeder_rate_presets, CTLFLAG_RD,
166     &feeder_rate_presets, 0, "compile-time rate presets");
167 SYSCTL_INT(_hw_snd, OID_AUTO, feeder_rate_polyphase_max, CTLFLAG_RWTUN,
168     &feeder_rate_polyphase_max, 0, "maximum allowable polyphase entries");
169 
170 static int
171 sysctl_hw_snd_feeder_rate_min(SYSCTL_HANDLER_ARGS)
172 {
173 	int err, val;
174 
175 	val = feeder_rate_min;
176 	err = sysctl_handle_int(oidp, &val, 0, req);
177 
178 	if (err != 0 || req->newptr == NULL || val == feeder_rate_min)
179 		return (err);
180 
181 	if (!(Z_FACTOR_SAFE(val) && val < feeder_rate_max))
182 		return (EINVAL);
183 
184 	feeder_rate_min = val;
185 
186 	return (0);
187 }
188 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_min,
189     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
190     sysctl_hw_snd_feeder_rate_min, "I",
191     "minimum allowable rate");
192 
193 static int
194 sysctl_hw_snd_feeder_rate_max(SYSCTL_HANDLER_ARGS)
195 {
196 	int err, val;
197 
198 	val = feeder_rate_max;
199 	err = sysctl_handle_int(oidp, &val, 0, req);
200 
201 	if (err != 0 || req->newptr == NULL || val == feeder_rate_max)
202 		return (err);
203 
204 	if (!(Z_FACTOR_SAFE(val) && val > feeder_rate_min))
205 		return (EINVAL);
206 
207 	feeder_rate_max = val;
208 
209 	return (0);
210 }
211 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_max,
212     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
213     sysctl_hw_snd_feeder_rate_max, "I",
214     "maximum allowable rate");
215 
216 static int
217 sysctl_hw_snd_feeder_rate_round(SYSCTL_HANDLER_ARGS)
218 {
219 	int err, val;
220 
221 	val = feeder_rate_round;
222 	err = sysctl_handle_int(oidp, &val, 0, req);
223 
224 	if (err != 0 || req->newptr == NULL || val == feeder_rate_round)
225 		return (err);
226 
227 	if (val < Z_ROUNDHZ_MIN || val > Z_ROUNDHZ_MAX)
228 		return (EINVAL);
229 
230 	feeder_rate_round = val - (val % Z_ROUNDHZ);
231 
232 	return (0);
233 }
234 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_round,
235     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
236     sysctl_hw_snd_feeder_rate_round, "I",
237     "sample rate converter rounding threshold");
238 
239 static int
240 sysctl_hw_snd_feeder_rate_quality(SYSCTL_HANDLER_ARGS)
241 {
242 	struct snddev_info *d;
243 	struct pcm_channel *c;
244 	struct pcm_feeder *f;
245 	int i, err, val;
246 
247 	val = feeder_rate_quality;
248 	err = sysctl_handle_int(oidp, &val, 0, req);
249 
250 	if (err != 0 || req->newptr == NULL || val == feeder_rate_quality)
251 		return (err);
252 
253 	if (val < Z_QUALITY_MIN || val > Z_QUALITY_MAX)
254 		return (EINVAL);
255 
256 	feeder_rate_quality = val;
257 
258 	/*
259 	 * Traverse all available channels on each device and try to
260 	 * set resampler quality if and only if it is exist as
261 	 * part of feeder chains and the channel is idle.
262 	 */
263 	for (i = 0; pcm_devclass != NULL &&
264 	    i < devclass_get_maxunit(pcm_devclass); i++) {
265 		d = devclass_get_softc(pcm_devclass, i);
266 		if (!PCM_REGISTERED(d))
267 			continue;
268 		PCM_LOCK(d);
269 		PCM_WAIT(d);
270 		PCM_ACQUIRE(d);
271 		CHN_FOREACH(c, d, channels.pcm) {
272 			CHN_LOCK(c);
273 			f = chn_findfeeder(c, FEEDER_RATE);
274 			if (f == NULL || f->data == NULL || CHN_STARTED(c)) {
275 				CHN_UNLOCK(c);
276 				continue;
277 			}
278 			(void)FEEDER_SET(f, FEEDRATE_QUALITY, val);
279 			CHN_UNLOCK(c);
280 		}
281 		PCM_RELEASE(d);
282 		PCM_UNLOCK(d);
283 	}
284 
285 	return (0);
286 }
287 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_quality,
288     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 0, sizeof(int),
289     sysctl_hw_snd_feeder_rate_quality, "I",
290     "sample rate converter quality ("__XSTRING(Z_QUALITY_MIN)"=low .. "
291     __XSTRING(Z_QUALITY_MAX)"=high)");
292 #endif	/* _KERNEL */
293 
294 
295 /*
296  * Resampler type.
297  */
298 #define Z_IS_ZOH(i)		((i)->quality == Z_QUALITY_ZOH)
299 #define Z_IS_LINEAR(i)		((i)->quality == Z_QUALITY_LINEAR)
300 #define Z_IS_SINC(i)		((i)->quality > Z_QUALITY_LINEAR)
301 
302 /*
303  * Macroses for accurate sample time drift calculations.
304  *
305  * gy2gx : given the amount of output, return the _exact_ required amount of
306  *         input.
307  * gx2gy : given the amount of input, return the _maximum_ amount of output
308  *         that will be generated.
309  * drift : given the amount of input and output, return the elapsed
310  *         sample-time.
311  */
312 #define _Z_GCAST(x)		((uint64_t)(x))
313 
314 #if defined(__GNUCLIKE_ASM) && defined(__i386__)
315 /*
316  * This is where i386 being beaten to a pulp. Fortunately this function is
317  * rarely being called and if it is, it will decide the best (hopefully)
318  * fastest way to do the division. If we can ensure that everything is dword
319  * aligned, letting the compiler to call udivdi3 to do the division can be
320  * faster compared to this.
321  *
322  * amd64 is the clear winner here, no question about it.
323  */
324 static __inline uint32_t
325 Z_DIV(uint64_t v, uint32_t d)
326 {
327 	uint32_t hi, lo, quo, rem;
328 
329 	hi = v >> 32;
330 	lo = v & 0xffffffff;
331 
332 	/*
333 	 * As much as we can, try to avoid long division like a plague.
334 	 */
335 	if (hi == 0)
336 		quo = lo / d;
337 	else
338 		__asm("divl %2"
339 		    : "=a" (quo), "=d" (rem)
340 		    : "r" (d), "0" (lo), "1" (hi));
341 
342 	return (quo);
343 }
344 #else
345 #define Z_DIV(x, y)		((x) / (y))
346 #endif
347 
348 #define _Z_GY2GX(i, a, v)						\
349 	Z_DIV(((_Z_GCAST((i)->z_gx) * (v)) + ((i)->z_gy - (a) - 1)),	\
350 	(i)->z_gy)
351 
352 #define _Z_GX2GY(i, a, v)						\
353 	Z_DIV(((_Z_GCAST((i)->z_gy) * (v)) + (a)), (i)->z_gx)
354 
355 #define _Z_DRIFT(i, x, y)						\
356 	((_Z_GCAST((i)->z_gy) * (x)) - (_Z_GCAST((i)->z_gx) * (y)))
357 
358 #define z_gy2gx(i, v)		_Z_GY2GX(i, (i)->z_alpha, v)
359 #define z_gx2gy(i, v)		_Z_GX2GY(i, (i)->z_alpha, v)
360 #define z_drift(i, x, y)	_Z_DRIFT(i, x, y)
361 
362 /*
363  * Macroses for SINC coefficients table manipulations.. whatever.
364  */
365 #define Z_SINC_COEFF_IDX(i)	((i)->quality - Z_QUALITY_LINEAR - 1)
366 
367 #define Z_SINC_LEN(i)							\
368 	((int32_t)(((uint64_t)z_coeff_tab[Z_SINC_COEFF_IDX(i)].len <<	\
369 	    Z_SHIFT) / (i)->z_dy))
370 
371 #define Z_SINC_BASE_LEN(i)						\
372 	((z_coeff_tab[Z_SINC_COEFF_IDX(i)].len - 1) >> (Z_DRIFT_SHIFT - 1))
373 
374 /*
375  * Macroses for linear delay buffer operations. Alignment is not
376  * really necessary since we're not using true circular buffer, but it
377  * will help us guard against possible trespasser. To be honest,
378  * the linear block operations does not need guarding at all due to
379  * accurate drifting!
380  */
381 #define z_align(i, v)		((v) & (i)->z_mask)
382 #define z_next(i, o, v)		z_align(i, (o) + (v))
383 #define z_prev(i, o, v)		z_align(i, (o) - (v))
384 #define z_fetched(i)		(z_align(i, (i)->z_pos - (i)->z_start) - 1)
385 #define z_free(i)		((i)->z_full - (i)->z_pos)
386 
387 /*
388  * Macroses for Bla Bla .. :)
389  */
390 #define z_copy(src, dst, sz)	(void)memcpy(dst, src, sz)
391 #define z_feed(...)		FEEDER_FEED(__VA_ARGS__)
392 
393 static __inline uint32_t
394 z_min(uint32_t x, uint32_t y)
395 {
396 
397 	return ((x < y) ? x : y);
398 }
399 
400 static int32_t
401 z_gcd(int32_t x, int32_t y)
402 {
403 	int32_t w;
404 
405 	while (y != 0) {
406 		w = x % y;
407 		x = y;
408 		y = w;
409 	}
410 
411 	return (x);
412 }
413 
414 static int32_t
415 z_roundpow2(int32_t v)
416 {
417 	int32_t i;
418 
419 	i = 1;
420 
421 	/*
422 	 * Let it overflow at will..
423 	 */
424 	while (i > 0 && i < v)
425 		i <<= 1;
426 
427 	return (i);
428 }
429 
430 /*
431  * Zero Order Hold, the worst of the worst, an insult against quality,
432  * but super fast.
433  */
434 static void
435 z_feed_zoh(struct z_info *info, uint8_t *dst)
436 {
437 #if 0
438 	z_copy(info->z_delay +
439 	    (info->z_start * info->channels * info->bps), dst,
440 	    info->channels * info->bps);
441 #else
442 	uint32_t cnt;
443 	uint8_t *src;
444 
445 	cnt = info->channels * info->bps;
446 	src = info->z_delay + (info->z_start * cnt);
447 
448 	/*
449 	 * This is a bit faster than doing bcopy() since we're dealing
450 	 * with possible unaligned samples.
451 	 */
452 	do {
453 		*dst++ = *src++;
454 	} while (--cnt != 0);
455 #endif
456 }
457 
458 /*
459  * Linear Interpolation. This at least sounds better (perceptually) and fast,
460  * but without any proper filtering which means aliasing still exist and
461  * could become worst with a right sample. Interpolation centered within
462  * Z_LINEAR_ONE between the present and previous sample and everything is
463  * done with simple 32bit scaling arithmetic.
464  */
465 #define Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)					\
466 static void									\
467 z_feed_linear_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)		\
468 {										\
469 	int32_t z;								\
470 	intpcm_t x, y;								\
471 	uint32_t ch;								\
472 	uint8_t *sx, *sy;							\
473 										\
474 	z = ((uint32_t)info->z_alpha * info->z_dx) >> Z_LINEAR_UNSHIFT;		\
475 										\
476 	sx = info->z_delay + (info->z_start * info->channels *			\
477 	    PCM_##BIT##_BPS);							\
478 	sy = sx - (info->channels * PCM_##BIT##_BPS);				\
479 										\
480 	ch = info->channels;							\
481 										\
482 	do {									\
483 		x = _PCM_READ_##SIGN##BIT##_##ENDIAN(sx);			\
484 		y = _PCM_READ_##SIGN##BIT##_##ENDIAN(sy);			\
485 		x = Z_LINEAR_INTERPOLATE_##BIT(z, x, y);			\
486 		_PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, x);			\
487 		sx += PCM_##BIT##_BPS;						\
488 		sy += PCM_##BIT##_BPS;						\
489 		dst += PCM_##BIT##_BPS;						\
490 	} while (--ch != 0);							\
491 }
492 
493 /*
494  * Userland clipping diagnostic check, not enabled in kernel compilation.
495  * While doing sinc interpolation, unrealistic samples like full scale sine
496  * wav will clip, but for other things this will not make any noise at all.
497  * Everybody should learn how to normalized perceived loudness of their own
498  * music/sounds/samples (hint: ReplayGain).
499  */
500 #ifdef Z_DIAGNOSTIC
501 #define Z_CLIP_CHECK(v, BIT)	do {					\
502 	if ((v) > PCM_S##BIT##_MAX) {					\
503 		fprintf(stderr, "Overflow: v=%jd, max=%jd\n",		\
504 		    (intmax_t)(v), (intmax_t)PCM_S##BIT##_MAX);		\
505 	} else if ((v) < PCM_S##BIT##_MIN) {				\
506 		fprintf(stderr, "Underflow: v=%jd, min=%jd\n",		\
507 		    (intmax_t)(v), (intmax_t)PCM_S##BIT##_MIN);		\
508 	}								\
509 } while (0)
510 #else
511 #define Z_CLIP_CHECK(...)
512 #endif
513 
514 #define Z_CLAMP(v, BIT)							\
515 	(((v) > PCM_S##BIT##_MAX) ? PCM_S##BIT##_MAX :			\
516 	(((v) < PCM_S##BIT##_MIN) ? PCM_S##BIT##_MIN : (v)))
517 
518 /*
519  * Sine Cardinal (SINC) Interpolation. Scaling is done in 64 bit, so
520  * there's no point to hold the plate any longer. All samples will be
521  * shifted to a full 32 bit, scaled and restored during write for
522  * maximum dynamic range (only for downsampling).
523  */
524 #define _Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, adv)			\
525 	c += z >> Z_SHIFT;						\
526 	z &= Z_MASK;							\
527 	coeff = Z_COEFF_INTERPOLATE(z, z_coeff[c], z_dcoeff[c]);	\
528 	x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);			\
529 	v += Z_NORM_##BIT((intpcm64_t)x * coeff);			\
530 	z += info->z_dy;						\
531 	p adv##= info->channels * PCM_##BIT##_BPS
532 
533 /*
534  * XXX GCC4 optimization is such a !@#$%, need manual unrolling.
535  */
536 #if defined(__GNUC__) && __GNUC__ >= 4
537 #define Z_SINC_ACCUMULATE(...)	do {					\
538 	_Z_SINC_ACCUMULATE(__VA_ARGS__);				\
539 	_Z_SINC_ACCUMULATE(__VA_ARGS__);				\
540 } while (0)
541 #define Z_SINC_ACCUMULATE_DECR		2
542 #else
543 #define Z_SINC_ACCUMULATE(...)	do {					\
544 	_Z_SINC_ACCUMULATE(__VA_ARGS__);				\
545 } while (0)
546 #define Z_SINC_ACCUMULATE_DECR		1
547 #endif
548 
549 #define Z_DECLARE_SINC(SIGN, BIT, ENDIAN)					\
550 static void									\
551 z_feed_sinc_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)		\
552 {										\
553 	intpcm64_t v;								\
554 	intpcm_t x;								\
555 	uint8_t *p;								\
556 	int32_t coeff, z, *z_coeff, *z_dcoeff;					\
557 	uint32_t c, center, ch, i;						\
558 										\
559 	z_coeff = info->z_coeff;						\
560 	z_dcoeff = info->z_dcoeff;						\
561 	center = z_prev(info, info->z_start, info->z_size);			\
562 	ch = info->channels * PCM_##BIT##_BPS;					\
563 	dst += ch;								\
564 										\
565 	do {									\
566 		dst -= PCM_##BIT##_BPS;						\
567 		ch -= PCM_##BIT##_BPS;						\
568 		v = 0;								\
569 		z = info->z_alpha * info->z_dx;					\
570 		c = 0;								\
571 		p = info->z_delay + (z_next(info, center, 1) *			\
572 		    info->channels * PCM_##BIT##_BPS) + ch;			\
573 		for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR) 	\
574 			Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, +);		\
575 		z = info->z_dy - (info->z_alpha * info->z_dx);			\
576 		c = 0;								\
577 		p = info->z_delay + (center * info->channels *			\
578 		    PCM_##BIT##_BPS) + ch;					\
579 		for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR) 	\
580 			Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, -);		\
581 		if (info->z_scale != Z_ONE)					\
582 			v = Z_SCALE_##BIT(v, info->z_scale);			\
583 		else								\
584 			v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;		\
585 		Z_CLIP_CHECK(v, BIT);						\
586 		_PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));	\
587 	} while (ch != 0);							\
588 }
589 
590 #define Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)				\
591 static void									\
592 z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)	\
593 {										\
594 	intpcm64_t v;								\
595 	intpcm_t x;								\
596 	uint8_t *p;								\
597 	int32_t ch, i, start, *z_pcoeff;					\
598 										\
599 	ch = info->channels * PCM_##BIT##_BPS;					\
600 	dst += ch;								\
601 	start = z_prev(info, info->z_start, (info->z_size << 1) - 1) * ch;	\
602 										\
603 	do {									\
604 		dst -= PCM_##BIT##_BPS;						\
605 		ch -= PCM_##BIT##_BPS;						\
606 		v = 0;								\
607 		p = info->z_delay + start + ch;					\
608 		z_pcoeff = info->z_pcoeff +					\
609 		    ((info->z_alpha * info->z_size) << 1);			\
610 		for (i = info->z_size; i != 0; i--) {				\
611 			x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);		\
612 			v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);		\
613 			z_pcoeff++;						\
614 			p += info->channels * PCM_##BIT##_BPS;			\
615 			x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);		\
616 			v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);		\
617 			z_pcoeff++;						\
618 			p += info->channels * PCM_##BIT##_BPS;			\
619 		}								\
620 		if (info->z_scale != Z_ONE)					\
621 			v = Z_SCALE_##BIT(v, info->z_scale);			\
622 		else								\
623 			v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;		\
624 		Z_CLIP_CHECK(v, BIT);						\
625 		_PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));	\
626 	} while (ch != 0);							\
627 }
628 
629 #define Z_DECLARE(SIGN, BIT, ENDIAN)					\
630 	Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)				\
631 	Z_DECLARE_SINC(SIGN, BIT, ENDIAN)				\
632 	Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)
633 
634 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
635 Z_DECLARE(S, 16, LE)
636 Z_DECLARE(S, 32, LE)
637 #endif
638 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
639 Z_DECLARE(S, 16, BE)
640 Z_DECLARE(S, 32, BE)
641 #endif
642 #ifdef SND_FEEDER_MULTIFORMAT
643 Z_DECLARE(S,  8, NE)
644 Z_DECLARE(S, 24, LE)
645 Z_DECLARE(S, 24, BE)
646 Z_DECLARE(U,  8, NE)
647 Z_DECLARE(U, 16, LE)
648 Z_DECLARE(U, 24, LE)
649 Z_DECLARE(U, 32, LE)
650 Z_DECLARE(U, 16, BE)
651 Z_DECLARE(U, 24, BE)
652 Z_DECLARE(U, 32, BE)
653 #endif
654 
655 enum {
656 	Z_RESAMPLER_ZOH,
657 	Z_RESAMPLER_LINEAR,
658 	Z_RESAMPLER_SINC,
659 	Z_RESAMPLER_SINC_POLYPHASE,
660 	Z_RESAMPLER_LAST
661 };
662 
663 #define Z_RESAMPLER_IDX(i)						\
664 	(Z_IS_SINC(i) ? Z_RESAMPLER_SINC : (i)->quality)
665 
666 #define Z_RESAMPLER_ENTRY(SIGN, BIT, ENDIAN)					\
667 	{									\
668 	    AFMT_##SIGN##BIT##_##ENDIAN,					\
669 	    {									\
670 		[Z_RESAMPLER_ZOH]    = z_feed_zoh,				\
671 		[Z_RESAMPLER_LINEAR] = z_feed_linear_##SIGN##BIT##ENDIAN,	\
672 		[Z_RESAMPLER_SINC]   = z_feed_sinc_##SIGN##BIT##ENDIAN,		\
673 		[Z_RESAMPLER_SINC_POLYPHASE]   =				\
674 		    z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN			\
675 	    }									\
676 	}
677 
678 static const struct {
679 	uint32_t format;
680 	z_resampler_t resampler[Z_RESAMPLER_LAST];
681 } z_resampler_tab[] = {
682 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
683 	Z_RESAMPLER_ENTRY(S, 16, LE),
684 	Z_RESAMPLER_ENTRY(S, 32, LE),
685 #endif
686 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
687 	Z_RESAMPLER_ENTRY(S, 16, BE),
688 	Z_RESAMPLER_ENTRY(S, 32, BE),
689 #endif
690 #ifdef SND_FEEDER_MULTIFORMAT
691 	Z_RESAMPLER_ENTRY(S,  8, NE),
692 	Z_RESAMPLER_ENTRY(S, 24, LE),
693 	Z_RESAMPLER_ENTRY(S, 24, BE),
694 	Z_RESAMPLER_ENTRY(U,  8, NE),
695 	Z_RESAMPLER_ENTRY(U, 16, LE),
696 	Z_RESAMPLER_ENTRY(U, 24, LE),
697 	Z_RESAMPLER_ENTRY(U, 32, LE),
698 	Z_RESAMPLER_ENTRY(U, 16, BE),
699 	Z_RESAMPLER_ENTRY(U, 24, BE),
700 	Z_RESAMPLER_ENTRY(U, 32, BE),
701 #endif
702 };
703 
704 #define Z_RESAMPLER_TAB_SIZE						\
705 	((int32_t)(sizeof(z_resampler_tab) / sizeof(z_resampler_tab[0])))
706 
707 static void
708 z_resampler_reset(struct z_info *info)
709 {
710 
711 	info->src = info->rsrc - (info->rsrc % ((feeder_rate_round > 0 &&
712 	    info->rsrc > feeder_rate_round) ? feeder_rate_round : 1));
713 	info->dst = info->rdst - (info->rdst % ((feeder_rate_round > 0 &&
714 	    info->rdst > feeder_rate_round) ? feeder_rate_round : 1));
715 	info->z_gx = 1;
716 	info->z_gy = 1;
717 	info->z_alpha = 0;
718 	info->z_resample = NULL;
719 	info->z_size = 1;
720 	info->z_coeff = NULL;
721 	info->z_dcoeff = NULL;
722 	if (info->z_pcoeff != NULL) {
723 		free(info->z_pcoeff, M_DEVBUF);
724 		info->z_pcoeff = NULL;
725 	}
726 	info->z_scale = Z_ONE;
727 	info->z_dx = Z_FULL_ONE;
728 	info->z_dy = Z_FULL_ONE;
729 #ifdef Z_DIAGNOSTIC
730 	info->z_cycle = 0;
731 #endif
732 	if (info->quality < Z_QUALITY_MIN)
733 		info->quality = Z_QUALITY_MIN;
734 	else if (info->quality > Z_QUALITY_MAX)
735 		info->quality = Z_QUALITY_MAX;
736 }
737 
738 #ifdef Z_PARANOID
739 static int32_t
740 z_resampler_sinc_len(struct z_info *info)
741 {
742 	int32_t c, z, len, lmax;
743 
744 	if (!Z_IS_SINC(info))
745 		return (1);
746 
747 	/*
748 	 * A rather careful (or useless) way to calculate filter length.
749 	 * Z_SINC_LEN() itself is accurate enough to do its job. Extra
750 	 * sanity checking is not going to hurt though..
751 	 */
752 	c = 0;
753 	z = info->z_dy;
754 	len = 0;
755 	lmax = z_coeff_tab[Z_SINC_COEFF_IDX(info)].len;
756 
757 	do {
758 		c += z >> Z_SHIFT;
759 		z &= Z_MASK;
760 		z += info->z_dy;
761 	} while (c < lmax && ++len > 0);
762 
763 	if (len != Z_SINC_LEN(info)) {
764 #ifdef _KERNEL
765 		printf("%s(): sinc l=%d != Z_SINC_LEN=%d\n",
766 		    __func__, len, Z_SINC_LEN(info));
767 #else
768 		fprintf(stderr, "%s(): sinc l=%d != Z_SINC_LEN=%d\n",
769 		    __func__, len, Z_SINC_LEN(info));
770 		return (-1);
771 #endif
772 	}
773 
774 	return (len);
775 }
776 #else
777 #define z_resampler_sinc_len(i)		(Z_IS_SINC(i) ? Z_SINC_LEN(i) : 1)
778 #endif
779 
780 #define Z_POLYPHASE_COEFF_SHIFT		0
781 
782 /*
783  * Pick suitable polynomial interpolators based on filter oversampled ratio
784  * (2 ^ Z_DRIFT_SHIFT).
785  */
786 #if !(defined(Z_COEFF_INTERP_ZOH) || defined(Z_COEFF_INTERP_LINEAR) ||		\
787     defined(Z_COEFF_INTERP_QUADRATIC) || defined(Z_COEFF_INTERP_HERMITE) ||	\
788     defined(Z_COEFF_INTER_BSPLINE) || defined(Z_COEFF_INTERP_OPT32X) ||		\
789     defined(Z_COEFF_INTERP_OPT16X) || defined(Z_COEFF_INTERP_OPT8X) ||		\
790     defined(Z_COEFF_INTERP_OPT4X) || defined(Z_COEFF_INTERP_OPT2X))
791 #if Z_DRIFT_SHIFT >= 6
792 #define Z_COEFF_INTERP_BSPLINE		1
793 #elif Z_DRIFT_SHIFT >= 5
794 #define Z_COEFF_INTERP_OPT32X		1
795 #elif Z_DRIFT_SHIFT == 4
796 #define Z_COEFF_INTERP_OPT16X		1
797 #elif Z_DRIFT_SHIFT == 3
798 #define Z_COEFF_INTERP_OPT8X		1
799 #elif Z_DRIFT_SHIFT == 2
800 #define Z_COEFF_INTERP_OPT4X		1
801 #elif Z_DRIFT_SHIFT == 1
802 #define Z_COEFF_INTERP_OPT2X		1
803 #else
804 #error "Z_DRIFT_SHIFT screwed!"
805 #endif
806 #endif
807 
808 /*
809  * In classic polyphase mode, the actual coefficients for each phases need to
810  * be calculated based on default prototype filters. For highly oversampled
811  * filter, linear or quadradatic interpolator should be enough. Anything less
812  * than that require 'special' interpolators to reduce interpolation errors.
813  *
814  * "Polynomial Interpolators for High-Quality Resampling of Oversampled Audio"
815  *    by Olli Niemitalo
816  *    - http://www.student.oulu.fi/~oniemita/dsp/deip.pdf
817  *
818  */
819 static int32_t
820 z_coeff_interpolate(int32_t z, int32_t *z_coeff)
821 {
822 	int32_t coeff;
823 #if defined(Z_COEFF_INTERP_ZOH)
824 
825 	/* 1-point, 0th-order (Zero Order Hold) */
826 	z = z;
827 	coeff = z_coeff[0];
828 #elif defined(Z_COEFF_INTERP_LINEAR)
829 	int32_t zl0, zl1;
830 
831 	/* 2-point, 1st-order Linear */
832 	zl0 = z_coeff[0];
833 	zl1 = z_coeff[1] - z_coeff[0];
834 
835 	coeff = Z_RSHIFT((int64_t)zl1 * z, Z_SHIFT) + zl0;
836 #elif defined(Z_COEFF_INTERP_QUADRATIC)
837 	int32_t zq0, zq1, zq2;
838 
839 	/* 3-point, 2nd-order Quadratic */
840 	zq0 = z_coeff[0];
841 	zq1 = z_coeff[1] - z_coeff[-1];
842 	zq2 = z_coeff[1] + z_coeff[-1] - (z_coeff[0] << 1);
843 
844 	coeff = Z_RSHIFT((Z_RSHIFT((int64_t)zq2 * z, Z_SHIFT) +
845 	    zq1) * z, Z_SHIFT + 1) + zq0;
846 #elif defined(Z_COEFF_INTERP_HERMITE)
847 	int32_t zh0, zh1, zh2, zh3;
848 
849 	/* 4-point, 3rd-order Hermite */
850 	zh0 = z_coeff[0];
851 	zh1 = z_coeff[1] - z_coeff[-1];
852 	zh2 = (z_coeff[-1] << 1) - (z_coeff[0] * 5) + (z_coeff[1] << 2) -
853 	    z_coeff[2];
854 	zh3 = z_coeff[2] - z_coeff[-1] + ((z_coeff[0] - z_coeff[1]) * 3);
855 
856 	coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zh3 * z, Z_SHIFT) +
857 	    zh2) * z, Z_SHIFT) + zh1) * z, Z_SHIFT + 1) + zh0;
858 #elif defined(Z_COEFF_INTERP_BSPLINE)
859 	int32_t zb0, zb1, zb2, zb3;
860 
861 	/* 4-point, 3rd-order B-Spline */
862 	zb0 = Z_RSHIFT(0x15555555LL * (((int64_t)z_coeff[0] << 2) +
863 	    z_coeff[-1] + z_coeff[1]), 30);
864 	zb1 = z_coeff[1] - z_coeff[-1];
865 	zb2 = z_coeff[-1] + z_coeff[1] - (z_coeff[0] << 1);
866 	zb3 = Z_RSHIFT(0x15555555LL * (((z_coeff[0] - z_coeff[1]) * 3) +
867 	    z_coeff[2] - z_coeff[-1]), 30);
868 
869 	coeff = (Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zb3 * z, Z_SHIFT) +
870 	    zb2) * z, Z_SHIFT) + zb1) * z, Z_SHIFT) + zb0 + 1) >> 1;
871 #elif defined(Z_COEFF_INTERP_OPT32X)
872 	int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
873 	int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
874 
875 	/* 6-point, 5th-order Optimal 32x */
876 	zoz = z - (Z_ONE >> 1);
877 	zoe1 = z_coeff[1] + z_coeff[0];
878 	zoe2 = z_coeff[2] + z_coeff[-1];
879 	zoe3 = z_coeff[3] + z_coeff[-2];
880 	zoo1 = z_coeff[1] - z_coeff[0];
881 	zoo2 = z_coeff[2] - z_coeff[-1];
882 	zoo3 = z_coeff[3] - z_coeff[-2];
883 
884 	zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
885 	    (0x00170c29LL * zoe3), 30);
886 	zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
887 	    (0x008cd4dcLL * zoo3), 30);
888 	zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
889 	    (0x0160b5d0LL * zoe3), 30);
890 	zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
891 	    (0x01cfe914LL * zoo3), 30);
892 	zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
893 	    (0x015508ddLL * zoe3), 30);
894 	zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
895 	    (0x0082d81aLL * zoo3), 30);
896 
897 	coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
898 	    (int64_t)zoc5 * zoz, Z_SHIFT) +
899 	    zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
900 	    zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
901 #elif defined(Z_COEFF_INTERP_OPT16X)
902 	int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
903 	int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
904 
905 	/* 6-point, 5th-order Optimal 16x */
906 	zoz = z - (Z_ONE >> 1);
907 	zoe1 = z_coeff[1] + z_coeff[0];
908 	zoe2 = z_coeff[2] + z_coeff[-1];
909 	zoe3 = z_coeff[3] + z_coeff[-2];
910 	zoo1 = z_coeff[1] - z_coeff[0];
911 	zoo2 = z_coeff[2] - z_coeff[-1];
912 	zoo3 = z_coeff[3] - z_coeff[-2];
913 
914 	zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
915 	    (0x00170c29LL * zoe3), 30);
916 	zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
917 	    (0x008cd4dcLL * zoo3), 30);
918 	zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
919 	    (0x0160b5d0LL * zoe3), 30);
920 	zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
921 	    (0x01cfe914LL * zoo3), 30);
922 	zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
923 	    (0x015508ddLL * zoe3), 30);
924 	zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
925 	    (0x0082d81aLL * zoo3), 30);
926 
927 	coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
928 	    (int64_t)zoc5 * zoz, Z_SHIFT) +
929 	    zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
930 	    zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
931 #elif defined(Z_COEFF_INTERP_OPT8X)
932 	int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
933 	int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
934 
935 	/* 6-point, 5th-order Optimal 8x */
936 	zoz = z - (Z_ONE >> 1);
937 	zoe1 = z_coeff[1] + z_coeff[0];
938 	zoe2 = z_coeff[2] + z_coeff[-1];
939 	zoe3 = z_coeff[3] + z_coeff[-2];
940 	zoo1 = z_coeff[1] - z_coeff[0];
941 	zoo2 = z_coeff[2] - z_coeff[-1];
942 	zoo3 = z_coeff[3] - z_coeff[-2];
943 
944 	zoc0 = Z_RSHIFT((0x1aa9b47dLL * zoe1) + (0x053d9944LL * zoe2) +
945 	    (0x0018b23fLL * zoe3), 30);
946 	zoc1 = Z_RSHIFT((0x14a104d1LL * zoo1) + (0x0d7d2504LL * zoo2) +
947 	    (0x0094b599LL * zoo3), 30);
948 	zoc2 = Z_RSHIFT((-0x0d22530bLL * zoe1) + (0x0bb37a2cLL * zoe2) +
949 	    (0x016ed8e0LL * zoe3), 30);
950 	zoc3 = Z_RSHIFT((-0x0d744b1cLL * zoo1) + (0x01649591LL * zoo2) +
951 	    (0x01dae93aLL * zoo3), 30);
952 	zoc4 = Z_RSHIFT((0x02a7ee1bLL * zoe1) + (-0x03fbdb24LL * zoe2) +
953 	    (0x0153ed07LL * zoe3), 30);
954 	zoc5 = Z_RSHIFT((0x04cf9b6cLL * zoo1) + (-0x0266b378LL * zoo2) +
955 	    (0x007a7c26LL * zoo3), 30);
956 
957 	coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
958 	    (int64_t)zoc5 * zoz, Z_SHIFT) +
959 	    zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
960 	    zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
961 #elif defined(Z_COEFF_INTERP_OPT4X)
962 	int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
963 	int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
964 
965 	/* 6-point, 5th-order Optimal 4x */
966 	zoz = z - (Z_ONE >> 1);
967 	zoe1 = z_coeff[1] + z_coeff[0];
968 	zoe2 = z_coeff[2] + z_coeff[-1];
969 	zoe3 = z_coeff[3] + z_coeff[-2];
970 	zoo1 = z_coeff[1] - z_coeff[0];
971 	zoo2 = z_coeff[2] - z_coeff[-1];
972 	zoo3 = z_coeff[3] - z_coeff[-2];
973 
974 	zoc0 = Z_RSHIFT((0x1a8eda43LL * zoe1) + (0x0556ee38LL * zoe2) +
975 	    (0x001a3784LL * zoe3), 30);
976 	zoc1 = Z_RSHIFT((0x143d863eLL * zoo1) + (0x0d910e36LL * zoo2) +
977 	    (0x009ca889LL * zoo3), 30);
978 	zoc2 = Z_RSHIFT((-0x0d026821LL * zoe1) + (0x0b837773LL * zoe2) +
979 	    (0x017ef0c6LL * zoe3), 30);
980 	zoc3 = Z_RSHIFT((-0x0cef1502LL * zoo1) + (0x01207a8eLL * zoo2) +
981 	    (0x01e936dbLL * zoo3), 30);
982 	zoc4 = Z_RSHIFT((0x029fe643LL * zoe1) + (-0x03ef3fc8LL * zoe2) +
983 	    (0x014f5923LL * zoe3), 30);
984 	zoc5 = Z_RSHIFT((0x043a9d08LL * zoo1) + (-0x02154febLL * zoo2) +
985 	    (0x00670dbdLL * zoo3), 30);
986 
987 	coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
988 	    (int64_t)zoc5 * zoz, Z_SHIFT) +
989 	    zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
990 	    zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
991 #elif defined(Z_COEFF_INTERP_OPT2X)
992 	int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
993 	int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
994 
995 	/* 6-point, 5th-order Optimal 2x */
996 	zoz = z - (Z_ONE >> 1);
997 	zoe1 = z_coeff[1] + z_coeff[0];
998 	zoe2 = z_coeff[2] + z_coeff[-1];
999 	zoe3 = z_coeff[3] + z_coeff[-2];
1000 	zoo1 = z_coeff[1] - z_coeff[0];
1001 	zoo2 = z_coeff[2] - z_coeff[-1];
1002 	zoo3 = z_coeff[3] - z_coeff[-2];
1003 
1004 	zoc0 = Z_RSHIFT((0x19edb6fdLL * zoe1) + (0x05ebd062LL * zoe2) +
1005 	    (0x00267881LL * zoe3), 30);
1006 	zoc1 = Z_RSHIFT((0x1223af76LL * zoo1) + (0x0de3dd6bLL * zoo2) +
1007 	    (0x00d683cdLL * zoo3), 30);
1008 	zoc2 = Z_RSHIFT((-0x0c3ee068LL * zoe1) + (0x0a5c3769LL * zoe2) +
1009 	    (0x01e2aceaLL * zoe3), 30);
1010 	zoc3 = Z_RSHIFT((-0x0a8ab614LL * zoo1) + (-0x0019522eLL * zoo2) +
1011 	    (0x022cefc7LL * zoo3), 30);
1012 	zoc4 = Z_RSHIFT((0x0276187dLL * zoe1) + (-0x03a801e8LL * zoe2) +
1013 	    (0x0131d935LL * zoe3), 30);
1014 	zoc5 = Z_RSHIFT((0x02c373f5LL * zoo1) + (-0x01275f83LL * zoo2) +
1015 	    (0x0018ee79LL * zoo3), 30);
1016 
1017 	coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
1018 	    (int64_t)zoc5 * zoz, Z_SHIFT) +
1019 	    zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
1020 	    zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
1021 #else
1022 #error "Interpolation type screwed!"
1023 #endif
1024 
1025 #if Z_POLYPHASE_COEFF_SHIFT > 0
1026 	coeff = Z_RSHIFT(coeff, Z_POLYPHASE_COEFF_SHIFT);
1027 #endif
1028 	return (coeff);
1029 }
1030 
1031 static int
1032 z_resampler_build_polyphase(struct z_info *info)
1033 {
1034 	int32_t alpha, c, i, z, idx;
1035 
1036 	/* Let this be here first. */
1037 	if (info->z_pcoeff != NULL) {
1038 		free(info->z_pcoeff, M_DEVBUF);
1039 		info->z_pcoeff = NULL;
1040 	}
1041 
1042 	if (feeder_rate_polyphase_max < 1)
1043 		return (ENOTSUP);
1044 
1045 	if (((int64_t)info->z_size * info->z_gy * 2) >
1046 	    feeder_rate_polyphase_max) {
1047 #ifndef _KERNEL
1048 		fprintf(stderr, "Polyphase entries exceed: [%d/%d] %jd > %d\n",
1049 		    info->z_gx, info->z_gy,
1050 		    (intmax_t)info->z_size * info->z_gy * 2,
1051 		    feeder_rate_polyphase_max);
1052 #endif
1053 		return (E2BIG);
1054 	}
1055 
1056 	info->z_pcoeff = malloc(sizeof(int32_t) *
1057 	    info->z_size * info->z_gy * 2, M_DEVBUF, M_NOWAIT | M_ZERO);
1058 	if (info->z_pcoeff == NULL)
1059 		return (ENOMEM);
1060 
1061 	for (alpha = 0; alpha < info->z_gy; alpha++) {
1062 		z = alpha * info->z_dx;
1063 		c = 0;
1064 		for (i = info->z_size; i != 0; i--) {
1065 			c += z >> Z_SHIFT;
1066 			z &= Z_MASK;
1067 			idx = (alpha * info->z_size * 2) +
1068 			    (info->z_size * 2) - i;
1069 			info->z_pcoeff[idx] =
1070 			    z_coeff_interpolate(z, info->z_coeff + c);
1071 			z += info->z_dy;
1072 		}
1073 		z = info->z_dy - (alpha * info->z_dx);
1074 		c = 0;
1075 		for (i = info->z_size; i != 0; i--) {
1076 			c += z >> Z_SHIFT;
1077 			z &= Z_MASK;
1078 			idx = (alpha * info->z_size * 2) + i - 1;
1079 			info->z_pcoeff[idx] =
1080 			    z_coeff_interpolate(z, info->z_coeff + c);
1081 			z += info->z_dy;
1082 		}
1083 	}
1084 
1085 #ifndef _KERNEL
1086 	fprintf(stderr, "Polyphase: [%d/%d] %d entries\n",
1087 	    info->z_gx, info->z_gy, info->z_size * info->z_gy * 2);
1088 #endif
1089 
1090 	return (0);
1091 }
1092 
1093 static int
1094 z_resampler_setup(struct pcm_feeder *f)
1095 {
1096 	struct z_info *info;
1097 	int64_t gy2gx_max, gx2gy_max;
1098 	uint32_t format;
1099 	int32_t align, i, z_scale;
1100 	int adaptive;
1101 
1102 	info = f->data;
1103 	z_resampler_reset(info);
1104 
1105 	if (info->src == info->dst)
1106 		return (0);
1107 
1108 	/* Shrink by greatest common divisor. */
1109 	i = z_gcd(info->src, info->dst);
1110 	info->z_gx = info->src / i;
1111 	info->z_gy = info->dst / i;
1112 
1113 	/* Too big, or too small. Bail out. */
1114 	if (!(Z_FACTOR_SAFE(info->z_gx) && Z_FACTOR_SAFE(info->z_gy)))
1115 		return (EINVAL);
1116 
1117 	format = f->desc->in;
1118 	adaptive = 0;
1119 	z_scale = 0;
1120 
1121 	/*
1122 	 * Setup everything: filter length, conversion factor, etc.
1123 	 */
1124 	if (Z_IS_SINC(info)) {
1125 		/*
1126 		 * Downsampling, or upsampling scaling factor. As long as the
1127 		 * factor can be represented by a fraction of 1 << Z_SHIFT,
1128 		 * we're pretty much in business. Scaling is not needed for
1129 		 * upsampling, so we just slap Z_ONE there.
1130 		 */
1131 		if (info->z_gx > info->z_gy)
1132 			/*
1133 			 * If the downsampling ratio is beyond sanity,
1134 			 * enable semi-adaptive mode. Although handling
1135 			 * extreme ratio is possible, the result of the
1136 			 * conversion is just pointless, unworthy,
1137 			 * nonsensical noises, etc.
1138 			 */
1139 			if ((info->z_gx / info->z_gy) > Z_SINC_DOWNMAX)
1140 				z_scale = Z_ONE / Z_SINC_DOWNMAX;
1141 			else
1142 				z_scale = ((uint64_t)info->z_gy << Z_SHIFT) /
1143 				    info->z_gx;
1144 		else
1145 			z_scale = Z_ONE;
1146 
1147 		/*
1148 		 * This is actually impossible, unless anything above
1149 		 * overflow.
1150 		 */
1151 		if (z_scale < 1)
1152 			return (E2BIG);
1153 
1154 		/*
1155 		 * Calculate sample time/coefficients index drift. It is
1156 		 * a constant for upsampling, but downsampling require
1157 		 * heavy duty filtering with possible too long filters.
1158 		 * If anything goes wrong, revisit again and enable
1159 		 * adaptive mode.
1160 		 */
1161 z_setup_adaptive_sinc:
1162 		if (info->z_pcoeff != NULL) {
1163 			free(info->z_pcoeff, M_DEVBUF);
1164 			info->z_pcoeff = NULL;
1165 		}
1166 
1167 		if (adaptive == 0) {
1168 			info->z_dy = z_scale << Z_DRIFT_SHIFT;
1169 			if (info->z_dy < 1)
1170 				return (E2BIG);
1171 			info->z_scale = z_scale;
1172 		} else {
1173 			info->z_dy = Z_FULL_ONE;
1174 			info->z_scale = Z_ONE;
1175 		}
1176 
1177 #if 0
1178 #define Z_SCALE_DIV	10000
1179 #define Z_SCALE_LIMIT(s, v)						\
1180 	((((uint64_t)(s) * (v)) + (Z_SCALE_DIV >> 1)) / Z_SCALE_DIV)
1181 
1182 		info->z_scale = Z_SCALE_LIMIT(info->z_scale, 9780);
1183 #endif
1184 
1185 		/* Smallest drift increment. */
1186 		info->z_dx = info->z_dy / info->z_gy;
1187 
1188 		/*
1189 		 * Overflow or underflow. Try adaptive, let it continue and
1190 		 * retry.
1191 		 */
1192 		if (info->z_dx < 1) {
1193 			if (adaptive == 0) {
1194 				adaptive = 1;
1195 				goto z_setup_adaptive_sinc;
1196 			}
1197 			return (E2BIG);
1198 		}
1199 
1200 		/*
1201 		 * Round back output drift.
1202 		 */
1203 		info->z_dy = info->z_dx * info->z_gy;
1204 
1205 		for (i = 0; i < Z_COEFF_TAB_SIZE; i++) {
1206 			if (Z_SINC_COEFF_IDX(info) != i)
1207 				continue;
1208 			/*
1209 			 * Calculate required filter length and guard
1210 			 * against possible abusive result. Note that
1211 			 * this represents only 1/2 of the entire filter
1212 			 * length.
1213 			 */
1214 			info->z_size = z_resampler_sinc_len(info);
1215 
1216 			/*
1217 			 * Multiple of 2 rounding, for better accumulator
1218 			 * performance.
1219 			 */
1220 			info->z_size &= ~1;
1221 
1222 			if (info->z_size < 2 || info->z_size > Z_SINC_MAX) {
1223 				if (adaptive == 0) {
1224 					adaptive = 1;
1225 					goto z_setup_adaptive_sinc;
1226 				}
1227 				return (E2BIG);
1228 			}
1229 			info->z_coeff = z_coeff_tab[i].coeff + Z_COEFF_OFFSET;
1230 			info->z_dcoeff = z_coeff_tab[i].dcoeff;
1231 			break;
1232 		}
1233 
1234 		if (info->z_coeff == NULL || info->z_dcoeff == NULL)
1235 			return (EINVAL);
1236 	} else if (Z_IS_LINEAR(info)) {
1237 		/*
1238 		 * Don't put much effort if we're doing linear interpolation.
1239 		 * Just center the interpolation distance within Z_LINEAR_ONE,
1240 		 * and be happy about it.
1241 		 */
1242 		info->z_dx = Z_LINEAR_FULL_ONE / info->z_gy;
1243 	}
1244 
1245 	/*
1246 	 * We're safe for now, lets continue.. Look for our resampler
1247 	 * depending on configured format and quality.
1248 	 */
1249 	for (i = 0; i < Z_RESAMPLER_TAB_SIZE; i++) {
1250 		int ridx;
1251 
1252 		if (AFMT_ENCODING(format) != z_resampler_tab[i].format)
1253 			continue;
1254 		if (Z_IS_SINC(info) && adaptive == 0 &&
1255 		    z_resampler_build_polyphase(info) == 0)
1256 			ridx = Z_RESAMPLER_SINC_POLYPHASE;
1257 		else
1258 			ridx = Z_RESAMPLER_IDX(info);
1259 		info->z_resample = z_resampler_tab[i].resampler[ridx];
1260 		break;
1261 	}
1262 
1263 	if (info->z_resample == NULL)
1264 		return (EINVAL);
1265 
1266 	info->bps = AFMT_BPS(format);
1267 	align = info->channels * info->bps;
1268 
1269 	/*
1270 	 * Calculate largest value that can be fed into z_gy2gx() and
1271 	 * z_gx2gy() without causing (signed) 32bit overflow. z_gy2gx() will
1272 	 * be called early during feeding process to determine how much input
1273 	 * samples that is required to generate requested output, while
1274 	 * z_gx2gy() will be called just before samples filtering /
1275 	 * accumulation process based on available samples that has been
1276 	 * calculated using z_gx2gy().
1277 	 *
1278 	 * Now that is damn confusing, I guess ;-) .
1279 	 */
1280 	gy2gx_max = (((uint64_t)info->z_gy * INT32_MAX) - info->z_gy + 1) /
1281 	    info->z_gx;
1282 
1283 	if ((gy2gx_max * align) > SND_FXDIV_MAX)
1284 		gy2gx_max = SND_FXDIV_MAX / align;
1285 
1286 	if (gy2gx_max < 1)
1287 		return (E2BIG);
1288 
1289 	gx2gy_max = (((uint64_t)info->z_gx * INT32_MAX) - info->z_gy) /
1290 	    info->z_gy;
1291 
1292 	if (gx2gy_max > INT32_MAX)
1293 		gx2gy_max = INT32_MAX;
1294 
1295 	if (gx2gy_max < 1)
1296 		return (E2BIG);
1297 
1298 	/*
1299 	 * Ensure that z_gy2gx() at its largest possible calculated value
1300 	 * (alpha = 0) will not cause overflow further late during z_gx2gy()
1301 	 * stage.
1302 	 */
1303 	if (z_gy2gx(info, gy2gx_max) > _Z_GCAST(gx2gy_max))
1304 		return (E2BIG);
1305 
1306 	info->z_maxfeed = gy2gx_max * align;
1307 
1308 #ifdef Z_USE_ALPHADRIFT
1309 	info->z_startdrift = z_gy2gx(info, 1);
1310 	info->z_alphadrift = z_drift(info, info->z_startdrift, 1);
1311 #endif
1312 
1313 	i = z_gy2gx(info, 1);
1314 	info->z_full = z_roundpow2((info->z_size << 1) + i);
1315 
1316 	/*
1317 	 * Too big to be true, and overflowing left and right like mad ..
1318 	 */
1319 	if ((info->z_full * align) < 1) {
1320 		if (adaptive == 0 && Z_IS_SINC(info)) {
1321 			adaptive = 1;
1322 			goto z_setup_adaptive_sinc;
1323 		}
1324 		return (E2BIG);
1325 	}
1326 
1327 	/*
1328 	 * Increase full buffer size if its too small to reduce cyclic
1329 	 * buffer shifting in main conversion/feeder loop.
1330 	 */
1331 	while (info->z_full < Z_RESERVOIR_MAX &&
1332 	    (info->z_full - (info->z_size << 1)) < Z_RESERVOIR)
1333 		info->z_full <<= 1;
1334 
1335 	/* Initialize buffer position. */
1336 	info->z_mask = info->z_full - 1;
1337 	info->z_start = z_prev(info, info->z_size << 1, 1);
1338 	info->z_pos = z_next(info, info->z_start, 1);
1339 
1340 	/*
1341 	 * Allocate or reuse delay line buffer, whichever makes sense.
1342 	 */
1343 	i = info->z_full * align;
1344 	if (i < 1)
1345 		return (E2BIG);
1346 
1347 	if (info->z_delay == NULL || info->z_alloc < i ||
1348 	    i <= (info->z_alloc >> 1)) {
1349 		if (info->z_delay != NULL)
1350 			free(info->z_delay, M_DEVBUF);
1351 		info->z_delay = malloc(i, M_DEVBUF, M_NOWAIT | M_ZERO);
1352 		if (info->z_delay == NULL)
1353 			return (ENOMEM);
1354 		info->z_alloc = i;
1355 	}
1356 
1357 	/*
1358 	 * Zero out head of buffer to avoid pops and clicks.
1359 	 */
1360 	memset(info->z_delay, sndbuf_zerodata(f->desc->out),
1361 	    info->z_pos * align);
1362 
1363 #ifdef Z_DIAGNOSTIC
1364 	/*
1365 	 * XXX Debuging mess !@#$%^
1366 	 */
1367 #define dumpz(x)	fprintf(stderr, "\t%12s = %10u : %-11d\n",	\
1368 			    "z_"__STRING(x), (uint32_t)info->z_##x,	\
1369 			    (int32_t)info->z_##x)
1370 	fprintf(stderr, "\n%s():\n", __func__);
1371 	fprintf(stderr, "\tchannels=%d, bps=%d, format=0x%08x, quality=%d\n",
1372 	    info->channels, info->bps, format, info->quality);
1373 	fprintf(stderr, "\t%d (%d) -> %d (%d), ",
1374 	    info->src, info->rsrc, info->dst, info->rdst);
1375 	fprintf(stderr, "[%d/%d]\n", info->z_gx, info->z_gy);
1376 	fprintf(stderr, "\tminreq=%d, ", z_gy2gx(info, 1));
1377 	if (adaptive != 0)
1378 		z_scale = Z_ONE;
1379 	fprintf(stderr, "factor=0x%08x/0x%08x (%f)\n",
1380 	    z_scale, Z_ONE, (double)z_scale / Z_ONE);
1381 	fprintf(stderr, "\tbase_length=%d, ", Z_SINC_BASE_LEN(info));
1382 	fprintf(stderr, "adaptive=%s\n", (adaptive != 0) ? "YES" : "NO");
1383 	dumpz(size);
1384 	dumpz(alloc);
1385 	if (info->z_alloc < 1024)
1386 		fprintf(stderr, "\t%15s%10d Bytes\n",
1387 		    "", info->z_alloc);
1388 	else if (info->z_alloc < (1024 << 10))
1389 		fprintf(stderr, "\t%15s%10d KBytes\n",
1390 		    "", info->z_alloc >> 10);
1391 	else if (info->z_alloc < (1024 << 20))
1392 		fprintf(stderr, "\t%15s%10d MBytes\n",
1393 		    "", info->z_alloc >> 20);
1394 	else
1395 		fprintf(stderr, "\t%15s%10d GBytes\n",
1396 		    "", info->z_alloc >> 30);
1397 	fprintf(stderr, "\t%12s   %10d (min output samples)\n",
1398 	    "",
1399 	    (int32_t)z_gx2gy(info, info->z_full - (info->z_size << 1)));
1400 	fprintf(stderr, "\t%12s   %10d (min allocated output samples)\n",
1401 	    "",
1402 	    (int32_t)z_gx2gy(info, (info->z_alloc / align) -
1403 	    (info->z_size << 1)));
1404 	fprintf(stderr, "\t%12s = %10d\n",
1405 	    "z_gy2gx()", (int32_t)z_gy2gx(info, 1));
1406 	fprintf(stderr, "\t%12s = %10d -> z_gy2gx() -> %d\n",
1407 	    "Max", (int32_t)gy2gx_max, (int32_t)z_gy2gx(info, gy2gx_max));
1408 	fprintf(stderr, "\t%12s = %10d\n",
1409 	    "z_gx2gy()", (int32_t)z_gx2gy(info, 1));
1410 	fprintf(stderr, "\t%12s = %10d -> z_gx2gy() -> %d\n",
1411 	    "Max", (int32_t)gx2gy_max, (int32_t)z_gx2gy(info, gx2gy_max));
1412 	dumpz(maxfeed);
1413 	dumpz(full);
1414 	dumpz(start);
1415 	dumpz(pos);
1416 	dumpz(scale);
1417 	fprintf(stderr, "\t%12s   %10f\n", "",
1418 	    (double)info->z_scale / Z_ONE);
1419 	dumpz(dx);
1420 	fprintf(stderr, "\t%12s   %10f\n", "",
1421 	    (double)info->z_dx / info->z_dy);
1422 	dumpz(dy);
1423 	fprintf(stderr, "\t%12s   %10d (drift step)\n", "",
1424 	    info->z_dy >> Z_SHIFT);
1425 	fprintf(stderr, "\t%12s   %10d (scaling differences)\n", "",
1426 	    (z_scale << Z_DRIFT_SHIFT) - info->z_dy);
1427 	fprintf(stderr, "\t%12s = %u bytes\n",
1428 	    "intpcm32_t", sizeof(intpcm32_t));
1429 	fprintf(stderr, "\t%12s = 0x%08x, smallest=%.16lf\n",
1430 	    "Z_ONE", Z_ONE, (double)1.0 / (double)Z_ONE);
1431 #endif
1432 
1433 	return (0);
1434 }
1435 
1436 static int
1437 z_resampler_set(struct pcm_feeder *f, int what, int32_t value)
1438 {
1439 	struct z_info *info;
1440 	int32_t oquality;
1441 
1442 	info = f->data;
1443 
1444 	switch (what) {
1445 	case Z_RATE_SRC:
1446 		if (value < feeder_rate_min || value > feeder_rate_max)
1447 			return (E2BIG);
1448 		if (value == info->rsrc)
1449 			return (0);
1450 		info->rsrc = value;
1451 		break;
1452 	case Z_RATE_DST:
1453 		if (value < feeder_rate_min || value > feeder_rate_max)
1454 			return (E2BIG);
1455 		if (value == info->rdst)
1456 			return (0);
1457 		info->rdst = value;
1458 		break;
1459 	case Z_RATE_QUALITY:
1460 		if (value < Z_QUALITY_MIN || value > Z_QUALITY_MAX)
1461 			return (EINVAL);
1462 		if (value == info->quality)
1463 			return (0);
1464 		/*
1465 		 * If we failed to set the requested quality, restore
1466 		 * the old one. We cannot afford leaving it broken since
1467 		 * passive feeder chains like vchans never reinitialize
1468 		 * itself.
1469 		 */
1470 		oquality = info->quality;
1471 		info->quality = value;
1472 		if (z_resampler_setup(f) == 0)
1473 			return (0);
1474 		info->quality = oquality;
1475 		break;
1476 	case Z_RATE_CHANNELS:
1477 		if (value < SND_CHN_MIN || value > SND_CHN_MAX)
1478 			return (EINVAL);
1479 		if (value == info->channels)
1480 			return (0);
1481 		info->channels = value;
1482 		break;
1483 	default:
1484 		return (EINVAL);
1485 		break;
1486 	}
1487 
1488 	return (z_resampler_setup(f));
1489 }
1490 
1491 static int
1492 z_resampler_get(struct pcm_feeder *f, int what)
1493 {
1494 	struct z_info *info;
1495 
1496 	info = f->data;
1497 
1498 	switch (what) {
1499 	case Z_RATE_SRC:
1500 		return (info->rsrc);
1501 		break;
1502 	case Z_RATE_DST:
1503 		return (info->rdst);
1504 		break;
1505 	case Z_RATE_QUALITY:
1506 		return (info->quality);
1507 		break;
1508 	case Z_RATE_CHANNELS:
1509 		return (info->channels);
1510 		break;
1511 	default:
1512 		break;
1513 	}
1514 
1515 	return (-1);
1516 }
1517 
1518 static int
1519 z_resampler_init(struct pcm_feeder *f)
1520 {
1521 	struct z_info *info;
1522 	int ret;
1523 
1524 	if (f->desc->in != f->desc->out)
1525 		return (EINVAL);
1526 
1527 	info = malloc(sizeof(*info), M_DEVBUF, M_NOWAIT | M_ZERO);
1528 	if (info == NULL)
1529 		return (ENOMEM);
1530 
1531 	info->rsrc = Z_RATE_DEFAULT;
1532 	info->rdst = Z_RATE_DEFAULT;
1533 	info->quality = feeder_rate_quality;
1534 	info->channels = AFMT_CHANNEL(f->desc->in);
1535 
1536 	f->data = info;
1537 
1538 	ret = z_resampler_setup(f);
1539 	if (ret != 0) {
1540 		if (info->z_pcoeff != NULL)
1541 			free(info->z_pcoeff, M_DEVBUF);
1542 		if (info->z_delay != NULL)
1543 			free(info->z_delay, M_DEVBUF);
1544 		free(info, M_DEVBUF);
1545 		f->data = NULL;
1546 	}
1547 
1548 	return (ret);
1549 }
1550 
1551 static int
1552 z_resampler_free(struct pcm_feeder *f)
1553 {
1554 	struct z_info *info;
1555 
1556 	info = f->data;
1557 	if (info != NULL) {
1558 		if (info->z_pcoeff != NULL)
1559 			free(info->z_pcoeff, M_DEVBUF);
1560 		if (info->z_delay != NULL)
1561 			free(info->z_delay, M_DEVBUF);
1562 		free(info, M_DEVBUF);
1563 	}
1564 
1565 	f->data = NULL;
1566 
1567 	return (0);
1568 }
1569 
1570 static uint32_t
1571 z_resampler_feed_internal(struct pcm_feeder *f, struct pcm_channel *c,
1572     uint8_t *b, uint32_t count, void *source)
1573 {
1574 	struct z_info *info;
1575 	int32_t alphadrift, startdrift, reqout, ocount, reqin, align;
1576 	int32_t fetch, fetched, start, cp;
1577 	uint8_t *dst;
1578 
1579 	info = f->data;
1580 	if (info->z_resample == NULL)
1581 		return (z_feed(f->source, c, b, count, source));
1582 
1583 	/*
1584 	 * Calculate sample size alignment and amount of sample output.
1585 	 * We will do everything in sample domain, but at the end we
1586 	 * will jump back to byte domain.
1587 	 */
1588 	align = info->channels * info->bps;
1589 	ocount = SND_FXDIV(count, align);
1590 	if (ocount == 0)
1591 		return (0);
1592 
1593 	/*
1594 	 * Calculate amount of input samples that is needed to generate
1595 	 * exact amount of output.
1596 	 */
1597 	reqin = z_gy2gx(info, ocount) - z_fetched(info);
1598 
1599 #ifdef Z_USE_ALPHADRIFT
1600 	startdrift = info->z_startdrift;
1601 	alphadrift = info->z_alphadrift;
1602 #else
1603 	startdrift = _Z_GY2GX(info, 0, 1);
1604 	alphadrift = z_drift(info, startdrift, 1);
1605 #endif
1606 
1607 	dst = b;
1608 
1609 	do {
1610 		if (reqin != 0) {
1611 			fetch = z_min(z_free(info), reqin);
1612 			if (fetch == 0) {
1613 				/*
1614 				 * No more free spaces, so wind enough
1615 				 * samples back to the head of delay line
1616 				 * in byte domain.
1617 				 */
1618 				fetched = z_fetched(info);
1619 				start = z_prev(info, info->z_start,
1620 				    (info->z_size << 1) - 1);
1621 				cp = (info->z_size << 1) + fetched;
1622 				z_copy(info->z_delay + (start * align),
1623 				    info->z_delay, cp * align);
1624 				info->z_start =
1625 				    z_prev(info, info->z_size << 1, 1);
1626 				info->z_pos =
1627 				    z_next(info, info->z_start, fetched + 1);
1628 				fetch = z_min(z_free(info), reqin);
1629 #ifdef Z_DIAGNOSTIC
1630 				if (1) {
1631 					static uint32_t kk = 0;
1632 					fprintf(stderr,
1633 					    "Buffer Move: "
1634 					    "start=%d fetched=%d cp=%d "
1635 					    "cycle=%u [%u]\r",
1636 					    start, fetched, cp, info->z_cycle,
1637 					    ++kk);
1638 				}
1639 				info->z_cycle = 0;
1640 #endif
1641 			}
1642 			if (fetch != 0) {
1643 				/*
1644 				 * Fetch in byte domain and jump back
1645 				 * to sample domain.
1646 				 */
1647 				fetched = SND_FXDIV(z_feed(f->source, c,
1648 				    info->z_delay + (info->z_pos * align),
1649 				    fetch * align, source), align);
1650 				/*
1651 				 * Prepare to convert fetched buffer,
1652 				 * or mark us done if we cannot fulfill
1653 				 * the request.
1654 				 */
1655 				reqin -= fetched;
1656 				info->z_pos += fetched;
1657 				if (fetched != fetch)
1658 					reqin = 0;
1659 			}
1660 		}
1661 
1662 		reqout = z_min(z_gx2gy(info, z_fetched(info)), ocount);
1663 		if (reqout != 0) {
1664 			ocount -= reqout;
1665 
1666 			/*
1667 			 * Drift.. drift.. drift..
1668 			 *
1669 			 * Notice that there are 2 methods of doing the drift
1670 			 * operations: The former is much cleaner (in a sense
1671 			 * of mathematical readings of my eyes), but slower
1672 			 * due to integer division in z_gy2gx(). Nevertheless,
1673 			 * both should give the same exact accurate drifting
1674 			 * results, so the later is favourable.
1675 			 */
1676 			do {
1677 				info->z_resample(info, dst);
1678 #if 0
1679 				startdrift = z_gy2gx(info, 1);
1680 				alphadrift = z_drift(info, startdrift, 1);
1681 				info->z_start += startdrift;
1682 				info->z_alpha += alphadrift;
1683 #else
1684 				info->z_alpha += alphadrift;
1685 				if (info->z_alpha < info->z_gy)
1686 					info->z_start += startdrift;
1687 				else {
1688 					info->z_start += startdrift - 1;
1689 					info->z_alpha -= info->z_gy;
1690 				}
1691 #endif
1692 				dst += align;
1693 #ifdef Z_DIAGNOSTIC
1694 				info->z_cycle++;
1695 #endif
1696 			} while (--reqout != 0);
1697 		}
1698 	} while (reqin != 0 && ocount != 0);
1699 
1700 	/*
1701 	 * Back to byte domain..
1702 	 */
1703 	return (dst - b);
1704 }
1705 
1706 static int
1707 z_resampler_feed(struct pcm_feeder *f, struct pcm_channel *c, uint8_t *b,
1708     uint32_t count, void *source)
1709 {
1710 	uint32_t feed, maxfeed, left;
1711 
1712 	/*
1713 	 * Split count to smaller chunks to avoid possible 32bit overflow.
1714 	 */
1715 	maxfeed = ((struct z_info *)(f->data))->z_maxfeed;
1716 	left = count;
1717 
1718 	do {
1719 		feed = z_resampler_feed_internal(f, c, b,
1720 		    z_min(maxfeed, left), source);
1721 		b += feed;
1722 		left -= feed;
1723 	} while (left != 0 && feed != 0);
1724 
1725 	return (count - left);
1726 }
1727 
1728 static struct pcm_feederdesc feeder_rate_desc[] = {
1729 	{ FEEDER_RATE, 0, 0, 0, 0 },
1730 	{ 0, 0, 0, 0, 0 },
1731 };
1732 
1733 static kobj_method_t feeder_rate_methods[] = {
1734 	KOBJMETHOD(feeder_init,		z_resampler_init),
1735 	KOBJMETHOD(feeder_free,		z_resampler_free),
1736 	KOBJMETHOD(feeder_set,		z_resampler_set),
1737 	KOBJMETHOD(feeder_get,		z_resampler_get),
1738 	KOBJMETHOD(feeder_feed,		z_resampler_feed),
1739 	KOBJMETHOD_END
1740 };
1741 
1742 FEEDER_DECLARE(feeder_rate, NULL);
1743