1 /****************************************************************************
2 **
3 ** Copyright (C) 2016 The Qt Company Ltd.
4 ** Contact: https://www.qt.io/licensing/
5 **
6 ** This file is part of the Qt Toolkit.
7 **
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** Commercial License Usage
10 ** Licensees holding valid commercial Qt licenses may use this file in
11 ** accordance with the commercial license agreement provided with the
12 ** Software or, alternatively, in accordance with the terms contained in
13 ** a written agreement between you and The Qt Company. For licensing terms
14 ** and conditions see https://www.qt.io/terms-conditions. For further
15 ** information use the contact form at https://www.qt.io/contact-us.
16 **
17 ** GNU Lesser General Public License Usage
18 ** Alternatively, this file may be used under the terms of the GNU Lesser
19 ** General Public License version 3 as published by the Free Software
20 ** Foundation and appearing in the file LICENSE.LGPL3 included in the
21 ** packaging of this file. Please review the following information to
22 ** ensure the GNU Lesser General Public License version 3 requirements
23 ** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24 **
25 ** GNU General Public License Usage
26 ** Alternatively, this file may be used under the terms of the GNU
27 ** General Public License version 2.0 or (at your option) the GNU General
28 ** Public license version 3 or any later version approved by the KDE Free
29 ** Qt Foundation. The licenses are as published by the Free Software
30 ** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31 ** included in the packaging of this file. Please review the following
32 ** information to ensure the GNU General Public License requirements will
33 ** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34 ** https://www.gnu.org/licenses/gpl-3.0.html.
35 **
36 ** $QT_END_LICENSE$
37 **
38 ****************************************************************************/
39
40 #include "qopenslesaudioinput.h"
41
42 #include "qopenslesengine.h"
43 #include <qbuffer.h>
44 #include <private/qaudiohelpers_p.h>
45 #include <qdebug.h>
46
47 #ifdef ANDROID
48 #include <SLES/OpenSLES_AndroidConfiguration.h>
49 #include <QtCore/private/qjnihelpers_p.h>
50 #include <QtCore/private/qjni_p.h>
51 #endif
52
53 QT_BEGIN_NAMESPACE
54
55 #define NUM_BUFFERS 2
56 #define DEFAULT_PERIOD_TIME_MS 50
57 #define MINIMUM_PERIOD_TIME_MS 5
58
59 #ifdef ANDROID
hasRecordingPermission()60 static bool hasRecordingPermission()
61 {
62 using namespace QtAndroidPrivate;
63 if (androidSdkVersion() < 23)
64 return true;
65
66 const QString key(QLatin1String("android.permission.RECORD_AUDIO"));
67 PermissionsResult res = checkPermission(key);
68 if (res == PermissionsResult::Granted) // Permission already granted?
69 return true;
70
71 QJNIEnvironmentPrivate env;
72 const auto &results = requestPermissionsSync(env, QStringList() << key);
73 if (!results.contains(key)) {
74 qWarning("No permission found for key: %s", qPrintable(key));
75 return false;
76 }
77
78 if (results[key] == PermissionsResult::Denied) {
79 qDebug("%s - Permission denied by user!", qPrintable(key));
80 return false;
81 }
82
83 return true;
84 }
85
bufferQueueCallback(SLAndroidSimpleBufferQueueItf,void * context)86 static void bufferQueueCallback(SLAndroidSimpleBufferQueueItf, void *context)
87 #else
88 static void bufferQueueCallback(SLBufferQueueItf, void *context)
89 #endif
90 {
91 // Process buffer in main thread
92 QMetaObject::invokeMethod(reinterpret_cast<QOpenSLESAudioInput*>(context), "processBuffer");
93 }
94
QOpenSLESAudioInput(const QByteArray & device)95 QOpenSLESAudioInput::QOpenSLESAudioInput(const QByteArray &device)
96 : m_device(device)
97 , m_engine(QOpenSLESEngine::instance())
98 , m_recorderObject(0)
99 , m_recorder(0)
100 , m_bufferQueue(0)
101 , m_pullMode(true)
102 , m_processedBytes(0)
103 , m_audioSource(0)
104 , m_bufferIODevice(0)
105 , m_errorState(QAudio::NoError)
106 , m_deviceState(QAudio::StoppedState)
107 , m_lastNotifyTime(0)
108 , m_volume(1.0)
109 , m_bufferSize(0)
110 , m_periodSize(0)
111 , m_intervalTime(1000)
112 , m_buffers(new QByteArray[NUM_BUFFERS])
113 , m_currentBuffer(0)
114 {
115 #ifdef ANDROID
116 if (qstrcmp(device, QT_ANDROID_PRESET_CAMCORDER) == 0)
117 m_recorderPreset = SL_ANDROID_RECORDING_PRESET_CAMCORDER;
118 else if (qstrcmp(device, QT_ANDROID_PRESET_VOICE_RECOGNITION) == 0)
119 m_recorderPreset = SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION;
120 else if (qstrcmp(device, QT_ANDROID_PRESET_VOICE_COMMUNICATION) == 0)
121 m_recorderPreset = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;
122 else
123 m_recorderPreset = SL_ANDROID_RECORDING_PRESET_GENERIC;
124 #endif
125 }
126
~QOpenSLESAudioInput()127 QOpenSLESAudioInput::~QOpenSLESAudioInput()
128 {
129 if (m_recorderObject)
130 (*m_recorderObject)->Destroy(m_recorderObject);
131 delete[] m_buffers;
132 }
133
error() const134 QAudio::Error QOpenSLESAudioInput::error() const
135 {
136 return m_errorState;
137 }
138
state() const139 QAudio::State QOpenSLESAudioInput::state() const
140 {
141 return m_deviceState;
142 }
143
setFormat(const QAudioFormat & format)144 void QOpenSLESAudioInput::setFormat(const QAudioFormat &format)
145 {
146 if (m_deviceState == QAudio::StoppedState)
147 m_format = format;
148 }
149
format() const150 QAudioFormat QOpenSLESAudioInput::format() const
151 {
152 return m_format;
153 }
154
start(QIODevice * device)155 void QOpenSLESAudioInput::start(QIODevice *device)
156 {
157 if (m_deviceState != QAudio::StoppedState)
158 stopRecording();
159
160 if (!m_pullMode && m_bufferIODevice) {
161 m_bufferIODevice->close();
162 delete m_bufferIODevice;
163 m_bufferIODevice = 0;
164 }
165
166 m_pullMode = true;
167 m_audioSource = device;
168
169 if (startRecording()) {
170 m_deviceState = QAudio::ActiveState;
171 } else {
172 m_deviceState = QAudio::StoppedState;
173 Q_EMIT errorChanged(m_errorState);
174 }
175
176 Q_EMIT stateChanged(m_deviceState);
177 }
178
start()179 QIODevice *QOpenSLESAudioInput::start()
180 {
181 if (m_deviceState != QAudio::StoppedState)
182 stopRecording();
183
184 m_audioSource = 0;
185
186 if (!m_pullMode && m_bufferIODevice) {
187 m_bufferIODevice->close();
188 delete m_bufferIODevice;
189 }
190
191 m_pullMode = false;
192 m_pushBuffer.clear();
193 m_bufferIODevice = new QBuffer(&m_pushBuffer);
194 m_bufferIODevice->open(QIODevice::ReadOnly);
195
196 if (startRecording()) {
197 m_deviceState = QAudio::IdleState;
198 } else {
199 m_deviceState = QAudio::StoppedState;
200 Q_EMIT errorChanged(m_errorState);
201 m_bufferIODevice->close();
202 delete m_bufferIODevice;
203 m_bufferIODevice = 0;
204 }
205
206 Q_EMIT stateChanged(m_deviceState);
207 return m_bufferIODevice;
208 }
209
startRecording()210 bool QOpenSLESAudioInput::startRecording()
211 {
212 if (!hasRecordingPermission())
213 return false;
214
215 m_processedBytes = 0;
216 m_clockStamp.restart();
217 m_lastNotifyTime = 0;
218
219 SLresult result;
220
221 // configure audio source
222 SLDataLocator_IODevice loc_dev = { SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT,
223 SL_DEFAULTDEVICEID_AUDIOINPUT, NULL };
224 SLDataSource audioSrc = { &loc_dev, NULL };
225
226 // configure audio sink
227 #ifdef ANDROID
228 SLDataLocator_AndroidSimpleBufferQueue loc_bq = { SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
229 NUM_BUFFERS };
230 #else
231 SLDataLocator_BufferQueue loc_bq = { SL_DATALOCATOR_BUFFERQUEUE, NUM_BUFFERS };
232 #endif
233
234 SLDataFormat_PCM format_pcm = QOpenSLESEngine::audioFormatToSLFormatPCM(m_format);
235 SLDataSink audioSnk = { &loc_bq, &format_pcm };
236
237 // create audio recorder
238 // (requires the RECORD_AUDIO permission)
239 #ifdef ANDROID
240 const SLInterfaceID id[2] = { SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_ANDROIDCONFIGURATION };
241 const SLboolean req[2] = { SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
242 #else
243 const SLInterfaceID id[1] = { SL_IID_BUFFERQUEUE };
244 const SLboolean req[1] = { SL_BOOLEAN_TRUE };
245 #endif
246
247 result = (*m_engine->slEngine())->CreateAudioRecorder(m_engine->slEngine(), &m_recorderObject,
248 &audioSrc, &audioSnk,
249 sizeof(req) / sizeof(SLboolean), id, req);
250 if (result != SL_RESULT_SUCCESS) {
251 m_errorState = QAudio::OpenError;
252 return false;
253 }
254
255 #ifdef ANDROID
256 // configure recorder source
257 SLAndroidConfigurationItf configItf;
258 result = (*m_recorderObject)->GetInterface(m_recorderObject, SL_IID_ANDROIDCONFIGURATION,
259 &configItf);
260 if (result != SL_RESULT_SUCCESS) {
261 m_errorState = QAudio::OpenError;
262 return false;
263 }
264
265 result = (*configItf)->SetConfiguration(configItf, SL_ANDROID_KEY_RECORDING_PRESET,
266 &m_recorderPreset, sizeof(SLuint32));
267
268 SLuint32 presetValue = SL_ANDROID_RECORDING_PRESET_NONE;
269 SLuint32 presetSize = 2*sizeof(SLuint32); // intentionally too big
270 result = (*configItf)->GetConfiguration(configItf, SL_ANDROID_KEY_RECORDING_PRESET,
271 &presetSize, (void*)&presetValue);
272
273 if (result != SL_RESULT_SUCCESS || presetValue == SL_ANDROID_RECORDING_PRESET_NONE) {
274 m_errorState = QAudio::OpenError;
275 return false;
276 }
277 #endif
278
279 // realize the audio recorder
280 result = (*m_recorderObject)->Realize(m_recorderObject, SL_BOOLEAN_FALSE);
281 if (result != SL_RESULT_SUCCESS) {
282 m_errorState = QAudio::OpenError;
283 return false;
284 }
285
286 // get the record interface
287 result = (*m_recorderObject)->GetInterface(m_recorderObject, SL_IID_RECORD, &m_recorder);
288 if (result != SL_RESULT_SUCCESS) {
289 m_errorState = QAudio::FatalError;
290 return false;
291 }
292
293 // get the buffer queue interface
294 #ifdef ANDROID
295 SLInterfaceID bufferqueueItfID = SL_IID_ANDROIDSIMPLEBUFFERQUEUE;
296 #else
297 SLInterfaceID bufferqueueItfID = SL_IID_BUFFERQUEUE;
298 #endif
299 result = (*m_recorderObject)->GetInterface(m_recorderObject, bufferqueueItfID, &m_bufferQueue);
300 if (result != SL_RESULT_SUCCESS) {
301 m_errorState = QAudio::FatalError;
302 return false;
303 }
304
305 // register callback on the buffer queue
306 result = (*m_bufferQueue)->RegisterCallback(m_bufferQueue, bufferQueueCallback, this);
307 if (result != SL_RESULT_SUCCESS) {
308 m_errorState = QAudio::FatalError;
309 return false;
310 }
311
312 if (m_bufferSize <= 0) {
313 m_bufferSize = m_format.bytesForDuration(DEFAULT_PERIOD_TIME_MS * 1000);
314 } else {
315 int minimumBufSize = m_format.bytesForDuration(MINIMUM_PERIOD_TIME_MS * 1000);
316 if (m_bufferSize < minimumBufSize)
317 m_bufferSize = minimumBufSize;
318 }
319
320 m_periodSize = m_bufferSize;
321
322 // enqueue empty buffers to be filled by the recorder
323 for (int i = 0; i < NUM_BUFFERS; ++i) {
324 m_buffers[i].resize(m_periodSize);
325
326 result = (*m_bufferQueue)->Enqueue(m_bufferQueue, m_buffers[i].data(), m_periodSize);
327 if (result != SL_RESULT_SUCCESS) {
328 m_errorState = QAudio::FatalError;
329 return false;
330 }
331 }
332
333 // start recording
334 result = (*m_recorder)->SetRecordState(m_recorder, SL_RECORDSTATE_RECORDING);
335 if (result != SL_RESULT_SUCCESS) {
336 m_errorState = QAudio::FatalError;
337 return false;
338 }
339
340 m_errorState = QAudio::NoError;
341
342 return true;
343 }
344
stop()345 void QOpenSLESAudioInput::stop()
346 {
347 if (m_deviceState == QAudio::StoppedState)
348 return;
349
350 m_deviceState = QAudio::StoppedState;
351
352 stopRecording();
353
354 m_errorState = QAudio::NoError;
355 Q_EMIT stateChanged(m_deviceState);
356 }
357
stopRecording()358 void QOpenSLESAudioInput::stopRecording()
359 {
360 flushBuffers();
361
362 (*m_recorder)->SetRecordState(m_recorder, SL_RECORDSTATE_STOPPED);
363 (*m_bufferQueue)->Clear(m_bufferQueue);
364
365 (*m_recorderObject)->Destroy(m_recorderObject);
366 m_recorderObject = 0;
367
368 for (int i = 0; i < NUM_BUFFERS; ++i)
369 m_buffers[i].clear();
370 m_currentBuffer = 0;
371
372 if (!m_pullMode && m_bufferIODevice) {
373 m_bufferIODevice->close();
374 delete m_bufferIODevice;
375 m_bufferIODevice = 0;
376 m_pushBuffer.clear();
377 }
378 }
379
suspend()380 void QOpenSLESAudioInput::suspend()
381 {
382 if (m_deviceState == QAudio::ActiveState) {
383 m_deviceState = QAudio::SuspendedState;
384 emit stateChanged(m_deviceState);
385
386 (*m_recorder)->SetRecordState(m_recorder, SL_RECORDSTATE_PAUSED);
387 }
388 }
389
resume()390 void QOpenSLESAudioInput::resume()
391 {
392 if (m_deviceState == QAudio::SuspendedState || m_deviceState == QAudio::IdleState) {
393 (*m_recorder)->SetRecordState(m_recorder, SL_RECORDSTATE_RECORDING);
394
395 m_deviceState = QAudio::ActiveState;
396 emit stateChanged(m_deviceState);
397 }
398 }
399
processBuffer()400 void QOpenSLESAudioInput::processBuffer()
401 {
402 if (m_deviceState == QAudio::StoppedState || m_deviceState == QAudio::SuspendedState)
403 return;
404
405 if (m_deviceState != QAudio::ActiveState) {
406 m_errorState = QAudio::NoError;
407 m_deviceState = QAudio::ActiveState;
408 emit stateChanged(m_deviceState);
409 }
410
411 QByteArray *processedBuffer = &m_buffers[m_currentBuffer];
412 writeDataToDevice(processedBuffer->constData(), processedBuffer->size());
413
414 // Re-enqueue the buffer
415 SLresult result = (*m_bufferQueue)->Enqueue(m_bufferQueue,
416 processedBuffer->data(),
417 processedBuffer->size());
418
419 m_currentBuffer = (m_currentBuffer + 1) % NUM_BUFFERS;
420
421 // If the buffer queue is empty (shouldn't happen), stop recording.
422 #ifdef ANDROID
423 SLAndroidSimpleBufferQueueState state;
424 #else
425 SLBufferQueueState state;
426 #endif
427 result = (*m_bufferQueue)->GetState(m_bufferQueue, &state);
428 if (result != SL_RESULT_SUCCESS || state.count == 0) {
429 stop();
430 m_errorState = QAudio::FatalError;
431 Q_EMIT errorChanged(m_errorState);
432 }
433 }
434
writeDataToDevice(const char * data,int size)435 void QOpenSLESAudioInput::writeDataToDevice(const char *data, int size)
436 {
437 m_processedBytes += size;
438
439 QByteArray outData;
440
441 // Apply volume
442 if (m_volume < 1.0f) {
443 outData.resize(size);
444 QAudioHelperInternal::qMultiplySamples(m_volume, m_format, data, outData.data(), size);
445 } else {
446 outData.append(data, size);
447 }
448
449 if (m_pullMode) {
450 // write buffer to the QIODevice
451 if (m_audioSource->write(outData) < 0) {
452 stop();
453 m_errorState = QAudio::IOError;
454 Q_EMIT errorChanged(m_errorState);
455 }
456 } else {
457 // emits readyRead() so user will call read() on QIODevice to get some audio data
458 if (m_bufferIODevice != 0) {
459 m_pushBuffer.append(outData);
460 Q_EMIT m_bufferIODevice->readyRead();
461 }
462 }
463
464 // Send notify signal if needed
465 qint64 processedMsecs = processedUSecs() / 1000;
466 if (m_intervalTime && (processedMsecs - m_lastNotifyTime) >= m_intervalTime) {
467 Q_EMIT notify();
468 m_lastNotifyTime = processedMsecs;
469 }
470 }
471
flushBuffers()472 void QOpenSLESAudioInput::flushBuffers()
473 {
474 SLmillisecond recorderPos;
475 (*m_recorder)->GetPosition(m_recorder, &recorderPos);
476 qint64 devicePos = processedUSecs();
477
478 qint64 delta = recorderPos * 1000 - devicePos;
479
480 if (delta > 0) {
481 const int writeSize = std::min(m_buffers[m_currentBuffer].size(),
482 m_format.bytesForDuration(delta));
483 writeDataToDevice(m_buffers[m_currentBuffer].constData(), writeSize);
484 }
485 }
486
bytesReady() const487 int QOpenSLESAudioInput::bytesReady() const
488 {
489 if (m_deviceState == QAudio::ActiveState || m_deviceState == QAudio::SuspendedState)
490 return m_bufferIODevice ? m_bufferIODevice->bytesAvailable() : m_periodSize;
491
492 return 0;
493 }
494
setBufferSize(int value)495 void QOpenSLESAudioInput::setBufferSize(int value)
496 {
497 m_bufferSize = value;
498 }
499
bufferSize() const500 int QOpenSLESAudioInput::bufferSize() const
501 {
502 return m_bufferSize;
503 }
504
periodSize() const505 int QOpenSLESAudioInput::periodSize() const
506 {
507 return m_periodSize;
508 }
509
setNotifyInterval(int ms)510 void QOpenSLESAudioInput::setNotifyInterval(int ms)
511 {
512 m_intervalTime = qMax(0, ms);
513 }
514
notifyInterval() const515 int QOpenSLESAudioInput::notifyInterval() const
516 {
517 return m_intervalTime;
518 }
519
processedUSecs() const520 qint64 QOpenSLESAudioInput::processedUSecs() const
521 {
522 return m_format.durationForBytes(m_processedBytes);
523 }
524
elapsedUSecs() const525 qint64 QOpenSLESAudioInput::elapsedUSecs() const
526 {
527 if (m_deviceState == QAudio::StoppedState)
528 return 0;
529
530 return m_clockStamp.elapsed() * qint64(1000);
531 }
532
setVolume(qreal vol)533 void QOpenSLESAudioInput::setVolume(qreal vol)
534 {
535 m_volume = vol;
536 }
537
volume() const538 qreal QOpenSLESAudioInput::volume() const
539 {
540 return m_volume;
541 }
542
reset()543 void QOpenSLESAudioInput::reset()
544 {
545 stop();
546 }
547
548 QT_END_NAMESPACE
549