1 /*****************************************************************************
2 * audiotoolbox_midi.c: Software MIDI synthesizer using AudioToolbox
3 *****************************************************************************
4 * Copyright (C) 2017 VLC authors and VideoLAN
5 * $Id: 8adf545980e4ea16e5f04ebbf0da565ec1581fa5 $
6 *
7 * Authors: Marvin Scholz <epirat07 at gmail dot com>
8 *
9 * Based on the fluidsynth module by Rémi Denis-Courmont
10 *
11 * This program is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU Lesser General Public License as published by
13 * the Free Software Foundation; either version 2.1 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU Lesser General Public License for more details.
20 *
21 * You should have received a copy of the GNU Lesser General Public License
22 * along with this program; if not, write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
24 *****************************************************************************/
25
26 #ifdef HAVE_CONFIG_H
27 # include "config.h"
28 #endif
29
30 #include <vlc_common.h>
31 #include <vlc_plugin.h>
32 #include <vlc_codec.h>
33 #include <vlc_dialog.h>
34
35 #include <CoreFoundation/CoreFoundation.h>
36 #include <AudioUnit/AudioUnit.h>
37 #include <AudioToolbox/AudioToolbox.h>
38
39 #include <TargetConditionals.h>
40
41 #ifndef on_err_goto
42 #define on_err_goto(errorCode, exceptionLabel) \
43 do { if ((errorCode) != noErr) goto exceptionLabel; \
44 } while ( 0 )
45 #endif
46
47 #define SOUNDFONT_TEXT N_("SoundFont file")
48 #define SOUNDFONT_LONGTEXT N_( \
49 "SoundFont file to use for software synthesis." )
50
51 static int Open (vlc_object_t *);
52 static void Close (vlc_object_t *);
53
54 #define CFG_PREFIX "aumidi-"
55
56 vlc_module_begin()
57 set_description(N_("AudioToolbox MIDI synthesizer"))
58 set_capability("audio decoder", 100)
59 set_shortname(N_("AUMIDI"))
60 set_category(CAT_INPUT)
61 set_subcategory(SUBCAT_INPUT_ACODEC)
62 set_callbacks(Open, Close)
63 add_loadfile(CFG_PREFIX "soundfont", "",
64 SOUNDFONT_TEXT, SOUNDFONT_LONGTEXT, false)
65 vlc_module_end()
66
67
68 struct decoder_sys_t
69 {
70 AUGraph graph;
71 AudioUnit synthUnit;
72 AudioUnit outputUnit;
73 date_t end_date;
74 };
75
76 static int DecodeBlock (decoder_t *p_dec, block_t *p_block);
77 static void Flush (decoder_t *);
78
79 /* MIDI constants */
80 enum
81 {
82 kMidiMessage_NoteOff = 0x80,
83 kMidiMessage_NoteOn = 0x90,
84 kMidiMessage_PolyPressure = 0xA0,
85 kMidiMessage_ControlChange = 0xB0,
86 kMidiMessage_ProgramChange = 0xC0,
87 kMidiMessage_ChannelPressure = 0xD0,
88 kMidiMessage_PitchWheel = 0xE0,
89 kMidiMessage_SysEx = 0xF0,
90
91 kMidiMessage_BankMSBControl = 0,
92 kMidiMessage_BankLSBControl = 32,
93
94 /* Values for kMidiMessage_ControlChange */
95 kMidiController_AllSoundOff = 0x78,
96 kMidiController_ResetAllControllers = 0x79,
97 kMidiController_AllNotesOff = 0x7B
98 };
99
100 /* Helper functions */
AddAppleAUNode(AUGraph graph,OSType type,OSType subtype,AUNode * node)101 static OSStatus AddAppleAUNode(AUGraph graph, OSType type, OSType subtype, AUNode *node)
102 {
103 AudioComponentDescription cDesc = {};
104 cDesc.componentType = type;
105 cDesc.componentSubType = subtype;
106 cDesc.componentManufacturer = kAudioUnitManufacturer_Apple;
107
108 return AUGraphAddNode(graph, &cDesc, node);
109 }
110
CreateAUGraph(AUGraph * outGraph,AudioUnit * outSynth,AudioUnit * outOut)111 static OSStatus CreateAUGraph(AUGraph *outGraph, AudioUnit *outSynth, AudioUnit *outOut)
112 {
113 OSStatus res;
114
115 // AudioUnit nodes
116 AUNode synthNode, limiterNode, outNode;
117
118 // Create the Graph to which we will add our nodes
119 on_err_goto(res = NewAUGraph(outGraph), bailout);
120
121 // Create/add the MIDI synthesizer node (DLS Synth)
122 #if TARGET_OS_IPHONE
123 // On iOS/tvOS use MIDISynth, DLSSynth does not exist there
124 on_err_goto(res = AddAppleAUNode(*outGraph,
125 kAudioUnitType_MusicDevice,
126 kAudioUnitSubType_MIDISynth,
127 &synthNode), bailout);
128 #else
129 // Prefer DLSSynth on macOS, as it has a better default behavior
130 on_err_goto(res = AddAppleAUNode(*outGraph,
131 kAudioUnitType_MusicDevice,
132 kAudioUnitSubType_DLSSynth,
133 &synthNode), bailout);
134 #endif
135
136 // Create/add the peak limiter node
137 on_err_goto(res = AddAppleAUNode(*outGraph,
138 kAudioUnitType_Effect,
139 kAudioUnitSubType_PeakLimiter,
140 &limiterNode), bailout);
141
142 // Create/add the output node (GenericOutput)
143 on_err_goto(res = AddAppleAUNode(*outGraph,
144 kAudioUnitType_Output,
145 kAudioUnitSubType_GenericOutput,
146 &outNode), bailout);
147
148 // Open the Graph, this opens the units that belong to the graph
149 // so that we can connect them
150 on_err_goto(res = AUGraphOpen(*outGraph), bailout);
151
152 // Connect the synthesizer node to the limiter
153 on_err_goto(res = AUGraphConnectNodeInput(*outGraph, synthNode, 0, limiterNode, 0), bailout);
154 // Connect the limiter node to the output
155 on_err_goto(res = AUGraphConnectNodeInput(*outGraph, limiterNode, 0, outNode, 0), bailout);
156
157 // Get reference to the synthesizer node
158 on_err_goto(res = AUGraphNodeInfo(*outGraph, synthNode, 0, outSynth), bailout);
159 // Get reference to the output node
160 on_err_goto(res = AUGraphNodeInfo(*outGraph, outNode, 0, outOut), bailout);
161
162 bailout:
163 return res;
164 }
165
SetSoundfont(decoder_t * p_dec,AudioUnit synthUnit,const char * sfPath)166 static int SetSoundfont(decoder_t *p_dec, AudioUnit synthUnit, const char *sfPath) {
167 if (!sfPath) {
168 msg_Dbg(p_dec, "using default soundfont");
169 return VLC_SUCCESS;
170 }
171
172 msg_Dbg(p_dec, "using custom soundfont: '%s'", sfPath);
173 CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault,
174 (const UInt8 *)sfPath,
175 strlen(sfPath), false);
176 if (unlikely(url == NULL))
177 return VLC_ENOMEM;
178
179 OSStatus status = AudioUnitSetProperty(synthUnit,
180 kMusicDeviceProperty_SoundBankURL,
181 kAudioUnitScope_Global, 0,
182 &url, sizeof(url));
183 CFRelease(url);
184
185 if (status != noErr) {
186 msg_Err(p_dec, "failed setting custom SoundFont for MIDI synthesis (%i)", status);
187 return VLC_EGENERIC;
188 }
189 return VLC_SUCCESS;
190 }
191
Open(vlc_object_t * p_this)192 static int Open(vlc_object_t *p_this)
193 {
194 decoder_t *p_dec = (decoder_t *)p_this;
195 OSStatus status = noErr;
196 int ret = VLC_SUCCESS;
197
198 if (p_dec->fmt_in.i_codec != VLC_CODEC_MIDI)
199 return VLC_EGENERIC;
200
201 decoder_sys_t *p_sys = malloc(sizeof (*p_sys));
202 if (unlikely(p_sys == NULL))
203 return VLC_ENOMEM;
204
205 p_sys->graph = NULL;
206 status = CreateAUGraph(&p_sys->graph, &p_sys->synthUnit, &p_sys->outputUnit);
207 if (unlikely(status != noErr)) {
208 msg_Err(p_dec, "failed to create audiograph (%i)", status);
209 ret = VLC_EGENERIC;
210 goto bailout;
211 }
212
213 // Set custom soundfont
214 char *sfPath = var_InheritString(p_dec, CFG_PREFIX "soundfont");
215 ret = SetSoundfont(p_dec, p_sys->synthUnit, sfPath);
216 free(sfPath);
217 if (unlikely(ret != VLC_SUCCESS))
218 goto bailout;
219
220 // Set VLC output audio format info
221 p_dec->fmt_out.i_codec = VLC_CODEC_FL32;
222 p_dec->fmt_out.audio.i_bitspersample = 32;
223 p_dec->fmt_out.audio.i_rate = 44100;
224 p_dec->fmt_out.audio.i_channels = 2;
225 p_dec->fmt_out.audio.i_physical_channels = AOUT_CHAN_LEFT | AOUT_CHAN_RIGHT;
226
227 if (decoder_UpdateAudioFormat(p_dec) < 0) {
228 ret = VLC_EGENERIC;
229 goto bailout;
230 }
231
232 // Prepare AudioUnit output audio format info
233 AudioStreamBasicDescription ASBD = {};
234 unsigned bytesPerSample = sizeof(Float32);
235 ASBD.mFormatID = kAudioFormatLinearPCM;
236 ASBD.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked;
237 ASBD.mSampleRate = 44100;
238 ASBD.mFramesPerPacket = 1;
239 ASBD.mChannelsPerFrame = 2;
240 ASBD.mBytesPerFrame = bytesPerSample * ASBD.mChannelsPerFrame;
241 ASBD.mBytesPerPacket = ASBD.mBytesPerFrame * ASBD.mFramesPerPacket;
242 ASBD.mBitsPerChannel = 8 * bytesPerSample;
243
244 // Set AudioUnit format
245 status = AudioUnitSetProperty(p_sys->outputUnit,
246 kAudioUnitProperty_StreamFormat,
247 kAudioUnitScope_Output, 0, &ASBD,
248 sizeof(AudioStreamBasicDescription));
249 if (unlikely(status != noErr)) {
250 msg_Err(p_dec, "failed setting output format for output unit (%i)", status);
251 ret = VLC_EGENERIC;
252 goto bailout;
253 }
254
255 // Prepare the AU
256 status = AUGraphInitialize (p_sys->graph);
257 if (unlikely(status != noErr)) {
258 if (status == kAudioUnitErr_InvalidFile)
259 msg_Err(p_dec, "failed initializing audiograph: invalid soundfont file");
260 else
261 msg_Err(p_dec, "failed initializing audiograph (%i)", status);
262 ret = VLC_EGENERIC;
263 goto bailout;
264 }
265
266 // Prepare MIDI soundbank
267 MusicDeviceMIDIEvent(p_sys->synthUnit,
268 kMidiMessage_ControlChange,
269 kMidiMessage_BankMSBControl, 0, 0);
270
271 // Start the AU
272 status = AUGraphStart(p_sys->graph);
273 if (unlikely(status != noErr)) {
274 msg_Err(p_dec, "failed starting audiograph (%i)", status);
275 ret = VLC_EGENERIC;
276 goto bailout;
277 }
278
279 // Initialize date (for PTS)
280 date_Init(&p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1);
281 date_Set(&p_sys->end_date, 0);
282
283 p_dec->p_sys = p_sys;
284 p_dec->pf_decode = DecodeBlock;
285 p_dec->pf_flush = Flush;
286
287 bailout:
288 // Cleanup if error occured
289 if (ret != VLC_SUCCESS) {
290 if (p_sys->graph)
291 DisposeAUGraph(p_sys->graph);
292 free(p_sys);
293 }
294 return ret;
295 }
296
297
Close(vlc_object_t * p_this)298 static void Close (vlc_object_t *p_this)
299 {
300 decoder_sys_t *p_sys = ((decoder_t *)p_this)->p_sys;
301
302 if (p_sys->graph) {
303 AUGraphStop(p_sys->graph);
304 DisposeAUGraph(p_sys->graph);
305 }
306 free(p_sys);
307 }
308
Flush(decoder_t * p_dec)309 static void Flush (decoder_t *p_dec)
310 {
311 decoder_sys_t *p_sys = p_dec->p_sys;
312
313 date_Set(&p_sys->end_date, VLC_TS_INVALID);
314
315 // Turn all sound on all channels off
316 // else 'old' notes could still be playing
317 for (unsigned channel = 0; channel < 16; channel++) {
318 MusicDeviceMIDIEvent(p_sys->synthUnit, kMidiMessage_ControlChange | channel, kMidiController_AllSoundOff, 0, 0);
319 }
320 }
321
DecodeBlock(decoder_t * p_dec,block_t * p_block)322 static int DecodeBlock (decoder_t *p_dec, block_t *p_block)
323 {
324 decoder_sys_t *p_sys = p_dec->p_sys;
325 block_t *p_out = NULL;
326 OSStatus status = noErr;
327
328 if (p_block == NULL) /* No Drain */
329 return VLCDEC_SUCCESS;
330
331 if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
332 Flush(p_dec);
333 if (p_block->i_flags & BLOCK_FLAG_CORRUPTED) {
334 block_Release(p_block);
335 return VLCDEC_SUCCESS;
336 }
337 }
338
339 if (p_block->i_pts > VLC_TS_INVALID && !date_Get(&p_sys->end_date)) {
340 date_Set(&p_sys->end_date, p_block->i_pts);
341 } else if (p_block->i_pts < date_Get(&p_sys->end_date)) {
342 msg_Warn(p_dec, "MIDI message in the past?");
343 goto drop;
344 }
345
346 if (p_block->i_buffer < 1)
347 goto drop;
348
349 uint8_t event = p_block->p_buffer[0];
350 uint8_t data1 = (p_block->i_buffer > 1) ? (p_block->p_buffer[1]) : 0;
351 uint8_t data2 = (p_block->i_buffer > 2) ? (p_block->p_buffer[2]) : 0;
352
353 switch (event & 0xF0)
354 {
355 case kMidiMessage_NoteOff:
356 case kMidiMessage_NoteOn:
357 case kMidiMessage_PolyPressure:
358 case kMidiMessage_ControlChange:
359 case kMidiMessage_ProgramChange:
360 case kMidiMessage_ChannelPressure:
361 case kMidiMessage_PitchWheel:
362 MusicDeviceMIDIEvent(p_sys->synthUnit, event, data1, data2, 0);
363 break;
364
365 case kMidiMessage_SysEx:
366 if (p_block->i_buffer < UINT32_MAX)
367 MusicDeviceSysEx(p_sys->synthUnit, p_block->p_buffer, (UInt32)p_block->i_buffer);
368 break;
369
370 default:
371 msg_Warn(p_dec, "unhandled MIDI event: %x", event & 0xF0);
372 break;
373 }
374
375 // Calculate frame count
376 // Simplification of 44100 / 1000000
377 // TODO: Other samplerates
378 unsigned frames =
379 (p_block->i_pts - date_Get(&p_sys->end_date)) * 441 / 10000;
380
381 if (frames == 0)
382 goto drop;
383
384 p_out = decoder_NewAudioBuffer(p_dec, frames);
385 if (p_out == NULL)
386 goto drop;
387
388 p_out->i_pts = date_Get(&p_sys->end_date );
389 p_out->i_length = date_Increment(&p_sys->end_date, frames)
390 - p_out->i_pts;
391
392 // Prepare Timestamp for the AudioUnit render call
393 AudioTimeStamp timestamp = {};
394 timestamp.mFlags = kAudioTimeStampWordClockTimeValid;
395 timestamp.mWordClockTime = p_out->i_pts;
396
397 // Prepare Buffer for the AudioUnit render call
398 AudioBufferList bufferList;
399 bufferList.mNumberBuffers = 1;
400 bufferList.mBuffers[0].mNumberChannels = 2;
401 bufferList.mBuffers[0].mDataByteSize = frames * sizeof(Float32) * 2;
402 bufferList.mBuffers[0].mData = p_out->p_buffer;
403
404 status = AudioUnitRender(p_sys->outputUnit,
405 NULL,
406 ×tamp, 0,
407 frames, &bufferList);
408
409 if (status != noErr) {
410 msg_Warn(p_dec, "rendering audio unit failed: %i", status);
411 block_Release(p_out);
412 p_out = NULL;
413 }
414
415 drop:
416 block_Release(p_block);
417 if (p_out != NULL)
418 decoder_QueueAudio(p_dec, p_out);
419 return VLCDEC_SUCCESS;
420 }
421