1 // Copyright 2018 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef SERVICES_AUDIO_LOOPBACK_STREAM_H_ 6 #define SERVICES_AUDIO_LOOPBACK_STREAM_H_ 7 8 #include <map> 9 #include <memory> 10 #include <utility> 11 #include <vector> 12 13 #include "base/callback.h" 14 #include "base/macros.h" 15 #include "base/memory/scoped_refptr.h" 16 #include "base/memory/weak_ptr.h" 17 #include "base/optional.h" 18 #include "base/sequence_checker.h" 19 #include "base/sequenced_task_runner.h" 20 #include "base/synchronization/lock.h" 21 #include "base/time/time.h" 22 #include "base/timer/timer.h" 23 #include "base/unguessable_token.h" 24 #include "media/base/audio_parameters.h" 25 #include "media/mojo/mojom/audio_data_pipe.mojom.h" 26 #include "media/mojo/mojom/audio_input_stream.mojom.h" 27 #include "mojo/public/cpp/bindings/pending_receiver.h" 28 #include "mojo/public/cpp/bindings/pending_remote.h" 29 #include "mojo/public/cpp/bindings/receiver.h" 30 #include "mojo/public/cpp/bindings/remote.h" 31 #include "services/audio/input_controller.h" 32 #include "services/audio/input_sync_writer.h" 33 #include "services/audio/loopback_coordinator.h" 34 #include "services/audio/loopback_group_member.h" 35 #include "services/audio/snooper_node.h" 36 37 namespace base { 38 class TickClock; 39 } // namespace base 40 41 namespace media { 42 class AudioBus; 43 } // namespace media 44 45 namespace audio { 46 47 // An AudioInputStream that provides the result of looping-back and 48 // mixing-together all current and future audio output streams in the same 49 // group. The loopback re-mixes the audio, if necessary, so that the resulting 50 // data stream's format matches the required AudioParameters. 51 // 52 // This is organized in three main components: 1) The LoopbackStream itself acts 53 // as a "shell" that manages mojo bindings and creates/controls the other 54 // components. 2) One or more SnooperNodes that buffer input data for each 55 // source OutputStream and format-convert it. 3) A "flow network" that runs via 56 // a different task runner, to take all the audio collected in the SnooperNodes 57 // and mix it into a single data stream. 58 class LoopbackStream : public media::mojom::AudioInputStream, 59 public LoopbackCoordinator::Observer { 60 public: 61 using CreatedCallback = 62 base::OnceCallback<void(media::mojom::ReadOnlyAudioDataPipePtr)>; 63 using BindingLostCallback = base::OnceCallback<void(LoopbackStream*)>; 64 65 LoopbackStream( 66 CreatedCallback created_callback, 67 BindingLostCallback binding_lost_callback, 68 scoped_refptr<base::SequencedTaskRunner> flow_task_runner, 69 mojo::PendingReceiver<media::mojom::AudioInputStream> receiver, 70 mojo::PendingRemote<media::mojom::AudioInputStreamClient> client, 71 mojo::PendingRemote<media::mojom::AudioInputStreamObserver> observer, 72 const media::AudioParameters& params, 73 uint32_t shared_memory_count, 74 LoopbackCoordinator* coordinator, 75 const base::UnguessableToken& group_id); 76 77 ~LoopbackStream() final; 78 is_recording()79 bool is_recording() const { return network_ && network_->is_started(); } 80 81 // media::mojom::AudioInputStream implementation. 82 void Record() final; 83 void SetVolume(double volume) final; 84 85 // LoopbackCoordinator::Observer implementation. When a member joins 86 // a group, a SnooperNode is created for it, and a loopback flow from 87 // LoopbackGroupMember → SnooperNode → FlowNetwork is built-up. 88 void OnMemberJoinedGroup(LoopbackGroupMember* member) final; 89 void OnMemberLeftGroup(LoopbackGroupMember* member) final; 90 91 // Overrides for unit testing. These must be called before Record(). set_clock_for_testing(const base::TickClock * clock)92 void set_clock_for_testing(const base::TickClock* clock) { 93 network_->set_clock_for_testing(clock); 94 } set_sync_writer_for_testing(std::unique_ptr<InputController::SyncWriter> writer)95 void set_sync_writer_for_testing( 96 std::unique_ptr<InputController::SyncWriter> writer) { 97 network_->set_writer_for_testing(std::move(writer)); 98 } 99 100 // Generally, a volume of 1.0 should be the maximum possible. However, there 101 // are cases where requests to amplify are made by specifying values higher 102 // than 1.0. 103 static constexpr double kMaxVolume = 2.0; 104 105 private: 106 // Drives all audio flows, re-mixing the audio from multiple SnooperNodes into 107 // a single audio stream. This class mainly operates on a separate task runner 108 // from LoopbackStream and can only be destroyed by scheduling it to occur on 109 // that same task runner. 110 class FlowNetwork { 111 public: 112 FlowNetwork(scoped_refptr<base::SequencedTaskRunner> flow_task_runner, 113 const media::AudioParameters& output_params, 114 std::unique_ptr<InputSyncWriter> writer); 115 116 // These must be called to override the Clock/SyncWriter before Start(). set_clock_for_testing(const base::TickClock * clock)117 void set_clock_for_testing(const base::TickClock* clock) { clock_ = clock; } set_writer_for_testing(std::unique_ptr<InputController::SyncWriter> writer)118 void set_writer_for_testing( 119 std::unique_ptr<InputController::SyncWriter> writer) { 120 writer_ = std::move(writer); 121 } 122 is_started()123 bool is_started() const { 124 DCHECK_CALLED_ON_VALID_SEQUENCE(control_sequence_); 125 return !!timer_; 126 } 127 output_params()128 const media::AudioParameters& output_params() const { 129 return output_params_; 130 } 131 132 // Add/Remove an input into this flow network. These may be called at any 133 // time, before or after Start(). All inputs must be removed before the 134 // FlowNetwork is scheduled for destruction. 135 void AddInput(SnooperNode* node); 136 void RemoveInput(SnooperNode* node); 137 138 // This may be called at any time, before or after Start(), to change the 139 // volume setting. 140 void SetVolume(double volume); 141 142 // Start generating audio data. This must only be called once, and there is 143 // no "stop" until destruction time. 144 void Start(); 145 146 private: 147 // Since this class guarantees its destructor will be called via the flow 148 // task runner, and destruction is carried out only by base::DeleteHelper, 149 // make the destructor is private. 150 friend class base::DeleteHelper<FlowNetwork>; 151 ~FlowNetwork(); 152 153 // Called periodically via the audio flow task runner to drive all the audio 154 // flows from the SnooperNodes, mix them together, and output to the 155 // AudioDataPipe. Each call schedules the next one until the |run_state_| 156 // becomes stopped. 157 void GenerateMoreAudio(); 158 159 const base::TickClock* clock_; 160 161 // Task runner that calls GenerateMoreAudio() to drive all the audio data 162 // flows. 163 const scoped_refptr<base::SequencedTaskRunner> flow_task_runner_; 164 165 // The audio parameters of the output. 166 const media::AudioParameters output_params_; 167 168 // Destination for the output of this FlowNetwork. 169 std::unique_ptr<InputController::SyncWriter> writer_; 170 171 // Ensures thread-safe access to changing the |inputs_| and |volume_| while 172 // running. 173 base::Lock lock_; 174 175 // The input nodes. 176 std::vector<SnooperNode*> inputs_; // Guarded by |lock_|. 177 178 // Current stream volume. The audio output from this FlowNetwork is scaled 179 // by this amount during mixing. 180 double volume_ = 1.0; // Guarded by |lock_|. 181 182 // This is set once Start() is called, and lives until this FlowNetwork is 183 // destroyed. It is used to schedule cancelable tasks run by the 184 // |flow_task_runner_|. 185 base::Optional<base::OneShotTimer> timer_; 186 187 // These are used to compute when the |timer_| fires and calls 188 // GenerateMoreAudio(). They ensure that each timer task is scheduled to 189 // fire with a delay that accounted for how much time was spent processing. 190 base::TimeTicks first_generate_time_; 191 int64_t frames_elapsed_ = 0; 192 base::TimeTicks next_generate_time_; 193 194 // The amount of time in the past from which to capture the audio. The audio 195 // recorded from each SnooperNode input is being generated with a target 196 // playout time in the near future (usually 1 to 20 ms). To avoid underflow, 197 // audio is always fetched from a safe position in the recent past. 198 // 199 // This is updated to match the SnooperNode whose recording is most delayed. 200 base::TimeDelta capture_delay_; 201 202 // Used to transfer the audio from each SnooperNode and mix them into a 203 // single audio signal. |transfer_bus_| is only allocated when first needed, 204 // but |mix_bus_| is allocated in the constructor because it is always 205 // needed. 206 std::unique_ptr<media::AudioBus> transfer_bus_; 207 const std::unique_ptr<media::AudioBus> mix_bus_; 208 209 SEQUENCE_CHECKER(control_sequence_); 210 211 DISALLOW_COPY_AND_ASSIGN(FlowNetwork); 212 }; 213 214 // Reports a fatal error to the client, and then runs the BindingLostCallback. 215 void OnError(); 216 217 // Run when any of |receiver_|, |client_|, or |observer_| are closed. This 218 // callback is generally used to automatically terminate this LoopbackStream. 219 BindingLostCallback binding_lost_callback_; 220 221 // Mojo bindings. If any of these is closed, the LoopbackStream will call 222 // OnError(), which will run the |binding_lost_callback_|. 223 mojo::Receiver<media::mojom::AudioInputStream> receiver_; 224 mojo::Remote<media::mojom::AudioInputStreamClient> client_; 225 mojo::Remote<media::mojom::AudioInputStreamObserver> observer_; 226 227 // Used for identifying group members and snooping on their audio data flow. 228 LoopbackCoordinator* const coordinator_; 229 const base::UnguessableToken group_id_; 230 231 // The snoopers associated with each group member. This is not a flat_map 232 // because SnooperNodes cannot move around in memory while in operation. 233 std::map<LoopbackGroupMember*, SnooperNode> snoopers_; 234 235 // The flow network that generates the single loopback result stream. It is 236 // owned by LoopbackStream, but it's destruction must be carried out by the 237 // flow task runner. This is never null, unless the system cannot support 238 // loopback (see constructor definition comments). 239 std::unique_ptr<FlowNetwork, base::OnTaskRunnerDeleter> network_; 240 241 SEQUENCE_CHECKER(sequence_checker_); 242 243 base::WeakPtrFactory<LoopbackStream> weak_factory_{this}; 244 245 DISALLOW_COPY_AND_ASSIGN(LoopbackStream); 246 }; 247 248 } // namespace audio 249 250 #endif // SERVICES_AUDIO_LOOPBACK_STREAM_H_ 251