Line data Source code
1 : /*
2 : * GPAC - Multimedia Framework C SDK
3 : *
4 : * Authors: Jean Le Feuvre
5 : * Copyright (c) Telecom ParisTech 2000-2021
6 : * All rights reserved
7 : *
8 : * This file is part of GPAC / Scene Compositor sub-project
9 : *
10 : * GPAC is free software; you can redistribute it and/or modify
11 : * it under the terms of the GNU Lesser General Public License as published by
12 : * the Free Software Foundation; either version 2, or (at your option)
13 : * any later version.
14 : *
15 : * GPAC is distributed in the hope that it will be useful,
16 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 : * GNU Lesser General Public License for more details.
19 : *
20 : * You should have received a copy of the GNU Lesser General Public
21 : * License along with this library; see the file COPYING. If not, write to
22 : * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 : *
24 : */
25 :
26 : #include <gpac/internal/compositor_dev.h>
27 :
28 20 : void gf_ar_rcfg_done(GF_Filter *filter, GF_FilterPid *pid, GF_FilterPacket *pck)
29 : {
30 : u32 size;
31 20 : GF_AudioRenderer *ar = (GF_AudioRenderer *) gf_filter_pck_get_data(pck, &size);
32 : assert(!size);
33 : assert(ar->wait_for_rcfg);
34 20 : ar->wait_for_rcfg --;
35 20 : if (ar->wait_for_rcfg) {
36 1 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfigure negociation %d still pending\n", ar->wait_for_rcfg));
37 : } else {
38 19 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfigure negociation done\n"));
39 : }
40 20 : }
41 :
42 623 : static GF_Err gf_ar_setup_output_format(GF_AudioRenderer *ar)
43 : {
44 : u32 freq, a_fmt, nb_chan;
45 : u64 ch_cfg;
46 : u32 bsize;
47 :
48 623 : freq = ar->compositor->asr;
49 623 : a_fmt = ar->compositor->afmt;
50 623 : nb_chan = ar->compositor->ach;
51 623 : ch_cfg = ar->compositor->alayout;
52 623 : bsize = ar->compositor->asize;
53 623 : if (!bsize) bsize = 1024;
54 :
55 623 : if (!freq || !a_fmt || !nb_chan || !ch_cfg) {
56 623 : gf_mixer_get_config(ar->mixer, &freq, &nb_chan, &a_fmt, &ch_cfg);
57 :
58 : /*user disabled multichannel audio*/
59 623 : if (!ar->compositor->amc && (nb_chan>2) ) nb_chan = 2;
60 : } else {
61 0 : if (!ar->config_forced) ar->config_forced++;
62 : }
63 :
64 :
65 623 : gf_mixer_set_config(ar->mixer, freq, nb_chan, a_fmt, ch_cfg);
66 :
67 623 : if (ar->samplerate) {
68 20 : ar->time_at_last_config_sr = ar->current_time_sr * freq / ar->samplerate;
69 : }
70 623 : if (!ar->compositor->abuf) {
71 : #ifdef GPAC_CONFIG_ANDROID
72 : ar->compositor->abuf = 200;
73 : #else
74 0 : ar->compositor->abuf = 100;
75 : #endif
76 : }
77 623 : ar->samplerate = freq;
78 623 : ar->bytes_per_samp = nb_chan * gf_audio_fmt_bit_depth(a_fmt) / 8;
79 623 : ar->bytes_per_second = freq * ar->bytes_per_samp;
80 623 : ar->max_bytes_out = ar->bytes_per_second * ar->compositor->abuf / 1000;
81 623 : while (ar->max_bytes_out % (2*ar->bytes_per_samp) ) ar->max_bytes_out++;
82 623 : ar->buffer_size = ar->bytes_per_samp * bsize;
83 :
84 623 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfigure audio to %d Hz %d channels %s\n", freq, nb_chan, gf_audio_fmt_name(a_fmt) ));
85 :
86 623 : if (ar->aout) {
87 20 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_SAMPLE_RATE, &PROP_UINT(freq) );
88 20 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_TIMESCALE, &PROP_UINT(freq) );
89 20 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_NUM_CHANNELS, &PROP_UINT(nb_chan) );
90 20 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_FORMAT, &PROP_UINT(a_fmt) );
91 20 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_CHANNEL_LAYOUT, &PROP_LONGUINT(ch_cfg) );
92 20 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_VOLUME, &PROP_UINT(ar->volume) );
93 20 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_PAN, &PROP_UINT(ar->pan) );
94 :
95 20 : gf_filter_pid_set_max_buffer(ar->aout, 1000*ar->compositor->abuf);
96 : }
97 :
98 623 : ar->time_at_last_config = ar->current_time;
99 623 : ar->bytes_requested = 0;
100 :
101 623 : if (ar->aout) {
102 : GF_FilterPacket *pck;
103 : //issue a dummy packet to tag the point at which we reconfigured
104 20 : pck = gf_filter_pck_new_shared(ar->aout, (u8 *) ar, 0, gf_ar_rcfg_done);
105 20 : if (!pck) return GF_OUT_OF_MEM;
106 20 : ar->wait_for_rcfg ++;
107 20 : gf_filter_pck_set_readonly(pck);
108 20 : gf_filter_pck_send(pck);
109 : }
110 : return GF_OK;
111 : }
112 :
113 668 : static void gf_ar_pause(GF_AudioRenderer *ar, Bool DoFreeze, Bool for_reconfig, Bool reset_hw_buffer)
114 : {
115 : GF_FilterEvent evt;
116 668 : gf_mixer_lock(ar->mixer, GF_TRUE);
117 668 : if (DoFreeze) {
118 333 : if (!ar->Frozen) {
119 333 : ar->freeze_time = gf_sys_clock_high_res();
120 333 : if (!for_reconfig && ar->aout) {
121 5 : GF_FEVT_INIT(evt, GF_FEVT_STOP, ar->aout);
122 5 : gf_filter_pid_send_event(ar->aout, &evt);
123 : }
124 333 : GF_LOG(GF_LOG_DEBUG, GF_LOG_SYNC, ("[Audio] pausing master clock - time "LLD" (sys time "LLD")\n", ar->freeze_time, gf_sys_clock_high_res()));
125 333 : ar->Frozen = GF_TRUE;
126 : }
127 : } else {
128 335 : if (ar->Frozen) {
129 333 : if (!for_reconfig && ar->aout) {
130 5 : GF_FEVT_INIT(evt, GF_FEVT_PLAY, ar->aout);
131 5 : evt.play.hw_buffer_reset = reset_hw_buffer ? 1 : 0;
132 5 : gf_filter_pid_send_event(ar->aout, &evt);
133 : }
134 :
135 333 : ar->start_time += gf_sys_clock_high_res() - ar->freeze_time;
136 333 : GF_LOG(GF_LOG_DEBUG, GF_LOG_SYNC, ("[Audio] resuming master clock - new time "LLD" (sys time "LLD") \n", ar->start_time, gf_sys_clock_high_res()));
137 333 : ar->Frozen = GF_FALSE;
138 : }
139 : }
140 668 : gf_mixer_lock(ar->mixer, GF_FALSE);
141 668 : }
142 :
143 :
144 605 : GF_AudioRenderer *gf_sc_ar_load(GF_Compositor *compositor, u32 init_flags)
145 : {
146 : GF_AudioRenderer *ar;
147 605 : ar = (GF_AudioRenderer *) gf_malloc(sizeof(GF_AudioRenderer));
148 : memset(ar, 0, sizeof(GF_AudioRenderer));
149 :
150 605 : ar->compositor = compositor;
151 :
152 605 : ar->mixer = gf_mixer_new(ar);
153 605 : ar->non_rt_output = GF_TRUE;
154 605 : ar->volume = MIN(100, compositor->avol);
155 605 : ar->pan = MIN(100, compositor->apan);
156 605 : if (! (init_flags & GF_TERM_NO_AUDIO) ) {
157 603 : gf_ar_setup_output_format(ar);
158 : }
159 605 : gf_mixer_set_max_speed(ar->mixer, compositor->max_aspeed);
160 605 : ar->current_time = 0;
161 605 : return ar;
162 : }
163 :
164 604 : void gf_sc_ar_del(GF_AudioRenderer *ar)
165 : {
166 604 : if (!ar) return;
167 :
168 604 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Destroying compositor\n"));
169 : /*resume if paused (might cause deadlock otherwise)*/
170 604 : if (ar->Frozen) gf_sc_ar_control(ar, GF_SC_AR_RESUME);
171 :
172 604 : gf_mixer_del(ar->mixer);
173 604 : gf_free(ar);
174 604 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Renderer destroyed\n"));
175 : }
176 :
177 :
178 2509 : void gf_sc_ar_reset(GF_AudioRenderer *ar)
179 : {
180 2509 : if (!ar) return;
181 :
182 2509 : gf_mixer_remove_all(ar->mixer);
183 2509 : if (ar->scene_ready) {
184 606 : ar->scene_ready = 0;
185 606 : ar->current_time = 0;
186 606 : ar->current_time_sr = 0;
187 606 : ar->bytes_requested = 0;
188 606 : ar->nb_audio_objects = 0;
189 : }
190 : }
191 :
192 628 : void gf_sc_ar_control(GF_AudioRenderer *ar, u32 PauseType)
193 : {
194 628 : gf_ar_pause(ar, (PauseType==GF_SC_AR_PAUSE) ? GF_TRUE : GF_FALSE, GF_FALSE, (PauseType==GF_SC_AR_RESET_HW_AND_PLAY) ? GF_TRUE : GF_FALSE);
195 628 : }
196 :
197 1204 : void gf_sc_ar_set_volume(GF_AudioRenderer *ar, u32 Volume)
198 : {
199 1204 : if (Volume>100) Volume=100;
200 1204 : if (ar->volume==Volume) return;
201 0 : ar->volume = Volume;
202 0 : if (ar->aout) gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_VOLUME, &PROP_UINT(ar->volume) );
203 : }
204 :
205 2 : void gf_sc_ar_mute(GF_AudioRenderer *ar, Bool mute)
206 : {
207 2 : ar->mute = mute;
208 2 : if (ar->aout) gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_VOLUME, &PROP_UINT(mute ? 0 : ar->volume) );
209 2 : }
210 :
211 1204 : void gf_sc_ar_set_pan(GF_AudioRenderer *ar, u32 Balance)
212 : {
213 1204 : if (Balance>100) Balance = 100;
214 1204 : if (ar->pan == Balance) return;
215 0 : ar->pan = Balance;
216 0 : if (ar->aout) gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_PAN, &PROP_UINT(ar->pan) );
217 : }
218 :
219 :
220 : void compositor_setup_aout(GF_Compositor *ctx);
221 21 : void gf_sc_ar_add_src(GF_AudioRenderer *ar, GF_AudioInterface *source)
222 : {
223 : Bool recfg;
224 21 : if (!ar) return;
225 :
226 21 : compositor_setup_aout(ar->compositor);
227 :
228 : /*lock mixer*/
229 21 : gf_mixer_lock(ar->mixer, GF_TRUE);
230 21 : gf_mixer_add_input(ar->mixer, source);
231 : /*if changed reconfig*/
232 21 : recfg = gf_mixer_reconfig(ar->mixer);
233 21 : if (!ar->need_reconfig) ar->need_reconfig = recfg;
234 :
235 21 : if (!gf_mixer_empty(ar->mixer) && ar->aout) {
236 : GF_FilterEvent evt;
237 21 : GF_FEVT_INIT(evt, GF_FEVT_PLAY, ar->aout);
238 21 : gf_filter_pid_send_event(ar->aout, &evt);
239 : }
240 :
241 : /*unlock mixer*/
242 21 : gf_mixer_lock(ar->mixer, GF_FALSE);
243 : }
244 :
245 21 : void gf_sc_ar_remove_src(GF_AudioRenderer *ar, GF_AudioInterface *source)
246 : {
247 21 : if (ar) {
248 21 : gf_mixer_remove_input(ar->mixer, source);
249 21 : if (gf_mixer_empty(ar->mixer) && ar->aout) {
250 : GF_FilterEvent evt;
251 21 : GF_FEVT_INIT(evt, GF_FEVT_STOP, ar->aout);
252 21 : gf_filter_pid_send_event(ar->aout, &evt);
253 : }
254 : }
255 21 : }
256 :
257 :
258 : #if 0 //unused
259 : void gf_sc_ar_set_priority(GF_AudioRenderer *ar, u32 priority)
260 : {
261 : if (ar->aout)
262 : gf_filter_pid_set_property(ar->aout, GF_PROP_PID_AUDIO_PRIORITY, &PROP_UINT(priority) );
263 : }
264 : #endif
265 :
266 22396 : void gf_sc_ar_update_video_clock(GF_AudioRenderer *ar, u32 video_ts)
267 : {
268 22396 : ar->video_ts = video_ts;
269 22396 : }
270 :
271 34299 : static void gf_ar_pck_done(GF_Filter *filter, GF_FilterPid *pid, GF_FilterPacket *pck)
272 : {
273 : u32 size;
274 34299 : GF_Compositor *compositor = gf_filter_pid_get_udta(pid);
275 34299 : /*data = */gf_filter_pck_get_data(pck, &size);
276 34325 : if (!size) return;
277 :
278 : assert(size <= compositor->audio_renderer->nb_bytes_out);
279 34273 : compositor->audio_renderer->nb_bytes_out -= size;
280 : }
281 :
282 124704 : void gf_ar_send_packets(GF_AudioRenderer *ar)
283 : {
284 : u32 written, max_send=100;
285 124704 : u64 now = gf_sys_clock_high_res();
286 :
287 124704 : if (!ar->aout) {
288 120981 : if (ar->compositor->player) {
289 54 : ar->current_time = (u32) ( (now - ar->start_time)/1000);
290 : }
291 : return;
292 : }
293 3723 : if (!ar->scene_ready) return;
294 3689 : if (ar->need_reconfig) return;
295 3689 : if (ar->Frozen) return;
296 :
297 : //reconfiguration is pending, wait for the packet issued at reconfig to be consumed before issuing any new frame
298 3689 : if (ar->wait_for_rcfg) {
299 77 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Waiting for audio output reconfiguration\n"));
300 : return;
301 : }
302 : if (ar->scene_ready) {
303 3612 : if (!ar->start_time) {
304 5 : ar->start_time = now;
305 : }
306 3612 : if (!ar->nb_audio_objects && !ar->non_rt_output) {
307 587 : ar->current_time = (u32) ( (now - ar->start_time)/1000);
308 587 : return;
309 : }
310 : }
311 :
312 34617 : while (max_send) {
313 : u32 delay_ms = 0;
314 : u8 *data;
315 : u32 dur;
316 : GF_FilterPacket *pck;
317 :
318 34302 : if (gf_filter_pid_would_block(ar->aout))
319 : break;
320 :
321 34299 : pck = gf_filter_pck_new_alloc_destructor(ar->aout, ar->buffer_size, &data, gf_ar_pck_done);
322 34299 : if (!pck) break;
323 :
324 34299 : if (ar->compositor->async) {
325 : GF_Fraction64 ref_ts;
326 34299 : gf_filter_get_clock_hint(ar->compositor->filter, NULL, &ref_ts);
327 : //valid clock hint, compute delay between last known playback point and current time
328 34299 : if (ref_ts.den) {
329 0 : if (ref_ts.den != ar->samplerate) {
330 0 : delay_ms = (u32) (1000 * ar->current_time_sr / ar->samplerate);
331 0 : delay_ms -= (u32) (1000 * ref_ts.num / ref_ts.den);
332 : } else {
333 0 : delay_ms = (u32) (1000 * (ar->current_time_sr - ref_ts.num) / ar->samplerate);
334 : }
335 : }
336 : //unknown clock hint, use number of bytes out as delay
337 : else {
338 34299 : delay_ms = (1000*ar->nb_bytes_out) / ar->bytes_per_second;
339 : }
340 : }
341 :
342 34299 : gf_mixer_lock(ar->mixer, GF_TRUE);
343 34299 : written = gf_mixer_get_output(ar->mixer, data, ar->buffer_size, delay_ms);
344 34299 : gf_mixer_lock(ar->mixer, GF_FALSE);
345 :
346 34299 : if (!written) {
347 28671 : if (!ar->non_rt_output) written = ar->buffer_size;
348 28671 : else if (ar->scene_ready && ar->nb_audio_objects && !gf_mixer_buffering(ar->mixer) ) written = ar->buffer_size;
349 : else {
350 26 : gf_filter_pck_truncate(pck, 0);
351 26 : gf_filter_pck_discard(pck);
352 26 : break;
353 : }
354 : }
355 :
356 34273 : if (written<ar->buffer_size) {
357 79 : gf_filter_pck_truncate(pck, written);
358 : }
359 34273 : gf_filter_pck_set_sap(pck, GF_FILTER_SAP_1);
360 34273 : gf_filter_pck_set_cts(pck, ar->current_time_sr);
361 34273 : dur = written / ar->bytes_per_samp;
362 34273 : gf_filter_pck_set_duration(pck, dur);
363 34273 : GF_LOG(GF_LOG_INFO, GF_LOG_AUDIO, ("[Compositor] Send audio frame TS "LLU" nb samples %d - AR clock %u - delay %d ms\n", ar->current_time_sr, dur, ar->current_time, delay_ms));
364 :
365 34273 : ar->nb_bytes_out += written;
366 34273 : gf_filter_pck_send(pck);
367 :
368 34273 : ar->bytes_requested += written;
369 34273 : ar->current_time_sr = ar->time_at_last_config_sr + (u32) (ar->bytes_requested / ar->bytes_per_samp);
370 34273 : ar->current_time = ar->time_at_last_config + (u32) (ar->bytes_requested * 1000 / ar->bytes_per_second);
371 :
372 34273 : max_send--;
373 :
374 : //this is a safety for non blocking mode, otherwise the pid_would_block is enough
375 34273 : if (ar->nb_bytes_out > ar->max_bytes_out)
376 : break;
377 : }
378 : }
379 :
380 :
381 124704 : void gf_sc_ar_send_or_reconfig(GF_AudioRenderer *ar)
382 : {
383 : Bool frozen;
384 124704 : if (ar->need_reconfig) {
385 20 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] Reconfiguring audio mixer\n"));
386 : /*lock mixer*/
387 20 : gf_mixer_lock(ar->mixer, GF_TRUE);
388 :
389 20 : frozen = ar->Frozen;
390 20 : if (!frozen )
391 20 : gf_ar_pause(ar, GF_TRUE, GF_TRUE, GF_FALSE);
392 :
393 20 : ar->need_reconfig = GF_FALSE;
394 20 : gf_ar_setup_output_format(ar);
395 :
396 20 : if (!frozen)
397 20 : gf_ar_pause(ar, GF_FALSE, GF_TRUE, GF_FALSE);
398 :
399 : /*unlock mixer*/
400 20 : gf_mixer_lock(ar->mixer, GF_FALSE);
401 : }
402 124704 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Compositor] sending audio packets\n"));
403 124704 : gf_ar_send_packets(ar);
404 124704 : }
405 :
406 : #if 0 //unused
407 : u32 gf_sc_ar_get_delay(GF_AudioRenderer *ar)
408 : {
409 : if (!ar->bytes_per_second) return 0;
410 : //try to FIXME, this is not as precise as what we have before using ar->audio_out->GetAudioDelay(ar->audio_out)
411 : // since we don't know how much of the first packet data out there has been consumed
412 : return 1000 * ar->nb_bytes_out / ar->bytes_per_second;
413 : }
414 : #endif
415 :
416 1688 : u32 gf_sc_ar_get_clock(GF_AudioRenderer *ar)
417 : {
418 1688 : return ar->current_time;
419 : }
420 :
421 :
422 :
|
