Line data Source code
1 : /*
2 : * GPAC - Multimedia Framework C SDK
3 : *
4 : * Authors: Jean Le Feuvre
5 : * Copyright (c) Telecom ParisTech 2000-2018
6 : * All rights reserved
7 : *
8 : * This file is part of GPAC / Scene Compositor sub-project
9 : *
10 : * GPAC is free software; you can redistribute it and/or modify
11 : * it under the terms of the GNU Lesser General Public License as published by
12 : * the Free Software Foundation; either version 2, or (at your option)
13 : * any later version.
14 : *
15 : * GPAC is distributed in the hope that it will be useful,
16 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 : * GNU Lesser General Public License for more details.
19 : *
20 : * You should have received a copy of the GNU Lesser General Public
21 : * License along with this library; see the file COPYING. If not, write to
22 : * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
23 : *
24 : */
25 :
26 : #include <gpac/internal/compositor_dev.h>
27 :
28 : #define ENABLE_EARLY_FRAME_DETECTION
29 :
30 : /*diff time in ms to consider an audio frame too late and drop it - we should try to dynamically figure this out
31 : since the drift may be high on TS for example, where PTS-PCR>500ms is quite common*/
32 : #define MAX_RESYNC_TIME 1000
33 : //if drift between audio object time and clock varies more is than this value (in ms) between two drift computation, clock is adjusted. We don't adjust for lower values otherwise we would
34 : //introduce oscillations in the clock and non-smooth playback
35 : #define MIN_DRIFT_ADJUST 75
36 :
37 :
38 39475 : static u8 *gf_audio_input_fetch_frame(void *callback, u32 *size, u32 *planar_size, u32 audio_delay_ms)
39 : {
40 : char *frame;
41 : u32 obj_time, ts;
42 : s32 drift;
43 : Fixed speed;
44 : Bool done;
45 : GF_AudioInput *ai = (GF_AudioInput *) callback;
46 : /*even if the stream is signaled as finished we must check it, because it may have been restarted by a mediaControl*/
47 39475 : if (!ai->stream) return NULL;
48 :
49 39475 : done = ai->stream_finished;
50 39475 : ai->input_ifce.is_buffering = GF_FALSE;
51 :
52 39475 : frame = gf_mo_fetch_data(ai->stream, ai->compositor->audio_renderer->non_rt_output ? GF_MO_FETCH_PAUSED : GF_MO_FETCH, 0, &ai->stream_finished, &ts, size, NULL, NULL, NULL, planar_size);
53 : /*invalidate scene on end of stream to refresh audio graph*/
54 39475 : if (done != ai->stream_finished) {
55 8 : gf_sc_invalidate(ai->compositor, NULL);
56 : }
57 :
58 : /*no more data or not enough data, reset syncro drift*/
59 39475 : if (!frame) {
60 28027 : if (!ai->stream_finished && gf_mo_is_started(ai->stream) && (ai->stream->odm->ck->speed == FIX_ONE)) {
61 17826 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Audio Input] No data in audio object\n"));
62 : }
63 28027 : gf_mo_adjust_clock(ai->stream, 0);
64 28027 : ai->input_ifce.is_buffering = gf_mo_is_buffering(ai->stream);
65 28027 : *size = 0;
66 28027 : return NULL;
67 : }
68 11448 : ai->need_release = GF_TRUE;
69 :
70 : //step mode, return the frame without sync check
71 11448 : if (ai->compositor->audio_renderer->non_rt_output) {
72 11448 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Audio Input] audio frame CTS %u %d bytes fetched\n", ts, *size));
73 : return frame;
74 : }
75 :
76 0 : speed = gf_mo_get_current_speed(ai->stream);
77 :
78 0 : gf_mo_get_object_time(ai->stream, &obj_time);
79 0 : obj_time += audio_delay_ms;
80 0 : if (ai->compositor->bench_mode) {
81 : drift = 0;
82 : } else {
83 0 : drift = (s32)obj_time;
84 0 : drift -= (s32)ts;
85 : }
86 0 : if (ai->stream->odm->prev_clock_at_discontinuity_plus_one) {
87 : s32 drift_old = drift;
88 : s32 diff;
89 0 : drift_old -= (s32) ai->stream->odm->ck->init_timestamp;
90 0 : drift_old += (s32) ai->stream->odm->prev_clock_at_discontinuity_plus_one - 1;
91 0 : diff = ABS(drift_old);
92 0 : diff -= ABS(drift);
93 0 : if (diff < 0) {
94 0 : GF_LOG(GF_LOG_INFO, GF_LOG_SYNC, ("[Audio Input] in clock discontinuity: drift old clock %d new clock %d - disabling clock adjustment\n", drift_old, drift));
95 : drift = 0;
96 : audio_delay_ms = 0;
97 : } else {
98 0 : GF_LOG(GF_LOG_INFO, GF_LOG_SYNC, ("[Audio Input] end of clock discontinuity: drift old clock %d new clock %d\n", drift_old, drift));
99 0 : ai->stream->odm->prev_clock_at_discontinuity_plus_one = 0;
100 0 : if (drift<0) {
101 : drift = 0;
102 : }
103 : }
104 : }
105 :
106 : #ifdef ENABLE_EARLY_FRAME_DETECTION
107 : /*too early (silence insertions), skip*/
108 0 : if (drift < 0) {
109 0 : GF_LOG(GF_LOG_INFO, GF_LOG_AUDIO, ("[Audio Input] audio too early of %d (CTS %u at OTB %u with audio delay %d ms)\n", drift + audio_delay_ms, ts, obj_time, audio_delay_ms));
110 0 : ai->need_release = GF_FALSE;
111 0 : gf_mo_release_data(ai->stream, 0, -1);
112 0 : *size = 0;
113 0 : return NULL;
114 : }
115 : #endif
116 : /*adjust drift*/
117 0 : if (audio_delay_ms) {
118 0 : s32 resync_delay = speed > 0 ? FIX2INT(speed * MAX_RESYNC_TIME) : FIX2INT(-speed * MAX_RESYNC_TIME);
119 : /*CU is way too late, discard and fetch a new one - this usually happen when media speed is more than 1*/
120 0 : if (drift>resync_delay) {
121 0 : GF_LOG(GF_LOG_INFO, GF_LOG_AUDIO, ("[Audio Input] Audio data too late obj time %d - CTS %d - drift %d ms - resync forced\n", obj_time - audio_delay_ms, ts, drift));
122 0 : gf_mo_release_data(ai->stream, *size, 2);
123 0 : ai->need_release = GF_FALSE;
124 0 : return gf_audio_input_fetch_frame(callback, size, planar_size, audio_delay_ms);
125 : }
126 0 : if (ai->stream->odm && ai->stream->odm->ck)
127 0 : resync_delay = ai->stream->odm->ck->audio_delay - drift;
128 : else
129 0 : resync_delay = -drift;
130 :
131 0 : if (resync_delay < 0) resync_delay = -resync_delay;
132 :
133 0 : if (resync_delay > MIN_DRIFT_ADJUST) {
134 0 : GF_LOG(GF_LOG_DEBUG, GF_LOG_AUDIO, ("[Audio Input] Audio clock: delay %d - obj time %d - audio delay %d - CTS %d - adjust drift %d\n", audio_delay_ms, obj_time, audio_delay_ms, ts, drift));
135 0 : gf_mo_adjust_clock(ai->stream, drift);
136 : }
137 : }
138 : return frame;
139 : }
140 :
141 11448 : static void gf_audio_input_release_frame(void *callback, u32 nb_bytes)
142 : {
143 : GF_AudioInput *ai = (GF_AudioInput *) callback;
144 11448 : if (!ai->stream) return;
145 11448 : gf_mo_release_data(ai->stream, nb_bytes, 1);
146 11448 : ai->need_release = GF_FALSE;
147 : }
148 :
149 36543 : static Fixed gf_audio_input_get_speed(void *callback)
150 : {
151 : GF_AudioInput *ai = (GF_AudioInput *) callback;
152 36543 : return gf_mo_get_current_speed(ai->stream);
153 : }
154 :
155 34018 : static Bool gf_audio_input_get_volume(void *callback, Fixed *vol)
156 : {
157 : GF_AudioInput *ai = (GF_AudioInput *) callback;
158 34018 : if (ai->snd && ai->snd->GetChannelVolume) {
159 33869 : return ai->snd->GetChannelVolume(ai->snd->owner, vol);
160 : } else {
161 : u32 i;
162 3576 : for (i=0; i<GF_AUDIO_MIXER_MAX_CHANNELS; i++)
163 3576 : vol[i] = ai->intensity;
164 :
165 149 : return (ai->intensity==FIX_ONE) ? GF_FALSE : GF_TRUE;
166 : }
167 : }
168 :
169 102433 : static Bool gf_audio_input_is_muted(void *callback)
170 : {
171 : GF_AudioInput *ai = (GF_AudioInput *) callback;
172 102433 : if (!ai->stream) return GF_TRUE;
173 :
174 102433 : if (ai->stream->odm->nb_buffering)
175 13619 : gf_odm_check_buffering(ai->stream->odm, NULL);
176 102433 : if (ai->is_muted)
177 : return GF_TRUE;
178 102433 : return gf_mo_is_muted(ai->stream);
179 : }
180 :
181 68576 : static Bool gf_audio_input_get_config(GF_AudioInterface *aifc, Bool for_recf)
182 : {
183 68576 : GF_AudioInput *ai = (GF_AudioInput *) aifc->callback;
184 68576 : if (!ai->stream) return GF_FALSE;
185 : /*watchout for object reuse*/
186 68507 : if (aifc->samplerate && !ai->stream->config_changed) return GF_TRUE;
187 :
188 77 : gf_mo_get_audio_info(ai->stream, &aifc->samplerate, &aifc->afmt , &aifc->chan, &aifc->ch_layout, &aifc->forced_layout);
189 :
190 77 : if (!for_recf)
191 0 : return aifc->samplerate ? GF_TRUE : GF_FALSE;
192 :
193 77 : if (aifc->samplerate && aifc->chan && aifc->afmt && ((aifc->chan<=2) || aifc->ch_layout)) {
194 21 : ai->stream->config_changed = GF_FALSE;
195 21 : return GF_TRUE;
196 : }
197 : //still not ready !
198 56 : ai->stream->config_changed=GF_TRUE;
199 56 : return GF_FALSE;
200 : }
201 :
202 : GF_EXPORT
203 236 : void gf_sc_audio_setup(GF_AudioInput *ai, GF_Compositor *compositor, GF_Node *node)
204 : {
205 : memset(ai, 0, sizeof(GF_AudioInput));
206 236 : ai->owner = node;
207 236 : ai->compositor = compositor;
208 236 : ai->stream = NULL;
209 : /*setup io interface*/
210 236 : ai->input_ifce.FetchFrame = gf_audio_input_fetch_frame;
211 236 : ai->input_ifce.ReleaseFrame = gf_audio_input_release_frame;
212 236 : ai->input_ifce.GetConfig = gf_audio_input_get_config;
213 236 : ai->input_ifce.GetChannelVolume = gf_audio_input_get_volume;
214 236 : ai->input_ifce.GetSpeed = gf_audio_input_get_speed;
215 236 : ai->input_ifce.IsMuted = gf_audio_input_is_muted;
216 236 : ai->input_ifce.callback = ai;
217 236 : ai->intensity = FIX_ONE;
218 :
219 236 : ai->speed = FIX_ONE;
220 :
221 236 : }
222 :
223 236 : void gf_sc_audio_predestroy(GF_AudioInput *ai)
224 : {
225 236 : gf_sc_audio_stop(ai);
226 236 : gf_sc_audio_unregister(ai);
227 236 : }
228 :
229 : GF_EXPORT
230 566 : GF_Err gf_sc_audio_open(GF_AudioInput *ai, MFURL *url, Double clipBegin, Double clipEnd, Bool lock_timeline)
231 : {
232 566 : if (ai->is_open) return GF_BAD_PARAM;
233 :
234 : /*get media object*/
235 566 : ai->stream = gf_mo_register(ai->owner, url, lock_timeline, GF_FALSE);
236 : /*bad URL*/
237 566 : if (!ai->stream) return GF_NOT_SUPPORTED;
238 :
239 : /*request play*/
240 22 : gf_mo_play(ai->stream, clipBegin, clipEnd, GF_FALSE);
241 :
242 22 : ai->stream_finished = GF_FALSE;
243 22 : ai->is_open = 1;
244 : //force reload of audio props
245 22 : ai->stream->config_changed = GF_TRUE;
246 :
247 22 : return GF_OK;
248 : }
249 :
250 : GF_EXPORT
251 240 : void gf_sc_audio_stop(GF_AudioInput *ai)
252 : {
253 240 : if (!ai->is_open) return;
254 :
255 : /*we must make sure audio mixer is not using the stream otherwise we may leave it dirty (with unrelease frame)*/
256 22 : gf_mixer_lock(ai->compositor->audio_renderer->mixer, GF_TRUE);
257 :
258 : assert(!ai->need_release);
259 :
260 22 : gf_mo_stop(&ai->stream);
261 22 : ai->is_open = 0;
262 22 : gf_mo_unregister(ai->owner, ai->stream);
263 22 : ai->stream = NULL;
264 :
265 22 : gf_mixer_lock(ai->compositor->audio_renderer->mixer, GF_FALSE);
266 :
267 : }
268 :
269 : GF_EXPORT
270 6 : void gf_sc_audio_restart(GF_AudioInput *ai)
271 : {
272 6 : if (!ai->is_open) return;
273 6 : if (ai->need_release) gf_mo_release_data(ai->stream, 0xFFFFFFFF, 2);
274 6 : ai->need_release = GF_FALSE;
275 6 : ai->stream_finished = GF_FALSE;
276 :
277 6 : gf_mo_restart(ai->stream);
278 : }
279 :
280 : GF_EXPORT
281 9 : Bool gf_sc_audio_check_url(GF_AudioInput *ai, MFURL *url)
282 : {
283 9 : if (!ai->stream) return url->count;
284 6 : return gf_mo_url_changed(ai->stream, url);
285 : }
286 :
287 : GF_EXPORT
288 1710 : void gf_sc_audio_register(GF_AudioInput *ai, GF_TraverseState *tr_state)
289 : {
290 : GF_AudioInterface *aifce;
291 : /*check interface is valid*/
292 1710 : if (!ai->input_ifce.FetchFrame
293 1710 : || !ai->input_ifce.GetChannelVolume
294 1710 : || !ai->input_ifce.GetConfig
295 1710 : || !ai->input_ifce.GetSpeed
296 1710 : || !ai->input_ifce.IsMuted
297 1710 : || !ai->input_ifce.ReleaseFrame
298 : ) return;
299 :
300 1710 : aifce = &ai->input_ifce;
301 :
302 1710 : if (tr_state->audio_parent) {
303 : /*this assume only one parent may use an audio node*/
304 16 : if (ai->register_with_parent) return;
305 1 : if (ai->register_with_renderer) {
306 0 : gf_sc_ar_remove_src(ai->compositor->audio_renderer, aifce);
307 0 : ai->register_with_renderer = GF_FALSE;
308 : }
309 1 : tr_state->audio_parent->add_source(tr_state->audio_parent, ai);
310 1 : ai->register_with_parent = GF_TRUE;
311 1 : ai->snd = tr_state->sound_holder;
312 1694 : } else if (!ai->register_with_renderer) {
313 :
314 21 : if (ai->register_with_parent) {
315 0 : ai->register_with_parent = GF_FALSE;
316 : /*if used in a parent audio group, do a complete traverse to rebuild the group*/
317 0 : gf_sc_invalidate(ai->compositor, NULL);
318 : }
319 :
320 21 : gf_sc_ar_add_src(ai->compositor->audio_renderer, aifce);
321 21 : ai->register_with_renderer = GF_TRUE;
322 21 : ai->snd = tr_state->sound_holder;
323 : }
324 : }
325 :
326 : GF_EXPORT
327 243 : void gf_sc_audio_unregister(GF_AudioInput *ai)
328 : {
329 243 : GF_AudioInterface *aifce = &ai->input_ifce;
330 :
331 243 : if (ai->register_with_renderer) {
332 21 : ai->register_with_renderer = GF_FALSE;
333 21 : gf_sc_ar_remove_src(ai->compositor->audio_renderer, aifce);
334 : } else {
335 : /*if used in a parent audio group, do a complete traverse to rebuild the group*/
336 222 : gf_sc_invalidate(ai->compositor, NULL);
337 : }
338 243 : }
339 :
|
