2 * FreeRDP: A Remote Desktop Protocol client.
3 * Video Redirection Virtual Channel - Media Container
5 * Copyright 2010-2011 Vic Lee
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
24 #include <freerdp/utils/memory.h>
25 #include <freerdp/utils/stream.h>
26 #include <freerdp/utils/list.h>
27 #include <freerdp/utils/thread.h>
28 #include <freerdp/utils/mutex.h>
29 #include <freerdp/utils/event.h>
30 #include <freerdp/utils/sleep.h>
31 #include <freerdp/plugins/tsmf.h>
33 #include "drdynvc_types.h"
34 #include "tsmf_constants.h"
35 #include "tsmf_types.h"
36 #include "tsmf_decoder.h"
37 #include "tsmf_audio.h"
38 #include "tsmf_main.h"
39 #include "tsmf_codec.h"
40 #include "tsmf_media.h"
42 #define AUDIO_TOLERANCE 10000000LL
44 struct _TSMF_PRESENTATION
46 uint8 presentation_id[GUID_SIZE];
48 const char* audio_name;
49 const char* audio_device;
56 uint16 last_num_rects;
63 uint16 output_num_rects;
64 RDP_RECT* output_rects;
66 IWTSVirtualChannelCallback* channel_callback;
68 uint64 audio_start_time;
69 uint64 audio_end_time;
71 /* The stream list could be accessed by differnt threads and need to be protected. */
81 TSMF_PRESENTATION* presentation;
83 ITSMFDecoder* decoder;
90 ITSMFAudioDevice* audio;
93 uint32 bits_per_sample;
95 /* The end_time of last played sample */
97 /* Next sample should not start before this system time. */
98 uint64 next_start_time;
100 freerdp_thread* thread;
104 /* The sample ack response queue will be accessed only by the stream thread. */
105 LIST* sample_ack_list;
121 IWTSVirtualChannelCallback* channel_callback;
125 static LIST* presentation_list = NULL;
127 static uint64 get_current_time(void)
131 gettimeofday(&tp, 0);
132 return ((uint64)tp.tv_sec) * 10000000LL + ((uint64)tp.tv_usec) * 10LL;
135 static TSMF_SAMPLE* tsmf_stream_pop_sample(TSMF_STREAM* stream, int sync)
140 boolean pending = false;
141 TSMF_PRESENTATION* presentation = stream->presentation;
143 if (!stream->sample_list->head)
148 if (stream->major_type == TSMF_MAJOR_TYPE_AUDIO)
150 /* Check if some other stream has earlier sample that needs to be played first */
151 if (stream->last_end_time > AUDIO_TOLERANCE)
153 freerdp_mutex_lock(presentation->mutex);
154 for (item = presentation->stream_list->head; item; item = item->next)
156 s = (TSMF_STREAM*) item->data;
157 if (s != stream && !s->eos && s->last_end_time &&
158 s->last_end_time < stream->last_end_time - AUDIO_TOLERANCE)
164 freerdp_mutex_unlock(presentation->mutex);
169 if (stream->last_end_time > presentation->audio_end_time)
178 freerdp_thread_lock(stream->thread);
179 sample = (TSMF_SAMPLE*) list_dequeue(stream->sample_list);
180 freerdp_thread_unlock(stream->thread);
182 if (sample && sample->end_time > stream->last_end_time)
183 stream->last_end_time = sample->end_time;
188 static void tsmf_sample_free(TSMF_SAMPLE* sample)
195 static void tsmf_sample_ack(TSMF_SAMPLE* sample)
197 tsmf_playback_ack(sample->channel_callback, sample->sample_id, sample->duration, sample->data_size);
200 static void tsmf_sample_queue_ack(TSMF_SAMPLE* sample)
202 TSMF_STREAM* stream = sample->stream;
204 list_enqueue(stream->sample_ack_list, sample);
207 static void tsmf_stream_process_ack(TSMF_STREAM* stream)
212 ack_time = get_current_time();
213 while (stream->sample_ack_list->head && !freerdp_thread_is_stopped(stream->thread))
215 sample = (TSMF_SAMPLE*) list_peek(stream->sample_ack_list);
216 if (sample->ack_time > ack_time)
219 sample = list_dequeue(stream->sample_ack_list);
220 tsmf_sample_ack(sample);
221 tsmf_sample_free(sample);
225 TSMF_PRESENTATION* tsmf_presentation_new(const uint8* guid, IWTSVirtualChannelCallback* pChannelCallback)
227 TSMF_PRESENTATION* presentation;
229 presentation = tsmf_presentation_find_by_id(guid);
232 DEBUG_WARN("duplicated presentation id!");
236 presentation = xnew(TSMF_PRESENTATION);
238 memcpy(presentation->presentation_id, guid, GUID_SIZE);
239 presentation->channel_callback = pChannelCallback;
241 presentation->mutex = freerdp_mutex_new();
242 presentation->stream_list = list_new();
244 list_enqueue(presentation_list, presentation);
249 TSMF_PRESENTATION* tsmf_presentation_find_by_id(const uint8* guid)
252 TSMF_PRESENTATION* presentation;
254 for (item = presentation_list->head; item; item = item->next)
256 presentation = (TSMF_PRESENTATION*) item->data;
257 if (memcmp(presentation->presentation_id, guid, GUID_SIZE) == 0)
263 static void tsmf_presentation_restore_last_video_frame(TSMF_PRESENTATION* presentation)
265 RDP_REDRAW_EVENT* revent;
267 if (presentation->last_width && presentation->last_height)
269 revent = (RDP_REDRAW_EVENT*) freerdp_event_new(RDP_EVENT_CLASS_TSMF, RDP_EVENT_TYPE_TSMF_REDRAW,
271 revent->x = presentation->last_x;
272 revent->y = presentation->last_y;
273 revent->width = presentation->last_width;
274 revent->height = presentation->last_height;
275 if (!tsmf_push_event(presentation->channel_callback, (RDP_EVENT*) revent))
277 freerdp_event_free((RDP_EVENT*) revent);
279 presentation->last_x = 0;
280 presentation->last_y = 0;
281 presentation->last_width = 0;
282 presentation->last_height = 0;
286 static void tsmf_sample_playback_video(TSMF_SAMPLE* sample)
289 RDP_VIDEO_FRAME_EVENT* vevent;
290 TSMF_STREAM* stream = sample->stream;
291 TSMF_PRESENTATION* presentation = stream->presentation;
293 DEBUG_DVC("MessageId %d EndTime %d data_size %d consumed.",
294 sample->sample_id, (int)sample->end_time, sample->data_size);
298 t = get_current_time();
299 if (stream->next_start_time > t &&
300 (sample->end_time >= presentation->audio_start_time ||
301 sample->end_time < stream->last_end_time))
303 freerdp_usleep((stream->next_start_time - t) / 10);
305 stream->next_start_time = t + sample->duration - 50000;
307 if (presentation->last_x != presentation->output_x ||
308 presentation->last_y != presentation->output_y ||
309 presentation->last_width != presentation->output_width ||
310 presentation->last_height != presentation->output_height ||
311 presentation->last_num_rects != presentation->output_num_rects ||
312 (presentation->last_rects && presentation->output_rects &&
313 memcmp(presentation->last_rects, presentation->output_rects,
314 presentation->last_num_rects * sizeof(RDP_RECT)) != 0))
316 tsmf_presentation_restore_last_video_frame(presentation);
318 presentation->last_x = presentation->output_x;
319 presentation->last_y = presentation->output_y;
320 presentation->last_width = presentation->output_width;
321 presentation->last_height = presentation->output_height;
323 if (presentation->last_rects)
325 xfree(presentation->last_rects);
326 presentation->last_rects = NULL;
328 presentation->last_num_rects = presentation->output_num_rects;
329 if (presentation->last_num_rects > 0)
331 presentation->last_rects = xzalloc(presentation->last_num_rects * sizeof(RDP_RECT));
332 memcpy(presentation->last_rects, presentation->output_rects,
333 presentation->last_num_rects * sizeof(RDP_RECT));
337 vevent = (RDP_VIDEO_FRAME_EVENT*) freerdp_event_new(RDP_EVENT_CLASS_TSMF, RDP_EVENT_TYPE_TSMF_VIDEO_FRAME,
339 vevent->frame_data = sample->data;
340 vevent->frame_size = sample->decoded_size;
341 vevent->frame_pixfmt = sample->pixfmt;
342 vevent->frame_width = sample->stream->width;
343 vevent->frame_height = sample->stream->height;
344 vevent->x = presentation->output_x;
345 vevent->y = presentation->output_y;
346 vevent->width = presentation->output_width;
347 vevent->height = presentation->output_height;
348 if (presentation->output_num_rects > 0)
350 vevent->num_visible_rects = presentation->output_num_rects;
351 vevent->visible_rects = (RDP_RECT*) xzalloc(presentation->output_num_rects * sizeof(RDP_RECT));
352 memcpy(vevent->visible_rects, presentation->output_rects,
353 presentation->output_num_rects * sizeof(RDP_RECT));
356 /* The frame data ownership is passed to the event object, and is freed after the event is processed. */
358 sample->decoded_size = 0;
360 if (!tsmf_push_event(sample->channel_callback, (RDP_EVENT*) vevent))
362 freerdp_event_free((RDP_EVENT*) vevent);
366 /* Dump a .ppm image for every 30 frames. Assuming the frame is in YUV format, we
367 extract the Y values to create a grayscale image. */
368 static int frame_id = 0;
371 if ((frame_id % 30) == 0)
373 snprintf(buf, sizeof(buf), "/tmp/FreeRDP_Frame_%d.ppm", frame_id);
374 fp = fopen(buf, "wb");
375 fwrite("P5\n", 1, 3, fp);
376 snprintf(buf, sizeof(buf), "%d %d\n", sample->stream->width, sample->stream->height);
377 fwrite(buf, 1, strlen(buf), fp);
378 fwrite("255\n", 1, 4, fp);
379 fwrite(sample->data, 1, sample->stream->width * sample->stream->height, fp);
388 static void tsmf_sample_playback_audio(TSMF_SAMPLE* sample)
391 TSMF_STREAM* stream = sample->stream;
393 DEBUG_DVC("MessageId %d EndTime %d consumed.",
394 sample->sample_id, (int)sample->end_time);
396 if (sample->stream->audio && sample->data)
398 sample->stream->audio->Play(sample->stream->audio,
399 sample->data, sample->decoded_size);
401 sample->decoded_size = 0;
403 if (stream->audio && stream->audio->GetLatency)
404 latency = stream->audio->GetLatency(stream->audio);
411 sample->ack_time = latency + get_current_time();
412 stream->last_end_time = sample->end_time + latency;
413 stream->presentation->audio_start_time = sample->start_time + latency;
414 stream->presentation->audio_end_time = sample->end_time + latency;
417 static void tsmf_sample_playback(TSMF_SAMPLE* sample)
423 TSMF_STREAM* stream = sample->stream;
426 ret = stream->decoder->Decode(stream->decoder, sample->data, sample->data_size, sample->extensions);
429 tsmf_sample_ack(sample);
430 tsmf_sample_free(sample);
437 if (stream->major_type == TSMF_MAJOR_TYPE_VIDEO)
439 if (stream->decoder->GetDecodedFormat)
441 pixfmt = stream->decoder->GetDecodedFormat(stream->decoder);
442 if (pixfmt == ((uint32) -1))
444 tsmf_sample_ack(sample);
445 tsmf_sample_free(sample);
448 sample->pixfmt = pixfmt;
451 if (stream->decoder->GetDecodedDimension)
452 ret = stream->decoder->GetDecodedDimension(stream->decoder, &width, &height);
453 if (ret && (width != stream->width || height != stream->height))
455 DEBUG_DVC("video dimension changed to %d x %d", width, height);
456 stream->width = width;
457 stream->height = height;
461 if (stream->decoder->GetDecodedData)
463 sample->data = stream->decoder->GetDecodedData(stream->decoder, &sample->decoded_size);
466 switch (sample->stream->major_type)
468 case TSMF_MAJOR_TYPE_VIDEO:
469 tsmf_sample_playback_video(sample);
470 tsmf_sample_ack(sample);
471 tsmf_sample_free(sample);
473 case TSMF_MAJOR_TYPE_AUDIO:
474 tsmf_sample_playback_audio(sample);
475 tsmf_sample_queue_ack(sample);
480 static void* tsmf_stream_playback_func(void* arg)
483 TSMF_STREAM* stream = (TSMF_STREAM*) arg;
484 TSMF_PRESENTATION* presentation = stream->presentation;
486 DEBUG_DVC("in %d", stream->stream_id);
488 if (stream->major_type == TSMF_MAJOR_TYPE_AUDIO &&
489 stream->sample_rate && stream->channels && stream->bits_per_sample)
491 stream->audio = tsmf_load_audio_device(
492 presentation->audio_name && presentation->audio_name[0] ? presentation->audio_name : NULL,
493 presentation->audio_device && presentation->audio_device[0] ? presentation->audio_device : NULL);
496 stream->audio->SetFormat(stream->audio,
497 stream->sample_rate, stream->channels, stream->bits_per_sample);
500 while (!freerdp_thread_is_stopped(stream->thread))
502 tsmf_stream_process_ack(stream);
503 sample = tsmf_stream_pop_sample(stream, 1);
505 tsmf_sample_playback(sample);
507 freerdp_usleep(5000);
509 if (stream->eos || presentation->eos)
511 while ((sample = tsmf_stream_pop_sample(stream, 1)) != NULL)
512 tsmf_sample_playback(sample);
516 stream->audio->Free(stream->audio);
517 stream->audio = NULL;
520 freerdp_thread_quit(stream->thread);
522 DEBUG_DVC("out %d", stream->stream_id);
527 static void tsmf_stream_start(TSMF_STREAM* stream)
529 if (!freerdp_thread_is_running(stream->thread))
531 freerdp_thread_start(stream->thread, tsmf_stream_playback_func, stream);
535 static void tsmf_stream_stop(TSMF_STREAM* stream)
537 if (freerdp_thread_is_running(stream->thread))
539 freerdp_thread_stop(stream->thread);
543 void tsmf_presentation_start(TSMF_PRESENTATION* presentation)
548 for (item = presentation->stream_list->head; item; item = item->next)
550 stream = (TSMF_STREAM*) item->data;
551 tsmf_stream_start(stream);
555 void tsmf_presentation_stop(TSMF_PRESENTATION* presentation)
560 tsmf_presentation_flush(presentation);
562 for (item = presentation->stream_list->head; item; item = item->next)
564 stream = (TSMF_STREAM*) item->data;
565 tsmf_stream_stop(stream);
568 tsmf_presentation_restore_last_video_frame(presentation);
569 if (presentation->last_rects)
571 xfree(presentation->last_rects);
572 presentation->last_rects = NULL;
574 presentation->last_num_rects = 0;
575 if (presentation->output_rects)
577 xfree(presentation->output_rects);
578 presentation->output_rects = NULL;
580 presentation->output_num_rects = 0;
583 void tsmf_presentation_set_geometry_info(TSMF_PRESENTATION* presentation,
584 uint32 x, uint32 y, uint32 width, uint32 height,
585 int num_rects, RDP_RECT* rects)
587 presentation->output_x = x;
588 presentation->output_y = y;
589 presentation->output_width = width;
590 presentation->output_height = height;
591 if (presentation->output_rects)
592 xfree(presentation->output_rects);
593 presentation->output_rects = rects;
594 presentation->output_num_rects = num_rects;
597 void tsmf_presentation_set_audio_device(TSMF_PRESENTATION* presentation, const char* name, const char* device)
599 presentation->audio_name = name;
600 presentation->audio_device = device;
603 static void tsmf_stream_flush(TSMF_STREAM* stream)
607 while ((sample = tsmf_stream_pop_sample(stream, 0)) != NULL)
608 tsmf_sample_free(sample);
610 while ((sample = list_dequeue(stream->sample_ack_list)) != NULL)
611 tsmf_sample_free(sample);
614 stream->audio->Flush(stream->audio);
617 stream->last_end_time = 0;
618 stream->next_start_time = 0;
619 if (stream->major_type == TSMF_MAJOR_TYPE_AUDIO)
621 stream->presentation->audio_start_time = 0;
622 stream->presentation->audio_end_time = 0;
626 void tsmf_presentation_flush(TSMF_PRESENTATION* presentation)
629 TSMF_STREAM * stream;
631 for (item = presentation->stream_list->head; item; item = item->next)
633 stream = (TSMF_STREAM*) item->data;
634 tsmf_stream_flush(stream);
637 presentation->eos = 0;
638 presentation->audio_start_time = 0;
639 presentation->audio_end_time = 0;
642 void tsmf_presentation_free(TSMF_PRESENTATION* presentation)
646 tsmf_presentation_stop(presentation);
647 list_remove(presentation_list, presentation);
649 while (presentation->stream_list->head)
651 stream = (TSMF_STREAM*) list_peek(presentation->stream_list);
652 tsmf_stream_free(stream);
654 list_free(presentation->stream_list);
656 freerdp_mutex_free(presentation->mutex);
661 TSMF_STREAM* tsmf_stream_new(TSMF_PRESENTATION* presentation, uint32 stream_id)
665 stream = tsmf_stream_find_by_id(presentation, stream_id);
668 DEBUG_WARN("duplicated stream id %d!", stream_id);
672 stream = xnew(TSMF_STREAM);
674 stream->stream_id = stream_id;
675 stream->presentation = presentation;
676 stream->thread = freerdp_thread_new();
677 stream->sample_list = list_new();
678 stream->sample_ack_list = list_new();
680 freerdp_mutex_lock(presentation->mutex);
681 list_enqueue(presentation->stream_list, stream);
682 freerdp_mutex_unlock(presentation->mutex);
687 TSMF_STREAM* tsmf_stream_find_by_id(TSMF_PRESENTATION* presentation, uint32 stream_id)
692 for (item = presentation->stream_list->head; item; item = item->next)
694 stream = (TSMF_STREAM*) item->data;
695 if (stream->stream_id == stream_id)
701 void tsmf_stream_set_format(TSMF_STREAM* stream, const char* name, STREAM* s)
703 TS_AM_MEDIA_TYPE mediatype;
707 DEBUG_WARN("duplicated call");
711 tsmf_codec_parse_media_type(&mediatype, s);
713 if (mediatype.MajorType == TSMF_MAJOR_TYPE_VIDEO)
715 DEBUG_DVC("video width %d height %d bit_rate %d frame_rate %f codec_data %d",
716 mediatype.Width, mediatype.Height, mediatype.BitRate,
717 (double)mediatype.SamplesPerSecond.Numerator / (double)mediatype.SamplesPerSecond.Denominator,
718 mediatype.ExtraDataSize);
720 else if (mediatype.MajorType == TSMF_MAJOR_TYPE_AUDIO)
722 DEBUG_DVC("audio channel %d sample_rate %d bits_per_sample %d codec_data %d",
723 mediatype.Channels, mediatype.SamplesPerSecond.Numerator, mediatype.BitsPerSample,
724 mediatype.ExtraDataSize);
725 stream->sample_rate = mediatype.SamplesPerSecond.Numerator;
726 stream->channels = mediatype.Channels;
727 stream->bits_per_sample = mediatype.BitsPerSample;
728 if (stream->bits_per_sample == 0)
729 stream->bits_per_sample = 16;
732 stream->major_type = mediatype.MajorType;
733 stream->width = mediatype.Width;
734 stream->height = mediatype.Height;
735 stream->decoder = tsmf_load_decoder(name, &mediatype);
738 void tsmf_stream_end(TSMF_STREAM* stream)
741 stream->presentation->eos = 1;
744 void tsmf_stream_free(TSMF_STREAM* stream)
746 TSMF_PRESENTATION* presentation = stream->presentation;
748 tsmf_stream_stop(stream);
749 tsmf_stream_flush(stream);
751 freerdp_mutex_lock(presentation->mutex);
752 list_remove(presentation->stream_list, stream);
753 freerdp_mutex_unlock(presentation->mutex);
755 list_free(stream->sample_list);
756 list_free(stream->sample_ack_list);
759 stream->decoder->Free(stream->decoder);
761 freerdp_thread_free(stream->thread);
766 void tsmf_stream_push_sample(TSMF_STREAM* stream, IWTSVirtualChannelCallback* pChannelCallback,
767 uint32 sample_id, uint64 start_time, uint64 end_time, uint64 duration, uint32 extensions,
768 uint32 data_size, uint8* data)
772 sample = xnew(TSMF_SAMPLE);
774 sample->sample_id = sample_id;
775 sample->start_time = start_time;
776 sample->end_time = end_time;
777 sample->duration = duration;
778 sample->extensions = extensions;
779 sample->stream = stream;
780 sample->channel_callback = pChannelCallback;
781 sample->data_size = data_size;
782 sample->data = xzalloc(data_size + TSMF_BUFFER_PADDING_SIZE);
783 memcpy(sample->data, data, data_size);
785 freerdp_thread_lock(stream->thread);
786 list_enqueue(stream->sample_list, sample);
787 freerdp_thread_unlock(stream->thread);
790 void tsmf_media_init(void)
792 if (presentation_list == NULL)
793 presentation_list = list_new();