2
* FreeRDP: A Remote Desktop Protocol client.
3
* Video Redirection Virtual Channel - FFmpeg Decoder
5
* Copyright 2010-2011 Vic Lee
7
* Licensed under the Apache License, Version 2.0 (the "License");
8
* you may not use this file except in compliance with the License.
9
* You may obtain a copy of the License at
11
* http://www.apache.org/licenses/LICENSE-2.0
13
* Unless required by applicable law or agreed to in writing, software
14
* distributed under the License is distributed on an "AS IS" BASIS,
15
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
* See the License for the specific language governing permissions and
17
* limitations under the License.
23
#include <freerdp/utils/memory.h>
24
#include <freerdp/utils/event.h>
25
#include <freerdp/plugins/tsmf.h>
26
#include <libavcodec/avcodec.h>
28
#include "tsmf_constants.h"
29
#include "tsmf_decoder.h"
31
/* Compatibility with older FFmpeg */
32
#if LIBAVUTIL_VERSION_MAJOR < 50
33
#define AVMEDIA_TYPE_VIDEO 0
34
#define AVMEDIA_TYPE_AUDIO 1
37
typedef struct _TSMFFFmpegDecoder
42
enum CodecID codec_id;
43
AVCodecContext* codec_context;
50
uint32 decoded_size_max;
53
static boolean tsmf_ffmpeg_init_context(ITSMFDecoder* decoder)
55
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
57
mdecoder->codec_context = avcodec_alloc_context();
58
if (!mdecoder->codec_context)
60
DEBUG_WARN("avcodec_alloc_context failed.");
67
static boolean tsmf_ffmpeg_init_video_stream(ITSMFDecoder* decoder, const TS_AM_MEDIA_TYPE* media_type)
69
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
71
mdecoder->codec_context->width = media_type->Width;
72
mdecoder->codec_context->height = media_type->Height;
73
mdecoder->codec_context->bit_rate = media_type->BitRate;
74
mdecoder->codec_context->time_base.den = media_type->SamplesPerSecond.Numerator;
75
mdecoder->codec_context->time_base.num = media_type->SamplesPerSecond.Denominator;
77
mdecoder->frame = avcodec_alloc_frame();
82
static boolean tsmf_ffmpeg_init_audio_stream(ITSMFDecoder* decoder, const TS_AM_MEDIA_TYPE* media_type)
84
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
86
mdecoder->codec_context->sample_rate = media_type->SamplesPerSecond.Numerator;
87
mdecoder->codec_context->bit_rate = media_type->BitRate;
88
mdecoder->codec_context->channels = media_type->Channels;
89
mdecoder->codec_context->block_align = media_type->BlockAlign;
91
#ifdef AV_CPU_FLAG_SSE2
92
mdecoder->codec_context->dsp_mask = AV_CPU_FLAG_SSE2 | AV_CPU_FLAG_MMX2;
94
#if LIBAVCODEC_VERSION_MAJOR < 53
95
mdecoder->codec_context->dsp_mask = FF_MM_SSE2 | FF_MM_MMXEXT;
97
mdecoder->codec_context->dsp_mask = FF_MM_SSE2 | FF_MM_MMX2;
104
static boolean tsmf_ffmpeg_init_stream(ITSMFDecoder* decoder, const TS_AM_MEDIA_TYPE* media_type)
106
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
111
mdecoder->codec = avcodec_find_decoder(mdecoder->codec_id);
112
if (!mdecoder->codec)
114
DEBUG_WARN("avcodec_find_decoder failed.");
118
mdecoder->codec_context->codec_id = mdecoder->codec_id;
119
mdecoder->codec_context->codec_type = mdecoder->media_type;
121
if (mdecoder->media_type == AVMEDIA_TYPE_VIDEO)
123
if (!tsmf_ffmpeg_init_video_stream(decoder, media_type))
126
else if (mdecoder->media_type == AVMEDIA_TYPE_AUDIO)
128
if (!tsmf_ffmpeg_init_audio_stream(decoder, media_type))
132
if (media_type->ExtraData)
134
if (media_type->SubType == TSMF_SUB_TYPE_AVC1 &&
135
media_type->FormatType == TSMF_FORMAT_TYPE_MPEG2VIDEOINFO)
137
/* The extradata format that FFmpeg uses is following CodecPrivate in Matroska.
138
See http://haali.su/mkv/codecs.pdf */
139
mdecoder->codec_context->extradata_size = media_type->ExtraDataSize + 8;
140
mdecoder->codec_context->extradata = xzalloc(mdecoder->codec_context->extradata_size);
141
p = mdecoder->codec_context->extradata;
142
*p++ = 1; /* Reserved? */
143
*p++ = media_type->ExtraData[8]; /* Profile */
144
*p++ = 0; /* Profile */
145
*p++ = media_type->ExtraData[12]; /* Level */
146
*p++ = 0xff; /* Flag? */
147
*p++ = 0xe0 | 0x01; /* Reserved | #sps */
148
s = media_type->ExtraData + 20;
149
size = ((uint32)(*s)) * 256 + ((uint32)(*(s + 1)));
150
memcpy(p, s, size + 2);
154
size = ((uint32)(*s)) * 256 + ((uint32)(*(s + 1)));
155
memcpy(p, s, size + 2);
159
/* Add a padding to avoid invalid memory read in some codec */
160
mdecoder->codec_context->extradata_size = media_type->ExtraDataSize + 8;
161
mdecoder->codec_context->extradata = xzalloc(mdecoder->codec_context->extradata_size);
162
memcpy(mdecoder->codec_context->extradata, media_type->ExtraData, media_type->ExtraDataSize);
163
memset(mdecoder->codec_context->extradata + media_type->ExtraDataSize, 0, 8);
167
if (mdecoder->codec->capabilities & CODEC_CAP_TRUNCATED)
168
mdecoder->codec_context->flags |= CODEC_FLAG_TRUNCATED;
173
static boolean tsmf_ffmpeg_prepare(ITSMFDecoder* decoder)
175
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
177
if (avcodec_open(mdecoder->codec_context, mdecoder->codec) < 0)
179
DEBUG_WARN("avcodec_open failed.");
183
mdecoder->prepared = 1;
188
static boolean tsmf_ffmpeg_set_format(ITSMFDecoder* decoder, TS_AM_MEDIA_TYPE* media_type)
190
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
192
switch (media_type->MajorType)
194
case TSMF_MAJOR_TYPE_VIDEO:
195
mdecoder->media_type = AVMEDIA_TYPE_VIDEO;
197
case TSMF_MAJOR_TYPE_AUDIO:
198
mdecoder->media_type = AVMEDIA_TYPE_AUDIO;
203
switch (media_type->SubType)
205
case TSMF_SUB_TYPE_WVC1:
206
mdecoder->codec_id = CODEC_ID_VC1;
208
case TSMF_SUB_TYPE_WMA2:
209
mdecoder->codec_id = CODEC_ID_WMAV2;
211
case TSMF_SUB_TYPE_WMA9:
212
mdecoder->codec_id = CODEC_ID_WMAPRO;
214
case TSMF_SUB_TYPE_MP3:
215
mdecoder->codec_id = CODEC_ID_MP3;
217
case TSMF_SUB_TYPE_MP2A:
218
mdecoder->codec_id = CODEC_ID_MP2;
220
case TSMF_SUB_TYPE_MP2V:
221
mdecoder->codec_id = CODEC_ID_MPEG2VIDEO;
223
case TSMF_SUB_TYPE_WMV3:
224
mdecoder->codec_id = CODEC_ID_WMV3;
226
case TSMF_SUB_TYPE_AAC:
227
mdecoder->codec_id = CODEC_ID_AAC;
228
/* For AAC the pFormat is a HEAACWAVEINFO struct, and the codec data
229
is at the end of it. See
230
http://msdn.microsoft.com/en-us/library/dd757806.aspx */
231
if (media_type->ExtraData)
233
media_type->ExtraData += 12;
234
media_type->ExtraDataSize -= 12;
237
case TSMF_SUB_TYPE_H264:
238
case TSMF_SUB_TYPE_AVC1:
239
mdecoder->codec_id = CODEC_ID_H264;
241
case TSMF_SUB_TYPE_AC3:
242
mdecoder->codec_id = CODEC_ID_AC3;
248
if (!tsmf_ffmpeg_init_context(decoder))
250
if (!tsmf_ffmpeg_init_stream(decoder, media_type))
252
if (!tsmf_ffmpeg_prepare(decoder))
258
static boolean tsmf_ffmpeg_decode_video(ITSMFDecoder* decoder, const uint8* data, uint32 data_size, uint32 extensions)
260
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
266
#if LIBAVCODEC_VERSION_MAJOR < 52 || (LIBAVCODEC_VERSION_MAJOR == 52 && LIBAVCODEC_VERSION_MINOR <= 20)
267
len = avcodec_decode_video(mdecoder->codec_context, mdecoder->frame, &decoded, data, data_size);
271
av_init_packet(&pkt);
272
pkt.data = (uint8*) data;
273
pkt.size = data_size;
274
if (extensions & TSMM_SAMPLE_EXT_CLEANPOINT)
275
pkt.flags |= AV_PKT_FLAG_KEY;
276
len = avcodec_decode_video2(mdecoder->codec_context, mdecoder->frame, &decoded, &pkt);
282
DEBUG_WARN("data_size %d, avcodec_decode_video failed (%d)", data_size, len);
287
DEBUG_WARN("data_size %d, no frame is decoded.", data_size);
292
DEBUG_DVC("linesize[0] %d linesize[1] %d linesize[2] %d linesize[3] %d "
293
"pix_fmt %d width %d height %d",
294
mdecoder->frame->linesize[0], mdecoder->frame->linesize[1],
295
mdecoder->frame->linesize[2], mdecoder->frame->linesize[3],
296
mdecoder->codec_context->pix_fmt,
297
mdecoder->codec_context->width, mdecoder->codec_context->height);
299
mdecoder->decoded_size = avpicture_get_size(mdecoder->codec_context->pix_fmt,
300
mdecoder->codec_context->width, mdecoder->codec_context->height);
301
mdecoder->decoded_data = xzalloc(mdecoder->decoded_size);
302
frame = avcodec_alloc_frame();
303
avpicture_fill((AVPicture *) frame, mdecoder->decoded_data,
304
mdecoder->codec_context->pix_fmt,
305
mdecoder->codec_context->width, mdecoder->codec_context->height);
307
av_picture_copy((AVPicture *) frame, (AVPicture *) mdecoder->frame,
308
mdecoder->codec_context->pix_fmt,
309
mdecoder->codec_context->width, mdecoder->codec_context->height);
317
static boolean tsmf_ffmpeg_decode_audio(ITSMFDecoder* decoder, const uint8* data, uint32 data_size, uint32 extensions)
319
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
328
LLOGLN(0, ("tsmf_ffmpeg_decode_audio: data_size %d", data_size));
330
for (i = 0; i < data_size; i++)
332
LLOG(0, ("%02X ", data[i]));
339
if (mdecoder->decoded_size_max == 0)
340
mdecoder->decoded_size_max = AVCODEC_MAX_AUDIO_FRAME_SIZE + 16;
341
mdecoder->decoded_data = xzalloc(mdecoder->decoded_size_max);
342
/* align the memory for SSE2 needs */
343
dst = (uint8*) (((uintptr_t)mdecoder->decoded_data + 15) & ~ 0x0F);
344
dst_offset = dst - mdecoder->decoded_data;
346
src_size = data_size;
350
/* Ensure enough space for decoding */
351
if (mdecoder->decoded_size_max - mdecoder->decoded_size < AVCODEC_MAX_AUDIO_FRAME_SIZE)
353
mdecoder->decoded_size_max = mdecoder->decoded_size_max * 2 + 16;
354
mdecoder->decoded_data = xrealloc(mdecoder->decoded_data, mdecoder->decoded_size_max);
355
dst = (uint8*) (((uintptr_t)mdecoder->decoded_data + 15) & ~ 0x0F);
356
if (dst - mdecoder->decoded_data != dst_offset)
358
/* re-align the memory if the alignment has changed after realloc */
359
memmove(dst, mdecoder->decoded_data + dst_offset, mdecoder->decoded_size);
360
dst_offset = dst - mdecoder->decoded_data;
362
dst += mdecoder->decoded_size;
364
frame_size = mdecoder->decoded_size_max - mdecoder->decoded_size;
365
#if LIBAVCODEC_VERSION_MAJOR < 52 || (LIBAVCODEC_VERSION_MAJOR == 52 && LIBAVCODEC_VERSION_MINOR <= 20)
366
len = avcodec_decode_audio2(mdecoder->codec_context,
367
(int16_t*) dst, &frame_size,
372
av_init_packet(&pkt);
373
pkt.data = (uint8*) src;
375
len = avcodec_decode_audio3(mdecoder->codec_context,
376
(int16_t*) dst, &frame_size, &pkt);
379
if (len <= 0 || frame_size <= 0)
381
DEBUG_WARN("error decoding");
386
mdecoder->decoded_size += frame_size;
390
if (mdecoder->decoded_size == 0)
392
xfree(mdecoder->decoded_data);
393
mdecoder->decoded_data = NULL;
397
/* move the aligned decoded data to original place */
398
memmove(mdecoder->decoded_data, mdecoder->decoded_data + dst_offset, mdecoder->decoded_size);
401
DEBUG_DVC("data_size %d decoded_size %d",
402
data_size, mdecoder->decoded_size);
407
static boolean tsmf_ffmpeg_decode(ITSMFDecoder* decoder, const uint8* data, uint32 data_size, uint32 extensions)
409
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
411
if (mdecoder->decoded_data)
413
xfree(mdecoder->decoded_data);
414
mdecoder->decoded_data = NULL;
416
mdecoder->decoded_size = 0;
418
switch (mdecoder->media_type)
420
case AVMEDIA_TYPE_VIDEO:
421
return tsmf_ffmpeg_decode_video(decoder, data, data_size, extensions);
422
case AVMEDIA_TYPE_AUDIO:
423
return tsmf_ffmpeg_decode_audio(decoder, data, data_size, extensions);
425
DEBUG_WARN("unknown media type.");
430
static uint8* tsmf_ffmpeg_get_decoded_data(ITSMFDecoder* decoder, uint32* size)
432
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
435
*size = mdecoder->decoded_size;
436
buf = mdecoder->decoded_data;
437
mdecoder->decoded_data = NULL;
438
mdecoder->decoded_size = 0;
442
static uint32 tsmf_ffmpeg_get_decoded_format(ITSMFDecoder* decoder)
444
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
446
switch (mdecoder->codec_context->pix_fmt)
448
case PIX_FMT_YUV420P:
449
return RDP_PIXFMT_I420;
452
DEBUG_WARN("unsupported pixel format %u",
453
mdecoder->codec_context->pix_fmt);
458
static boolean tsmf_ffmpeg_get_decoded_dimension(ITSMFDecoder* decoder, uint32* width, uint32* height)
460
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
462
if (mdecoder->codec_context->width > 0 && mdecoder->codec_context->height > 0)
464
*width = mdecoder->codec_context->width;
465
*height = mdecoder->codec_context->height;
474
static void tsmf_ffmpeg_free(ITSMFDecoder* decoder)
476
TSMFFFmpegDecoder* mdecoder = (TSMFFFmpegDecoder*) decoder;
479
av_free(mdecoder->frame);
480
if (mdecoder->decoded_data)
481
xfree(mdecoder->decoded_data);
482
if (mdecoder->codec_context)
484
if (mdecoder->prepared)
485
avcodec_close(mdecoder->codec_context);
486
if (mdecoder->codec_context->extradata)
487
xfree(mdecoder->codec_context->extradata);
488
av_free(mdecoder->codec_context);
493
static boolean initialized = false;
496
TSMFDecoderEntry(void)
498
TSMFFFmpegDecoder * decoder;
503
avcodec_register_all();
507
decoder = xnew(TSMFFFmpegDecoder);
509
decoder->iface.SetFormat = tsmf_ffmpeg_set_format;
510
decoder->iface.Decode = tsmf_ffmpeg_decode;
511
decoder->iface.GetDecodedData = tsmf_ffmpeg_get_decoded_data;
512
decoder->iface.GetDecodedFormat = tsmf_ffmpeg_get_decoded_format;
513
decoder->iface.GetDecodedDimension = tsmf_ffmpeg_get_decoded_dimension;
514
decoder->iface.Free = tsmf_ffmpeg_free;
516
return (ITSMFDecoder*) decoder;