2
* Copyright (C) 2016 Canonical, Ltd.
4
* This program is free software: you can redistribute it and/or modify it
5
* under the terms of the GNU General Public License version 3, as published
6
* by the Free Software Foundation.
8
* This program is distributed in the hope that it will be useful, but
9
* WITHOUT ANY WARRANTY; without even the implied warranties of
10
* MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
11
* PURPOSE. See the GNU General Public License for more details.
13
* You should have received a copy of the GNU General Public License along
14
* with this program. If not, see <http://www.gnu.org/licenses/>.
18
// Ignore all warnings coming from the external Android headers as
19
// we don't control them and also don't want to get any warnings
20
// from them which will only polute our build output.
21
#pragma GCC diagnostic push
22
#pragma GCC diagnostic warning "-w"
23
#include <system/window.h>
24
#pragma GCC diagnostic pop
26
#include <boost/concept_check.hpp>
28
#include "mcs/logger.h"
29
#include "mcs/keep_alive.h"
31
#include "mcs/video/statistics.h"
33
#include "mcs/android/h264encoder.h"
36
static constexpr const char *kEncoderThreadName{"H264Encoder"};
37
static constexpr const char *kH264MimeType{"video/avc"};
38
static constexpr const char *kRawMimeType{"video/raw"};
39
// From frameworks/native/include/media/openmax/OMX_IVCommon.h
40
static constexpr int32_t kOMXColorFormatAndroidOpaque = 0x7F000789;
41
static constexpr int32_t kOMXVideoIntraRefreshCyclic = 0;
42
// From frameworks/native/include/media/openmax/OMX_Video.h
43
static constexpr int32_t kOMXVideoControlRateConstant = 2;
44
// From frameworks/native/include/media/hardware/MetadataBufferType.h
45
static constexpr uint32_t kMetadataBufferTypeGrallocSource = 1;
46
// Supplying -1 as framerate means the encoder decides on which framerate
48
static constexpr int32_t kAnyFramerate = -1;
49
// Default is a bitrate of 5 MBit/s
50
static constexpr int32_t kDefaultBitrate = 5000000;
51
// By default send an I frame every 15 seconds which is the
52
// same Android currently configures in its WiFi Display code path.
53
static constexpr std::chrono::seconds kDefaultIFrameInterval{15};
54
// From frameworks/av/include/media/stagefright/MediaErrors.h
55
enum AndroidMediaError {
56
kAndroidMediaErrorBase = -1000,
57
kAndroidMediaErrorNotConnected = kAndroidMediaErrorBase - 1,
58
kAndroidMediaErrorBufferTooSmall = kAndroidMediaErrorBase - 9,
59
kAndroidMediaErrorEndOfStream = kAndroidMediaErrorBase - 11,
61
// Constants for all the fields we're putting into the AMessage
62
// structure to configure the MediaCodec instance for our needs.
63
static constexpr const char *kFormatKeyMime{"mime"};
64
static constexpr const char *kFormatKeyStoreMetaDataInBuffers{"store-metadata-in-buffers"};
65
static constexpr const char *kFormatKeyStoreMetaDataInBuffersOutput{"store-metadata-in-buffers-output"};
66
static constexpr const char *kFormatKeyWidth{"width"};
67
static constexpr const char *kFormatKeyHeight{"height"};
68
static constexpr const char *kFormatKeyStride{"stride"};
69
static constexpr const char *kFormatKeySliceHeight{"slice-height"};
70
static constexpr const char *kFormatKeyColorFormat{"color-format"};
71
static constexpr const char *kFormatKeyBitrate{"bitrate"};
72
static constexpr const char *kFormatKeyBitrateMode{"bitrate-mode"};
73
static constexpr const char *kFormatKeyFramerate{"frame-rate"};
74
static constexpr const char *kFormatKeyIntraRefreshMode{"intra-refresh-mode"};
75
static constexpr const char *kFormatKeyIntraRefreshCIRMbs{"intra-refresh-CIR-mbs"};
76
static constexpr const char *kFormatKeyIFrameInterval{"i-frame-interval"};
77
static constexpr const char *kFormatKeyProfileIdc{"profile-idc"};
78
static constexpr const char *kFormatKeyLevelIdc{"level-idc"};
79
static constexpr const char *kFormatKeyConstraintSet{"constraint-set"};
80
static constexpr const char *kFormatKeyPrependSpsPpstoIdrFrames{"prepend-sps-pps-to-idr-frames"};
86
class MediaSourceBuffer : public video::Buffer
89
typedef std::shared_ptr<MediaSourceBuffer> Ptr;
91
~MediaSourceBuffer() {
95
const auto ref_count = media_buffer_get_refcount(buffer_);
97
// If someone has set a reference on the buffer we just have to
98
// release it here and the other one will take care about actually
101
media_buffer_release(buffer_);
103
media_buffer_destroy(buffer_);
107
static MediaSourceBuffer::Ptr Create(MediaBufferWrapper *buffer) {
108
const auto sp = std::shared_ptr<MediaSourceBuffer>(new MediaSourceBuffer);
109
sp->buffer_ = buffer;
110
sp->ExtractTimestamp();
114
virtual uint32_t Length() const {
115
return media_buffer_get_size(buffer_);
118
virtual uint8_t* Data() {
119
return static_cast<uint8_t*>(media_buffer_get_data(buffer_));
122
virtual bool IsValid() const {
123
return buffer_ != nullptr;
127
void ExtractTimestamp() {
128
const auto meta_data = media_buffer_get_meta_data(buffer_);
132
const uint32_t key_time = media_meta_data_get_key_id(MEDIA_META_DATA_KEY_TIME);
134
media_meta_data_find_int64(meta_data, key_time, &time_us);
136
SetTimestamp(time_us);
140
MediaBufferWrapper *buffer_;
143
video::BaseEncoder::Config H264Encoder::DefaultConfiguration() {
145
config.framerate = kAnyFramerate;
146
config.bitrate = kDefaultBitrate;
147
config.i_frame_interval = kDefaultIFrameInterval.count();
148
config.intra_refresh_mode = kOMXVideoIntraRefreshCyclic;
152
video::BaseEncoder::Ptr H264Encoder::Create() {
153
return std::shared_ptr<H264Encoder>(new H264Encoder);
156
H264Encoder::H264Encoder() :
159
source_format_(nullptr),
162
input_queue_(mcs::video::BufferQueue::Create()),
167
H264Encoder::~H264Encoder() {
171
media_codec_source_release(encoder_);
174
media_source_release(source_);
177
media_message_release(format_);
180
media_meta_data_release(source_format_);
183
bool H264Encoder::Configure(const Config &config) {
187
MCS_DEBUG("configuring with %dx%d@%d", config.width, config.height, config.framerate);
189
auto format = media_message_create();
193
media_message_set_string(format, kFormatKeyMime, kH264MimeType, 0);
195
media_message_set_int32(format, kFormatKeyStoreMetaDataInBuffers, true);
196
media_message_set_int32(format, kFormatKeyStoreMetaDataInBuffersOutput, false);
198
media_message_set_int32(format, kFormatKeyWidth, config.width);
199
media_message_set_int32(format, kFormatKeyHeight, config.height);
200
media_message_set_int32(format, kFormatKeyStride, config.width);
201
media_message_set_int32(format, kFormatKeySliceHeight, config.width);
203
media_message_set_int32(format, kFormatKeyColorFormat, kOMXColorFormatAndroidOpaque);
205
media_message_set_int32(format, kFormatKeyBitrate, config.bitrate);
206
media_message_set_int32(format, kFormatKeyBitrateMode, kOMXVideoControlRateConstant);
207
media_message_set_int32(format, kFormatKeyFramerate, config.framerate);
209
media_message_set_int32(format, kFormatKeyIntraRefreshMode, 0);
211
// Update macroblocks in a cyclic fashion with 10% of all MBs within
212
// frame gets updated at one time. It takes about 10 frames to
213
// completely update a whole video frame. If the frame rate is 30,
214
// it takes about 333 ms in the best case (if next frame is not an IDR)
215
// to recover from a lost/corrupted packet.
216
const int32_t mbs = (((config.width + 15) / 16) * ((config.height + 15) / 16) * 10) / 100;
217
media_message_set_int32(format, kFormatKeyIntraRefreshCIRMbs, mbs);
219
if (config.i_frame_interval > 0)
220
media_message_set_int32(format, kFormatKeyIFrameInterval, config.i_frame_interval);
222
if (config.profile_idc > 0)
223
media_message_set_int32(format, kFormatKeyProfileIdc, config.profile_idc);
225
if (config.level_idc > 0)
226
media_message_set_int32(format, kFormatKeyLevelIdc, config.level_idc);
228
if (config.constraint_set > 0)
229
media_message_set_int32(format, kFormatKeyConstraintSet, config.constraint_set);
231
// FIXME we need to find a way to check if the encoder supports prepending
232
// SPS/PPS to the buffers it is producing or if we have to manually do that
233
media_message_set_int32(format, kFormatKeyPrependSpsPpstoIdrFrames, 1);
235
source_ = media_source_create();
237
MCS_ERROR("Failed to create media input source for encoder");
238
media_message_release(format);
242
auto source_format = media_meta_data_create();
243
if (!source_format) {
244
MCS_ERROR("Failed to create media meta data for encoder source");
245
media_message_release(format);
246
media_source_release(source_);
251
// Notice that we're passing video/raw as mime type here which is quite
252
// important to let the encoder do the right thing with the incoming data
253
media_meta_data_set_cstring(source_format,
254
media_meta_data_get_key_id(MEDIA_META_DATA_KEY_MIME),
257
// We're setting the opaque color format here as the encoder is then
258
// meant to figure out the color format from the GL frames itself.
259
media_meta_data_set_int32(source_format,
260
media_meta_data_get_key_id(MEDIA_META_DATA_KEY_COLOR_FORMAT),
261
kOMXColorFormatAndroidOpaque);
263
media_meta_data_set_int32(source_format,
264
media_meta_data_get_key_id(MEDIA_META_DATA_KEY_WIDTH),
266
media_meta_data_set_int32(source_format,
267
media_meta_data_get_key_id(MEDIA_META_DATA_KEY_HEIGHT),
269
media_meta_data_set_int32(source_format,
270
media_meta_data_get_key_id(MEDIA_META_DATA_KEY_STRIDE),
272
media_meta_data_set_int32(source_format,
273
media_meta_data_get_key_id(MEDIA_META_DATA_KEY_SLICE_HEIGHT),
275
media_meta_data_set_int32(source_format,
276
media_meta_data_get_key_id(MEDIA_META_DATA_KEY_FRAMERATE),
279
media_source_set_format(source_, source_format);
281
media_source_set_start_callback(source_, &H264Encoder::OnSourceStart, this);
282
media_source_set_stop_callback(source_, &H264Encoder::OnSourceStop, this);
283
media_source_set_read_callback(source_, &H264Encoder::OnSourceRead, this);
284
media_source_set_pause_callback(source_, &H264Encoder::OnSourcePause, this);
286
encoder_ = media_codec_source_create(format, source_, 0);
288
MCS_ERROR("Failed to create encoder instance");
289
media_meta_data_release(source_format);
290
media_message_release(format);
291
media_source_release(source_);
298
source_format_ = source_format;
300
MCS_DEBUG("Configured encoder succesfully");
305
bool H264Encoder::Start() {
306
if (!encoder_ || running_)
309
// We have to set us to running before we start the media
310
// codec source as that will directly call OnSourceRead
311
// which will fail if running_ isn't set to true.
314
if (!media_codec_source_start(encoder_)) {
315
MCS_ERROR("Failed to start encoder");
320
MCS_DEBUG("Started encoder");
325
int H264Encoder::OnSourceStart(MediaMetaDataWrapper *meta, void *user_data) {
326
boost::ignore_unused_variable_warning(meta);
327
boost::ignore_unused_variable_warning(user_data);
334
int H264Encoder::OnSourceStop(void *user_data) {
335
boost::ignore_unused_variable_warning(user_data);
342
int H264Encoder::OnSourcePause(void *user_data) {
343
boost::ignore_unused_variable_warning(user_data);
350
MediaBufferWrapper* H264Encoder::PackBuffer(const mcs::video::Buffer::Ptr &input_buffer, const mcs::TimestampUs timestamp) {
351
if (!input_buffer->NativeHandle()) {
352
MCS_WARNING("Ignoring buffer without native handle");
356
const auto anwb = reinterpret_cast<ANativeWindowBuffer*>(input_buffer->NativeHandle());
358
uint32_t type = kMetadataBufferTypeGrallocSource;
359
const size_t size = sizeof(buffer_handle_t) + sizeof(type);
361
// We let the media buffer allocate the memory here to let it keep
362
// the ownership and release the memory once its destroyed.
363
auto buffer = media_buffer_create(size);
367
auto data = media_buffer_get_data(buffer);
369
// We're passing the buffer handle directly as part of the buffer data
370
// here to the encoder and it will figure out it has to deal with a
371
// buffer with the key value we put in front of it. See also
372
// frameworks/av/media/libstagefright/SurfaceMediaSource.cpp for more
373
// details about this.
374
memcpy(data, &type, sizeof(type));
375
memcpy(data + sizeof(type), &anwb->handle, sizeof(buffer_handle_t));
377
media_buffer_set_return_callback(buffer, &H264Encoder::OnBufferReturned, this);
379
// We need to put a reference on the buffer here if we want the
380
// callback we set above being called.
381
media_buffer_ref(buffer);
383
auto meta = media_buffer_get_meta_data(buffer);
384
const auto key_time = media_meta_data_get_key_id(MEDIA_META_DATA_KEY_TIME);
385
media_meta_data_set_int64(meta, key_time, timestamp);
387
pending_buffers_.push_back(BufferItem{input_buffer, buffer});
392
int H264Encoder::OnSourceRead(MediaBufferWrapper **buffer, void *user_data) {
393
auto thiz = static_cast<H264Encoder*>(user_data);
395
if (!thiz || !thiz->running_)
396
return kAndroidMediaErrorNotConnected;
399
return kAndroidMediaErrorBufferTooSmall;
401
const auto input_buffer = thiz->input_queue_->Next();
403
const auto next_buffer = thiz->PackBuffer(input_buffer, input_buffer->Timestamp());
406
return kAndroidMediaErrorEndOfStream;
408
*buffer = next_buffer;
413
void H264Encoder::OnBufferReturned(MediaBufferWrapper *buffer, void *user_data) {
414
auto thiz = static_cast<H264Encoder*>(user_data);
419
// Find the right pending buffer matching the returned one
420
auto iter = thiz->pending_buffers_.begin();
421
for (; iter != thiz->pending_buffers_.end(); ++iter) {
422
if (iter->media_buffer == buffer)
426
if (iter == thiz->pending_buffers_.end()) {
427
MCS_WARNING("Didn't remember returned buffer!?");
431
// Unset observer to be able to call release on the MediaBuffer
432
// and reduce its reference count. It has an internal check if
433
// an observer is still set or not before it will actually release
435
media_buffer_set_return_callback(iter->media_buffer, nullptr, nullptr);
436
media_buffer_release(iter->media_buffer);
438
auto buf = iter->buffer;
439
thiz->pending_buffers_.erase(iter);
441
// After we've cleaned up everything we can send the buffer
442
// back to the producer which then can reuse it.
446
bool H264Encoder::DoesBufferContainCodecConfig(MediaBufferWrapper *buffer) {
447
auto meta_data = media_buffer_get_meta_data(buffer);
451
uint32_t key_is_codec_config = media_meta_data_get_key_id(MEDIA_META_DATA_KEY_IS_CODEC_CONFIG);
452
int32_t is_codec_config = 0;
453
media_meta_data_find_int32(meta_data, key_is_codec_config, &is_codec_config);
454
return static_cast<bool>(is_codec_config);
457
bool H264Encoder::Execute() {
459
MCS_ERROR("Tried to execute encoder while not started");
463
MediaBufferWrapper *buffer = nullptr;
464
if (!media_codec_source_read(encoder_, &buffer)) {
465
MCS_ERROR("Failed to read a new buffer from encoder");
469
auto mbuf = MediaSourceBuffer::Create(buffer);
471
if (mbuf->Timestamp() > 0) {
472
const int64_t now = mcs::Utils::GetNowUs();
473
const int64_t diff = (now - mbuf->Timestamp()) / 1000ll;
474
video::Statistics::Instance()->RecordEncoderBufferOut(diff);
477
if (DoesBufferContainCodecConfig(buffer)) {
478
if (auto sp = delegate_.lock())
479
sp->OnBufferWithCodecConfig(mbuf);
482
if (auto sp = delegate_.lock())
483
sp->OnBufferAvailable(mbuf);
488
bool H264Encoder::Stop() {
489
if (!encoder_ || !running_)
492
if (!media_codec_source_stop(encoder_))
500
void H264Encoder::QueueBuffer(const video::Buffer::Ptr &buffer) {
504
input_queue_->Push(buffer);
507
void* H264Encoder::NativeWindowHandle() const {
511
return media_codec_source_get_native_window_handle(encoder_);
514
video::BaseEncoder::Config H264Encoder::Configuration() const {
518
void H264Encoder::SendIDRFrame() {
524
media_codec_source_request_idr_frame(encoder_);
527
} // namespace android
added
removed
src/mcs/android/h264encoder.cpp
