OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "content/common/gpu/media/vt_video_encode_accelerator.h" | |
6 | |
7 #include "base/thread_task_runner_handle.h" | |
8 #include "media/base/mac/coremedia_glue.h" | |
9 #include "media/base/mac/corevideo_glue.h" | |
10 #include "media/base/mac/video_frame_mac.h" | |
11 | |
12 namespace content { | |
13 | |
14 namespace { | |
15 | |
16 // Subjectively chosen. | |
17 // TODO(emircan): Check if we can find the actual system capabilities via | |
18 // creating VTCompressionSessions with varying requirements. | |
19 // See crbug.com/584784. | |
20 const size_t kNumInputBuffers = 4; | |
21 const size_t kMaxFrameRateNumerator = 30; | |
22 const size_t kMaxFrameRateDenominator = 1; | |
23 const size_t kMaxResolutionWidth = 4096; | |
24 const size_t kMaxResolutionHeight = 2160; | |
25 // The ratio of |input_visible_size| area to the max expected output | |
26 // BitstreamBuffer size in bytes. VideoToolbox returns variable sized encoded | |
27 // data whereas media::VideoEncodeAccelerator provides a uniform BitstreamBuffer | |
28 // size to fill this data into. This ratio is used to determine a size that | |
29 // would ideally be big enough to fit all frames. | |
30 const size_t kOutputBufferSizeRatio = 10; | |
31 | |
32 // Container for the associated data of a video frame being processed. | |
33 struct InProgressFrameEncode { | |
Pawel Osciak
2016/02/08 04:33:42
Could this be a private substruct of VTVEA?
emircan
2016/02/08 23:41:23
Done.
| |
34 const base::TimeDelta timestamp; | |
35 const base::TimeTicks reference_time; | |
36 | |
37 InProgressFrameEncode(base::TimeDelta rtp_timestamp, base::TimeTicks ref_time) | |
38 : timestamp(rtp_timestamp), reference_time(ref_time) {} | |
39 | |
40 private: | |
41 DISALLOW_IMPLICIT_CONSTRUCTORS(InProgressFrameEncode); | |
42 }; | |
43 | |
44 } // namespace | |
45 | |
46 struct VTVideoEncodeAccelerator::BitstreamBufferRef { | |
47 BitstreamBufferRef(int32_t id, | |
48 scoped_ptr<base::SharedMemory> shm, | |
Pawel Osciak
2016/02/08 04:33:42
const& perhaps?
emircan
2016/02/08 23:41:23
I cannot use std::move on const scoped_ptr&.
| |
49 size_t size) | |
50 : id(id), shm(std::move(shm)), size(size) {} | |
51 const int32_t id; | |
52 const scoped_ptr<base::SharedMemory> shm; | |
53 const size_t size; | |
54 | |
55 private: | |
56 DISALLOW_IMPLICIT_CONSTRUCTORS(BitstreamBufferRef); | |
57 }; | |
58 | |
59 VTVideoEncodeAccelerator::VTVideoEncodeAccelerator() | |
60 : videotoolbox_glue_(VideoToolboxGlue::Get()), | |
Pawel Osciak
2016/02/08 04:33:43
We used to check this for != nullptr (https://code
emircan
2016/02/08 23:41:23
Hmm we dont have a similar IsSupported() call thou
| |
61 client_task_runner_(base::ThreadTaskRunnerHandle::Get()) { | |
62 } | |
63 | |
64 VTVideoEncodeAccelerator::~VTVideoEncodeAccelerator() { | |
65 DCHECK(thread_checker_.CalledOnValidThread()); | |
66 } | |
67 | |
68 media::VideoEncodeAccelerator::SupportedProfiles | |
69 VTVideoEncodeAccelerator::GetSupportedProfiles() { | |
70 DVLOG(3) << __FUNCTION__; | |
71 DCHECK(thread_checker_.CalledOnValidThread()); | |
72 | |
73 SupportedProfiles profiles; | |
74 SupportedProfile profile; | |
75 profile.profile = media::H264PROFILE_BASELINE; | |
76 profile.max_framerate_numerator = kMaxFrameRateNumerator; | |
77 profile.max_framerate_denominator = kMaxFrameRateDenominator; | |
78 profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight); | |
79 profiles.push_back(profile); | |
80 return profiles; | |
81 } | |
82 | |
83 bool VTVideoEncodeAccelerator::Initialize( | |
84 media::VideoPixelFormat format, | |
85 const gfx::Size& input_visible_size, | |
86 media::VideoCodecProfile output_profile, | |
87 uint32_t initial_bitrate, | |
88 Client* client) { | |
89 DVLOG(3) << __FUNCTION__ | |
90 << ": input_format=" << media::VideoPixelFormatToString(format) | |
91 << ", input_visible_size=" << input_visible_size.ToString() | |
92 << ", output_profile=" << output_profile | |
93 << ", initial_bitrate=" << initial_bitrate; | |
94 DCHECK(thread_checker_.CalledOnValidThread()); | |
95 DCHECK(client); | |
96 DCHECK_EQ(media::PIXEL_FORMAT_I420, format); | |
Pawel Osciak
2016/02/08 04:33:42
I'd suggest if()s and careful check for argument v
emircan
2016/02/08 23:41:23
Done.
| |
97 DCHECK_EQ(media::H264PROFILE_BASELINE, output_profile); | |
Pawel Osciak
2016/02/08 04:33:42
I'd prefer we if()'d profile here nevertheless. Th
emircan
2016/02/08 23:41:24
Done.
| |
98 | |
99 bitrate_ = initial_bitrate; | |
100 input_visible_size_ = input_visible_size; | |
101 | |
102 if (!ResetCompressionSession()) { | |
103 DLOG(ERROR) << "Failed creating compression session"; | |
104 return false; | |
105 } | |
106 | |
107 client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client)); | |
Pawel Osciak
2016/02/08 04:33:42
I'd suggest doing this before calling any methods,
emircan
2016/02/08 23:41:23
Ok, moving it right after the initial ifs.
| |
108 client_ = client_ptr_factory_->GetWeakPtr(); | |
109 client_->RequireBitstreamBuffers( | |
110 kNumInputBuffers, input_visible_size_, | |
111 input_visible_size_.GetArea() / kOutputBufferSizeRatio); | |
112 return true; | |
113 } | |
114 | |
115 void VTVideoEncodeAccelerator::Encode( | |
116 const scoped_refptr<media::VideoFrame>& frame, | |
117 bool force_keyframe) { | |
118 DVLOG(3) << __FUNCTION__; | |
119 DCHECK(thread_checker_.CalledOnValidThread()); | |
120 DCHECK(compression_session_); | |
121 DCHECK(frame); | |
122 | |
123 base::TimeTicks ref_time; | |
124 if (!frame->metadata()->GetTimeTicks( | |
125 media::VideoFrameMetadata::REFERENCE_TIME, &ref_time)) { | |
126 ref_time = base::TimeTicks::Now(); | |
127 } | |
128 auto timestamp_cm = CoreMediaGlue::CMTimeMake( | |
129 frame->timestamp().InMicroseconds(), USEC_PER_SEC); | |
130 // Wrap information we'll need after the frame is encoded in a heap object. | |
131 // We'll get the pointer back from the VideoToolbox completion callback. | |
132 scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode( | |
133 frame->timestamp(), ref_time)); | |
134 | |
135 // TODO(emircan): See if we can eliminate a copy here by using | |
136 // CVPixelBufferPool for the allocation of incoming VideoFrames. | |
137 base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer = | |
138 media::WrapVideoFrameInCVPixelBuffer(*frame); | |
139 base::ScopedCFTypeRef<CFDictionaryRef> frame_props = | |
140 media::video_toolbox::DictionaryWithKeyValue( | |
141 videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(), | |
142 force_keyframe ? kCFBooleanTrue : kCFBooleanFalse); | |
143 | |
144 OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame( | |
145 compression_session_, pixel_buffer, timestamp_cm, | |
146 CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props, | |
147 reinterpret_cast<void*>(request.release()), nullptr); | |
148 DLOG_IF(ERROR, status != noErr) | |
Pawel Osciak
2016/02/08 04:33:42
No need to NOTIFY_ERROR? Can we continue from next
emircan
2016/02/08 23:41:23
Thanks for pointing out. I realize I haven't used
| |
149 << " VTCompressionSessionEncodeFrame failed: " << status; | |
150 } | |
151 | |
152 void VTVideoEncodeAccelerator::UseOutputBitstreamBuffer( | |
153 const media::BitstreamBuffer& buffer) { | |
154 DVLOG(3) << __FUNCTION__; | |
155 DCHECK(thread_checker_.CalledOnValidThread()); | |
156 DCHECK_GE(buffer.size(), static_cast<size_t>(input_visible_size_.GetArea() / | |
Pawel Osciak
2016/02/08 04:33:42
I think it'd be better to if() here please.
emircan
2016/02/08 23:41:23
Done.
| |
157 kOutputBufferSizeRatio)); | |
158 | |
159 scoped_ptr<base::SharedMemory> shm( | |
160 new base::SharedMemory(buffer.handle(), false)); | |
161 if (!shm->Map(buffer.size())) { | |
162 DLOG(ERROR) << "Failed mapping shared memory."; | |
Pawel Osciak
2016/02/08 04:33:42
NOTIFY_ERROR?
emircan
2016/02/08 23:41:23
Done.
| |
163 return; | |
164 } | |
165 | |
166 scoped_ptr<BitstreamBufferRef> buffer_ref( | |
167 new BitstreamBufferRef(buffer.id(), std::move(shm), buffer.size())); | |
168 encoder_output_queue_.push_back(std::move(buffer_ref)); | |
Pawel Osciak
2016/02/08 04:33:42
Do we need to wake something up here? If we got En
emircan
2016/02/08 23:41:23
Replied to the later comment.
| |
169 } | |
170 | |
171 void VTVideoEncodeAccelerator::RequestEncodingParametersChange( | |
172 uint32_t bitrate, | |
173 uint32_t framerate) { | |
Pawel Osciak
2016/02/08 04:33:42
If this class cannot handle changing framerate, sh
emircan
2016/02/08 23:41:23
Actually both changes aren't supported, but at lea
| |
174 DVLOG(3) << __FUNCTION__; | |
175 DCHECK(thread_checker_.CalledOnValidThread()); | |
176 | |
177 bitrate_ = bitrate; | |
Pawel Osciak
2016/02/08 04:33:42
We should preferably check input values here.
emircan
2016/02/08 23:41:24
What kind of checks? I found (bitrate < 1) checks
| |
178 if (!compression_session_) | |
Pawel Osciak
2016/02/08 04:33:42
NOTIFY_ERROR?
emircan
2016/02/08 23:41:24
Done.
| |
179 return; | |
180 | |
181 session_property_setter_->SetSessionProperty( | |
Pawel Osciak
2016/02/08 04:33:42
SetSessionProperty() methods are bool, but we are
emircan
2016/02/08 23:41:24
I will go through them.
| |
182 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(), | |
183 static_cast<int32_t>(bitrate_)); | |
184 } | |
185 | |
186 void VTVideoEncodeAccelerator::Destroy() { | |
187 DVLOG(3) << __FUNCTION__; | |
188 DCHECK(thread_checker_.CalledOnValidThread()); | |
189 | |
190 DestroyCompressionSession(); | |
191 delete this; | |
192 } | |
193 | |
194 // static | |
195 void VTVideoEncodeAccelerator::CompressionCallback(void* encoder_opaque, | |
Pawel Osciak
2016/02/08 04:33:42
Do we know what thread this is called on?
Should
emircan
2016/02/08 23:41:23
No. To quote the documentation: "This function may
| |
196 void* request_opaque, | |
197 OSStatus status, | |
198 VTEncodeInfoFlags info, | |
199 CMSampleBufferRef sbuf) { | |
200 DVLOG(3) << __FUNCTION__; | |
201 | |
202 if (status != noErr) | |
203 DLOG(ERROR) << " encode failed: " << status; | |
Pawel Osciak
2016/02/08 04:33:43
NOTIFY_ERROR?
emircan
2016/02/08 23:41:23
Done.
| |
204 | |
205 if (info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped) { | |
206 DVLOG(2) << " frame dropped"; | |
207 return; | |
208 } | |
209 | |
210 auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex( | |
211 CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0)); | |
212 const bool keyframe = | |
213 !CFDictionaryContainsKey(sample_attachments, | |
214 CoreMediaGlue::kCMSampleAttachmentKey_NotSync()); | |
215 auto encoder = reinterpret_cast<VTVideoEncodeAccelerator*>(encoder_opaque); | |
216 DCHECK(encoder); | |
217 if (encoder->encoder_output_queue_.empty()) { | |
218 DLOG(ERROR) << "No more bitstream buffer to encode into."; | |
Pawel Osciak
2016/02/08 04:33:42
Please see my previous comment, but I think this c
emircan
2016/02/08 23:41:23
Thanks for pointing out. I didn't know Encode() an
| |
219 encoder->client_task_runner_->PostTask( | |
220 FROM_HERE, base::Bind(&Client::NotifyError, encoder->client_, | |
221 kPlatformFailureError)); | |
222 return; | |
223 } | |
224 scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref = | |
225 std::move(encoder->encoder_output_queue_.front()); | |
226 encoder->encoder_output_queue_.pop_front(); | |
227 | |
228 size_t used_buffer_size = 0; | |
229 const bool copy_rv = media::video_toolbox::CopySampleBufferToAnnexBBuffer( | |
230 sbuf, reinterpret_cast<uint8_t*>(buffer_ref->shm->memory()), | |
231 buffer_ref->size, keyframe, &used_buffer_size); | |
232 if (!copy_rv) { | |
233 DLOG(ERROR) << "Cannot copy output from SampleBuffer to AnnexBBuffer."; | |
234 encoder->encoder_output_queue_.push_back(std::move(buffer_ref)); | |
235 return; | |
236 } | |
237 | |
238 // This method is NOT called on |client_task_runner_|, so we still need to | |
Pawel Osciak
2016/02/08 04:33:42
Uhh, if so, I don't think we can access encoder at
emircan
2016/02/08 23:41:23
I am posting a task as you suggest.
But why woul
Pawel Osciak
2016/02/18 11:16:14
Concurrent read-only calls to containers are ok, b
| |
239 // post a task back to it to reach |client_|. | |
240 encoder->client_task_runner_->PostTask( | |
241 FROM_HERE, base::Bind(&Client::BitstreamBufferReady, encoder->client_, | |
242 buffer_ref->id, used_buffer_size, keyframe)); | |
243 } | |
244 | |
245 bool VTVideoEncodeAccelerator::ResetCompressionSession() { | |
246 DCHECK(thread_checker_.CalledOnValidThread()); | |
247 | |
248 DestroyCompressionSession(); | |
249 | |
250 base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec = | |
251 media::video_toolbox::DictionaryWithKeyValue(videotoolbox_glue_ | |
252 ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(), | |
253 kCFBooleanTrue); | |
254 | |
255 // Keep these in-sync with those in ConfigureSession(). | |
Pawel Osciak
2016/02/08 04:33:42
Where is ConfigureSession() ?
emircan
2016/02/08 23:41:23
Changed it to ConfigureCompressionSession().
| |
256 CFTypeRef attributes_keys[] = { | |
257 #if defined(OS_IOS) | |
258 kCVPixelBufferOpenGLESCompatibilityKey, | |
259 #else | |
260 kCVPixelBufferOpenGLCompatibilityKey, | |
261 #endif | |
262 kCVPixelBufferIOSurfacePropertiesKey, | |
263 kCVPixelBufferPixelFormatTypeKey | |
264 }; | |
265 const int format[] = { | |
Pawel Osciak
2016/02/08 04:33:42
Is this a fourcc? If so uint32_t please.
emircan
2016/02/08 23:41:24
We need to create CFArrayRef<int> from it.
| |
266 CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange}; | |
267 CFTypeRef attributes_values[] = { | |
268 kCFBooleanTrue, | |
269 media::video_toolbox::DictionaryWithKeysAndValues(nullptr, nullptr, 0) | |
270 .release(), | |
271 media::video_toolbox::ArrayWithIntegers(format, arraysize(format)) | |
272 .release()}; | |
273 const base::ScopedCFTypeRef<CFDictionaryRef> attributes = | |
274 media::video_toolbox::DictionaryWithKeysAndValues( | |
275 attributes_keys, attributes_values, arraysize(attributes_keys)); | |
276 for (auto& v : attributes_values) | |
277 CFRelease(v); | |
278 | |
279 // Create the compression session. | |
280 OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate( | |
281 kCFAllocatorDefault, | |
282 input_visible_size_.width(), | |
283 input_visible_size_.height(), | |
284 CoreMediaGlue::kCMVideoCodecType_H264, | |
285 encoder_spec, | |
286 attributes, | |
287 nullptr /* compressedDataAllocator */, | |
288 &VTVideoEncodeAccelerator::CompressionCallback, | |
289 reinterpret_cast<void*>(this), | |
290 compression_session_.InitializeInto()); | |
291 if (status != noErr) { | |
292 DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status; | |
293 return false; | |
294 } | |
295 ConfigureCompressionSession(); | |
296 return true; | |
297 } | |
298 | |
299 void VTVideoEncodeAccelerator::ConfigureCompressionSession() { | |
300 DCHECK(thread_checker_.CalledOnValidThread()); | |
301 DCHECK(compression_session_); | |
302 | |
303 session_property_setter_.reset( | |
304 new media::video_toolbox::SessionPropertySetter(compression_session_, | |
305 videotoolbox_glue_)); | |
306 session_property_setter_->SetSessionProperty( | |
307 videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(), | |
308 videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel()); | |
Pawel Osciak
2016/02/08 04:33:42
Is baseline preferred by us? Normally constrained
emircan
2016/02/08 23:41:24
I discussed this with WebRTC people. I learned tha
Pawel Osciak
2016/02/18 11:16:14
Are we sure that was Baseline, and not Constrained
emircan
2016/03/03 19:18:19
As far as I got answers from WebRTC folks, it is B
| |
309 session_property_setter_->SetSessionProperty( | |
310 videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true); | |
311 session_property_setter_->SetSessionProperty( | |
312 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(), | |
313 static_cast<int32_t>(bitrate_)); | |
314 session_property_setter_->SetSessionProperty( | |
315 videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(), | |
316 false); | |
317 } | |
318 | |
319 void VTVideoEncodeAccelerator::DestroyCompressionSession() { | |
320 DCHECK(thread_checker_.CalledOnValidThread()); | |
321 | |
322 if (compression_session_) { | |
323 videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_); | |
324 compression_session_.reset(); | |
325 } | |
326 } | |
327 | |
328 } // namespace content | |
OLD | NEW |