Add texture support to HardwareVideoEncoder.

webrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

 /*
  *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 package org.webrtc;
 
 import android.annotation.TargetApi;
+import android.graphics.Matrix;
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
+import android.opengl.GLES20;
 import android.os.Bundle;
+import android.view.Surface;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Deque;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.LinkedBlockingDeque;
 
 /** Android hardware video encoder. */
 @TargetApi(19)
@@ skipping 38 matching lines @@
   // Thread that delivers encoded frames to the user callback.
   private Thread outputThread;
 
   // Whether the encoder is running.  Volatile so that the output thread can watch this value and
   // exit when the encoder stops.
   private volatile boolean running = false;
   // Any exception thrown during shutdown.  The output thread releases the MediaCodec and uses this
   // value to send exceptions thrown during release back to the encoder thread.
   private volatile Exception shutdownException = null;
 
+  // Surface objects for texture-mode encoding.
+
+  // EGL context shared with the application.  Used to access texture inputs.
+  private EglBase14.Context textureContext;
+  // EGL base wrapping the shared texture context.  Holds hooks to both the shared context and the
+  // input surface.  Making this base current allows textures from the context to be drawn onto the
+  // surface.
+  private EglBase14 textureEglBase;
+  // Input surface for the codec.  The encoder will draw input textures onto this surface.
+  private Surface textureInputSurface;
+  // Drawer used to draw input textures onto the codec's input surface.
+  private GlRectDrawer textureDrawer;
+
   private MediaCodec codec;
   private Callback callback;
 
   private int width;
   private int height;
 
   // Contents of the last observed config frame output by the MediaCodec. Used by H.264.
   private ByteBuffer configBuffer = null;
 
   /**
    * Creates a new HardwareVideoEncoder with the given codecName, codecType, colorFormat, key frame
    * intervals, and bitrateAdjuster.
    *
    * @param codecName the hardware codec implementation to use
    * @param codecType the type of the given video codec (eg. VP8, VP9, or H264)
    * @param colorFormat color format used by the input buffer
    * @param keyFrameIntervalSec interval in seconds between key frames; used to initialize the codec
    * @param forceKeyFrameIntervalMs interval at which to force a key frame if one is not requested;
    *     used to reduce distortion caused by some codec implementations
    * @param bitrateAdjuster algorithm used to correct codec implementations that do not produce the
    *     desired bitrates
    * @throws IllegalArgumentException if colorFormat is unsupported
    */
   public HardwareVideoEncoder(String codecName, VideoCodecType codecType, int colorFormat,
-      int keyFrameIntervalSec, int forceKeyFrameIntervalMs, BitrateAdjuster bitrateAdjuster) {
+      int keyFrameIntervalSec, int forceKeyFrameIntervalMs, BitrateAdjuster bitrateAdjuster,
+      EglBase14.Context textureContext) {
     this.codecName = codecName;
     this.codecType = codecType;
     this.colorFormat = colorFormat;
-    this.inputColorFormat = ColorFormat.valueOf(colorFormat);
+    if (textureContext == null) {
+      this.inputColorFormat = ColorFormat.valueOf(colorFormat);
+    } else {
+      // ColorFormat copies bytes between buffers.  It is not used in texture mode.
+      this.inputColorFormat = null;
+    }
     this.keyFrameIntervalSec = keyFrameIntervalSec;
     this.forcedKeyFrameMs = forceKeyFrameIntervalMs;
     this.bitrateAdjuster = bitrateAdjuster;
     this.outputBuilders = new LinkedBlockingDeque<>();
+    this.textureContext = textureContext;
   }
 
   @Override
   public VideoCodecStatus initEncode(Settings settings, Callback callback) {
     return initEncodeInternal(
         settings.width, settings.height, settings.startBitrate, settings.maxFramerate, callback);
   }
 
   private VideoCodecStatus initEncodeInternal(
       int width, int height, int bitrateKbps, int fps, Callback callback) {
@@ skipping 18 matching lines @@
     }
     try {
       MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height);
       format.setInteger(MediaFormat.KEY_BIT_RATE, adjustedBitrate);
       format.setInteger(KEY_BITRATE_MODE, VIDEO_ControlRateConstant);
       format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
       format.setInteger(MediaFormat.KEY_FRAME_RATE, bitrateAdjuster.getAdjustedFramerate());
       format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
       Logging.d(TAG, "Format: " + format);
       codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+
+      if (textureContext != null) {
+        // Texture mode.
+        textureEglBase = new EglBase14(textureContext, EglBase.CONFIG_RECORDABLE);
+        textureInputSurface = codec.createInputSurface();
+        textureEglBase.createSurface(textureInputSurface);
+        textureDrawer = new GlRectDrawer();
+      }
+
       codec.start();
     } catch (IllegalStateException e) {
       Logging.e(TAG, "initEncode failed", e);
       release();
       return VideoCodecStatus.ERROR;
     }
 
     running = true;
     outputThread = createOutputThread();
     outputThread.start();
 
     return VideoCodecStatus.OK;
   }
 
   @Override
   public VideoCodecStatus release() {
     try {
+      if (outputThread == null) {
+        return VideoCodecStatus.OK;
+      }
       // The outputThread actually stops and releases the codec once running is false.
       running = false;
       if (!ThreadUtils.joinUninterruptibly(outputThread, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) {
         Logging.e(TAG, "Media encoder release timeout");
         return VideoCodecStatus.TIMEOUT;
       }
       if (shutdownException != null) {
         // Log the exception and turn it into an error.
         Logging.e(TAG, "Media encoder release exception", shutdownException);
         return VideoCodecStatus.ERROR;
       }
     } finally {
       codec = null;
       outputThread = null;
       outputBuilders.clear();
+
+      if (textureDrawer != null) {
+        textureDrawer.release();
+        textureDrawer = null;
+      }
+      if (textureEglBase != null) {
+        textureEglBase.release();
+        textureEglBase = null;
+      }
+      if (textureInputSurface != null) {
+        textureInputSurface.release();
+        textureInputSurface = null;
+      }
     }
     return VideoCodecStatus.OK;
   }
 
   @Override
   public VideoCodecStatus encode(VideoFrame videoFrame, EncodeInfo encodeInfo) {
     if (codec == null) {
       return VideoCodecStatus.UNINITIALIZED;
     }
 
     // If input resolution changed, restart the codec with the new resolution.
     int frameWidth = videoFrame.getWidth();
     int frameHeight = videoFrame.getHeight();
     if (frameWidth != width || frameHeight != height) {
       VideoCodecStatus status = resetCodec(frameWidth, frameHeight);
       if (status != VideoCodecStatus.OK) {
         return status;
       }
     }
 
-    // No timeout.  Don't block for an input buffer, drop frames if the encoder falls behind.
-    int index;
-    try {
-      index = codec.dequeueInputBuffer(0 /* timeout */);
-    } catch (IllegalStateException e) {
-      Logging.e(TAG, "dequeueInputBuffer failed", e);
-      return VideoCodecStatus.FALLBACK_SOFTWARE;
-    }
-
-    if (index == -1) {
-      // Encoder is falling behind.  No input buffers available.  Drop the frame.
-      Logging.e(TAG, "Dropped frame, no input buffers available");
-      return VideoCodecStatus.OK; // See webrtc bug 2887.
-    }
     if (outputBuilders.size() > MAX_ENCODER_Q_SIZE) {
       // Too many frames in the encoder.  Drop this frame.
       Logging.e(TAG, "Dropped frame, encoder queue full");
       return VideoCodecStatus.OK; // See webrtc bug 2887.
     }
 
-    // TODO(mellem):  Add support for input surfaces and textures.
-    ByteBuffer buffer;
-    try {
-      buffer = codec.getInputBuffers()[index];
-    } catch (IllegalStateException e) {
-      Logging.e(TAG, "getInputBuffers failed", e);
-      return VideoCodecStatus.FALLBACK_SOFTWARE;
-    }
-    VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420();
-    inputColorFormat.fillBufferFromI420(buffer, i420);
-
     boolean requestedKeyFrame = false;
     for (EncodedImage.FrameType frameType : encodeInfo.frameTypes) {
       if (frameType == EncodedImage.FrameType.VideoFrameKey) {
         requestedKeyFrame = true;
       }
     }
 
     // Frame timestamp rounded to the nearest microsecond and millisecond.
     long presentationTimestampUs = (videoFrame.getTimestampNs() + 500) / 1000;
     long presentationTimestampMs = (presentationTimestampUs + 500) / 1000;
     if (requestedKeyFrame || shouldForceKeyFrame(presentationTimestampMs)) {
       requestKeyFrame(presentationTimestampMs);
     }
 
+    VideoFrame.Buffer videoFrameBuffer = videoFrame.getBuffer();
     // Number of bytes in the video buffer. Y channel is sampled at one byte per pixel; U and V are
     // subsampled at one byte per four pixels.
-    int bufferSize = videoFrame.getBuffer().getHeight() * videoFrame.getBuffer().getWidth() * 3 / 2;
+    int bufferSize = videoFrameBuffer.getHeight() * videoFrameBuffer.getWidth() * 3 / 2;
     EncodedImage.Builder builder = EncodedImage.builder()
                                        .setCaptureTimeMs(presentationTimestampMs)
                                        .setCompleteFrame(true)
                                        .setEncodedWidth(videoFrame.getWidth())
                                        .setEncodedHeight(videoFrame.getHeight())
                                        .setRotation(videoFrame.getRotation());
     outputBuilders.offer(builder);
+
+    if (textureContext != null) {
+      if (!(videoFrameBuffer instanceof VideoFrame.TextureBuffer)) {
+        Logging.e(TAG, "Cannot encode non-texture buffer in texture mode");
+        return VideoCodecStatus.ERROR;
+      }
+      VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) videoFrameBuffer;
+      return encodeTextureBuffer(videoFrame, textureBuffer);
+    } else {
+      if (videoFrameBuffer instanceof VideoFrame.TextureBuffer) {
+        Logging.w(TAG, "Encoding texture buffer in byte mode; this may be inefficient");
+      }
+      return encodeByteBuffer(videoFrame, videoFrameBuffer, bufferSize, presentationTimestampUs);
+    }
+  }
+
+  private VideoCodecStatus encodeTextureBuffer(
+      VideoFrame videoFrame, VideoFrame.TextureBuffer textureBuffer) {
+    Matrix matrix = videoFrame.getTransformMatrix();
+    float[] transformationMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(matrix);
+
+    try {
+      textureEglBase.makeCurrent();
+      // TODO(perkj): glClear() shouldn't be necessary since every pixel is covered anyway,
+      // but it's a workaround for bug webrtc:5147.
+      GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+      switch (textureBuffer.getType()) {
+        case OES:
+          textureDrawer.drawOes(textureBuffer.getTextureId(), transformationMatrix, width, height,
+              0, 0, width, height);
+          break;
+        case RGB:
+          textureDrawer.drawRgb(textureBuffer.getTextureId(), transformationMatrix, width, height,
+              0, 0, width, height);
+          break;
+      }
+      textureEglBase.swapBuffers(videoFrame.getTimestampNs());
+    } catch (RuntimeException e) {
+      Logging.e(TAG, "encodeTexture failed", e);
+      // Keep the output builders in sync with buffers in the codec.
+      outputBuilders.pollLast();
+      return VideoCodecStatus.ERROR;
+    }
+    return VideoCodecStatus.OK;
+  }
+
+  private VideoCodecStatus encodeByteBuffer(VideoFrame videoFrame,
+      VideoFrame.Buffer videoFrameBuffer, int bufferSize, long presentationTimestampUs) {
+    // No timeout.  Don't block for an input buffer, drop frames if the encoder falls behind.
+    int index;
+    try {
+      index = codec.dequeueInputBuffer(0 /* timeout */);
+    } catch (IllegalStateException e) {
+      Logging.e(TAG, "dequeueInputBuffer failed", e);
+      return VideoCodecStatus.FALLBACK_SOFTWARE;
+    }
+
+    if (index == -1) {
+      // Encoder is falling behind.  No input buffers available.  Drop the frame.
+      Logging.e(TAG, "Dropped frame, no input buffers available");
+      return VideoCodecStatus.OK; // See webrtc bug 2887.
+    }
+
+    ByteBuffer buffer;
+    try {
+      buffer = codec.getInputBuffers()[index];
+    } catch (IllegalStateException e) {
+      Logging.e(TAG, "getInputBuffers failed", e);
+      return VideoCodecStatus.ERROR;
+    }
+    VideoFrame.I420Buffer i420 = videoFrameBuffer.toI420();
+    inputColorFormat.fillBufferFromI420(buffer, i420);
+    i420.release();
+
     try {
       codec.queueInputBuffer(
           index, 0 /* offset */, bufferSize, presentationTimestampUs, 0 /* flags */);
     } catch (IllegalStateException e) {
       Logging.e(TAG, "queueInputBuffer failed", e);
       // Keep the output builders in sync with buffers in the codec.
       outputBuilders.pollLast();
       // IllegalStateException thrown when the codec is in the wrong state.
-      return VideoCodecStatus.FALLBACK_SOFTWARE;
+      return VideoCodecStatus.ERROR;
     }
     return VideoCodecStatus.OK;
   }
 
   @Override
   public VideoCodecStatus setChannelParameters(short packetLoss, long roundTripTimeMs) {
     // No op.
     return VideoCodecStatus.OK;
   }
 
@@ skipping 179 matching lines @@
         case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
         case MediaCodecInfo.CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar:
         case MediaCodecUtils.COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m:
           return NV12;
         default:
           throw new IllegalArgumentException("Unsupported colorFormat: " + colorFormat);
       }
     }
   }
 }