android - 使用 Twilio 视频流式传输 CustomView ARcore

Question

当我想使用 twilio video api 和 ARcore 流式传输自定义 View 时遇到问题，基本上它流式传输黑屏。我使用示例中的 ViewCapturer 类到此链接 https://github.com/twilio/video-quickstart-android/tree/master/exampleCustomVideoCapturer来自官方文档，但不适用于 arcore，可能是由于 arFragment 中存在表面 View 。

感谢您的支持。

activity_camera.xml

<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:id="@+id/container"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context=".CameraARActivity">

    <fragment
        android:id="@+id/ux_fragment"
        android:name="com.google.ar.sceneform.ux.ArFragment"
        android:layout_width="match_parent"
        android:layout_height="match_parent" />

    <android.support.v7.widget.RecyclerView
        android:id="@+id/recycler_view"
        android:layout_width="match_parent"
        android:layout_height="100dp"
        android:layout_alignParentBottom="true"
        android:background="#c100a5a0"
        android:visibility="gone" />

    <ImageButton
        android:id="@+id/btnCloseChat"
        android:layout_width="24dp"
        android:layout_height="24dp"
        android:layout_alignParentBottom="true"
        android:layout_alignParentEnd="true"
        android:layout_marginBottom="86dp"
        android:layout_marginEnd="13dp"
        android:background="@android:color/transparent"
        android:contentDescription="Close chat button"
        android:src="@drawable/ic_close_black_24dp"
        android:visibility="gone" />

</RelativeLayout>

本地视频创建行:

screenVideoTrack = LocalVideoTrack.create(CameraARActivity.this, true, new ViewCapturer(mArFragment.getArSceneView()));

和 ViewCapturer 类

import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.view.View;

import com.twilio.video.VideoCapturer;
import com.twilio.video.VideoDimensions;
import com.twilio.video.VideoFormat;
import com.twilio.video.VideoFrame;
import com.twilio.video.VideoPixelFormat;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * ViewCapturer demonstrates how to implement a custom {@link VideoCapturer}. This class
 * captures the contents of a provided view and signals the {@link VideoCapturer.Listener} when
 * the frame is available.
 */
public class ViewCapturer implements VideoCapturer {
    private static final int VIEW_CAPTURER_FRAMERATE_MS = 100;

    private final View view;
    private Handler handler = new Handler(Looper.getMainLooper());
    private VideoCapturer.Listener videoCapturerListener;
    private AtomicBoolean started = new AtomicBoolean(false);
    private final Runnable viewCapturer = new Runnable() {
        @Override
        public void run() {
            boolean dropFrame = view.getWidth() == 0 || view.getHeight() == 0;

            // Only capture the view if the dimensions have been established
            if (!dropFrame) {
                // Draw view into bitmap backed canvas
                int measuredWidth = View.MeasureSpec.makeMeasureSpec(view.getWidth(),
                        View.MeasureSpec.EXACTLY);
                int measuredHeight = View.MeasureSpec.makeMeasureSpec(view.getHeight(),
                        View.MeasureSpec.EXACTLY);
                view.measure(measuredWidth, measuredHeight);
                view.layout(0, 0, view.getMeasuredWidth(), view.getMeasuredHeight());

                Bitmap viewBitmap = Bitmap.createBitmap(view.getWidth(), view.getHeight(),
                        Bitmap.Config.ARGB_8888);
                Canvas viewCanvas = new Canvas(viewBitmap);
                view.draw(viewCanvas);

                // Extract the frame from the bitmap
                int bytes = viewBitmap.getByteCount();
                ByteBuffer buffer = ByteBuffer.allocate(bytes);
                viewBitmap.copyPixelsToBuffer(buffer);
                byte[] array = buffer.array();
                final long captureTimeNs =
                        TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());

                // Create video frame
                VideoDimensions dimensions = new VideoDimensions(view.getWidth(), view.getHeight());
                VideoFrame videoFrame = new VideoFrame(array,
                        dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs);

                // Notify the listener
                if (started.get()) {
                    videoCapturerListener.onFrameCaptured(videoFrame);
                }
            }

            // Schedule the next capture
            if (started.get()) {
                handler.postDelayed(this, VIEW_CAPTURER_FRAMERATE_MS);
            }
        }
    };

    public ViewCapturer(View view) {
        this.view = view;
    }

    /**
     * Returns the list of supported formats for this view capturer. Currently, only supports
     * capturing to RGBA_8888 bitmaps.
     *
     * @return list of supported formats.
     */
    @Override
    public List<VideoFormat> getSupportedFormats() {
        List<VideoFormat> videoFormats = new ArrayList<>();
        VideoDimensions videoDimensions = new VideoDimensions(view.getWidth(), view.getHeight());
        VideoFormat videoFormat = new VideoFormat(videoDimensions, 30, VideoPixelFormat.RGBA_8888);

        videoFormats.add(videoFormat);

        return videoFormats;
    }

    /**
     * Returns true because we are capturing screen content.
     */
    @Override
    public boolean isScreencast() {
        return true;
    }

    /**
     * This will be invoked when it is time to start capturing frames.
     *
     * @param videoFormat the video format of the frames to be captured.
     * @param listener capturer listener.
     */
    @Override
    public void startCapture(VideoFormat videoFormat, Listener listener) {
        // Store the capturer listener
        this.videoCapturerListener = listener;
        this.started.set(true);

        // Notify capturer API that the capturer has started
        boolean capturerStarted = handler.postDelayed(viewCapturer,
                VIEW_CAPTURER_FRAMERATE_MS);
        this.videoCapturerListener.onCapturerStarted(capturerStarted);
    }

    /**
     * Stop capturing frames. Note that the SDK cannot receive frames once this has been invoked.
     */
    @Override
    public void stopCapture() {
        this.started.set(false);
        handler.removeCallbacks(viewCapturer);
    }
}

解决方案

package com.bitdrome.dionigi.eragle.utils;

import android.graphics.Bitmap;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.view.PixelCopy;
import android.view.SurfaceView;
import android.view.View;

import com.twilio.video.VideoCapturer;
import com.twilio.video.VideoDimensions;
import com.twilio.video.VideoFormat;
import com.twilio.video.VideoFrame;
import com.twilio.video.VideoPixelFormat;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * ViewCapturer demonstrates how to implement a custom {@link 
 VideoCapturer}. This class
 * captures the contents of a provided view and signals the {@link 
 VideoCapturer.Listener} when
 * the frame is available.
 */
public class ViewCapturer implements VideoCapturer, 
    PixelCopy.OnPixelCopyFinishedListener {
    private static int VIEW_CAPTURER_FRAMERATE_MS = 10;

private final View view;
private Bitmap viewBitmap;
private Handler handler = new Handler(Looper.getMainLooper());
private Handler handlerPixelCopy = new Handler(Looper.getMainLooper());
private VideoCapturer.Listener videoCapturerListener;
private AtomicBoolean started = new AtomicBoolean(false);

public ViewCapturer(View view) {
    this(view, 24);
}

public ViewCapturer(View view, int framePerSecond) {
    if (framePerSecond <= 0)
        throw new IllegalArgumentException("framePersecond must be greater than 0");
    this.view = view;
    float tmp = (1f / framePerSecond) * 1000;
    VIEW_CAPTURER_FRAMERATE_MS = Math.round(tmp);
}

private final Runnable viewCapturer = new Runnable() {
    @Override
    public void run() {
        boolean dropFrame = view.getWidth() == 0 || view.getHeight() == 0;

        // Only capture the view if the dimensions have been established
        if (!dropFrame) {
            // Draw view into bitmap backed canvas
            int measuredWidth = View.MeasureSpec.makeMeasureSpec(view.getWidth(),
                    View.MeasureSpec.EXACTLY);
            int measuredHeight = View.MeasureSpec.makeMeasureSpec(view.getHeight(),
                    View.MeasureSpec.EXACTLY);
            view.measure(measuredWidth, measuredHeight);
            view.layout(0, 0, view.getMeasuredWidth(), view.getMeasuredHeight());

            viewBitmap = Bitmap.createBitmap(view.getWidth(), view.getHeight(), Bitmap.Config.ARGB_8888);
            try {
                PixelCopy.request((SurfaceView) view, viewBitmap, ViewCapturer.this, handlerPixelCopy);
            } catch (IllegalArgumentException e) {
            }
        }
    }
};

/**
 * Returns the list of supported formats for this view capturer. Currently, only supports
 * capturing to RGBA_8888 bitmaps.
 *
 * @return list of supported formats.
 */
@Override
public List<VideoFormat> getSupportedFormats() {
    List<VideoFormat> videoFormats = new ArrayList<>();
    VideoDimensions videoDimensions = new VideoDimensions(view.getWidth(), view.getHeight());
    VideoFormat videoFormat = new VideoFormat(videoDimensions, 30, VideoPixelFormat.RGBA_8888);

    videoFormats.add(videoFormat);

    return videoFormats;
}

/**
 * Returns true because we are capturing screen content.
 */
@Override
public boolean isScreencast() {
    return true;
}

/**
 * This will be invoked when it is time to start capturing frames.
 *
 * @param videoFormat the video format of the frames to be captured.
 * @param listener    capturer listener.
 */
@Override
public void startCapture(VideoFormat videoFormat, Listener listener) {
    // Store the capturer listener
    this.videoCapturerListener = listener;
    this.started.set(true);

    // Notify capturer API that the capturer has started
    boolean capturerStarted = handler.postDelayed(viewCapturer,
            VIEW_CAPTURER_FRAMERATE_MS);
    this.videoCapturerListener.onCapturerStarted(capturerStarted);
}

/**
 * Stop capturing frames. Note that the SDK cannot receive frames once this has been invoked.
 */
@Override
public void stopCapture() {
    this.started.set(false);
    handler.removeCallbacks(viewCapturer);
}

@Override
public void onPixelCopyFinished(int i) {

    // Extract the frame from the bitmap
    int bytes = viewBitmap.getByteCount();
    ByteBuffer buffer = ByteBuffer.allocate(bytes);
    viewBitmap.copyPixelsToBuffer(buffer);
    byte[] array = buffer.array();
    final long captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());

    // Create video frame
    VideoDimensions dimensions = new VideoDimensions(view.getWidth(), view.getHeight());
    VideoFrame videoFrame = new VideoFrame(array,
            dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs);

    // Notify the listener
    if (started.get()) {
        videoCapturerListener.onFrameCaptured(videoFrame);
    }
    if (started.get()) {
        handler.postDelayed(viewCapturer, VIEW_CAPTURER_FRAMERATE_MS);
    }
}
}

Answer 1

对于必须使用 Twilio Video 流式传输 ARCore 的人

在您的 ARCore 渲染类中。

@Override
public void onDrawFrame(GL10 gl) {
     ....
     this.takeLastFrame();
}

private byte[] takeLastFrame() {
    int height = this.mFrameHeight;
    int width = this.mFrameWidth;

    Mat input = new Mat(height, width, CvType.CV_8UC4);

    ByteBuffer buffer = ByteBuffer.allocate(input.rows() * input.cols() * input.channels());

    GLES20.glReadPixels(0, 0, width, height,
            GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buffer);
    input.put(0, 0, buffer.array());

    Core.rotate(input, input, Core.ROTATE_180);
    Core.flip(input, input, 1);

    return convertMatToBytes(input);
}

private byte[] convertMatToBytes(Mat image) {
    int bufferSize = image.channels() * image.cols() * image.rows();
    byte[] b = new byte[bufferSize];
    image.get(0, 0, b);
    return b;
}

在您的自定义捕获器类中

byte[] array = view.takeLastFrame();
if (array != null && array.length > 0) {
    final long captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());

    // Create video frame
    VideoDimensions dimensions = new VideoDimensions(view.getFrameWidth(), view.getFrameHeight());
    VideoFrame videoFrame = new VideoFrame(array,
                        dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs);

    // Notify the listener
    if (started.get()) {
        videoCapturerListener.onFrameCaptured(videoFrame);
    }
}