如何实现RTMP或RTSP播放端回调YUV/RGB数据？

如何实现RTMP或RTSP播放端回调YUV/RGB数据？_daniusdk的技术博客_51CTO博客

今天某乎收到个问题推荐，如何实现RTSP回调YUV数据，用于二次处理？

正好前些年我们做RTSP和RTMP直播播放的时候，实现过相关的需求，本文就以Android为例，大概说说具体实现吧。

先说回调yuv或rgb这块意义吧，不管是RTSP还是RTMP直播播放模块，解码后的yuv/rgb数据，可以实现比如快照（编码保存png或jpeg）、回调给第三方用于比如视频分析、亦或比如回调给Unity，实现Unity平台下的绘制。

为了图文并茂，让大家有个基本的认识，先上张图，demo展示的是本地播放的同时，可把yuv或rgb回上来，供上层做二次处理：

我们把协议栈这块处理，放到JNI下，播放之前，设置回调：

I420ExternalRender()具体实现：

/*
* SmartPlayer.java
* SmartPlayer
*
* Github: https://github.com/daniulive/SmarterStreaming
*
* Created by DaniuLive on 2015/09/26.
*/

class I420ExternalRender implements NTExternalRender {
// public static final int NT_FRAME_FORMAT_RGBA = 1;
// public static final int NT_FRAME_FORMAT_ABGR = 2;
// public static final int NT_FRAME_FORMAT_I420 = 3;

private int width_ = 0;
private int height_ = 0;

private int y_row_bytes_ = 0;
private int u_row_bytes_ = 0;
private int v_row_bytes_ = 0;

private ByteBuffer y_buffer_ = null;
private ByteBuffer u_buffer_ = null;
private ByteBuffer v_buffer_ = null;

@Override
public int getNTFrameFormat() {
Log.i(TAG, "I420ExternalRender::getNTFrameFormat return "
+ NT_FRAME_FORMAT_I420);
return NT_FRAME_FORMAT_I420;
}

@Override
public void onNTFrameSizeChanged(int width, int height) {
width_ = width;
height_ = height;

y_row_bytes_ = (width_ + 15) & (~15);
u_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
v_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);

y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_ * height_);
u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_
* ((height_ + 1) / 2));
v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_
* ((height_ + 1) / 2));

Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_="
+ width_ + " height_=" + height_ + " y_row_bytes_="
+ y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_
+ " v_row_bytes_=" + v_row_bytes_);
}

@Override
public ByteBuffer getNTPlaneByteBuffer(int index) {
if (index == 0) {
return y_buffer_;
} else if (index == 1) {
return u_buffer_;
} else if (index == 2) {
return v_buffer_;
} else {
Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
return null;
}
}

@Override
public int getNTPlanePerRowBytes(int index) {
if (index == 0) {
return y_row_bytes_;
} else if (index == 1) {
return u_row_bytes_;
} else if (index == 2) {
return v_row_bytes_;
} else {
Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
return 0;
}
}

public void onNTRenderFrame(int width, int height, long timestamp)
{
if ( y_buffer_ == null )
return;

if ( u_buffer_ == null )
return;

if ( v_buffer_ == null )
return;


y_buffer_.rewind();

u_buffer_.rewind();

v_buffer_.rewind();

/*
if ( !is_saved_image )
{
is_saved_image = true;

int y_len = y_row_bytes_*height_;

int u_len = u_row_bytes_*((height_+1)/2);
int v_len = v_row_bytes_*((height_+1)/2);

int data_len = y_len + (y_row_bytes_*((height_+1)/2));

byte[] nv21_data = new byte[data_len];

byte[] u_data = new byte[u_len];
byte[] v_data = new byte[v_len];

y_buffer_.get(nv21_data, 0, y_len);
u_buffer_.get(u_data, 0, u_len);
v_buffer_.get(v_data, 0, v_len);

int[] strides = new int[2];
strides[0] = y_row_bytes_;
strides[1] = y_row_bytes_;


int loop_row_c = ((height_+1)/2);
int loop_c = ((width_+1)/2);

int dst_row = y_len;
int src_v_row = 0;
int src_u_row = 0;

for ( int i = 0; i < loop_row_c; ++i)
{
int dst_pos = dst_row;

for ( int j = 0; j <loop_c; ++j )
{
nv21_data[dst_pos++] = v_data[src_v_row + j];
nv21_data[dst_pos++] = u_data[src_u_row + j];
}

dst_row += y_row_bytes_;
src_v_row += v_row_bytes_;
src_u_row += u_row_bytes_;
}

String imagePath = "/sdcard" + "/" + "testonv21" + ".jpeg";

Log.e(TAG, "I420ExternalRender::begin test save iamge++ image_path:" + imagePath);

try
{
File file = new File(imagePath);

FileOutputStream image_os = new FileOutputStream(file);

YuvImage image = new YuvImage(nv21_data, ImageFormat.NV21, width_, height_, strides);

image.compressToJpeg(new android.graphics.Rect(0, 0, width_, height_), 50, image_os);

image_os.flush();
image_os.close();
}
catch(IOException e)
{
e.printStackTrace();
}

Log.e(TAG, "I420ExternalRender::begin test save iamge--");
}

*/


Log.i(TAG, "I420ExternalRender::onNTRenderFrame w=" + width + " h=" + height + " timestamp=" + timestamp);

// copy buffer

// test
// byte[] test_buffer = new byte[16];
// y_buffer_.get(test_buffer);

// Log.i(TAG, "I420ExternalRender::onNTRenderFrame y data:" + bytesToHexString(test_buffer));

// u_buffer_.get(test_buffer);
// Log.i(TAG, "I420ExternalRender::onNTRenderFrame u data:" + bytesToHexString(test_buffer));

// v_buffer_.get(test_buffer);
// Log.i(TAG, "I420ExternalRender::onNTRenderFrame v data:" + bytesToHexString(test_buffer));
}
}