技术背景
我们在对接RTSP播放器相关的技术诉求的时候，遇到这样的需求，客户做特种设备巡检的，需要把摄像头拍到的RTSP流拉下来，然后添加动态水印后，再生成新的RTSP URL，供平台调用。真个流程需要延迟尽可能的低，分辨率要支持到1080p，并需要把添加过动态水印的数据，保存到本地。
技术实现
在此之前，大牛直播SDK有非常成熟的RTSP播放、轻量级RTSP服务和录像模块，要做的就是，拉取到RTSP流后，把解码后的YUV或RGB回调给上层，上层通过图层的形式，添加动态文字水印（图片水印亦可），然后，投递给轻量级RTSP服务，RTSP服务对外提供个拉流的RTSP URL，无图无真相：
左侧就是我们基于Windows平台C#的播放器的demo，二次开发的，添加了软、硬编码设置（考虑到分辨率比较高，添加支持了硬编码选项设置）、动态水印设置、轻量级RTSP服务、实时录像和RTMP推送。
先说数据回调，本文以回调yuv数据为例：
代码语言：c#复制video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32, IntPtr.Zero, video_frame_call_back_);
回调后的数据，投递到轻量级RTSP服务模块。
代码语言：c#复制public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
{
if (frame == IntPtr.Zero)
{
return;
}
NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
if (publisher_wrapper_ != null)
{
if (publisher_wrapper_.IsPublisherHandleAvailable())
{
if (publisher_wrapper_.IsPublishing() || publisher_wrapper_.IsRecording() || publisher_wrapper_.IsRTSPPublisherRunning())
{
//publisher_wrapper_.OnPostRGB32Data(0, video_frame.plane0_, video_frame.width_ * 4 * video_frame.height_, video_frame.stride0_,
// video_frame.width_, video_frame.height_);
publisher_wrapper_.OnPostYUVData(0, video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
video_frame.plane2_, video_frame.stride2_,
video_frame.width_, video_frame.height_);
}
}
}
}音频由于暂时不要二次处理，直接投递过去，如果需要处理的话，处理后再投递给publisher wrapper：
代码语言：c#复制public void SetAudioPCMFrameCallBack(IntPtr handle, IntPtr user_data,
UInt32 status, IntPtr data, UInt32 size,
Int32 sample_rate, Int32 channel, Int32 per_channel_sample_number)
{
if (data == IntPtr.Zero || size == 0)
{
return;
}
if (publisher_wrapper_ != null)
{
if (publisher_wrapper_.IsPublisherHandleAvailable())
{
if (publisher_wrapper_.IsPublishing() || publisher_wrapper_.IsRecording() || publisher_wrapper_.IsRTSPPublisherRunning())
{
publisher_wrapper_.OnPostAudioPCMData(data, size, 0, sample_rate, channel, per_channel_sample_number);
}
}
}
}设置文字水印字体：
代码语言：c#复制 private void btn_set_font_Click(object sender, EventArgs e)
{
FontDialog font_dlg = new FontDialog();
DialogResult result = font_dlg.ShowDialog();
if (result == DialogResult.OK)
{
// 获取用户所选字体
Font selectedFont = font_dlg.Font;
btn_set_font.Text = "" selectedFont.Name ", " selectedFont.Size "pt";
selected_osd_font_ = new Font(selectedFont.Name, selectedFont.Size, FontStyle.Regular, GraphicsUnit.Point);
}
}动态设置文字水印：
代码语言：c#复制 private async void btn_text_osd_Click(object sender, EventArgs e)
{
string format = "yyyy-MM-dd HH:mm:ss.fff";
StringBuilder sb = new StringBuilder();
sb.Append("施工单位：上海视沃信息科技有限公司(daniusdk.com)");
sb.Append("rn");
sb.Append("施工时间：");
sb.Append(DateTime.Now.DayOfWeek.ToString());
sb.Append(" ");
sb.Append(DateTime.Now.ToString(format));
sb.Append("rn");
sb.Append("当前位置：上海市");
string str = sb.ToString();
Bitmap bmp = GenerateBitmap(str);
int index = 1;
int x = 0;
int y = 200;
UpdateLayerRegion(index, x, y, bmp);
await Task.Delay(30);
UpdateARGBBitmap(index, bmp);
}如果需要硬编码：
代码语言：c#复制 if (btn_check_video_hardware_encoder_.Checked)
{
is_hw_encoder = true;
}
Int32 cur_sel_encoder_id = 0;
Int32 cur_sel_gpu = 0;
if (is_hw_encoder)
{
int cur_sel_hw = combobox_video_encoders_.SelectedIndex;
if (cur_sel_hw >= 0)
{
cur_sel_encoder_id = Convert.ToInt32(combobox_video_encoders_.SelectedValue);
cur_sel_gpu = -1;
int cur_sel_hw_dev = combobox_video_hardware_encoder_devices_.SelectedIndex;
if (cur_sel_hw_dev >= 0)
{
cur_sel_gpu = Convert.ToInt32(combobox_video_hardware_encoder_devices_.SelectedValue);
}
}
else
{
is_hw_encoder = false;
}
}
if (!is_hw_encoder)
{
if ((int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id)
{
cur_sel_encoder_id = btn_check_openh264_encoder_.Checked ? 1 : 0;
}
}
publisher_wrapper_.SetVideoEncoder((int)(is_hw_encoder ? 1 : 0), (int)cur_sel_encoder_id, (uint)cur_video_codec_id, (int)cur_sel_gpu);
publisher_wrapper_.SetVideoQualityV2(publisher_wrapper_.CalVideoQuality(width_, height_, is_h264_encoder));
publisher_wrapper_.SetVideoBitRate(publisher_wrapper_.CalBitRate(video_fps_, width_, height_));
publisher_wrapper_.SetVideoMaxBitRate(publisher_wrapper_.CalMaxKBitRate(video_fps_, width_, height_, false));
publisher_wrapper_.SetVideoKeyFrameInterval(key_frame_interval_);
if (is_h264_encoder)
{
publisher_wrapper_.SetVideoEncoderProfile(1);
}
publisher_wrapper_.SetVideoEncoderSpeed(publisher_wrapper_.CalVideoEncoderSpeed(width_, height_, is_h264_encoder));启动停止RTSP服务：
代码语言：c#复制 private void btn_rtsp_service_Click(object sender, EventArgs e)
{
if(publisher_wrapper_.IsRTSPSerivceRunning())
{
publisher_wrapper_.StopRtspService();
btn_rtsp_service.Text = "启动RTSP服务";
btn_rtsp_stream.Enabled = false;
}
else
{
if(publisher_wrapper_.StartRtspService())
{
btn_rtsp_service.Text = "停止RTSP服务";
btn_rtsp_stream.Enabled = true;
}
}
}发布RTSP流：
代码语言：c#复制 private void btn_rtsp_stream_Click(object sender, EventArgs e)
{
if (publisher_wrapper_.IsRTSPPublisherRunning())
{
publisher_wrapper_.StopRtspStream();
btn_rtsp_stream.Text = "发布RTSP流";
btn_get_rtsp_session_numbers.Enabled = false;
btn_rtsp_service.Enabled = true;
}
else
{
if (!publisher_wrapper_.IsPublisherHandleAvailable())
{
if (!OpenPublisherHandle())
{
return;
}
}
if (publisher_wrapper_.GetPublisherHandleCount() < 1)
{
SetCommonOptionToPublisherSDK();
}
if (!publisher_wrapper_.StartRtspStream())
{
MessageBox.Show("调用StartRtspStream失败..");
return;
}
btn_rtsp_stream.Text = "停止RTSP流";
btn_get_rtsp_session_numbers.Enabled = true;
btn_rtsp_service.Enabled = false;
}
}获取RTSP会话数：
代码语言：c#复制 private void btn_get_rtsp_session_numbers_Click(object sender, EventArgs e)
{
if (publisher_wrapper_.IsRTSPPublisherRunning())
{
int session_numbers = publisher_wrapper_.GetRtspSessionNumbers();
MessageBox.Show(session_numbers.ToString(), "当前RTSP连接会话数");
}
}本地录像：
代码语言：c#复制 private void btn_start_recorder_Click(object sender, EventArgs e)
{
if (!publisher_wrapper_.IsPublisherHandleAvailable())
{
if (!OpenPublisherHandle())
{
return;
}
}
if (publisher_wrapper_.GetPublisherHandleCount() < 1)
{
SetCommonOptionToPublisherSDK();
}
if (!publisher_wrapper_.StartRecorder())
{
MessageBox.Show("调用StartRecorder失败..");
return;
}
btn_start_recorder.Enabled = false;
btn_stop_recorder.Enabled = true;
}
private void btn_stop_recorder_Click(object sender, EventArgs e)
{
if (!publisher_wrapper_.IsPublisherHandleAvailable())
return;
if (publisher_wrapper_.IsRecording())
{
publisher_wrapper_.StopRecorder();
btn_start_recorder.Enabled = true;
btn_stop_recorder.Enabled = false;
}
}暂停录像、恢复录像：
代码语言：c#复制 private void btn_pause_recorder_Click(object sender, EventArgs e)
{
if (!publisher_wrapper_.IsPublisherHandleAvailable())
{
return;
}
String btn_pause_rec_text = btn_pause_recorder.Text;
if ("暂停录像" == btn_pause_rec_text)
{
UInt32 ret = publisher_wrapper_.PauseRecorder(true);
if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
{
MessageBox.Show("暂停录像失败, 请重新尝试!");
return;
}
else if (NTBaseCodeDefine.NT_ERC_OK == ret)
{
btn_pause_recorder.Text = "恢复录像";
}
}
else
{
UInt32 ret = publisher_wrapper_.PauseRecorder(false);
if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
{
MessageBox.Show("恢复录像失败, 请重新尝试!");
return;
}
else if (NTBaseCodeDefine.NT_ERC_OK == ret)
{
btn_pause_recorder.Text = "暂停录像";
}
}
}推送RTMP：
代码语言：c#复制 private void btn_publish_rtmp_Click(object sender, EventArgs e)
{
if (!publisher_wrapper_.IsPublisherHandleAvailable())
{
if (!OpenPublisherHandle())
{
return;
}
}
if (publisher_wrapper_.GetPublisherHandleCount() < 1)
{
SetCommonOptionToPublisherSDK();
}
String url = "rtmp://192.168.0.108:1935/hls/stream1";
if (url.Length < 8)
{
publisher_wrapper_.Close();
MessageBox.Show("请输入推送地址");
return;
}
if (!publisher_wrapper_.StartPublisher(url))
{
MessageBox.Show("调用StartPublisher失败..");
return;
}
btn_publish_rtmp.Enabled = false;
btn_stop_publish_rtmp.Enabled = true;
}
private void btn_stop_publish_rtmp_Click(object sender, EventArgs e)
{
if (!publisher_wrapper_.IsPublisherHandleAvailable())
return;
if (publisher_wrapper_.IsPublishing())
{
publisher_wrapper_.StopPublisher();
btn_publish_rtmp.Enabled = true;
btn_stop_publish_rtmp.Enabled = false;
}
}图层设计，目前设计两个图层，一个是原始YUV底层，另外一个是文字水印图层，如果需要动态去除文字水印，只要index为1的图层，enable设置为0即可。
代码语言：c#复制 NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0,
0, IntPtr.Zero);
if (video_option_ == (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
{
NT_PB_ExternalVideoFrameLayerConfig external_layer_c0 = new NT_PB_ExternalVideoFrameLayerConfig();
external_layer_c0.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
external_layer_c0.base_.index_ = 0;
external_layer_c0.base_.enable_ = 1;
external_layer_c0.base_.region_.x_ = 0;
external_layer_c0.base_.region_.y_ = 0;
external_layer_c0.base_.region_.width_ = video_width_;
external_layer_c0.base_.region_.height_ = video_height_;
external_layer_c0.base_.offset_ = Marshal.OffsetOf(external_layer_c0.GetType(), "base_").ToInt32();
external_layer_c0.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c0);
IntPtr external_layer_conf0 = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c0));
Marshal.StructureToPtr(external_layer_c0, external_layer_conf0, true);
UInt32 external_r0 = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
external_layer_conf0, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,
0, IntPtr.Zero);
Marshal.FreeHGlobal(external_layer_conf0);
//OSD水印层
NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 = new NT_PB_ExternalVideoFrameLayerConfig();
external_layer_c1.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
external_layer_c1.base_.index_ = 1;
external_layer_c1.base_.enable_ = 1;
external_layer_c1.base_.region_.x_ = 0;
external_layer_c1.base_.region_.y_ = 200;
external_layer_c1.base_.region_.width_ = 200;
external_layer_c1.base_.region_.height_ = 200;
external_layer_c1.base_.offset_ = Marshal.OffsetOf(external_layer_c1.GetType(), "base_").ToInt32();
external_layer_c1.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c1);
IntPtr external_layer_conf = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1));
Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true);
UInt32 external_r1 = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,
0, IntPtr.Zero);
Marshal.FreeHGlobal(external_layer_conf);
//end
}总结
RTSP拉流二次编码，整体逻辑不复杂，就是把数据回调后，二次处理，我们推送端设计的是图层的形式，所以，回调后的数据，直接作为第0层，文字水印作为第一层，如果需要图片水印，图片水印作为第三层即可。RTSP拉流二次编码，如果做到客户端尽量无感知，需要尽可能的压缩整体处理的延迟，确保从数据采集，到二次处理，到再次播放出来毫秒级，满足绝大多数场景下的技术需求。
