背景
随着智能门禁等物联网产品的普及，越来越多的开发者对音视频互动体验提出了更高的要求。目前市面上大多一对一互动都是基于WebRTC，优点不再赘述，我们这里先说说可能需要面临的问题：WebRTC的服务器部署非常复杂，可以私有部署，但是非常复杂。传输基于UDP，很难保证传输质量，由于UDP是不可靠的传输协议，在复杂的公网网络环境下，各种突发流量、偶尔的传输错误、网络抖动、超时等等都会引起丢包异常，都会在一定程度上影响音视频通信的质量，难以应对复杂的互联网环境，如跨区跨运营商、低带宽、高丢包等场景，行话说的好：从demo到实用,中间还差1万个WebRTC。
其他技术方案
- 内网环境下的RTSP轻量级服务；
- 基于RTMP的公网或内网技术方案。
本方案系基于现有RTMP或内置RTSP服务、RTMP/RTSP直播播放模块，产品稳定度高，在保证超低延迟的基础上，加入噪音抑制、回音消除、自动增益控制等特性，确保通话体验（如需更好的消除效果，亦可考虑如麦克风阵列等技术方案），采用通用的RTMP服务器（如nginx、SRS）或自身的轻量级RTSP服务，更有利于私有部署，便于支持H.264的扩展SEI消息发送机制，方便扩展特定机型H.265编码支持。
技术实现
废话不多说，先上图：
关键demo代码说明：
拉流播放：
代码语言：javascript复制 btnPlaybackStartStopPlayback.setOnClickListener(new Button.OnClickListener()
{
// @Override
public void onClick(View v) {
if(isPlaybackViewStarted)
{
Log.i(PLAY_TAG, "Stop playback stream ");
btnPlaybackStartStopPlayback.setText("开始播放 ");
//btnPopInputText.setEnabled(true);
btnPlaybackPopInputUrl.setEnabled(true);
btnPlaybackHardwareDecoder.setEnabled(true);
btnPlaybackSetPlayBuffer.setEnabled(true);
btnPlaybackFastStartup.setEnabled(true);
if ( playerHandle != 0 )
{
libPlayer.SmartPlayerStopPlay(playerHandle);
libPlayer.SmartPlayerClose(playerHandle);
playerHandle = 0;
}
isPlaybackViewStarted = false;
Log.i(PLAY_TAG, "Stop playback stream--");
}
else
{
Log.i(PLAY_TAG, "Start playback stream ");
playerHandle = libPlayer.SmartPlayerOpen(curContext);
if(playerHandle == 0)
{
Log.e(PLAY_TAG, "surfaceHandle with nil..");
return;
}
libPlayer.SetSmartPlayerEventCallbackV2(playerHandle,
new EventHandePlayerV2());
libPlayer.SmartPlayerSetSurface(playerHandle, playerSurfaceView); //if set the second param with null, it means it will playback audio only..
// libPlayer.SmartPlayerSetSurface(playerHandle, null);
libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1);
// External Render test
//libPlayer.SmartPlayerSetExternalRender(playerHandle, new RGBAExternalRender());
//libPlayer.SmartPlayerSetExternalRender(playerHandle, new I420ExternalRender());
libPlayer.SmartPlayerSetExternalAudioOutput(playerHandle, new PlayerExternalPcmOutput());
libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1);
libPlayer.SmartPlayerSetBuffer(playerHandle, playbackBuffer);
libPlayer.SmartPlayerSetFastStartup(playerHandle, isPlaybackFastStartup?1:0);
if ( isPlaybackMute )
{
libPlayer.SmartPlayerSetMute(playerHandle, isPlaybackMute?1:0);
}
if (isPlaybackHardwareDecoder) {
int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle,1);
int isSupportH264HwDecoder = libPlayer
.SetSmartPlayerVideoHWDecoder(playerHandle,1);
Log.i(TAG, "isSupportH264HwDecoder: " isSupportH264HwDecoder ", isSupportHevcHwDecoder: " isSupportHevcHwDecoder);
}
if( playbackUrl == null )
{
Log.e(PLAY_TAG, "playback URL with NULL...");
return;
}
libPlayer.SmartPlayerSetAudioVolume(playerHandle, curAudioVolume);
libPlayer.SmartPlayerSetUrl(playerHandle, playbackUrl);
int iPlaybackRet = libPlayer.SmartPlayerStartPlay(playerHandle);
if( iPlaybackRet != 0 )
{
libPlayer.SmartPlayerClose(playerHandle);
playerHandle = 0;
Log.e(PLAY_TAG, "StartPlayback strem failed..");
return;
}
btnPlaybackStartStopPlayback.setText("停止播放 ");
btnPlaybackPopInputUrl.setEnabled(false);
btnPlaybackHardwareDecoder.setEnabled(false);
btnPlaybackSetPlayBuffer.setEnabled(false);
btnPlaybackFastStartup.setEnabled(false);
isPlaybackViewStarted = true;
Log.i(PLAY_TAG, "Start playback stream--");
}
}
});拉流端实时音量调节：
代码语言：javascript复制 audioVolumeBar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() {
@Override
public void onProgressChanged(SeekBar seekBar, int i, boolean b) {
}
@Override
public void onStartTrackingTouch(SeekBar seekBar) {
//Log.i(TAG, "开始拖动");
}
@Override
public void onStopTrackingTouch(SeekBar seekBar) {
Log.i(TAG, "停止拖动, CurProgress: " seekBar.getProgress());
curAudioVolume = seekBar.getProgress();
audioVolumeText.setText("当前音量: " curAudioVolume);
if(playerHandle != 0)
{
libPlayer.SmartPlayerSetAudioVolume(playerHandle, curAudioVolume);
}
}
});
}回调后的PCM数据，传给推送端，用于音频处理
代码语言：javascript复制 class NTAudioRecordV2CallbackImpl implements NTAudioRecordV2Callback
{
@Override
public void onNTAudioRecordV2Frame(ByteBuffer data, int size, int sampleRate, int channel, int per_channel_sample_number)
{
/*
Log.i(TAG, "onNTAudioRecordV2Frame size=" size " sampleRate=" sampleRate " channel=" channel
" per_channel_sample_number=" per_channel_sample_number);
*/
if ( (isPushingRtmp || isRTSPPublisherRunning) && publisherHandle != 0 )
{
libPublisher.SmartPublisherOnPCMData(publisherHandle, data, size, sampleRate, channel, per_channel_sample_number);
}
}
} class PlayerExternalPcmOutput implements NTExternalAudioOutput
{
private int sample_rate_ = 0;
private int channel_ = 0;
private int sample_size = 0;
private int buffer_size = 0;
private ByteBuffer pcm_buffer_ = null;
@Override
public ByteBuffer getPcmByteBuffer(int size)
{
//Log.i("getPcmByteBuffer", "size: " size);
if(size < 1)
{
return null;
}
if(buffer_size != size)
{
buffer_size = size;
pcm_buffer_ = ByteBuffer.allocateDirect(buffer_size);
}
return pcm_buffer_;
}
public void onGetPcmFrame(int ret, int sampleRate, int channel, int sampleSize, int is_low_latency)
{
/*Log.i("onGetPcmFrame", "ret: " ret ", sampleRate: " sampleRate ", channel: " channel ", sampleSize: " sampleSize
",is_low_latency:" is_low_latency " buffer_size：" buffer_size);*/
if ( pcm_buffer_ == null)
return;
pcm_buffer_.rewind();
if ( ret == 0 && (isPushingRtmp || isRTSPPublisherRunning))
{
libPublisher.SmartPublisherOnFarEndPCMData(publisherHandle, pcm_buffer_, sampleRate, channel, sampleSize, is_low_latency);
if (is_audio_mix_)
{
libPublisher.SmartPublisherOnMixPCMData(publisherHandle, 1, pcm_buffer_, 0, buffer_size, sampleRate, channel, sampleSize);
}
/*
java.nio.ByteOrder old_order = pcm_buffer_.order();
pcm_buffer_.order(java.nio.ByteOrder.nativeOrder());
java.nio.ShortBuffer short_buffer = pcm_buffer_.asShortBuffer();
pcm_buffer_.order(old_order);
short[] short_array = new short[short_buffer.remaining()];
short_buffer.get(short_array);
libPublisher.SmartPublisherOnMixPCMShortArray(publisherHandle, 1, short_array, 0, short_array.length, sampleRate, channel, sampleSize);
*/
}
// test
/*
byte[] test_buffer = new byte[16];
pcm_buffer_.get(test_buffer);
Log.i(TAG, "onGetPcmFrame data:" bytesToHexString(test_buffer));
*/
}
}推送端：
RTMP推送：
代码语言：javascript复制 class ButtonPushStartListener implements OnClickListener
{
public void onClick(View v)
{
if (isPushingRtmp)
{
stopPush();
if (!isRTSPPublisherRunning) {
ConfigControlEnable(true);
}
btnPushStartStop.setText("推送RTMP");
isPushingRtmp = false;
return;
}
Log.i(PUSH_TAG, "onClick start push rtmp..");
if (libPublisher == null)
return;
if (!isRTSPPublisherRunning) {
InitPusherAndSetConfig();
}
if ( inputPushURL != null && inputPushURL.length() > 1 )
{
publishURL = inputPushURL;
Log.i(PUSH_TAG, "start, input publish url:" publishURL);
}
else
{
publishURL = basePushURL String.valueOf((int)( System.currentTimeMillis() % 1000000));
Log.i(PUSH_TAG, "start, generate random url:" publishURL);
}
printPushText = "URL:" publishURL;
Log.i(PUSH_TAG, printPushText);
textPushCurURL = (TextView)findViewById(R.id.txt_push_cur_url);
textPushCurURL.setText(printPushText);
Log.i(PUSH_TAG, "videoWidth: " pushVideoWidth " videoHeight: " pushVideoHeight " pushType:" pushType);
if ( libPublisher.SmartPublisherSetURL(publisherHandle, publishURL) != 0 )
{
Log.e(PUSH_TAG, "Failed to set rtmp pusher URL..");
}
int startRet = libPublisher.SmartPublisherStartPublisher(publisherHandle);
if (startRet != 0) {
isPushingRtmp = false;
Log.e(TAG, "Failed to start push stream..");
return;
}
if ( !isRTSPPublisherRunning ) {
if (pushType == 0 || pushType == 1) {
CheckInitAudioRecorder(); //enable pure video publisher..
}
ConfigControlEnable(false);
}
btnPushStartStop.setText("停止推送 ");
isPushingRtmp = true;
}
};轻量级RTSP服务模式：
代码语言：javascript复制 //启动/停止RTSP服务
class ButtonRtspServiceListener implements OnClickListener {
public void onClick(View v) {
if (isRTSPServiceRunning) {
stopRtspService();
btnRtspService.setText("启动RTSP服务");
btnRtspPublisher.setEnabled(false);
isRTSPServiceRunning = false;
return;
}
Log.i(TAG, "onClick start rtsp service..");
rtsp_handle_ = libPublisher.OpenRtspServer(0);
if (rtsp_handle_ == 0) {
Log.e(TAG, "创建rtsp server实例失败! 请检查SDK有效性");
} else {
int port = 8554;
if (libPublisher.SetRtspServerPort(rtsp_handle_, port) != 0) {
libPublisher.CloseRtspServer(rtsp_handle_);
rtsp_handle_ = 0;
Log.e(TAG, "创建rtsp server端口失败! 请检查端口是否重复或者端口不在范围内!");
}
//String user_name = "admin";
//String password = "12345";
//libPublisher.SetRtspServerUserNamePassword(rtsp_handle_, user_name, password);
if (libPublisher.StartRtspServer(rtsp_handle_, 0) == 0) {
Log.i(TAG, "启动rtsp server 成功!");
} else {
libPublisher.CloseRtspServer(rtsp_handle_);
rtsp_handle_ = 0;
Log.e(TAG, "启动rtsp server失败! 请检查设置的端口是否被占用!");
}
btnRtspService.setText("停止RTSP服务");
btnRtspPublisher.setEnabled(true);
isRTSPServiceRunning = true;
}
}
}
//发布/停止RTSP流
class ButtonRtspPublisherListener implements OnClickListener {
public void onClick(View v) {
if (isRTSPPublisherRunning) {
stopRtspPublisher();
if (!isPushingRtmp) {
ConfigControlEnable(true);
}
btnRtspPublisher.setText("发布RTSP流");
btnGetRtspSessionNumbers.setEnabled(false);
btnRtspService.setEnabled(true);
isRTSPPublisherRunning = false;
return;
}
Log.i(TAG, "onClick start rtsp publisher..");
if (!isPushingRtmp) {
InitPusherAndSetConfig();
}
if (publisherHandle == 0) {
Log.e(TAG, "Start rtsp publisher, publisherHandle is null..");
return;
}
String rtsp_stream_name = "stream1";
libPublisher.SetRtspStreamName(publisherHandle, rtsp_stream_name);
libPublisher.ClearRtspStreamServer(publisherHandle);
libPublisher.AddRtspStreamServer(publisherHandle, rtsp_handle_, 0);
if (libPublisher.StartRtspStream(publisherHandle, 0) != 0) {
Log.e(TAG, "调用发布rtsp流接口失败!");
return;
}
if (!isPushingRtmp) {
if (pushType == 0 || pushType == 1) {
CheckInitAudioRecorder(); //enable pure video publisher..
}
ConfigControlEnable(false);
}
btnRtspPublisher.setText("停止RTSP流");
btnGetRtspSessionNumbers.setEnabled(true);
btnRtspService.setEnabled(false);
isRTSPPublisherRunning = true;
}
}
;RTMP推送和轻量级RTSP服务，可以在一个实例里面处理，所以推送参数的初始化，只需要调用一次即可。
代码语言：javascript复制 private void InitPusherAndSetConfig() {
Log.i(TAG, "videoWidth: " pushVideoWidth " videoHeight: " pushVideoHeight
" pushType:" pushType);
int audio_opt = 1;
int video_opt = 1;
if ( pushType == 1 )
{
video_opt = 0;
}
else if (pushType == 2 )
{
audio_opt = 0;
}
publisherHandle = libPublisher.SmartPublisherOpen(curContext, audio_opt, video_opt,
pushVideoWidth, pushVideoHeight);
if ( publisherHandle == 0 )
{
return;
}
if(videoEncodeType == 1)
{
int h264HWKbps = setHardwareEncoderKbps(true, pushVideoWidth,
pushVideoHeight);
Log.i(TAG, "h264HWKbps: " h264HWKbps);
int isSupportH264HWEncoder = libPublisher
.SetSmartPublisherVideoHWEncoder(publisherHandle, h264HWKbps);
if (isSupportH264HWEncoder == 0) {
Log.i(TAG, "Great, it supports h.264 hardware encoder!");
}
}
else if (videoEncodeType == 2)
{
int hevcHWKbps = setHardwareEncoderKbps(false, pushVideoWidth,
pushVideoHeight);
Log.i(TAG, "hevcHWKbps: " hevcHWKbps);
int isSupportHevcHWEncoder = libPublisher
.SetSmartPublisherVideoHevcHWEncoder(publisherHandle, hevcHWKbps);
if (isSupportHevcHWEncoder == 0) {
Log.i(TAG, "Great, it supports hevc hardware encoder!");
}
}
if(is_sw_vbr_mode)
{
int is_enable_vbr = 1;
int video_quality = CalVideoQuality(pushVideoWidth,
pushVideoHeight, true);
int vbr_max_bitrate = CalVbrMaxKBitRate(pushVideoWidth,
pushVideoHeight);
libPublisher.SmartPublisherSetSwVBRMode(publisherHandle, is_enable_vbr, video_quality, vbr_max_bitrate);
}
libPublisher.SetSmartPublisherEventCallbackV2(publisherHandle, new EventHandePublisherV2());
//如果想和时间显示在同一行，请去掉'n'
String watermarkText = "大牛直播(daniulive)nn";
String path = pushLogoPath;
if( pushWatemarkType == 0 )
{
if ( isPushWritelogoFileSuccess )
libPublisher.SmartPublisherSetPictureWatermark(publisherHandle, path, WATERMARK.WATERMARK_POSITION_TOPRIGHT, 160, 160, 10, 10);
}
else if( pushWatemarkType == 1 )
{
if ( isPushWritelogoFileSuccess )
libPublisher.SmartPublisherSetPictureWatermark(publisherHandle, path, WATERMARK.WATERMARK_POSITION_TOPRIGHT, 160, 160, 10, 10);
libPublisher.SmartPublisherSetTextWatermark(publisherHandle, watermarkText, 1, WATERMARK.WATERMARK_FONTSIZE_BIG, WATERMARK.WATERMARK_POSITION_BOTTOMRIGHT, 10, 10);
//libPublisher.SmartPublisherSetTextWatermarkFontFileName("/system/fonts/DroidSansFallback.ttf");
//libPublisher.SmartPublisherSetTextWatermarkFontFileName("/sdcard/DroidSansFallback.ttf");
}
else if(pushWatemarkType == 2)
{
libPublisher.SmartPublisherSetTextWatermark(publisherHandle, watermarkText, 1, WATERMARK.WATERMARK_FONTSIZE_BIG, WATERMARK.WATERMARK_POSITION_BOTTOMRIGHT, 10, 10);
//libPublisher.SmartPublisherSetTextWatermarkFontFileName("/system/fonts/DroidSansFallback.ttf");
}
else
{
Log.i(TAG, "no watermark settings..");
}
//end
if ( !is_push_speex )
{
// set AAC encoder
libPublisher.SmartPublisherSetAudioCodecType(publisherHandle, 1);
}
else
{
// set Speex encoder
libPublisher.SmartPublisherSetAudioCodecType(publisherHandle, 2);
libPublisher.SmartPublisherSetSpeexEncoderQuality(publisherHandle, 8);
}
libPublisher.SmartPublisherSetNoiseSuppression(publisherHandle, is_push_noise_suppression?1:0);
libPublisher.SmartPublisherSetAGC(publisherHandle, is_push_agc?1:0);
libPublisher.SmartPublisherSetEchoCancellation(publisherHandle, 1, echoCancelDelay);
libPublisher.SmartPublisherSetAudioMix(publisherHandle, is_audio_mix_?1:0);
libPublisher.SmartPublisherSetInputAudioVolume(publisherHandle, 0 , mic_audio_volume_);
if ( is_audio_mix_ )
{
libPublisher.SmartPublisherSetInputAudioVolume(publisherHandle, 1 , mix_audio_volume_);
}
libPublisher.SmartPublisherSetClippingMode(publisherHandle, 0);
libPublisher.SmartPublisherSetSWVideoEncoderProfile(publisherHandle, push_sw_video_encoder_profile);
//libPublisher.SetRtmpPublishingType(0);
//libPublisher.SmartPublisherSetGopInterval(publisherHandle, 18*3);
//libPublisher.SmartPublisherSetFPS(publisherHandle, 18);
libPublisher.SmartPublisherSetSWVideoEncoderSpeed(publisherHandle, sw_video_encoder_speed);
//libPublisher.SmartPublisherSetSWVideoBitRate(600, 1200);
}相关封装：
代码语言：javascript复制 //停止rtmp推送
private void stopPush() {
if(!isPushingRtmp)
{
return;
}
if ( !isRTSPPublisherRunning) {
if (audioRecord_ != null) {
Log.i(TAG, "stopPush, call audioRecord_.StopRecording..");
audioRecord_.Stop();
if (audioRecordCallback_ != null) {
audioRecord_.RemoveCallback(audioRecordCallback_);
audioRecordCallback_ = null;
}
audioRecord_ = null;
}
}
if (libPublisher != null) {
libPublisher.SmartPublisherStopPublisher(publisherHandle);
}
if (!isRTSPPublisherRunning) {
if (publisherHandle != 0) {
if (libPublisher != null) {
libPublisher.SmartPublisherClose(publisherHandle);
publisherHandle = 0;
}
}
}
}
//停止发布RTSP流
private void stopRtspPublisher() {
if(!isRTSPPublisherRunning)
{
return;
}
if (!isPushingRtmp) {
if (audioRecord_ != null) {
Log.i(TAG, "stopRtspPublisher, call audioRecord_.StopRecording..");
audioRecord_.Stop();
if (audioRecordCallback_ != null) {
audioRecord_.RemoveCallback(audioRecordCallback_);
audioRecordCallback_ = null;
}
audioRecord_ = null;
}
}
if (libPublisher != null) {
libPublisher.StopRtspStream(publisherHandle);
}
if (!isPushingRtmp) {
if (publisherHandle != 0) {
if (libPublisher != null) {
libPublisher.SmartPublisherClose(publisherHandle);
publisherHandle = 0;
}
}
}
}
//停止RTSP服务
private void stopRtspService() {
if(!isRTSPServiceRunning)
{
return;
}
if (libPublisher != null && rtsp_handle_ != 0) {
libPublisher.StopRtspServer(rtsp_handle_);
libPublisher.CloseRtspServer(rtsp_handle_);
rtsp_handle_ = 0;
}
}传递采集到的视频数据，摄像头数据采集，也可选用camera2的接口，对焦和体验更好：
代码语言：javascript复制 @Override
public void onPreviewFrame(byte[] data, Camera camera) {
pushFrameCount ;
if ( pushFrameCount % 3000 == 0 )
{
Log.i("OnPre", "gc ");
System.gc();
Log.i("OnPre", "gc-");
}
if (data == null) {
Parameters params = camera.getParameters();
Size size = params.getPreviewSize();
int bufferSize = (((size.width|0x1f) 1) * size.height * ImageFormat.getBitsPerPixel(params.getPreviewFormat())) / 8;
camera.addCallbackBuffer(new byte[bufferSize]);
}
else
{
if(isPushingRtmp || isRTSPPublisherRunning)
{
libPublisher.SmartPublisherOnCaptureVideoData(publisherHandle, data, data.length, pushCurrentCameraType, currentPushOrigentation);
}
camera.addCallbackBuffer(data);
}
} 如果内网环境下，用轻量级RTSP服务的话，需判断对方有没有播放自己的流数据的话，可以通过获取RTSP会话数来判断是否链接。
代码语言：javascript复制 //当前RTSP会话数弹出框
private void PopRtspSessionNumberDialog(int session_numbers) {
final EditText inputUrlTxt = new EditText(this);
inputUrlTxt.setFocusable(true);
inputUrlTxt.setEnabled(false);
String session_numbers_tag = "RTSP服务当前客户会话数: " session_numbers;
inputUrlTxt.setText(session_numbers_tag);
AlertDialog.Builder builderUrl = new AlertDialog.Builder(this);
builderUrl
.setTitle("内置RTSP服务")
.setView(inputUrlTxt).setNegativeButton("确定", null);
builderUrl.show();
}
//获取RTSP会话数
class ButtonGetRtspSessionNumbersListener implements OnClickListener {
public void onClick(View v) {
if (libPublisher != null && rtsp_handle_ != 0) {
int session_numbers = libPublisher.GetRtspServerClientSessionNumbers(rtsp_handle_);
Log.i(TAG, "GetRtspSessionNumbers: " session_numbers);
PopRtspSessionNumberDialog(session_numbers);
}
}
};总结
Android平台的一对一互动，除了WebRTC外，在保证低延迟的前提下，RTMP或RTSP技术方案也是非常不错的选择。
