Windows平台实现Unity下窗体|摄像头|屏幕采集推送

技术背景

随着Unity3D的应用范围越来越广,越来越多的行业开始基于Unity3D开发产品,如传统行业中虚拟仿真教育、航空工业、室内设计、城市规划、工业仿真等领域。

基于此,好多开发者苦于在Unity环境下,没有低延迟的推拉流解决方案,前几年,我们在Unity环境下推出了跨平台低延迟的RTMP|RTSP直播播放器,很好的解决了好多对延迟要求苛刻的使用场景。

随着时间的推移,越来越多的开发者联系我们,希望我们能推出Unity环境下的RTMP推送模块,获取到unity的实时数据,更低延迟更高效率的实现数据传输推送,基于此,我们发布了Unity环境下的RTMP推送模块。

本文以Windows平台为例,数据源分别为Unity的窗口、摄像头或整个屏幕,编码传输模块,还是调用大牛直播SDK(官方)的原生接口,简单界面先睹为快:

技术实现

1. 基础初始化


        private bool InitSDK()
        {
            if (!is_pusher_sdk_init_)
            {
                // 设置日志路径(请确保目录存在)
                String log_path = "D:\\pulisherlog";
                NTSmartLog.NT_SL_SetPath(log_path);

                UInt32 isInited = NTSmartPublisherSDK.NT_PB_Init(0, IntPtr.Zero);

                if (isInited != 0)
                {
                    Debug.Log("调用NT_PB_Init失败..");
                    return false;
                }

                is_pusher_sdk_init_ = true;
            }

            return true;
        }

2. 调用Open()接口,获取推送实例

       public bool OpenPublisherHandle(uint video_option, uint audio_option)
        {
            if (publisher_handle_ != IntPtr.Zero)
            {
                return true;
            }

            publisher_handle_count_ = 0;

            if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_Open(out publisher_handle_,
                video_option, audio_option, 0, IntPtr.Zero))
            {
                return false;
            }

            if (publisher_handle_ != IntPtr.Zero)
            {
                pb_event_call_back_ = new NT_PB_SDKEventCallBack(PbEventCallBack);

                NTSmartPublisherSDK.NT_PB_SetEventCallBack(publisher_handle_, IntPtr.Zero, pb_event_call_back_);

                return true;
            }
            else
            {
                return false;
            }
        }

3. 初始化参数配置

这里需要注意下,如果要采集unity窗口,需要设置图层模式,先填充一层RGBA黑色背景,然后再添加一层,用于叠加外部数据。

       private void SetCommonOptionToPublisherSDK()
        {
            if (!IsPublisherHandleAvailable())
            {
                Debug.Log("SetCommonOptionToPublisherSDK, publisher handle with null..");
                return;
            }

            NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0,
                            0, IntPtr.Zero);

            if (video_option == NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
            {
                // 第0层填充RGBA矩形, 目的是保证帧率, 颜色就填充全黑
                int red = 0;
                int green = 0;
                int blue = 0;
                int alpha = 255;

                NT_PB_RGBARectangleLayerConfig rgba_layer_c0 = new NT_PB_RGBARectangleLayerConfig();

                rgba_layer_c0.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
                rgba_layer_c0.base_.index_ = 0;
                rgba_layer_c0.base_.enable_ = 1;
                rgba_layer_c0.base_.region_.x_ = 0;
                rgba_layer_c0.base_.region_.y_ = 0;
                rgba_layer_c0.base_.region_.width_ = video_width_;
                rgba_layer_c0.base_.region_.height_ = video_height_;

                rgba_layer_c0.base_.offset_ = Marshal.OffsetOf(rgba_layer_c0.GetType(), "base_").ToInt32();
                rgba_layer_c0.base_.cb_size_ = (uint)Marshal.SizeOf(rgba_layer_c0);

                rgba_layer_c0.red_ = System.BitConverter.GetBytes(red)[0];
                rgba_layer_c0.green_ = System.BitConverter.GetBytes(green)[0];
                rgba_layer_c0.blue_ = System.BitConverter.GetBytes(blue)[0];
                rgba_layer_c0.alpha_ = System.BitConverter.GetBytes(alpha)[0];

                IntPtr rgba_conf = Marshal.AllocHGlobal(Marshal.SizeOf(rgba_layer_c0));

                Marshal.StructureToPtr(rgba_layer_c0, rgba_conf, true);

                UInt32 rgba_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
                                rgba_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE,
                                0, IntPtr.Zero);

                Marshal.FreeHGlobal(rgba_conf);

                NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 = new NT_PB_ExternalVideoFrameLayerConfig();

                external_layer_c1.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
                external_layer_c1.base_.index_ = 1;
                external_layer_c1.base_.enable_ = 1;
                external_layer_c1.base_.region_.x_ = 0;
                external_layer_c1.base_.region_.y_ = 0;
                external_layer_c1.base_.region_.width_ = video_width_;
                external_layer_c1.base_.region_.height_ = video_height_;

                external_layer_c1.base_.offset_ = Marshal.OffsetOf(external_layer_c1.GetType(), "base_").ToInt32();
                external_layer_c1.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c1);

                IntPtr external_layer_conf = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1));

                Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true);

                UInt32 external_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0,
                                external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME,
                                0, IntPtr.Zero);

                Marshal.FreeHGlobal(external_layer_conf);

            }
            else if (video_option == NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA)
            {
                CameraInfo camera = cameras_[cur_sel_camera_index_];
                NT_PB_VideoCaptureCapability cap = camera.capabilities_[cur_sel_camera_resolutions_index_];

                SetVideoCaptureDeviceBaseParameter(camera.id_.ToString(), (UInt32)cap.width_, (UInt32)cap.height_);
            }

            SetFrameRate((UInt32)CalBitRate(edit_key_frame_, video_width_, video_height_));

            Int32 type = 0;   //软编码
            Int32 encoder_id = 1;
            UInt32 codec_id = (UInt32)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;
            Int32 param1 = 0;

            SetVideoEncoder(type, encoder_id, codec_id, param1);

            SetVideoQualityV2(CalVideoQuality(video_width_, video_height_, is_h264_encoder));

            SetVideoMaxBitRate((CalMaxKBitRate(edit_key_frame_, video_width_, video_height_, false)));

            SetVideoKeyFrameInterval((edit_key_frame_));

            if (is_h264_encoder)
            {
                SetVideoEncoderProfile(1);
            }

            SetVideoEncoderSpeed(CalVideoEncoderSpeed(video_width_, video_height_, is_h264_encoder));

            // 音频相关设置

            SetAuidoInputDeviceId(0);
            SetPublisherAudioCodecType(1);
            SetPublisherMute(is_mute);
            SetEchoCancellation(0, 0);
            SetNoiseSuppression(0);
            SetAGC(0);
            SetVAD(0);
            SetInputAudioVolume(Convert.ToSingle(edit_audio_input_volume_));
        }

4. 数据采集

摄像头和屏幕的数据采集,还是调用原生的SDK接口,本文不再赘述,如果需要采集Unity窗体的数据,可以用参考以下代码:

        if ( texture_ == null || video_width_ != Screen.width || video_height_ != Screen.height)
        {
            Debug.Log("OnPostRender screen changed++ scr_width: " + Screen.width + " scr_height: " + Screen.height);

            if (screen_image_ != IntPtr.Zero)
            {
                Marshal.FreeHGlobal(screen_image_);
                screen_image_ = IntPtr.Zero;
            }

            if (texture_ !=  null)
            {
                UnityEngine.Object.Destroy(texture_);
                texture_ = null;
            }

            video_width_ = Screen.width;
            video_height_ = Screen.height;

            texture_ = new Texture2D(video_width_, video_height_, TextureFormat.BGRA32, false);

            screen_image_ = Marshal.AllocHGlobal(video_width_ * 4 * video_height_);

            Debug.Log("OnPostRender screen changed--");

            return;
        }

        texture_.ReadPixels(new Rect(0, 0, video_width_, video_height_), 0, 0, false);
        texture_.Apply();

从 texture里面,通过调用 GetRawTextureData(),获取到原始数据。

5. 数据对接

数据对接,通过调用以下接口:

       public void OnPostRGBAData(IntPtr data, int length, int stride, int width, int height)
        {
            NT_PB_Image pb_image = new NT_PB_Image();
            
            pb_image.format_ = (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32;
            pb_image.width_ = width;
            pb_image.height_ = height;
            pb_image.timestamp_ = 0;
            pb_image.cb_size_ = (UInt32)Marshal.SizeOf(pb_image);   

            pb_image.stride_ = new Int32[16];
            pb_image.stride_[0] = stride;

            pb_image.plane_size_ = new Int32[16];
            pb_image.plane_size_[0] = pb_image.stride_[0] * pb_image.height_;         

            pb_image.plane_ = new IntPtr[16];
            pb_image.plane_[0] = data;

            IntPtr image_data = Marshal.AllocHGlobal(Marshal.SizeOf(pb_image));

            Marshal.StructureToPtr(pb_image, image_data, true);

            NTSmartPublisherSDK.NT_PB_PostLayerImage(publisher_handle_, 0,
                            1, image_data, 0, IntPtr.Zero);

            Marshal.FreeHGlobal(image_data);
        }

6. 本地数据预览

        public bool StartPreview()
        {
            if(CheckPublisherHandleAvailable() == false)
                return false;

            video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);

            NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);

            if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_, 0, IntPtr.Zero))
            {
                if (0 == publisher_handle_count_)
                {
                    NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
                    publisher_handle_ = IntPtr.Zero;
                }

                return false;
            }

            publisher_handle_count_++;

            is_previewing_ = true;

            return true;
        }

设置preview后,处理preview的数据回调

        //预览数据回调
        public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
        {
            NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));

            NT_VideoFrame pVideoFrame = new NT_VideoFrame();

            pVideoFrame.width_ = pb_image.width_;
            pVideoFrame.height_ = pb_image.height_;

            pVideoFrame.stride_ = pb_image.stride_[0];

            Int32 argb_size = pb_image.stride_[0] * pb_image.height_;

            pVideoFrame.plane_data_ = new byte[argb_size];
            
            if (argb_size > 0)
            {
                Marshal.Copy(pb_image.plane_[0],pVideoFrame.plane_data_,0, argb_size);
            }

            {
                cur_image_ = pVideoFrame;
            }
        }      

7. 相关event回调处理

        private void PbEventCallBack(IntPtr handle, IntPtr user_data, 
            UInt32 event_id,
            Int64 param1,
            Int64 param2,
            UInt64 param3,
            UInt64 param4,
            [MarshalAs(UnmanagedType.LPStr)] String param5,
            [MarshalAs(UnmanagedType.LPStr)] String param6,
            IntPtr param7)
        {
            String event_log = "";

            switch (event_id)
            {
                case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTING:
                    event_log = "连接中";
                    if (!String.IsNullOrEmpty(param5))
                    {
                        event_log = event_log + " url:" + param5;
                    }
                    break;

                case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTION_FAILED:
                    event_log = "连接失败";
                    if (!String.IsNullOrEmpty(param5))
                    {
                        event_log = event_log + " url:" + param5;
                    }
                    break;

                case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTED:
                    event_log = "已连接";
                    if (!String.IsNullOrEmpty(param5))
                    {
                        event_log = event_log + " url:" + param5;
                    }
                    break;

                case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_DISCONNECTED:
                    event_log = "断开连接";
                    if (!String.IsNullOrEmpty(param5))
                    {
                        event_log = event_log + " url:" + param5;
                    }
                    break;

                default:
                    break;
            }

            if(OnLogEventMsg != null) OnLogEventMsg.Invoke(event_id, event_log);
        }

8. 开始推送、停止推送

       public bool StartPublisher(String url)
        {
            if (CheckPublisherHandleAvailable() == false) return false;

            if (publisher_handle_ == IntPtr.Zero)
            {
                return false;
            }
            if (!String.IsNullOrEmpty(url))
            {
                NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero);
            }

            if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero))
            {
                if (0 == publisher_handle_count_)
                {
                    NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
                    publisher_handle_ = IntPtr.Zero;
                }

                is_publishing_ = false;

                return false;
            }

            publisher_handle_count_++;

            is_publishing_ = true;

            return true;
        }

        public void StopPublisher()
        {
            if (is_publishing_ == false) return;

            publisher_handle_count_--;
            NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_);

            if (0 == publisher_handle_count_)
            {
                NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
                publisher_handle_ = IntPtr.Zero;
            }

            is_publishing_ = false;
        }

9. 关闭实例

        public void Close()
        {
            if (0 == publisher_handle_count_)
            {
                NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
                publisher_handle_ = IntPtr.Zero;
            }
        }

总结

经测试,Unity环境下,通过高效率的数据采集、编码和推送,配合SmartPlayer播放器播放,整体延迟可控制在毫秒级,可适用于大多数Unity环境下对延迟和稳定性要求苛刻的场景。