概述

音视频直播两种方式
- 实时互动
  - 会议
  - webRtc
- 流媒体分发
  - 娱乐直播
  - RTMP,HTTP-FLV,HLS

两种指标

延迟指标
音视频服务质量指标
- 分辨率
- 帧率
- 码率
- MOS
  - 服务质量评估

压缩算法

H264
H265
AVI

直播架构

音视频采集模块
音视频编码模块
网络传输模块
音视频解码模块
音视频渲染模块

WebRtc客户端架构

接口层
- Web接口
  - 浏览器
- Native接口
  - C++,Android,OC
Session层
- 媒体协商
- 收集Candidate。。。
核心引擎层
- 音频引擎
- 视频引擎
- 网络传输
设备层
- 硬件采集和播放

构建一对一信令服务器

组成
- 2个WebRtc终端
- 1个信令服务器
- 1个中继服务器(STUN/TURN)
  - 获取各自外网ip和端口
- 2个NAT

信令设计

客户端
- join
  - 用户加入房间
- leave
  - 用户离开房间
- message
  - 发送端到端消息
服务器
- joined
  - 用户已加入
- leaved
  - 用户已离开
- other_joined
  - 其他用户加入
- bye
  - 其他用户离开
- full
  - 房间已满

代码实现

socket
- socket.emit(‘xxx’)
  - 发送消息
- socket.emit(‘xxx’,xxx1,xx2)
  - 发送带参消息
- socket.to(room).emit(‘xxx’)
  - 给房间内所有人发消息
- socket.on(‘xxx’,function(){})
  - 接收消息
- socket.on(‘xxx’,function(arg){})
  - 接收带参数消息
HTTP服务器

const http = require('http')
const express = require('express')const app = express()
const http_server = http.createServer(app)http_server.listen(8081,'0.0.0.0')

完整代码

'use strict'const log4js = require('log4js');
const fs = require('fs');
const http = require('http');
const https = require('https');
const {Server} = require('socket.io')
const path = require('path')
const express = require('express');
const app = express();
app.use(express.static(path.join(__dirname, 'public')))
const USERCOUNT = 3;log4js.configure({appenders: {file: {type: 'file',filename: 'app.log',layout: {type: 'pattern',pattern: '%r %p - %m',}}},categories: {default: {appenders: ['file'],level: 'debug'}}
});
let logger = log4js.getLogger();let options = {key:fs.readFileSync('privkey.pem'),cert:fs.readFileSync('cacert.pem')
}
let https_server = https.createServer(options, app);
let http_server = http.createServer(app);
https_server.listen(8080, '0.0.0.0');
http_server.listen(8081, '0.0.0.0');
let httpsIo = new Server(https_server);
let httpIo = new Server(http_server);httpsIo.sockets.on('connection',(socket) =>{// console.log("connection")socket.on('message',(room,data) =>{console.log("message:"+data)// 中转消息socket.to(room).emit('message',room,data);});socket.on('join',(room) =>{socket.join(room);let myRoom=httpsIo.sockets.adapter.rooms[room];let users=(myRoom)?Object.keys(myRoom.sockets).length:0;logger.info('the user number of room is :'+users);if(users < USERCOUNT){// 给自己发joinsocket.emit('joined',room,socket.id);if(users > 1){// 给其他人发other_joinedsocket.to(room).emit('other_joined',room,socket.id);}}else{socket.leave(room);// 给自己发fullsocket.emit('full',room,socket.id);}});socket.on('level', (room)=> {socket.leave(room);// let myRoom=io.sockets.adapter.rooms[room];// let users=(myRoom)?Object.keys(myRoom.sockets).length:0;// 给其他人发byesocket.to(room).emit('bye',room,socket.id);// 给自己发levelsocket.emit('leaved',room,socket.id);});});httpIo.sockets.on('connection',(socket) =>{socket.on('message',(room,message) =>{console.log("message:"+message)// 转发给房间内所有人socket.to(room).emit('message',message);});socket.on('join',(room) =>{socket.join(room);let myRoom=httpIo.sockets.adapter.rooms[room];let users=(myRoom)?Object.keys(myRoom.sockets).length:0;logger.info('the user number of room is :'+users);if(users < USERCOUNT){// 给自己发joinsocket.emit('joined',room,socket.id);if(users > 1){// 给其他人发other_joinedsocket.to(room).emit('other_joined',room,socket.id);}}else{// 给自己发fullsocket.emit('full',room,socket.id);socket.leave(room);}});socket.on('level', function(room) {socket.leave(room);// let myRoom=io.sockets.adapter.rooms[room];// let users=(myRoom)?Object.keys(myRoom.sockets).length:0;// 给其他人发byesocket.to(room).emit('bye',room,socket.id);// 给自己发levelsocket.emit('left',room,socket.id);});});

浏览器webrtc

遍历音视频设备

遍历音视频设备
- navigater.mediaDevices.enumerateDevices()
- MediaDeviceInfo
  - deviceId
  - kind
  - label
  - groupId

function handleError(error){console.log('err:',error)
}function gotDevices(deviceInfos){for(let i=0;i<deviceInfo.length;i++){const deviceInfo = deviceInfos[i]}
}// 遍历
navigator.mediaDevices.enumerateDevices().then(gotDevices).catch(handleError)

采集音视频数据

采集音视频数据
- navigator.mediaDevices.getUserMedia(MediaStreamConstrains)
- MediaStreamConstrains
  - video:true/false/MediaStreamConstrainsSet
  - audio:true/false/MediaStreamConstrainsSet
MdeiaStream
- 流
MediaStreamTrack
- 轨

function gotMediaStream(stream){}let deviceId = 'xxx'let constraints = {videa:{width:640,height:480,frameRate:15,                               // 帧率15帧/秒facingMode:'environment',                   // 后置摄像头deviceId:deviceId?{exact:deviceId}:undefined},radio:false
}// 开始采集数据
navigator.mediaDevices.getUserMedia(constraints).then(gotMediaStream).catch((handleError))

本地视频预览

<video autoplay playsinline></video>

const lv = document.querySelector('video')
const contrains = {vide:true,audio:true
}function gotLocalStream(mediaStream){lv.srcObject = mediaStream
}function handleLocalStreamError(error){console.log('err:',error)
}navigator.mediaDevices.getUserMedia(contrains).then(gotLoaclStram).catch(handleLocalStreamError)

信令状态机

init
joined
joined_conn
joined_unbind

RTCPeerConnection

const configuration = {iceServers:[{urls:'stun:stun.example.org'}]
}
let pc = new RTCPeerConnection(configuration)

绑定Track

    // ls getUserMedia()获取到的MediaStreamfunction bindTracks(){ls.getTracks().forEach((track)=>{pc.addTrack(track,ls)})}

媒体协商

SDP
交换本地软硬件编码相关信息
createOffer
setLocalDescription
sendOffer
setRemoteDescription

ICE

服务器相关信息
IceCandidate
- candidate
  - address
  - port
  - protocol
- sdpMid
- sdpMLineIndex
连接步骤
- 收集candidate
- 交换candidate
- 尝试连接

// 获取本地candidate
pc.onicecandidate = (e)=>{if(e.candidate){}
}

SDP与ICE中转

客户端发送

function sendMessage(roomid,data){socket.emit('message',roomid,data)
}

服务端接收转发

socket.on('message',(room,data) =>{// 中转消息socket.to(room).emit('message',room,data);
});

客户端接收

    // 客户端接收socket.on('message',(room,data) =>{console.log("message:"+data)if(data.hasOwnProperty('type')&&data.type==='offer'){}else if(data.hasOwnProperty('type')&&data.type==='answer'){}else if(data.hasOwnProperty('type')&&data.type==='candidate'){}else{}});

远端获取音视频流

function getRemoteStream(e){}
let pc = new RTCPeerConnection(...)
pc.ontrack = getRemoteStream()

完成代码

<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><title>WebRtc</title><link href="css/client2.css" rel="stylesheet"></head>
<body>
<div><div><button id="connserver">ConnServer</button><button id="leave" disabled>Leave</button></div><div id="preview"><div><h2>Local</h2><video id="localVideo" autoplay playsinline muted></video><h2>Offer SDP</h2><textarea id="offer"></textarea></div><div><h2>Remote</h2><video id="remoteVideo" autoplay playsinline></video><h2>Answer SDP</h2><textarea id="answer"></textarea></div></div>
</div>
<!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.0.3/socket.io.js"/> -->
<!-- <script src="https://webrtc.github.io/adapter/adapter-lastest.js"/> --><script src="./js/socket_io.js"></script>
<script src="./js/adapter.js"></script>
<script src="./js/client2.js"></script></body>
</html>

button{margin: 10px 20px 25px 0;vertical-align: top;width: 134px;
}/*table{*/
/*    margin: 200px (50% - 100px) 0 0;*/
/*}*/textarea{color:#444;font-size: 0.9em;font-weight: 300;height: 20.0em;padding: 5px;width: calc(100% - 10px);
}div#getUserMedia{padding: 0 0 8px 0;
}div#preview{border-bottom: 1px solid #eee;margin: 0 0 1em 0;padding: 0 0 0.5em 0;
}div#preview>div{display: inline-block;vertical-align: top;width: calc(50% - 12px);
}video{background: #222;margin: 0 0 0 0;--width:100%;width: var(--width);height: 225px;
}@media screen and (max-width:720px){button{font-weight: 500;height: 56px;line-height: 1.3em;width: 90px;}div#getUserMedia{padding: 0 0 40px 0;}}

`use strict`var localVideo = document.querySelector('video#localVideo')
var remoteVideo = document.querySelector('video#remoteVideo')var btnConn = document.querySelector('button#connserver')
var btnLeave = document.querySelector('button#leave')var offer = document.querySelector('textarea#offer')
var answer = document.querySelector('text#answer')let pcConfig = {iceServers:[{urls:'stun:stun.l.google.com:19302'}]
}let localStream = null
let remoteStream = nulllet pc = nulllet roomid = null
let socket = nulllet offerdesc = null;
let state = 'init'function isPc(){let userAgentInfo = navigator.userAgentlet agents = ['Android','iPhone','iPad','iP']let flag = truefor(let i=0;i<agents.length;i++){if(userAgentInfo.indexOf(agents[i])>0){flag = falsebreak}}return flag
}function isAndroid(){let u = navigator.userAgentlet app = navigator.appVersionlet isAndroid = u.indexOf('Android') > -1 || u.indexOf('Linux') > -1let isIOS = !!u.match(/\(i[^;]+;( U;)? CPU.+Mac OS X/)if(isAndroid){return true;}if(isIOS){return false}
}function getQueryVariable(variable){let query = window.location.search.substring(1)let vars = query.split("&")for(let i=0;i<vars.length;i++){let pair = vars[i].split("=")if(pair[0]===variable){return pair[1]}}return false
}function sendMessage(roomid,data){console.log('send message to other end',roomid,data)if(!socket){console.log("socket is null")}socket.emit('message',roomid,data)
}function conn(){// 连接信令服务器socket = io.connect("https://192.168.1.9:8080")socket.on('joined',(roomid,id)=>{console.log('receive joined message',roomid,id)// 状态机变更state = 'joined'// 创建PeerConnection并绑定音视频轨createPeerConnection()bindTracks()// 设置button状态btnConn.disabled = truebtnLeave.disabled = falseconsole.log('reveive joined message,state=',state)})socket.on('other_joined',(roomid)=>{console.log('receive joined message:',roomid,state)if(state==='joined_unbind'){createPeerConnection()bindTracks()}// 更新状态机state = 'joined_conn'// 开始呼叫对方call()console.log('receive other_join message,state=',state)})socket.on('full',(roomid,id)=>{console.log('receive fuu message',roomid,id)socket.disconnect()// 挂断hangup()// 关闭本地媒体closeLocalMedia()// 状态机变更state = 'leaved'console.log('receive full message,state=',state)alert('this room is full')})socket.on('leaved',(roomid,id)=>{console.log('receive leaved message',roomid,id)// 状态机变更state = 'leaved'socket.disconnect()console.log('receive leaved message,state=',state)// 改变button状态btnConn.disabled = falsebtnLeave.disabled = true})socket.on('bye',(roomid,id)=>{console.log('receive bye message',roomid,id)// 状态机变更state = 'joined_unbind'// 挂断hangup()offer.value = ''answer.value = ''console.log('receive bye message,state=',state)})socket.on('disconnect',(socket)=>{console.log('receive disconnect message',roomid)if(!(state==='leaved')){hangup()closeLocalMedia()}state = 'leaved'})socket.on('message',(roomid,data) =>{console.log("receive message!",roomid,data)if(data===null||data===undefined){console.log('the message is invalid')return}if(data.hasOwnProperty('type')&&data.type==='offer'){// 收到SDP是offeroffer.value = data.sdp// 进行媒体协商pc.setRemoteDescription(new RTCSessionDescription(data))// 创建answerpc.createAnswer().then(getAnswer).catch(handleAnswerError)}else if(data.hasOwnProperty('type')&&data.type==='answer'){// 收到SDP是answeranswer.value = data.sdp// 进行媒体协商pc.setRemoteDescription(new RTCSessionDescription(data))}else if(data.hasOwnProperty('type')&&data.type==='candidate'){// 收到candidatelet candidate = new RTCIceCandidate({sdpMLineIndex:data.label,candidate:data.candidate})// 将远端Candidate消息添加到PeerConnectionpc.addIceCandidate(candidate)}else{console.log('the message is invalid',data)}});roomid = getQueryVariable('room')socket.emit('join',roomid)return true
}// 打开音视频成功的回调函数
function getMediaStream(stream){if(localStream){stream.getAudioTracks().forEach((track)=>{localStream.addTrack(track)stream.removeTrack(track)})}else{localStream = stream}localVideo.srcObject = localStreamconn()
}
// 错误处理函数
function handleError(err){console.log('Failed to get Media Stream',err)
}function start(){if(!navigator.mediaDevices||!navigator.mediaDevices.getUserMedia){console.log('the getUserMedia is not support')return}else{let constraints = {video:true,audio:{echoCancellation:true,noiseSuppression:true,autoGainControl:true}}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).catch(handleError)}}
// 获取远端媒体流
function getRemoteStream(e){remoteStream = e.streams[0]remoteVideo.srcObject = e.streams[0]
}// 处理Offer错误
function handleOfferError(err){console.log('Failed to create offer',err)
}
// 处理answer错误
function handleAnswerError(err){console.log('Failed to create answer',err)
}// 获取answer sdp
function getAnswer(desc){answer.value = desc.sdpsendMessage(roomid,desc)
}// 获取offer sdp
function getOffer(dess){pc.setLocalDescription(desc)offer.value = desc.sdpofferdesc = descsendMessage(roomid,offerdesc)
}function createPeerConnection(){console.log('create PeerConnection')if(!pc){pc = new RTCPeerConnection(pcConfig)// 当收集到Candidatepc.onicecandidate = (e)=>{if(e.candidate){console.log("candidate "+ JSON.stringify(e.candidate.toJSON()))sendMessage(roomid,{type:'candidate',label:event.candidate.sdpMLineIndex,id:event.candidate.sdpMid,candidate:event.candidate.candidate})}else{console.log('this is the end candidate')}}pc.ontrack = getRemoteStream}else{console.log('the pc have be created')}return
}// 将音视频track绑定到PeerConnection对象中
function bindTracks(){console.log('bind tracks into RTCPeerConnection')if(pc===null&&localStream==undefined){console.log('ps ic null or undefined')return}// 将本地音视频流添加到RTCPeerConnectionlocalStream.getTracks().forEach((track)=>{pc.addTrack(track,localStream)})
}// 呼叫
function call(){if(state==='joined_conn'){let offerOptions= {offerToReceiveVideo:1,offerToReceiveAudio:1}pc.createOffer(offerOptions).then(getOffer).catch(handleOfferError)}
}
// 挂断
function hangup(){if(!pc){return}offerdesc = nullpc.close()pc = null
}// 关闭本地媒体
function closeLocalMedia(){if(!(localStream===null||localStream===undefined)){localStream.getTracks().forEach((track)=>{track.stop()})}localStream = null
}// 打开音视频设备，连接信令服务器
function starConn(){// 开启本地视频start()return true
}// 离开
function leave(){socket.emit('leave',roomid)hangup()closeLocalMedia()offer.value=''answer.value= ''btnConn.disabled = falsebtnLeave.disabled = true
}btnConn.onclick = starConn
btnLeave.onclick = leave

学习笔记-webrtc相关推荐

FFmpeg基础到工程-多路H265监控录放开发学习笔记
多路H265监控录放开发学习笔记课程涉及:FFmpeg,WebRTC,SRS,Nginx,Darwin,Live555,等.包括:音视频.流媒体.直播.Android.视频监控28181.等. 具体 ...
PyTorch 学习笔记（六）：PyTorch hook 和关于 PyTorch backward 过程的理解 call
您的位置首页 PyTorch 学习笔记系列 PyTorch 学习笔记(六):PyTorch hook 和关于 PyTorch backward 过程的理解发布: 2017年8月4日 7,195阅读 ...
容器云原生DevOps学习笔记——第三期：从零搭建CI/CD系统标准化交付流程
暑期实习期间,所在的技术中台-效能研发团队规划设计并结合公司开源协同实现符合DevOps理念的研发工具平台,实现研发过程自动化.标准化: 实习期间对DevOps的理解一直懵懵懂懂,最近观看了阿里专家带 ...
容器云原生DevOps学习笔记——第二期：如何快速高质量的应用容器化迁移
暑期实习期间,所在的技术中台-效能研发团队规划设计并结合公司开源协同实现符合DevOps理念的研发工具平台,实现研发过程自动化.标准化: 实习期间对DevOps的理解一直懵懵懂懂,最近观看了阿里专家带 ...
2020年Yann Lecun深度学习笔记（下）
2020年Yann Lecun深度学习笔记(下)
2020年Yann Lecun深度学习笔记（上）
2020年Yann Lecun深度学习笔记(上)
知识图谱学习笔记（1）
知识图谱学习笔记第一部分,包含RDF介绍,以及Jena RDF API使用知识图谱的基石:RDF RDF(Resource Description Framework),即资源描述框架,其本质是一个 ...
计算机基础知识第十讲,计算机文化基础（第十讲）学习笔记
计算机文化基础(第十讲)学习笔记采样和量化PictureElement Pixel(像素)(链接: 采样的实质就是要用多少点(这个点我们叫像素)来描述一张图像,比如,一幅420x570的图像,就表示 ...
Go 学习推荐 —（Go by example 中文版、Go 构建 Web 应用、Go 学习笔记、Golang常见错误、Go 语言四十二章经、Go 语言高级编程）
Go by example 中文版 Go 构建 Web 应用 Go 学习笔记:无痕 Go 标准库中文文档 Golang开发新手常犯的50个错误 50 Shades of Go: Traps, Gotc ...

学习笔记-webrtc

概述

两种指标

压缩算法

直播架构

WebRtc客户端架构

构建一对一信令服务器

信令设计

代码实现

浏览器webrtc

遍历音视频设备

采集音视频数据

本地视频预览

信令状态机

RTCPeerConnection

绑定Track

媒体协商

ICE

SDP与ICE中转

远端获取音视频流

完成代码

学习笔记-webrtc相关推荐

最新文章

热门文章