前言
直播、短视频、在线会议等应用越来越多地进入人们的生活，随之诞生的是丰富的各类创意玩法与新鲜体验，其中大量应用了以AI检测和图形渲染为基础的AR技术。
而随着Web技术的不断成熟，AR技术在Web上的实现成为了一种可能。今天就总结了在Web端实现此功能的几个技术要点，跟大家一起探讨一下。
架构和概念
抽象整体的实现思路如下
#bytemd-mermaid-1671094042756-0{font-family:"trebuchet ms",verdana,arial,sans-serif;font-size:16px;fill:#333;}#bytemd-mermaid-1671094042756-0 .error-icon{fill:#552222;}#bytemd-mermaid-1671094042756-0 .error-text{fill:#552222;stroke:#552222;}#bytemd-mermaid-1671094042756-0 .edge-thickness-normal{stroke-width:2px;}#bytemd-mermaid-1671094042756-0 .edge-thickness-thick{stroke-width:3.5px;}#bytemd-mermaid-1671094042756-0 .edge-pattern-solid{stroke-dasharray:0;}#bytemd-mermaid-1671094042756-0 .edge-pattern-dashed{stroke-dasharray:3;}#bytemd-mermaid-1671094042756-0 .edge-pattern-dotted{stroke-dasharray:2;}#bytemd-mermaid-1671094042756-0 .marker{fill:#333333;stroke:#333333;}#bytemd-mermaid-1671094042756-0 .marker.cross{stroke:#333333;}#bytemd-mermaid-1671094042756-0 svg{font-family:"trebuchet ms",verdana,arial,sans-serif;font-size:16px;}#bytemd-mermaid-1671094042756-0 .label{font-family:"trebuchet ms",verdana,arial,sans-serif;color:#333;}#bytemd-mermaid-1671094042756-0 .cluster-label text{fill:#333;}#bytemd-mermaid-1671094042756-0 .cluster-label span{color:#333;}#bytemd-mermaid-1671094042756-0 .label text,#bytemd-mermaid-1671094042756-0 span{fill:#333;color:#333;}#bytemd-mermaid-1671094042756-0 .node rect,#bytemd-mermaid-1671094042756-0 .node circle,#bytemd-mermaid-1671094042756-0 .node ellipse,#bytemd-mermaid-1671094042756-0 .node polygon,#bytemd-mermaid-1671094042756-0 .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#bytemd-mermaid-1671094042756-0 .node .label{text-align:center;}#bytemd-mermaid-1671094042756-0 .node.clickable{cursor:pointer;}#bytemd-mermaid-1671094042756-0 .arrowheadPath{fill:#333333;}#bytemd-mermaid-1671094042756-0 .edgePath .path{stroke:#333333;stroke-width:1.5px;}#bytemd-mermaid-1671094042756-0 .flowchart-link{stroke:#333333;fill:none;}#bytemd-mermaid-1671094042756-0 .edgeLabel{background-color:#e8e8e8;text-align:center;}#bytemd-mermaid-1671094042756-0 .edgeLabel rect{opacity:0.5;background-color:#e8e8e8;fill:#e8e8e8;}#bytemd-mermaid-1671094042756-0 .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#bytemd-mermaid-1671094042756-0 .cluster text{fill:#333;}#bytemd-mermaid-1671094042756-0 .cluster span{color:#333;}#bytemd-mermaid-1671094042756-0 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:"trebuchet ms",verdana,arial,sans-serif;font-size:12px;background:hsl(80,100%,96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#bytemd-mermaid-1671094042756-0:root{--mermaid-font-family:"trebuchet ms",verdana,arial,sans-serif;}#bytemd-mermaid-1671094042756-0 flowchart{fill:apa;}
调取Camera获得相机画面
使用tensorflow加载人脸识别模型生成FaceMesh
根据FaceMesh生成三角网格并进行UV贴图
FaceMesh
MediaPipe Face Mesh是一种脸部几何解决方案，即使在移动设备上，也可以实时估计468个3D脸部界标。它采用 机器学习 （ML）来推断3D表面几何形状，只需要单个摄像机输入，而无需专用的深度传感器。该解决方案利用轻量级的模型架构以及整个管线中的GPU加速，可提供对实时体验至关重要的实时性能。
UVMap
UV是二维纹理坐标，U代表水平方向，V代表垂直方向。UV Map用来描述三维物体表面与图像纹理(Texture) 的映射关系，有了UV Map，我们就可以将二维的图像纹理粘贴到三维的物体表面。
矩形贴图和球面的映射图
技术实现
调取Camera获得相机画面
通过navigator.mediaDevices.getUserMedia获取stream，放到video查看。
async function setupWebcam() {
return new Promise( ( resolve, reject ) => {
const webcamElement = document.getElementById( "webcam" );
const navigatorAny = navigator;
navigator.getUserMedia = navigator.getUserMedia ||
navigatorAny.webkitGetUserMedia || navigatorAny.mozGetUserMedia ||
navigatorAny.msGetUserMedia;
if( navigator.getUserMedia ) {
navigator.getUserMedia( { video: true },
stream => {
webcamElement.srcObject = stream;
webcamElement.addEventListener( "loadeddata", resolve, false );
},
error => reject());
}
else {
reject();
}
});
}
人脸识别
代码语言：javascript复制//创建模型
createModel() {
return new Promise(async resolve => {
await tf.setBackend('webgl')
const model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
const detectorConfig = {
maxFaces: 1, //检测到的最大面部数量
refineLandmarks: true, //可以完善眼睛和嘴唇周围的地标坐标，并在虹膜周围输出其他地标
runtime: 'mediapipe',
solutionPath: 'https://unpkg.com/@mediapipe/face_mesh', //WASM二进制文件和模型文件所在的路径
};
this.model = await faceLandmarksDetection.createDetector(model, detectorConfig);
resolve(this.model);
})
},
//识别
async recognition() {
try {
const video = this.$refs.video;
const faces = await this.model.estimateFaces(video, {
flipHorizontal: false, //镜像
});
if (faces.length > 0) {
const keypoints = faces[0].keypoints;
this.render3D({
scaledMesh:keypoints.reduce((acc, pos) =>{
acc.push([pos.x,pos.y,pos.z])
return acc
}, [])
});
}else{
this.render3D({scaledMesh:[]})
}
} catch (error) {
console.log(error);
}
}
3D场景贴图
- TRIANGULATION
- UV_COORDS
//3D场景
const scene = new THREE.Scene();
//添加一些光照
scene.add( new THREE.AmbientLight( 0xcccccc, 0.4 ) );
camera.add( new THREE.PointLight( 0xffffff, 0.8 ) );
//正交相机
scene camera = new THREE.PerspectiveCamera( 45, 1, 0.1, 2000 );
camera.position.x = videoWidth / 2;
camera.position.y = -videoHeight / 2;
camera.position.z = -( videoHeight / 2 ) / Math.tan( 45 / 2 )
scene.add( camera );
//渲染器
const renderer = new THREE.WebGLRenderer({
canvas: document.getElementById( "overlay" ),
alpha: true
});
//创建geometry，将468个人脸特征点按照一定的顺序(TRIANGULATION)组成三角网格，并加载UV_COORDS
const geometry = new THREE.BufferGeometry()
geometry.setIndex(TRIANGULATION)
geometry.setAttribute('uv', new THREE.Float32BufferAttribute(UV_COORDS.map((item, index) => index % 2 ? item : 1 - item), 2))
geometry.computeVertexNormals()
//创建material
const textureLoader = new THREE.TextureLoader();
const meshImg = this.meshList[meshIndex].src;//材质图片地址
textureLoader.load(meshImg,texture=>{
texture.encoding = THREE.sRGBEncoding
texture.anisotropy = 16
const material = new THREE.MeshBasicMaterial({
map: texture,
transparent: true,
color: new THREE.Color(0xffffff),
reflectivity: 0.5
});
const mesh = new THREE.Mesh(geometry, material)
scene.add(mesh)
})
// 根据face mesh实时更新geometry
updateGeometry(prediction){
let w = canvasWidth;
let h = canvasWidth;
const faceMesh = resolveMesh(prediction.scaledMesh, w, h)
const positionBuffer = faceMesh.reduce((acc, pos) => acc.concat(pos), [])
geometry.setAttribute('position', new THREE.Float32BufferAttribute(positionBuffer, 3))
geometry.attributes.position.needsUpdate = true
}
resolveMesh(faceMesh, vw, vh){
return faceMesh.map(p => [p[0] - vw / 2, vh / 2 - p[1], -p[2]])
}
//渲染
render3D(prediction){
if (prediction) {
updateGeometry(prediction)
}
renderer.render(scene, threeCamera)
}
加载3D模型
代码语言：javascript复制//加载3D模型
const loader = new GLTFLoader();
const Object3D = new THREE.Object3D();
loader.load(modelUrl, (gltf) => {
const object = gltf.scene
const box = new THREE.Box3().setFromObject(object)
const size = box.getSize(new THREE.Vector3()).length()
const center = box.getCenter(new THREE.Vector3())
object.position.x = (object.position.x - center.x);
object.position.y = (object.position.y - center.y 1);
object.position.z = (object.position.z - center.z - 15);
Object3D.add(object)
this.scene.add(Object3D)
})
//计算Matrix
const position = prediction.midwayBetweenEyes[0]
const scale = this.getScale(prediction.scaledMesh, 234, 454)
const rotation = this.getRotation(prediction.scaledMesh, 10, 50, 280)
object.position.set(...position)
object.scale.setScalar(scale / 20)
object.scale.x *= -1
object.rotation.setFromRotationMatrix(rotation)
object.rotation.y = -object.rotation.y
object.rotateZ(Math.PI)
object.rotateX(-Math.PI * .05)
if (this.morphTarget) {
// flipped
this.morphTarget['leftEye'] && this.morphTarget['leftEye'](1 - prediction.faceRig.eye.r)
this.morphTarget['rightEye'] && this.morphTarget['rightEye'](1 - prediction.faceRig.eye.l)
this.morphTarget['mouth'] && this.morphTarget['mouth'](prediction.faceRig.mouth.shape.A)
}
