前言直播、短视频、网络会议等应用越来越多地走进人们的生活，随之而来的是各种创意玩法和新鲜体验，其中大量应用AR基于AI检测和图形渲染的技术。随着Web技术的不断成熟，AR技术在Web上的实现成为可能。今天总结了Web端实现该功能的几个技术要点，与大家共同探讨。架构及概念抽象整体实现思路如下调用Camera获取摄像头图像使用tensorflow加载人脸识别模型生成FaceMesh在FaceMesh的基础上生成三角网格并进行UV映射FaceMeshMediaPipeFaceMesh是一种人脸几何解决方案，即使在移动设备上，也可以实时估计468个3D面部特征点。它采用机器学习(ML)来推断3D表面几何形状，并且只需要一个摄像头输入，而无需专用的深度传感器。该解决方案利用轻量级模型架构与整个管道中的GPU加速相结合，以提供对实时体验至关重要的实时性能。UVMapUV是二维纹理坐标，U代表水平方向，V代表垂直方向。UVMap用于描述三维物体表面与图像纹理（Texture）之间的映射关系。借助UVMap，我们可以将二维图像纹理粘贴到三维物体表面。image.png矩形纹理和球面贴图技术实现调用摄像头获取摄像头画面通过navigator.mediaDevices.getUserMedia获取流放到视频中查看。asyncfunctionsetupWebcam(){returnnewPromise((resolve,reject)=>{constwebcamElement=document.getElementById("webcam");constnavigatorAny=navigator;navigator.getUserMedia=navigator.getUserMedia||navigatorAny.webkitGetUserMedia||navigatorAny...error=>reject());}else{reject();}});}复制代码人脸识别//创建模型createModel(){returnnewPromise(asyncresolve=>{awaittf.setBackend('webgl')constmodel=faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;constdetectorConfig={maxFaces:1,//最大检测人脸数refineLandmarks:true,//可以提高眼睛和嘴唇周围的界标坐标，并且在输出iris运行时周围的其他地标：'mediapipe',solutionPath:'https://unpkg.com/@mediapipe/face_mesh',//WASM二进制文件和模型文件所在的路径}；this.model=awaitfaceLandmarksDetection.createDetector(model,detectorConfig);resolve(this.model);})},//识别异步识别(){try{constvideo=this.$refs.video;constfaces=awaitthis.model.estimateFaces(video,{flipHorizo??ntal:false,//mirror});如果(faces.length>0){constkeypoints=faces[0].keypoints;this.render3D({scaledMesh:keypoints.reduce((acc,pos)=>{acc.push([pos.x,pos.y,pos.z])returnacc},[])});}else{this.render3D({scaledMesh:[]})}}catch(error){console.log(error);}}复制代码3D场景贴图TRIANGULATIONUV_COORDS//3D场景constscene=newTHREE.Scene();//添加一些光照scene.add(newTHREE.AmbientLight(0xcccccc,0.4));camera.add(newTHREE.PointLight(0xffffff,0.8));//正交相机场景camera=newTHREE.PerspectiveCamera(45,1,0.1,2000);camera.position.x=videoWidth/2;camera.position.y=-videoHeight/2;camera.position.z=-(videoHeight/2)/数学。tan(45/2)scene.add(相机);//Rendererconstrenderer=newTHREE.WebGLRenderer({canvas:document.getElementById("overlay"),alpha:true});//创建几何体，468个人脸特征点按一定顺序组成三角形网格（TRIANGULATION），并加载UV_COORDSconstgeometry=newTHREE.BufferGeometry()geometry.setIndex(TRIANGULATION)geometry.setAttribute('uv',newTHREE.Float32BufferAttribute(UV_COORDS.map((item,index)=>index%2?item:1-item),2))geometry.computeVertexNormals()//创建材质consttextureLoader=newTHREE.TextureLoader();constmeshImg=this.meshList[meshIndex].src;//材质图像地址textureLoader.load(meshImg,texture=>{texture.encoding=THREE.sRGBEncodingtexture.anisotropy=16constmaterial=newTHREE.MeshBasicMaterial({贴图：纹理，透明：真，颜色：新的THREE.Color(0xffffff)，反射率：0.5});constmesh=newTHREE.Mesh(geometry,material)scene.add(mesh)})//根据面网格实时更新geometryupdateGeometry(prediction){letw=canvasWidth;让h=画布宽度；constfaceMesh=resolveMesh(prediction.scaledMesh,w,h)constpositionBuffer=faceMesh.reduce((acc,pos)=>acc.concat(pos),[])geometry.setAttribute('position',newTHREE.Float32BufferAttribute(positionBuffer,3))geometry.attributes.position.needsUpdate=true}resolveMesh(faceMesh,vw,vh){返回faceMesh.map(p=>[p[0]-vw/2,vh/2-p[1],-p[2]])}//流染render3D(prediction){if(prediction){updateGeometry(prediction)}renderer.render(scene,threeCamera)}复制代码加载3D模型//加载3D模型constloader=newGLTFLoader();constObject3D=newTHREE.Object3D();loader.load(modelUrl,(gltf)=>{constobject=gltf.sceneconstbox=newTHREE.Box3().setFromObject(object)const大小=box.getSize(newTHREE.Vector3()).length()constcenter=box.getCenter(newTHREE.Vector3())object.position.x+=(object.position.x-center.x);对象.position.y+=(object.position.y-center.y+1);object.position.z+=(object.position.z-center.z-15);Object3D.add(object)this.scene.add(Object3D)})//计算矩阵constposition=prediction.midwayBetweenEyes[0]constscale=this.getScale(prediction.scaledMesh,234,454)constrotation=this.getRotation(prediction.scaledMesh,10,50,280)object.position.set(...position)object.scale.setScalar(scale/20)object.scale.x*=-1object.rotation.setFromRotationMatrix(rotation)object.rotation.y=-object.rotation.yobject.rotateZ(Math.PI)object.rotateX(-Math.PI*.05)if(this.morphTarget){//flippedthis.morphTarget['leftEye']&&this.morphTarget['leftEye'](1-预测。faceRig.eye.r)this.morphTarget['rightEye']&&this.morphTarget['rightEye'](1-prediction.faceRig.eye.l)this.morphTarget['mouth']&&this.morphTarget['mouth'](prediction.faceRig.mouth.shape.A)}
