最近项目里有人脸捕捉的需求，刚开始时参考的下面这篇文章，使用官方发布的Facial AR Remote，需要我们自己构建IOS客户端，因此需要准备包括MacOS操作系统、Xcode等开发环境，在Unity构建出Xcode工程后，还要考虑开发许可证等问题，而且在尝试时，我使用的Xcode13版本，在编译上还有一些问题，比较麻烦。
https://www.163.com/dy/article/E70U8CLT0526E124.html
随后发现了另一个解决方案，即Live Capture，IOS客户端已经发布于App Store中，名称Unity Face Capture：
Live Capture在Package Manager中通过git url的方式进行添加，地址：
http://com.unity.live-capture
Live Capture官方手册地址：
https://docs.unity.cn/Packages/com.unity.live-capture@1.0/manual/index.html
PDF文档下载地址：
https://link.csdn.net/?target=https://forum.unity.com/attachments/live-capture-apps-startup-guide-pdf.961348/
文档很详细，也可以参考下面的步骤：
1.新建空物体，添加Take Recoder组件，它依赖Playable Director组件，添加Take Recorder时自动添加Playable Director：
2.将制作好的包含人脸的模型拖入场景，挂载ARKit Face Actor组件，然后将其做成Prefab预制体。
3.在Take Recoder组件下方点击 按钮，添加ARKit Face Device，并将其Actor设为步骤2中挂载了ARKit Face Actor组件的人脸模型：
4.在Project窗口右键，Create / Live Capture / ARKit Face Capture / Mapper，创建一个Head Mapper资产，将其Rig Prefab设为步骤2中生成的Prefab，并设置人脸模型中对应的的Left Eye、RightEye、Head骨骼节点：
5.点击Head Mapper资产中的Add Renderer按钮，绑定BlendShape，名称如果与ARKit中要求一致则会自动绑定好，否则需要一一对应设置：
6.将编辑好的Head Mapper资产赋值给步骤2挂载的ARKit Face Actor组件中的Mapper：
7.Window / Live Capture / Connections 打开Connections窗口，创建服务器，点击Start即可启动：
8.启动后打开IOS客户端，点击Connect进行连接，若连接不上，检查一下手机和电脑是否处于同一网段，检查电脑防火墙，最好都关掉。
另外值得注意的是，服务端的启动是通过在Connections编辑器窗口中点击Start开启的，因此它的使用环境是在Unity编辑器环境中，如果想打包后运行时使用，需要将其从Package Manager中迁移到工程Assets目录下，并创建脚本编写启动方法：
创建LiveCaptureServer类，继承Server：
代码语言：javascript复制using System;
using System.IO;
using System.Linq;
using System.Text;
using UnityEngine;
using System.Collections.Generic;
using Unity.LiveCapture.Networking;
using Unity.LiveCapture.Networking.Discovery;
namespace Unity.LiveCapture.CompanionApp
{
[CreateAssetMenu(menuName = "Live Capture/Server")]
public class LiveCaptureServer : Server
{
const int k_DefaultPort = 9000;
/// <summary>
/// The server executes this event when a client has connected.
/// </summary>
public static event Action<ICompanionAppClient> ClientConnected = delegate { };
/// <summary>
/// The server executes this event when a client has disconnected.
/// </summary>
public static event Action<ICompanionAppClient> ClientDisconnected = delegate { };
struct ConnectHandler
{
public string Name;
public DateTime Time;
public Func<ICompanionAppClient, bool> Handler;
}
readonly Dictionary<string, Type> s_TypeToClientType = new Dictionary<string, Type>();
static readonly List<ConnectHandler> s_ClientConnectHandlers = new List<ConnectHandler>();
/// <summary>
/// Adds a callback used to take ownership of a client that has connected.
/// </summary>
/// <param name="handler">The callback function. It must return true if it takes ownership of a client.</param>
/// <param name="name">The name of the client to prefer. If set, this handler has priority over clients that have the given name.</param>
/// <param name="time">The time used to determine the priority of handlers when many are listening for the same
/// client <paramref name="name"/>. More recent values have higher priority.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="handler"/> is null.</exception>
public static void RegisterClientConnectHandler(Func<ICompanionAppClient, bool> handler, string name, DateTime time)
{
if (handler == null)
throw new ArgumentNullException(nameof(handler));
DeregisterClientConnectHandler(handler);
s_ClientConnectHandlers.Add(new ConnectHandler
{
Name = name,
Time = time,
Handler = handler,
});
}
/// <summary>
/// Removes a client connection callback.
/// </summary>
/// <param name="handler">The callback to remove.</param>>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="handler"/> is null.</exception>
public static void DeregisterClientConnectHandler(Func<ICompanionAppClient, bool> handler)
{
if (handler == null)
throw new ArgumentNullException(nameof(handler));
for (var i = 0; i < s_ClientConnectHandlers.Count; i )
{
if (s_ClientConnectHandlers[i].Handler == handler)
{
s_ClientConnectHandlers.RemoveAt(i);
}
}
}
public void Init()
{
foreach (var (type, attributes) in AttributeUtility.GetAllTypes<ClientAttribute>())
{
if (!typeof(CompanionAppClient).IsAssignableFrom(type))
{
Debug.LogError($"{type.FullName} must be assignable from {nameof(CompanionAppClient)} to use the {nameof(ClientAttribute)} attribute.");
continue;
}
foreach (var attribute in attributes)
{
s_TypeToClientType[attribute.Type] = type;
}
}
}
[SerializeField, Tooltip("The TCP port on which the server will listen for incoming connections. Changes to the port only take effect after restarting the server.")]
int m_Port = k_DefaultPort;
[SerializeField, Tooltip("Start the server automatically after entering play mode.")]
bool m_AutoStartOnPlay = true;
readonly DiscoveryServer m_Discovery = new DiscoveryServer();
readonly NetworkServer m_Server = new NetworkServer();
readonly Dictionary<Remote, ICompanionAppClient> m_RemoteToClient = new Dictionary<Remote, ICompanionAppClient>();
/// <summary>
/// The TCP port on which the server will listen for incoming connections.
/// </summary>
/// <remarks>
/// Changes to the port only take effect after restarting the server.
/// </remarks>
public int Port
{
get => m_Port;
set
{
if (m_Port != value)
{
m_Port = value;
OnServerChanged(true);
}
}
}
/// <summary>
/// Start the server automatically after entering play mode.
/// </summary>
public bool AutoStartOnPlay
{
get => m_AutoStartOnPlay;
set
{
if (m_AutoStartOnPlay != value)
{
m_AutoStartOnPlay = value;
OnServerChanged(true);
}
}
}
/// <summary>
/// Are clients able to connect to the server.
/// </summary>
public bool IsRunning => m_Server.IsRunning;
/// <summary>
/// The number of clients currently connected to the server.
/// </summary>
public int ClientCount => m_RemoteToClient.Count;
/// <inheritdoc/>
protected override void OnEnable()
{
base.OnEnable();
m_Server.RemoteConnected = OnClientConnected;
m_Server.RemoteDisconnected = OnClientDisconnected;
}
/// <inheritdoc/>
protected override void OnDisable()
{
base.OnDisable();
m_Discovery.Stop();
m_Server.Stop();
m_Server.RemoteConnected -= OnClientConnected;
m_Server.RemoteDisconnected -= OnClientDisconnected;
}
/// <summary>
/// Gets the currently connected clients.
/// </summary>
/// <returns>A new collection containing the client handles.</returns>
public IEnumerable<ICompanionAppClient> GetClients()
{
return m_RemoteToClient.Values;
}
/// <inheritdoc />
public override string GetName() => "Companion App Server";
/// <summary>
/// Start listening for clients connections.
/// </summary>
public void StartServer()
{
if (!NetworkUtilities.IsPortAvailable(m_Port))
{
Debug.LogError($"Unable to start server: Port {m_Port} is in use by another program! Close the other program, or assign a free port using the Live Capture Window.");
return;
}
if (m_Server.StartServer(m_Port))
{
// start server discovery
var config = new ServerData(
"Live Capture",
Environment.MachineName,
m_Server.ID,
PackageUtility.GetVersion(LiveCaptureInfo.Version)
);
var endPoints = m_Server.EndPoints.ToArray();
m_Discovery.Start(config, endPoints);
}
OnServerChanged(false);
}
/// <summary>
/// Disconnects all clients and stop listening for new connections.
/// </summary>
public void StopServer()
{
m_Server.Stop();
m_Discovery.Stop();
OnServerChanged(false);
}
/// <inheritdoc/>
public override void OnUpdate()
{
m_Server.Update();
m_Discovery.Update();
}
void OnClientConnected(Remote remote)
{
m_Server.RegisterMessageHandler(remote, InitializeClient, false);
}
void OnClientDisconnected(Remote remote, DisconnectStatus status)
{
if (m_RemoteToClient.TryGetValue(remote, out var client))
{
try
{
ClientDisconnected.Invoke(client);
}
catch (Exception e)
{
Debug.LogError(e);
}
m_RemoteToClient.Remove(remote);
OnServerChanged(false);
}
}
void InitializeClient(Message message)
{
try
{
if (message.ChannelType != ChannelType.ReliableOrdered)
{
return;
}
var streamReader = new StreamReader(message.Data, Encoding.UTF8);
var json = streamReader.ReadToEnd();
var data = default(ClientInitialization);
try
{
data = JsonUtility.FromJson<ClientInitialization>(json);
}
catch (Exception)
{
Debug.LogError($"{nameof(CompanionAppServer)} failed to initialize client connection! Could not parse JSON: {json}");
return;
}
if (!s_TypeToClientType.TryGetValue(data.Type, out var clientType))
{
Debug.LogError($"Unknown client type "{data.Type}" connected to {nameof(CompanionAppServer)}!");
return;
}
var remote = message.Remote;
var client = Activator.CreateInstance(clientType, m_Server, remote, data) as CompanionAppClient;
client.SendProtocol();
m_RemoteToClient.Add(remote, client);
AssignOwner(client);
ClientConnected.Invoke(client);
OnServerChanged(false);
}
catch (Exception e)
{
Debug.LogException(e);
}
finally
{
message.Dispose();
}
}
void AssignOwner(ICompanionAppClient client)
{
// connect to the registered handler that was most recently used with this client if possible
foreach (var handler in s_ClientConnectHandlers.OrderByDescending(h => h.Time.Ticks))
{
try
{
if (handler.Name == client.Name)
{
if (handler.Handler(client))
return;
}
}
catch (Exception e)
{
Debug.LogException(e);
}
}
// fall back to the first free device that is compatible with the client
foreach (var handler in s_ClientConnectHandlers)
{
try
{
if (handler.Handler(client))
return;
}
catch (Exception e)
{
Debug.LogException(e);
}
}
}
}
}
更改CompanionAppDevice类：
代码语言：javascript复制using System;
using UnityEngine;
namespace Unity.LiveCapture.CompanionApp
{
/// <summary>
/// A type of <see cref="LiveCaptureDevice"/> that uses a <see cref="ICompanionAppClient"/> for communication.
/// </summary>
interface ICompanionAppDevice
{
/// <summary>
/// Clears the client assigned to this device.
/// </summary>
void ClearClient();
}
/// <summary>
/// A type of <see cref="LiveCaptureDevice"/> that uses a <see cref="ICompanionAppClient"/> for communication.
/// </summary>
/// <typeparam name="TClient">The type of client this device communicates with.</typeparam>
public abstract class CompanionAppDevice<TClient> : LiveCaptureDevice, ICompanionAppDevice where TClient : class, ICompanionAppClient
{
bool m_ClientRegistered;
bool m_Recording;
TClient m_Client;
readonly SlateChangeTracker m_SlateChangeTracker = new SlateChangeTracker();
readonly TakeNameFormatter m_TakeNameFormatter = new TakeNameFormatter();
string m_LastAssetName;
bool TryGetInternalClient(out ICompanionAppClientInternal client)
{
client = m_Client as ICompanionAppClientInternal;
return client != null;
}
/// <summary>
/// This function is called when the object becomes enabled and active.
/// </summary>
protected virtual void OnEnable()
{
CompanionAppServer.ClientDisconnected = OnClientDisconnected;
LiveCaptureServer.ClientDisconnected = OnClientDisconnected;
RegisterClient();
}
/// <summary>
/// This function is called when the behaviour becomes disabled.
/// </summary>
/// <remaks>
/// This is also called when the object is destroyed and can be used for any cleanup code.
/// When scripts are reloaded after compilation has finished, OnDisable will be called, followed by an OnEnable after the script has been loaded.
/// </remaks>
protected virtual void OnDisable()
{
CompanionAppServer.ClientDisconnected -= OnClientDisconnected;
CompanionAppServer.DeregisterClientConnectHandler(OnClientConnected);
LiveCaptureServer.ClientConnected -= OnClientDisconnected;
LiveCaptureServer.DeregisterClientConnectHandler(OnClientConnected);
StopRecording();
UnregisterClient();
}
/// <summary>
/// This function is called when the behaviour gets destroyed.
/// </summary>
protected override void OnDestroy()
{
base.OnDestroy();
ClearClient();
}
/// <inheritdoc/>
public override bool IsReady()
{
return m_Client != null;
}
/// <inheritdoc/>
public override bool IsRecording()
{
return m_Recording;
}
/// <inheritdoc/>
public override void StartRecording()
{
if (!m_Recording)
{
m_Recording = true;
OnRecordingChanged();
SendRecordingState();
}
}
/// <inheritdoc/>
public override void StopRecording()
{
if (m_Recording)
{
m_Recording = false;
OnRecordingChanged();
SendRecordingState();
}
}
/// <summary>
/// Gets the client currently assigned to this device.
/// </summary>
/// <returns>The assigned client, or null if none is assigned.</returns>
public TClient GetClient()
{
return m_Client;
}
/// <summary>
/// Assigns a client to this device.
/// </summary>
/// <param name="client">The client to assign, or null to clear the assigned client.</param>
/// <param name="rememberAssignment">Try to auto-assign the client to this device when it reconnects in the future.</param>
public void SetClient(TClient client, bool rememberAssignment)
{
if (m_Client != client)
{
UnregisterClient();
if (m_Client != null)
{
ClientMappingDatabase.DeregisterClientAssociation(this, m_Client, rememberAssignment);
}
m_Client = client;
if (m_Client != null)
{
// if any device is also using this client, we must clear the client from the previous device.
if (ClientMappingDatabase.TryGetDevice(client, out var previousDevice))
{
previousDevice.ClearClient();
}
ClientMappingDatabase.RegisterClientAssociation(this, m_Client, rememberAssignment);
}
RegisterClient();
}
}
void RegisterClient()
{
if (!isActiveAndEnabled || m_ClientRegistered)
{
return;
}
LiveCaptureServer.DeregisterClientConnectHandler(OnClientConnected);
CompanionAppServer.DeregisterClientConnectHandler(OnClientConnected);
m_SlateChangeTracker.Reset();
if (TryGetInternalClient(out var client))
{
client.SetDeviceMode = ClientSetDeviceMode;
client.StartRecording = ClientStartRecording;
client.StopRecording = ClientStopRecording;
client.StartPlayer = ClientStartPlayer;
client.StopPlayer = ClientStopPlayer;
client.PausePlayer = ClientPausePlayer;
client.SetPlayerTime = ClientSetPlayerTime;
client.SetSelectedTake = ClientSetSelectedTake;
client.SetTakeData = ClientSetTakeData;
client.DeleteTake = ClientDeleteTake;
client.SetIterationBase = ClientSetIterationBase;
client.ClearIterationBase = ClientClearIterationBase;
client.TexturePreviewRequested = OnTexturePreviewRequested;
OnClientAssigned();
client.SendInitialize();
UpdateClient();
m_ClientRegistered = true;
}
else
{
ClientMappingDatabase.TryGetClientAssignment(this, out var clientName, out var time);
LiveCaptureServer.RegisterClientConnectHandler(OnClientConnected, clientName, time);
CompanionAppServer.RegisterClientConnectHandler(OnClientConnected, clientName, time);
}
}
void UnregisterClient()
{
if (!m_ClientRegistered)
{
return;
}
if (TryGetInternalClient(out var client))
{
OnClientUnassigned();
client.SendEndSession();
client.SetDeviceMode -= ClientSetDeviceMode;
client.StartRecording -= ClientStartRecording;
client.StopRecording -= ClientStopRecording;
client.StartPlayer -= ClientStartPlayer;
client.StopPlayer -= ClientStopPlayer;
client.PausePlayer -= ClientPausePlayer;
client.SetPlayerTime -= ClientSetPlayerTime;
client.SetSelectedTake -= ClientSetSelectedTake;
client.SetTakeData -= ClientSetTakeData;
client.DeleteTake -= ClientDeleteTake;
client.SetIterationBase -= ClientSetIterationBase;
client.ClearIterationBase -= ClientClearIterationBase;
client.TexturePreviewRequested -= OnTexturePreviewRequested;
m_ClientRegistered = false;
}
}
/// <inheritdoc />
public void ClearClient()
{
SetClient(null, true);
}
/// <summary>
/// Called to send the device state to the client.
/// </summary>
public virtual void UpdateClient()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
SendDeviceState(takeRecorder.IsLive());
var slate = takeRecorder.GetActiveSlate();
var hasSlate = slate != null;
var slateChanged = m_SlateChangeTracker.Changed(slate);
var take = hasSlate ? slate.Take : null;
var assetName = GetAssetName();
var assetNameChanged = assetName != m_LastAssetName;
m_LastAssetName = assetName;
if (TryGetInternalClient(out var client))
{
client.SendFrameRate(takeRecorder.IsLive() || take == null ? takeRecorder.FrameRate : take.FrameRate);
client.SendHasSlate(hasSlate);
client.SendSlateDuration(hasSlate ? slate.Duration : 0d);
client.SendSlateIsPreviewing(takeRecorder.IsPreviewPlaying());
client.SendSlatePreviewTime(takeRecorder.GetPreviewTime());
if (slateChanged || assetNameChanged)
{
if (hasSlate)
m_TakeNameFormatter.ConfigureTake(slate.SceneNumber, slate.ShotName, slate.TakeNumber);
else
m_TakeNameFormatter.ConfigureTake(0, "Shot", 0);
client.SendNextTakeName(m_TakeNameFormatter.GetTakeName());
client.SendNextAssetName(m_TakeNameFormatter.GetAssetName());
}
}
if (slateChanged)
{
SendSlateDescriptor(slate);
}
}
SendRecordingState();
}
/// <summary>
/// Gets the name used for the take asset name.
/// </summary>
/// <returns>The name of the asset.</returns>
protected virtual string GetAssetName() { return name; }
/// <summary>
/// The device calls this method when a new client is assigned.
/// </summary>
protected virtual void OnClientAssigned() {}
/// <summary>
/// The device calls this method when the client is unassigned.
/// </summary>
protected virtual void OnClientUnassigned() {}
/// <summary>
/// The device calls this method when the recording state has changed.
/// </summary>
protected virtual void OnRecordingChanged() {}
/// <summary>
/// The device calls this method when the slate has changed.
/// </summary>
protected virtual void OnSlateChanged(ISlate slate) {}
void ClientStartRecording()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.StartRecording();
}
Refresh();
}
void ClientStopRecording()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.StopRecording();
}
Refresh();
}
void ClientSetDeviceMode(DeviceMode deviceMode)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.SetLive(deviceMode == DeviceMode.LiveStream);
SendDeviceState(takeRecorder.IsLive());
}
}
void ClientStartPlayer()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.PlayPreview();
}
Refresh();
}
void ClientStopPlayer()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.PausePreview();
takeRecorder.SetPreviewTime(0d);
}
Refresh();
}
void ClientPausePlayer()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.PausePreview();
}
Refresh();
}
void ClientSetPlayerTime(double time)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.SetPreviewTime(time);
}
Refresh();
}
void SendDeviceState()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
SendDeviceState(takeRecorder.IsLive());
}
}
void SendDeviceState(bool isLive)
{
if (TryGetInternalClient(out var client))
{
client.SendDeviceMode(isLive ? DeviceMode.LiveStream : DeviceMode.Playback);
}
}
void SendRecordingState()
{
if (TryGetInternalClient(out var client))
{
client.SendRecordingState(IsRecording());
}
}
void SendSlateDescriptor()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
SendSlateDescriptor(takeRecorder.GetActiveSlate());
}
}
void SendSlateDescriptor(ISlate slate)
{
if (TryGetInternalClient(out var client))
{
client.SendSlateDescriptor(SlateDescriptor.Create(slate));
}
OnSlateChanged(slate);
}
void ClientSetSelectedTake(Guid guid)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
TakeManager.Default.SelectTake(takeRecorder.GetActiveSlate(), guid);
SendSlateDescriptor();
Refresh();
}
}
void ClientSetTakeData(TakeDescriptor descriptor)
{
TakeManager.Default.SetTakeData(descriptor);
SendSlateDescriptor();
Refresh();
}
void ClientDeleteTake(Guid guid)
{
TakeManager.Default.DeleteTake(guid);
SendSlateDescriptor();
Refresh();
}
void ClientSetIterationBase(Guid guid)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
var slate = takeRecorder.GetActiveSlate();
TakeManager.Default.SetIterationBase(slate, guid);
SendSlateDescriptor(slate);
Refresh();
}
}
void ClientClearIterationBase()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
var slate = takeRecorder.GetActiveSlate();
TakeManager.Default.ClearIterationBase(slate);
SendSlateDescriptor(slate);
Refresh();
}
}
void OnTexturePreviewRequested(Guid guid)
{
var texture = TakeManager.Default.GetAssetPreview<Texture2D>(guid);
if (texture != null && TryGetInternalClient(out var client))
{
client.SendTexturePreview(guid, texture);
}
}
bool OnClientConnected(ICompanionAppClient client)
{
if (m_Client == null && client is TClient c && (!ClientMappingDatabase.TryGetClientAssignment(this, out var clientName, out _) || c.Name == clientName))
{
SetClient(c, false);
return true;
}
return false;
}
void OnClientDisconnected(ICompanionAppClient client)
{
if (m_Client == client)
{
SetClient(null, false);
}
}
}
}本人打包后成功运行，测试脚本：
代码语言：javascript复制using UnityEngine;
using Unity.LiveCapture.CompanionApp;
public class LiveCaptureExample : MonoBehaviour
{
LiveCaptureServer server;
private void Awake()
{
server = Resources.Load<LiveCaptureServer>("Live Capture Server");
server.Init();
server.StartServer();
}
private void Update()
{
server.OnUpdate();
}
private void OnDestroy()
{
server.StopServer();
}
}
