[人工智能]hololens凝视物体、单击双击、语音控制、蓝牙

1、用unity开发
2、在HoloLens toolkit中找到TextToSpeech添加到场景中
3、添加KeywordManager.cs到场景中

using orange;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.Windows.Speech;

/// <summary>
/// 语音控制类，需要在start方法中初始化。
/// 使用方法：
/// 1、在场景对象中添加KeywordManager脚本
/// 2、在需要接收语音控制的脚本中，获取到KeywordManager实例，然后注册 OnSpeeked 事件
/// </summary>
public class KeywordManager : MonoBehaviour
{
    public delegate void OnSpeekHandler(SpeechKeys key);

    private KeywordRecognizer keywordRecognizer;
    private List<string> keys;

    public event OnSpeekHandler OnSpeeked;

    private void Start()
    {
        //从枚举对象中获取识别命令
        string[] values = System.Enum.GetNames(typeof(SpeechKeys));
        keywordRecognizer = new KeywordRecognizer(values);
        keys = new List<string>();
        keys.AddRange(values);
        keywordRecognizer.OnPhraseRecognized += KeywordRecognizer_OnPhraseRecognized;
        keywordRecognizer.Start();
    }

    void OnDestroy()
    {
        if (keywordRecognizer != null)
        {
            StopKeywordRecognizer();
            keywordRecognizer.OnPhraseRecognized -= KeywordRecognizer_OnPhraseRecognized;
            keywordRecognizer.Dispose();
        }
    }

    /// <summary>
    /// 识别完成
    /// </summary>
    /// <param name="args"></param>
    private void KeywordRecognizer_OnPhraseRecognized(PhraseRecognizedEventArgs args)
    {
        if (keys.Contains(args.text))
        {
            //转换为枚举类型，触发事件
            SpeechKeys key = (SpeechKeys)System.Enum.Parse(typeof(SpeechKeys), args.text);
            if (OnSpeeked != null)
            {
                OnSpeeked.Invoke(key);
            }
        }
    }

    /// <summary>  
    /// Make sure the keyword recognizer is off, then start it.  
    /// Otherwise, leave it alone because it's already in the desired state.  
    /// </summary>  
    public void StartKeywordRecognizer()
    {
        if (keywordRecognizer != null && !keywordRecognizer.IsRunning)
        {
            keywordRecognizer.Start();
        }
    }

    /// <summary>  
    /// Make sure the keyword recognizer is on, then stop it.  
    /// Otherwise, leave it alone because it's already in the desired state.  
    /// </summary>  
    public void StopKeywordRecognizer()
    {
        if (keywordRecognizer != null && keywordRecognizer.IsRunning)
        {
            keywordRecognizer.Stop();
        }
    }
}

4、添加GestureManager.cs到场景中

using orange;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.XR.WSA.Input;

public class GestureManager : MonoBehaviour
{

    private GestureRecognizer recognizer;

    public delegate void OnGestureClick();
    public delegate void OnGestureDoubleClick();

    private List<OnGestureClick> onClicks = new List<OnGestureClick>();
    private List<OnGestureDoubleClick> onDoubleClicks = new List<OnGestureDoubleClick>();

    public void AddGestureClick(OnGestureClick onClick)
    {
        if (onClick == null || onClicks.Contains(onClick)) return;
        onClicks.Add(onClick);
    }

    public void RemoveGestureClick(OnGestureClick onClick)
    {
        if (onClick != null && onClicks.Contains(onClick))
        {
            onClicks.Remove(onClick);
        }
    }
    public void AddGestureDoubleClick(OnGestureDoubleClick onDoubleClick)
    {
        if (onDoubleClick == null || onDoubleClicks.Contains(onDoubleClick)) return;
        onDoubleClicks.Add(onDoubleClick);
    }

    public void RemoveGestureDoubleOnClick(OnGestureDoubleClick onDoubleClick)
    {
        if (onDoubleClick != null && onDoubleClicks.Contains(onDoubleClick))
        {
            onDoubleClicks.Remove(onDoubleClick);
        }
    }

    void Start()
    {
        // 初始化开始
        // 创建手势识别对象
        recognizer = new GestureRecognizer();
        // 设置手势识别的类型
        recognizer.SetRecognizableGestures(GestureSettings.Tap | GestureSettings.Hold | GestureSettings.DoubleTap);
        // 添加手势识别的事件
        recognizer.TappedEvent += Recognizer_TappedEvent;
        recognizer.HoldStartedEvent += Recognizer_HoldStartedEvent;
        recognizer.HoldCompletedEvent += Recognizer_HoldCompletedEvent;
        recognizer.HoldCanceledEvent += Recognizer_HoldCanceledEvent;
        // 开启手势识别
        recognizer.StartCapturingGestures();
        Debug.Log("初始化完成~");
    }

    private void Recognizer_HoldCanceledEvent(InteractionSourceKind source, Ray headRay)
    {
    }

    private void Recognizer_HoldCompletedEvent(InteractionSourceKind source, Ray headRay)
    {
    }

    private void Recognizer_HoldStartedEvent(InteractionSourceKind source, Ray headRay)
    {
    }

    private void Recognizer_TappedEvent(InteractionSourceKind source, int tapCount, Ray headRay)
    {
        if (tapCount == 1)
        {
            //单击
            for (int i = 0; i < onClicks.Count; i++)
            {
                onClicks[i].Invoke();
            }
        }
        else if (tapCount == 2)
        {
            //双击
            for (int i = 0; i < onDoubleClicks.Count; i++)
            {
                onDoubleClicks[i].Invoke();
            }
        }
    }

    void OnDestroy()
    {
        // 销毁注册的事件
        recognizer.TappedEvent -= Recognizer_TappedEvent;
        recognizer.HoldStartedEvent -= Recognizer_HoldStartedEvent;
        recognizer.HoldCompletedEvent -= Recognizer_HoldCompletedEvent;
        recognizer.HoldCanceledEvent -= Recognizer_HoldCanceledEvent;
    }

}

5、主程序入口

using HoloToolkit.Unity;
using HoloToolkit.Unity.InputModule;
using UnityEngine;
using UnityEngine.UI;
using ZhiHui;

public class APP : MonoBehaviour
{
    public Transform modeles;
    public Material raycastMaterial;
    //
    public TextToSpeech textToSpeech;
    public KeywordManager keywordManager;
    //
    public GestureManager gestureManager;
    private GestureHandler gestureHandler;
    //
    public Canvas infoWindow;
    public Canvas jumpWindow;
    private InfoWindow mInfoWindow;
    private JumpWindow mJumpWondow;
    //
    private KeywordHandler keywordHandler;

    void Start()
    {
        //UI
        mInfoWindow = new InfoWindow(infoWindow);
        mJumpWondow = new JumpWindow(jumpWindow);
        //语音识别
        SpeechHandler.textToSpeech = textToSpeech;
        //设置视线检测
        RaycastHandler.raycastMaterial = raycastMaterial;
        //语音控制
        keywordHandler = new KeywordHandler(keywordManager);
        keywordHandler.infoWindow = mInfoWindow;
        keywordHandler.jumpWindow = mJumpWondow;
        //手势控制
        gestureHandler = new GestureHandler(gestureManager);
        gestureHandler.infoWindow = mInfoWindow;
        gestureHandler.jumpWindow = mJumpWondow;
        gestureHandler.Init();
    }

    void Update()
    {
        RaycastHandler.OnRaycast();
    }

    private void OnDestroy()
    {
        keywordHandler.Destroy();
        gestureHandler.Destroy();

        mInfoWindow.Destroy();
        mJumpWondow.Destroy();
    }

}

6、RaycastHandler.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;

namespace ZhiHui
{
    class RaycastHandler
    {
        public static GameObject raycastObject;
        private static GameObject raycastObjectTmp;

        public static Material raycastMaterial;
        private static Material raycastMaterialTmp;

        /// <summary>
        /// 射线检测委托
        /// </summary>
        /// <param name="hit"></param>
        public delegate void RaycastCall(RaycastHit hit);

        private static List<RaycastCall> racastCalls = new List<RaycastCall>();

        /// <summary>
        /// 添加回调
        /// </summary>
        /// <param name="racastCall"></param>
        public static void AddRaycastCall(RaycastCall racastCall)
        {
            if (racastCall == null || racastCalls.Contains(racastCall)) return;
            racastCalls.Add(racastCall);
        }

        /// <summary>
        /// 移除回调
        /// </summary>
        /// <param name="racastCall"></param>
        public static void RemoveRaycastCall(RaycastCall racastCall)
        {
            if (racastCall != null && racastCalls.Contains(racastCall))
            {
                racastCalls.Remove(racastCall);
            }
        }

        /// <summary>
        /// 选中物体
        /// </summary>
        /// <param name="objName"></param>
        public static void SelectObjectByName(string objName)
        {
            SetHightlightBack();
            SetHightlight(GameObject.Find(objName));
        }

        /// <summary>
        /// 恢复材质
        /// </summary>
        private static void SetHightlightBack()
        {
            if (raycastObjectTmp != null && raycastMaterialTmp != null)
            {
                Renderer r = raycastObjectTmp.GetComponent<Renderer>();
                if (r)
                {
                    r.material = raycastMaterialTmp;
                }
            }
        }

        /// <summary>
        /// 改变材质
        /// </summary>
        private static void SetHightlight(GameObject obj)
        {
            if (obj != null && raycastMaterial != null)
            {
                Renderer r = obj.GetComponent<Renderer>();
                if (r)
                {
                    raycastMaterialTmp = r.material;
                    r.material = raycastMaterial;
                }
            }
            raycastObjectTmp = obj;
        }

        /// <summary>
        /// 射线检测
        /// </summary>
        public static void OnRaycast()
        {
            //恢复材质
            SetHightlightBack();
            //
            RaycastHit hit;
            if (Physics.Raycast(Camera.main.transform.position, Camera.main.transform.forward, out hit))
            {
                raycastObject = hit.transform.gameObject;
                //改变材质
                SetHightlight(raycastObject);
            }
            else
            {
                raycastMaterialTmp = null;
                raycastObjectTmp = null;
            }
            //回调
            if (racastCalls != null)
            {
                for (int i = 0; i < racastCalls.Count; i++)
                {
                    racastCalls[i].Invoke(hit);
                }
            }
        }


        /// <summary>
        /// 获取物体LookAt后的旋转值
        /// </summary>
        /// <param name="originalObj"></param>
        /// <param name="targetPoint"></param>
        /// <returns></returns>
        public static Quaternion GetLookAtEuler(Vector3 originPoint, Vector3 targetPoint)
        {
            //计算物体在朝向某个向量后的正前方
            Vector3 forwardDir = targetPoint - originPoint;

            //计算朝向这个正前方时的物体四元数值
            Quaternion lookAtRot = Quaternion.LookRotation(forwardDir);

            //把四元数值转换成角度
            //Vector3 resultEuler = lookAtRot.eulerAngles;

            return lookAtRot;
        }

        /// <summary>
        /// 获取距离
        /// </summary>
        /// <returns></returns>
        public static float GetDistanceBetween(Vector3 v1, Vector3 v2)
        {
            float x = Mathf.Pow((v2.x - v1.x), 2);
            float y = Mathf.Pow((v2.y - v1.y), 2);
            float z = Mathf.Pow((v2.z - v1.z), 2);
            return Mathf.Sqrt(x + y + z);
        }

    }
}

7、SpeechHandler.cs

using HoloToolkit.Unity;
using HoloToolkit.Unity.InputModule;

namespace ZhiHui
{
    class SpeechHandler
    {
        public static TextToSpeech textToSpeech;

        /// <summary>
        /// 文字转语音
        /// </summary>
        /// <param name="txt"></param>
        public static void Speak(string txt)
        {
            if (textToSpeech == null) return;
            if (textToSpeech.isActiveAndEnabled)
            {
                if (!textToSpeech.IsSpeaking())
                {
                    textToSpeech.StartSpeaking(txt);
                }
            }
        }

        /// <summary>
        /// 文字转语音(要打断当前说话吗)
        /// </summary>
        /// <param name="txt"></param>
        /// <param name="breaking"></param>
        public static void Speak(string txt, bool breaking)
        {
            if (textToSpeech == null) return;
            if (textToSpeech.isActiveAndEnabled)
            {
                if (textToSpeech.IsSpeaking())
                {
                    if (breaking)
                    {
                        StopSpeaking();
                        textToSpeech.StartSpeaking(txt);
                    }
                }
                else
                {
                    textToSpeech.StartSpeaking(txt);
                }
            }
        }

        /// <summary>
        /// 停止说话
        /// </summary>
        public static void StopSpeaking()
        {
            if (textToSpeech == null) return;
            textToSpeech.StopSpeaking();
        }
    }
}

8、蓝牙扫描

using System;
using UnityEngine;
using UnityEngine.UI;
#if NETFX_CORE
using Windows.Devices.Bluetooth;
using Windows.Devices.Bluetooth.GenericAttributeProfile;
using Windows.Devices.Enumeration;
using Windows.Foundation;
using Windows.System;
using Windows.UI.Popups;
#endif

public class e2_bluetooth : MonoBehaviour
{

    public Text showText;

    void Start()
    {
#if NETFX_CORE
        ScanDevices();
#endif
    }

    void Update()
    {

    }
#if NETFX_CORE
    /// <summary>
    /// 扫描所有BLE设备
    /// </summary>
    async void ScanDevices()
    {
        try
        {
            DeviceInformationCollection bleDevices = await DeviceInformation.FindAllAsync(BluetoothLEDevice.GetDeviceSelector());
            if (bleDevices.Count == 0)
            {
                await new MessageDialog("没有找到蓝牙设备，请确认是否已配对Hololens!").ShowAsync();
                return;
            }
            else
            {
                for (int i = 0; i < bleDevices.Count; i++)
                {
                    showText.text += bleDevices[i].Name + "\n";
                }
            }
        }
        catch (Exception ex)
        {
            await new MessageDialog("扫描失败: " + ex.Message).ShowAsync();
        }

    }
#endif

}

9、蓝牙接收数据

#if NETFX_CORE
using System;
using System.Collections.Generic;
using System.Linq;

using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Devices.Bluetooth.Advertisement;

using System.Text;
using UnityEngine;
using UnityEngine.UI;


namespace ZhiHui
{
    class BluetoothReceiver
    {
        BluetoothLEAdvertisementWatcher watcher;
        EventProcessor eventProcessor;
        public ushort BEACON_ID = 1775;

        public delegate void OnSucceedCallBack(GPSDataPacket gpsDataPacket);
        private OnSucceedCallBack onSucceedCallBack;

        public BluetoothReceiver(EventProcessor processor)
        {
            eventProcessor = processor;
        }

        public void SetOnSucceedCallBack(OnSucceedCallBack call)
        {
            onSucceedCallBack = call;
        }

        public void Start()
        {
            if (watcher != null)
            {
                watcher.Stop();
            }
            watcher = new BluetoothLEAdvertisementWatcher();
            var manufacturerData = new BluetoothLEManufacturerData
            {
                CompanyId = BEACON_ID
            };
            watcher.AdvertisementFilter.Advertisement.ManufacturerData.Add(manufacturerData);
            watcher.Received += Watcher_Received;
            watcher.Start();
        }

        private async void Watcher_Received(BluetoothLEAdvertisementWatcher sender, BluetoothLEAdvertisementReceivedEventArgs args)
        {
            ushort identifier = args.Advertisement.ManufacturerData.First().CompanyId;
            byte[] data = args.Advertisement.ManufacturerData.First().Data.ToArray();

            eventProcessor.QueueEvent(() =>
            {
                //Unity UI ingestion here
                GPSDataPacket gpsDataPacket = new GPSDataPacket(data);
                if (gpsDataPacket != null)
                {
                    if (gpsDataPacket.Latitude == 0
                    || gpsDataPacket.Longitude == 0)
                    {
                        return;
                    }
                    if (onSucceedCallBack != null)
                    {
                        onSucceedCallBack.Invoke(gpsDataPacket);
                    }
                }
            });
        }

    }
}
#endif