From 68c302ae2d3f26a3045bf05c01dcc1da867fc38c Mon Sep 17 00:00:00 2001 From: DebrisHauler Date: Mon, 14 Aug 2023 21:26:23 -0400 Subject: [PATCH] Added voice commands demo. --- .../6 - Voice Commands.unity | 1038 +++++++++++++++++ .../6 - Voice Commands.unity.meta | 7 + .../6 - Voice Commands/VoiceCommandsDemo.cs | 56 + .../VoiceCommandsDemo.cs.meta | 11 + .../VoiceCommandsManager.cs | 353 ++++++ .../VoiceCommandsManager.cs.meta | 11 + .../Runtime/Utils/LoopingMicrophone.cs | 114 ++ .../Runtime/Utils/LoopingMicrophone.cs.meta | 11 + 8 files changed, 1601 insertions(+) create mode 100644 Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity create mode 100644 Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity.meta create mode 100644 Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs create mode 100644 Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs.meta create mode 100644 Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs create mode 100644 Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs.meta create mode 100644 Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs create mode 100644 Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs.meta diff --git a/Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity b/Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity new file mode 100644 index 00000000..c63f067d --- /dev/null +++ b/Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity @@ -0,0 +1,1038 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!29 &1 +OcclusionCullingSettings: + m_ObjectHideFlags: 0 + serializedVersion: 2 + m_OcclusionBakeSettings: + smallestOccluder: 5 + smallestHole: 0.25 + backfaceThreshold: 100 + m_SceneGUID: 00000000000000000000000000000000 + m_OcclusionCullingData: {fileID: 0} +--- !u!104 &2 +RenderSettings: + m_ObjectHideFlags: 0 + serializedVersion: 9 + m_Fog: 0 + m_FogColor: {r: 0.5, g: 0.5, b: 0.5, a: 1} + m_FogMode: 3 + m_FogDensity: 0.01 + m_LinearFogStart: 0 + m_LinearFogEnd: 300 + m_AmbientSkyColor: {r: 0.212, g: 0.227, b: 0.259, a: 1} + m_AmbientEquatorColor: {r: 0.114, g: 0.125, b: 0.133, a: 1} + m_AmbientGroundColor: {r: 0.047, g: 0.043, b: 0.035, a: 1} + m_AmbientIntensity: 1 + m_AmbientMode: 0 + m_SubtractiveShadowColor: {r: 0.42, g: 0.478, b: 0.627, a: 1} + m_SkyboxMaterial: {fileID: 10304, guid: 0000000000000000f000000000000000, type: 0} + m_HaloStrength: 0.5 + m_FlareStrength: 1 + m_FlareFadeSpeed: 3 + m_HaloTexture: {fileID: 0} + m_SpotCookie: {fileID: 10001, guid: 0000000000000000e000000000000000, type: 0} + m_DefaultReflectionMode: 0 + m_DefaultReflectionResolution: 128 + m_ReflectionBounces: 1 + m_ReflectionIntensity: 1 + m_CustomReflection: {fileID: 0} + m_Sun: {fileID: 0} + m_IndirectSpecularColor: {r: 0.44657898, g: 0.4964133, b: 0.5748178, a: 1} + m_UseRadianceAmbientProbe: 0 +--- !u!157 &3 +LightmapSettings: + m_ObjectHideFlags: 0 + serializedVersion: 12 + m_GIWorkflowMode: 1 + m_GISettings: + serializedVersion: 2 + m_BounceScale: 1 + m_IndirectOutputScale: 1 + m_AlbedoBoost: 1 + m_EnvironmentLightingMode: 0 + m_EnableBakedLightmaps: 1 + m_EnableRealtimeLightmaps: 0 + m_LightmapEditorSettings: + serializedVersion: 12 + m_Resolution: 2 + m_BakeResolution: 40 + m_AtlasSize: 1024 + m_AO: 0 + m_AOMaxDistance: 1 + m_CompAOExponent: 1 + m_CompAOExponentDirect: 0 + m_ExtractAmbientOcclusion: 0 + m_Padding: 2 + m_LightmapParameters: {fileID: 0} + m_LightmapsBakeMode: 1 + m_TextureCompression: 1 + m_FinalGather: 0 + m_FinalGatherFiltering: 1 + m_FinalGatherRayCount: 256 + m_ReflectionCompression: 2 + m_MixedBakeMode: 2 + m_BakeBackend: 1 + m_PVRSampling: 1 + m_PVRDirectSampleCount: 32 + m_PVRSampleCount: 512 + m_PVRBounces: 2 + m_PVREnvironmentSampleCount: 256 + m_PVREnvironmentReferencePointCount: 2048 + m_PVRFilteringMode: 1 + m_PVRDenoiserTypeDirect: 1 + m_PVRDenoiserTypeIndirect: 1 + m_PVRDenoiserTypeAO: 1 + m_PVRFilterTypeDirect: 0 + m_PVRFilterTypeIndirect: 0 + m_PVRFilterTypeAO: 0 + m_PVREnvironmentMIS: 1 + m_PVRCulling: 1 + m_PVRFilteringGaussRadiusDirect: 1 + m_PVRFilteringGaussRadiusIndirect: 5 + m_PVRFilteringGaussRadiusAO: 2 + m_PVRFilteringAtrousPositionSigmaDirect: 0.5 + m_PVRFilteringAtrousPositionSigmaIndirect: 2 + m_PVRFilteringAtrousPositionSigmaAO: 1 + m_ExportTrainingData: 0 + m_TrainingDataDestination: TrainingData + m_LightProbeSampleCountMultiplier: 4 + m_LightingDataAsset: {fileID: 0} + m_LightingSettings: {fileID: 0} +--- !u!196 &4 +NavMeshSettings: + serializedVersion: 2 + m_ObjectHideFlags: 0 + m_BuildSettings: + serializedVersion: 2 + agentTypeID: 0 + agentRadius: 0.5 + agentHeight: 2 + agentSlope: 45 + agentClimb: 0.4 + ledgeDropHeight: 0 + maxJumpAcrossDistance: 0 + minRegionArea: 2 + manualCellSize: 0 + cellSize: 0.16666667 + manualTileSize: 0 + tileSize: 256 + accuratePlacement: 0 + maxJobWorkers: 0 + preserveTilesOutsideBounds: 0 + debug: + m_Flags: 0 + m_NavMeshData: {fileID: 0} +--- !u!1 &15072618 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 15072620} + - component: {fileID: 15072619} + m_Layer: 0 + m_Name: Directional Light + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!108 &15072619 +Light: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 15072618} + m_Enabled: 1 + serializedVersion: 10 + m_Type: 1 + m_Shape: 0 + m_Color: {r: 1, g: 0.95686275, b: 0.8392157, a: 1} + m_Intensity: 1 + m_Range: 10 + m_SpotAngle: 30 + m_InnerSpotAngle: 21.80208 + m_CookieSize: 10 + m_Shadows: + m_Type: 2 + m_Resolution: -1 + m_CustomResolution: -1 + m_Strength: 1 + m_Bias: 0.05 + m_NormalBias: 0.4 + m_NearPlane: 0.2 + m_CullingMatrixOverride: + e00: 1 + e01: 0 + e02: 0 + e03: 0 + e10: 0 + e11: 1 + e12: 0 + e13: 0 + e20: 0 + e21: 0 + e22: 1 + e23: 0 + e30: 0 + e31: 0 + e32: 0 + e33: 1 + m_UseCullingMatrixOverride: 0 + m_Cookie: {fileID: 0} + m_DrawHalo: 0 + m_Flare: {fileID: 0} + m_RenderMode: 0 + m_CullingMask: + serializedVersion: 2 + m_Bits: 4294967295 + m_RenderingLayerMask: 1 + m_Lightmapping: 4 + m_LightShadowCasterMode: 0 + m_AreaSize: {x: 1, y: 1} + m_BounceIntensity: 1 + m_ColorTemperature: 6570 + m_UseColorTemperature: 0 + m_BoundingSphereOverride: {x: 0, y: 0, z: 0, w: 0} + m_UseBoundingSphereOverride: 0 + m_UseViewFrustumForShadowCasterCull: 1 + m_ShadowRadius: 0 + m_ShadowAngle: 0 +--- !u!4 &15072620 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 15072618} + m_LocalRotation: {x: 0.40821788, y: -0.23456968, z: 0.10938163, w: 0.8754261} + m_LocalPosition: {x: 0, y: 3, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 1 + m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0} +--- !u!1 &292323135 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 292323138} + - component: {fileID: 292323137} + - component: {fileID: 292323136} + m_Layer: 0 + m_Name: Main Camera + m_TagString: MainCamera + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!81 &292323136 +AudioListener: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 292323135} + m_Enabled: 1 +--- !u!20 &292323137 +Camera: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 292323135} + m_Enabled: 1 + serializedVersion: 2 + m_ClearFlags: 1 + m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0} + m_projectionMatrixMode: 1 + m_GateFitMode: 2 + m_FOVAxisMode: 0 + m_SensorSize: {x: 36, y: 24} + m_LensShift: {x: 0, y: 0} + m_FocalLength: 50 + m_NormalizedViewPortRect: + serializedVersion: 2 + x: 0 + y: 0 + width: 1 + height: 1 + near clip plane: 0.3 + far clip plane: 1000 + field of view: 60 + orthographic: 0 + orthographic size: 5 + m_Depth: -1 + m_CullingMask: + serializedVersion: 2 + m_Bits: 4294967295 + m_RenderingPath: -1 + m_TargetTexture: {fileID: 0} + m_TargetDisplay: 0 + m_TargetEye: 3 + m_HDR: 1 + m_AllowMSAA: 1 + m_AllowDynamicResolution: 0 + m_ForceIntoRT: 0 + m_OcclusionCulling: 1 + m_StereoConvergence: 10 + m_StereoSeparation: 0.022 +--- !u!4 &292323138 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 292323135} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 1, z: -10} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1 &483366481 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 483366484} + - component: {fileID: 483366483} + - component: {fileID: 483366482} + m_Layer: 5 + m_Name: Confidence + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &483366482 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 483366481} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 25 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 2 + m_MaxSize: 40 + m_Alignment: 0 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: Commands here... +--- !u!222 &483366483 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 483366481} + m_CullTransparentMesh: 1 +--- !u!224 &483366484 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 483366481} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1171553769} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.030541068, y: 0.05} + m_AnchorMax: {x: 0.9771207, y: 0.95900005} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!1 &629435050 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 629435053} + - component: {fileID: 629435052} + - component: {fileID: 629435051} + m_Layer: 0 + m_Name: EventSystem + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &629435051 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 629435050} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 4f231c4fb786f3946a6b90b886c48677, type: 3} + m_Name: + m_EditorClassIdentifier: + m_HorizontalAxis: Horizontal + m_VerticalAxis: Vertical + m_SubmitButton: Submit + m_CancelButton: Cancel + m_InputActionsPerSecond: 10 + m_RepeatDelay: 0.5 + m_ForceModuleActive: 0 +--- !u!114 &629435052 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 629435050} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 76c392e42b5098c458856cdf6ecaaaa1, type: 3} + m_Name: + m_EditorClassIdentifier: + m_FirstSelected: {fileID: 0} + m_sendNavigationEvents: 1 + m_DragThreshold: 10 +--- !u!4 &629435053 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 629435050} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 3 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1 &841073212 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 841073213} + - component: {fileID: 841073215} + - component: {fileID: 841073214} + m_Layer: 5 + m_Name: Continuous Transcription + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &841073213 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 841073212} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1836568107} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.01642512, y: 0.062171213} + m_AnchorMax: {x: 0.9806763, y: 0.92830557} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &841073214 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 841073212} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 40 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 4 + m_MaxSize: 65 + m_Alignment: 0 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: Transcription here... +--- !u!222 &841073215 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 841073212} + m_CullTransparentMesh: 1 +--- !u!1 &941271491 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 941271495} + - component: {fileID: 941271494} + - component: {fileID: 941271493} + - component: {fileID: 941271492} + m_Layer: 5 + m_Name: Canvas + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &941271492 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 941271491} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: dc42784cf147c0c48a680349fa168899, type: 3} + m_Name: + m_EditorClassIdentifier: + m_IgnoreReversedGraphics: 1 + m_BlockingObjects: 0 + m_BlockingMask: + serializedVersion: 2 + m_Bits: 4294967295 +--- !u!114 &941271493 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 941271491} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 0cd44c1031e13a943bb63640046fad76, type: 3} + m_Name: + m_EditorClassIdentifier: + m_UiScaleMode: 0 + m_ReferencePixelsPerUnit: 100 + m_ScaleFactor: 1 + m_ReferenceResolution: {x: 800, y: 600} + m_ScreenMatchMode: 0 + m_MatchWidthOrHeight: 0 + m_PhysicalUnit: 3 + m_FallbackScreenDPI: 96 + m_DefaultSpriteDPI: 96 + m_DynamicPixelsPerUnit: 1 + m_PresetInfoIsWorld: 0 +--- !u!223 &941271494 +Canvas: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 941271491} + m_Enabled: 1 + serializedVersion: 3 + m_RenderMode: 0 + m_Camera: {fileID: 0} + m_PlaneDistance: 100 + m_PixelPerfect: 0 + m_ReceivesEvents: 1 + m_OverrideSorting: 0 + m_OverridePixelPerfect: 0 + m_SortingBucketNormalizedSize: 0 + m_AdditionalShaderChannelsFlag: 0 + m_SortingLayerID: 0 + m_SortingOrder: 0 + m_TargetDisplay: 0 +--- !u!224 &941271495 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 941271491} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 0, y: 0, z: 0} + m_ConstrainProportionsScale: 0 + m_Children: + - {fileID: 1171553769} + - {fileID: 1896100825} + - {fileID: 1836568107} + m_Father: {fileID: 0} + m_RootOrder: 2 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 0, y: 0} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0, y: 0} +--- !u!1 &1171553768 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1171553769} + - component: {fileID: 1171553771} + - component: {fileID: 1171553770} + m_Layer: 5 + m_Name: Panel + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &1171553769 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1171553768} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: + - {fileID: 483366484} + m_Father: {fileID: 941271495} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0.41} + m_AnchorMax: {x: 0.5, y: 1} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &1171553770 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1171553768} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.745283, g: 0.745283, b: 0.745283, a: 0.392} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 10907, guid: 0000000000000000f000000000000000, type: 0} + m_Type: 1 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 +--- !u!222 &1171553771 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1171553768} + m_CullTransparentMesh: 1 +--- !u!1 &1622069615 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1622069618} + - component: {fileID: 1622069617} + - component: {fileID: 1622069616} + - component: {fileID: 1622069619} + m_Layer: 0 + m_Name: Demo + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &1622069616 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1622069615} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: ba593f48e9338834d825d2d7b0aa5bec, type: 3} + m_Name: + m_EditorClassIdentifier: + SimilarityThreshold: 0.75 + modelPath: Whisper/ggml-tiny.bin + isModelPathInStreamingAssets: 1 + initOnAwake: 1 + language: en + translateToEnglish: 0 + strategy: 0 + noContext: 1 + singleSegment: 0 + enableTokens: 0 + initialPrompt: + tokensTimestamps: 0 + speedUp: 0 + audioCtx: 0 + transcriptionText: {fileID: 841073214} + confidenceText: {fileID: 483366482} + microphone: {fileID: 1622069619} +--- !u!114 &1622069617 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1622069615} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: d81cde0116fb0014eaaff38cf61500b3, type: 3} + m_Name: + m_EditorClassIdentifier: + VC: {fileID: 1622069616} + resultText: {fileID: 2119194294} +--- !u!4 &1622069618 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1622069615} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 517.5, y: 104.54999, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 4 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &1622069619 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1622069615} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 78bcd74d2f8ce2046babf0385f6d5d15, type: 3} + m_Name: + m_EditorClassIdentifier: + maxLengthSec: 10 + frequency: 16000 + evaluationTime: 0.5 + microphoneDropdown: {fileID: 0} + microphoneDefaultLabel: Default microphone +--- !u!1 &1836568106 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1836568107} + - component: {fileID: 1836568109} + - component: {fileID: 1836568108} + m_Layer: 5 + m_Name: Panel + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &1836568107 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1836568106} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: + - {fileID: 841073213} + m_Father: {fileID: 941271495} + m_RootOrder: 2 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 1, y: 0.41} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0} +--- !u!114 &1836568108 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1836568106} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 0.392} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 10907, guid: 0000000000000000f000000000000000, type: 0} + m_Type: 1 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 +--- !u!222 &1836568109 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1836568106} + m_CullTransparentMesh: 1 +--- !u!1 &1896100824 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1896100825} + - component: {fileID: 1896100827} + - component: {fileID: 1896100826} + m_Layer: 5 + m_Name: Panel + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &1896100825 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1896100824} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: + - {fileID: 2119194293} + m_Father: {fileID: 941271495} + m_RootOrder: 1 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.5, y: 0.41} + m_AnchorMax: {x: 1, y: 1} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &1896100826 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1896100824} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.745283, g: 0.745283, b: 0.745283, a: 0.392} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 10907, guid: 0000000000000000f000000000000000, type: 0} + m_Type: 1 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 +--- !u!222 &1896100827 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1896100824} + m_CullTransparentMesh: 1 +--- !u!1 &2119194292 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 2119194293} + - component: {fileID: 2119194295} + - component: {fileID: 2119194294} + m_Layer: 5 + m_Name: Reult + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &2119194293 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2119194292} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1896100825} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.017135268, y: 0.033676635} + m_AnchorMax: {x: 0.97325593, y: 0.9633234} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &2119194294 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2119194292} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 25 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 2 + m_MaxSize: 40 + m_Alignment: 0 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: +--- !u!222 &2119194295 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 2119194292} + m_CullTransparentMesh: 1 diff --git a/Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity.meta b/Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity.meta new file mode 100644 index 00000000..fe1ed183 --- /dev/null +++ b/Assets/Samples/6 - Voice Commands/6 - Voice Commands.unity.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: c6bf10504e2620f4e8de89e2bf26bcb4 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs b/Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs new file mode 100644 index 00000000..067280e5 --- /dev/null +++ b/Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs @@ -0,0 +1,56 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.UI; + +public class VoiceCommandsDemo : MonoBehaviour +{ + public VoiceCommandsManager VC; + + public delegate void thenDoDelegate(string command); + + public Text resultText; + + // Start is called before the first frame update + void Start() + { + RegisterCommands(); + + } + + public void RegisterCommands() + { + VC.RegisterCommand("Try saying this!", (thenDoDelegate)thenDoA); + VC.RegisterCommand("Here's my choice.", (thenDoDelegate)thenDoB); + VC.RegisterCommand("Where am I?", (thenDoDelegate)thenDoC); + } + + public void thenDoA(string command) + { + resultText.text += "You said \""+command+"\"\n"; + + VC?.UnregisterAll();//Unregistering is also important to clear transcription. + + RegisterCommands(); + } + + public void thenDoB(string command) + { + resultText.text += "You said \"" + command + "\"\n"; + + VC?.UnregisterAll();//Unregistering is also important to clear transcription. + + VC.RegisterCommand("I'll have some eggs and bacon", (thenDoDelegate)thenDoA); + VC.RegisterCommand("Not hungry, thanks.", (thenDoDelegate)thenDoA); + } + + public void thenDoC(string command) + { + resultText.text += "You said \"" + command + "\"\n"; + + VC?.UnregisterAll();//Unregistering is also important to clear transcription. + + //capitalizaiton and punctuation are totally forgiving... + VC.RegisterCommand("oh i remember now... I'm SITTING at my desk!", (thenDoDelegate)thenDoA); + } +} diff --git a/Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs.meta b/Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs.meta new file mode 100644 index 00000000..705acd35 --- /dev/null +++ b/Assets/Samples/6 - Voice Commands/VoiceCommandsDemo.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d81cde0116fb0014eaaff38cf61500b3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs b/Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs new file mode 100644 index 00000000..e86ce4b1 --- /dev/null +++ b/Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs @@ -0,0 +1,353 @@ +using System.Collections; +using System.Collections.Generic; +using UnityEngine; +using Whisper; +using Whisper.Utils; +using System.Threading.Tasks; +using System.Text.RegularExpressions; +using System; +using Whisper.Native; +using System.Linq; +using UnityEngine.UI; + +public class VoiceCommandsManager : MonoBehaviour +{ + private class command + { + public string text; + public Delegate thenDo; + public float score; + } + + private List commands; + + [Range(0f, 1f)] + [Tooltip("How similar must the user's transcription be to trigger a match?")] + public float SimilarityThreshold; + + [SerializeField] + [Tooltip("Path to model weights file")] + private string modelPath = "Whisper/ggml-tiny.bin"; + + [SerializeField] + [Tooltip("Determines whether the StreamingAssets folder should be prepended to the model path")] + private bool isModelPathInStreamingAssets = true; + + [SerializeField] + [Tooltip("Should model weights be loaded on awake?")] + private bool initOnAwake = true; + + [SerializeField] + [Header("Language")] + [Tooltip("Output text language. Use empty or \"auto\" for auto-detection.")] + public string language = "en"; + + [Tooltip("Force output text to English translation. Improves translation quality.")] + public bool translateToEnglish; + + + [Header("Advanced settings")] + [SerializeField] + private WhisperSamplingStrategy strategy = WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY; + + [Tooltip("Do not use past transcription (if any) as initial prompt for the decoder.")] + public bool noContext = true; + + [Tooltip("Force single segment output (useful for streaming).")] + public bool singleSegment; + + + [Tooltip("Output tokens with their confidence in each segment.")] + public bool enableTokens; + + [Tooltip("Initial prompt as a string variable. " + + "It should improve transcription quality or guide it to the right direction.")] + [TextArea] + public string initialPrompt; + + [Header("Experimental settings")] + [Tooltip("[EXPERIMENTAL] Output timestamps for each token. Need enabled tokens to work.")] + public bool tokensTimestamps; + + [Tooltip("[EXPERIMENTAL] Speed-up the audio by 2x using Phase Vocoder. " + + "These can significantly reduce the quality of the output.")] + public bool speedUp = false; + + [Tooltip("[EXPERIMENTAL] Overwrite the audio context size (0 = use default). " + + "These can significantly reduce the quality of the output.")] + public int audioCtx; + + public Text transcriptionText; + public Text confidenceText; + + public LoopingMicrophone microphone; + private WhisperWrapper _whisper; + private WhisperParams _params; + + private Task _task; + + public void RegisterCommand(string command, Delegate thenDo) + { + initializeCommands(); + command c = new command(); + c.text = CleanText(command); + c.thenDo = thenDo; + c.score = 0f; + commands.Add(c); + } + + private void initializeCommands() + { + if (commands == null) + { + commands = new List(); + } + } + + public void UnregisterAll() + { + commands.Clear();//no more commands. + microphone.StartRecord();//this will reinstantiate an audio clip. + //effectively clearing out the transcription. + } + + public bool IsModelPathInStreamingAssets + { + get => isModelPathInStreamingAssets; + set + { + if (IsLoaded || IsLoading) + { + throw new InvalidOperationException("Cannot change model path after loading the model"); + } + + isModelPathInStreamingAssets = value; + } + } + + /// + /// Checks if whisper weights are loaded and ready to be used. + /// + public bool IsLoaded => _whisper != null; + + /// + /// Checks if whisper weights are still loading and not ready. + /// + public bool IsLoading { get; private set; } + + private async void Awake() + { + if (!initOnAwake) + return; + await InitModel(); + if (microphone != null) + { + microphone.OnEvaluate += Evaluate; + microphone.OnRecordStop += MicrophoneOnRecordStop; + } + + initializeCommands(); + + microphone.StartRecord(); + } + + private void Start() + { + + } + + /// + /// Load model and default parameters. Prepare it for text transcription. + /// + public async Task InitModel() + { + // check if model is already loaded or actively loading + if (IsLoaded) + { + Debug.LogWarning("Whisper model is already loaded and ready for use!"); + return; + } + + if (IsLoading) + { + Debug.LogWarning("Whisper model is already loading!"); + return; + } + + // load model and default params + IsLoading = true; + try + { + var path = IsModelPathInStreamingAssets + ? Application.streamingAssetsPath + "/" + modelPath + : modelPath; + _whisper = await WhisperWrapper.InitFromFileAsync(path); + _params = WhisperParams.GetDefaultParams(strategy); + UpdateParams(); + } + catch (Exception e) + { + Debug.LogException(e); + } + + IsLoading = false; + } + private void UpdateParams() + { + _params.Language = language; + _params.Translate = translateToEnglish; + _params.NoContext = noContext; + _params.SingleSegment = singleSegment; + _params.SpeedUp = speedUp; + _params.AudioCtx = audioCtx; + _params.EnableTokens = enableTokens; + _params.TokenTimestamps = tokensTimestamps; + _params.InitialPrompt = initialPrompt; + } + + + private async void Evaluate(AudioClip clip) + { + if (_task != null && !_task.IsCompleted) + return; + + if (commands.Count == 0) + return; + + _task = _whisper.GetTextAsync(microphone.GetData(), clip.frequency, + clip.channels, _params); + + // append current transcription into temporary output + var res = await _task; + + + string confidenceReadout = ""; + string cleanTranscription = CleanText(res.Result); + + for(int i = 0; i < commands.Count; i++) + { + string transcription = cleanTranscription; + int LCS = LongestCommonSubsequence(commands[i].text, transcription); + float prevScore = 0f; + float Score = (2f * LCS) / (commands[i].text.Length + transcription.Length); + while ((transcription.Length > commands[i].text.Length) && Score > prevScore) + { + string[] pieces = transcription.Split(' '); + if (pieces.Length > 1) + { + // Join the pieces excluding the first piece (index 0) + transcription = string.Join(" ", pieces, 1, pieces.Length - 1); + LCS = LongestCommonSubsequence(commands[i].text, transcription); + prevScore = Score; + Score = (2f * LCS) / (commands[i].text.Length + transcription.Length); + } + else + { + break; + } + } + commands[i].score = Score; + confidenceReadout += Score.ToString("F2") + " : " + commands[i].text +"\n"; + } + Debug.Log(res.Result); + DisplayResult(cleanTranscription, confidenceReadout); + + command maxInstance = commands.OrderByDescending(x => x.score).FirstOrDefault(); + if(maxInstance.score > SimilarityThreshold) + { + maxInstance.thenDo.DynamicInvoke(maxInstance.text); + } + } + + private void MicrophoneOnRecordStop() + { + //Maybe start it again... + } + + private void DisplayResult(string cleanTranscription, string confidenceReadout) + { + //Debug.Log(result); + transcriptionText.text = cleanTranscription; + confidenceText.text = confidenceReadout; + } + + static string CleanText(string input) + { + input = RemoveEncapsulatedText('[', ']', input); + input = RemoveEncapsulatedText('(', ')', input); + input = RemoveEncapsulatedText('*', '*', input); + input = RemoveMultipleSpaces(input); + input = input.ToUpper(); + input = KeepOnlyAllowedCharacters(input); + return input; + } + + static string KeepOnlyAllowedCharacters(string input) + { + string pattern = @"[^A-Z ]"; + Regex regex = new Regex(pattern); + + string result = regex.Replace(input, ""); + + return result; + } + + static string RemoveEncapsulatedText(char start, char end, string input) + { + // Define the regular expression pattern to match text within square brackets + string pattern = @"\" + start + @"[^\" + end + @"]*\" + end; + + // Create a regular expression object + Regex regex = new Regex(pattern); + + // Use the regular expression to replace matches with an empty string + string result = regex.Replace(input, ""); + + return result; + } + + static string RemoveMultipleSpaces(string input) + { + // Define the regular expression pattern to match multiple spaces + string pattern = @"\s+"; + + // Create a regular expression object + Regex regex = new Regex(pattern); + + // Use the regular expression to replace matches with a single space + string result = regex.Replace(input, " "); + + return result; + } + + static int LongestCommonSubsequence(string str1, string str2) + { + int m = str1.Length; + int n = str2.Length; + + // Create a 2D array to store LCS lengths for subproblems + int[,] dp = new int[m + 1, n + 1]; + + // Build the LCS array in a bottom-up manner + for (int i = 0; i <= m; i++) + { + for (int j = 0; j <= n; j++) + { + if (i == 0 || j == 0) + { + dp[i, j] = 0; // Base case: LCS of an empty string and any other string is 0 + } + else if (str1[i - 1] == str2[j - 1]) + { + dp[i, j] = dp[i - 1, j - 1] + 1; // Characters match, extend LCS by 1 + } + else + { + dp[i, j] = Math.Max(dp[i - 1, j], dp[i, j - 1]); // Characters don't match, get the max LCS from the previous cells + } + } + } + + return dp[m, n]; // The bottom-right cell contains the length of the LCS + } +} \ No newline at end of file diff --git a/Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs.meta b/Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs.meta new file mode 100644 index 00000000..5733eb34 --- /dev/null +++ b/Assets/Samples/6 - Voice Commands/VoiceCommandsManager.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: ba593f48e9338834d825d2d7b0aa5bec +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs b/Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs new file mode 100644 index 00000000..affa2ab7 --- /dev/null +++ b/Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using JetBrains.Annotations; +using UnityEngine; +using UnityEngine.UI; + +namespace Whisper.Utils +{ + public class LoopingMicrophone : MonoBehaviour + { + public int maxLengthSec = 10; + public int frequency = 16000; + public float evaluationTime = 0.5f; + + private float elapsedTime = 0f; + + [Header("Microphone selection (optional)")] + [CanBeNull] public Dropdown microphoneDropdown; + public string microphoneDefaultLabel = "Default microphone"; + + private AudioClip _clip; + + private string _selectedMicDevice; + public string SelectedMicDevice + { + get => _selectedMicDevice; + set + { + if (value != null && !AvailableMicDevices.Contains(value)) + throw new ArgumentException("Microphone device not found"); + _selectedMicDevice = value; + } + } + + public string RecordStartMicDevice { get; private set; } + public bool IsRecording { get; private set; } + + public IEnumerable AvailableMicDevices => Microphone.devices; + + public delegate void OnEvaluateDelegate(AudioClip clip); + public delegate void OnRecordStopDelegate(); + public event OnEvaluateDelegate OnEvaluate; + public event OnRecordStopDelegate OnRecordStop; + + private void Awake() + { + IsRecording = false; + if (microphoneDropdown != null) + { + microphoneDropdown.options = AvailableMicDevices + .Prepend(microphoneDefaultLabel) + .Select(text => new Dropdown.OptionData(text)) + .ToList(); + microphoneDropdown.value = microphoneDropdown.options + .FindIndex(op => op.text == microphoneDefaultLabel); + microphoneDropdown.onValueChanged.AddListener(OnMicrophoneChanged); + } + } + + private void Update() + { + if (!IsRecording) + return; + + elapsedTime += Time.deltaTime; + if (elapsedTime >= evaluationTime) + elapsedTime -= evaluationTime; + OnEvaluate?.Invoke(_clip); + } + + private void OnMicrophoneChanged(int ind) + { + if (microphoneDropdown == null) return; + var opt = microphoneDropdown.options[ind]; + SelectedMicDevice = opt.text == microphoneDefaultLabel ? null : opt.text; + } + + public void StartRecord() + { + RecordStartMicDevice = SelectedMicDevice; + _clip = Microphone.Start(RecordStartMicDevice, true, maxLengthSec, frequency); + IsRecording = true; + } + + public void StopRecord() + { + if (!IsRecording) + return; + + Microphone.End(RecordStartMicDevice); + IsRecording = false; + + OnRecordStop?.Invoke(); + } + + public float[] GetData() + { + int pos = Microphone.GetPosition(RecordStartMicDevice); + if(pos == 0) + { + float[] arr = new float[_clip.samples]; + _clip.GetData(arr, 0); + return arr; + } + float[] arr1 = new float[_clip.samples - pos]; + float[] arr2 = new float[pos]; + _clip.GetData(arr1, pos); + _clip.GetData(arr2, 0); + return arr1.Concat(arr2).ToArray(); + } + + } +} \ No newline at end of file diff --git a/Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs.meta b/Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs.meta new file mode 100644 index 00000000..138231db --- /dev/null +++ b/Packages/com.whisper.unity/Runtime/Utils/LoopingMicrophone.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 78bcd74d2f8ce2046babf0385f6d5d15 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: