Live Translations in the Metaverse

/// <summary>
/// Creates and formats Amazon Transcribe URL Parameters
/// </summary>
/// <param name="dateTimeString">transcribe formatted DateTime.Now string</param>
/// <param name="credentialScope">scope for aws region, service, and terminator</param>
/// <param name="languageCode">transcribe language id (default en-US)</param>
/// <param name="mediaEncoding">audio format</param>
/// <param name="sampleRate">audio rate</param>
private string GenerateQueryParams(string dateTimeString, string credentialScope, string languageCode = "en-US", string mediaEncoding = "pcm", string sampleRate = "16000")
{
    var credentials = $"{accessKeyId}/{credentialScope}";
    var result = new Dictionary<string, string>
    {
        {"X-Amz-Algorithm", "AWS4-HMAC-SHA256"},
        {"X-Amz-Credential", credentials},
        {"X-Amz-Date", dateTimeString},
        {"X-Amz-Expires", "300"},
        {"X-Amz-Security-Token", <SESSION_TOKEN>},
        {"X-Amz-SignedHeaders", "host"},
        {"language-code", languageCode},
        {"media-encoding", mediaEncoding},
        {"sample-rate", sampleRate},
        {"transfer-encoding", "chunked"}
    };
    return string.Join("&", result.Select(x => $"{x.Key}={Uri.EscapeDataString(x.Value)}"));
}

/// <summary>
/// Creates the request and signatures follow AWS Signature Version 4
/// </summary>
/// <param name="host"></param>
/// <param name="dateString"></param>
/// <param name="dateTimeString"></param>
/// <param name="credentialScope"></param>
/// <returns></returns>
private string GenerateSignature(string host, string dateString, string dateTimeString, string credentialScope)
{
    var canonicalRequest = CanonicalizeRequest("/stream-transcription-websocket", host, dateTimeString, credentialScope);
    var canonicalRequestHashBytes = GetHash(canonicalRequest);

    // construct the string to be signed
    var stringToSign = new StringBuilder();
    stringToSign.AppendFormat(
"{0}-{1}\n{2}\n{3}\n", 
“AWS4”, 
"HMAC-SHA256", 
dateTimeString, 
credentialScope
);
    stringToSign.Append(ToHex(canonicalRequestHashBytes, true));

    var kha = KeyedHashAlgorithm.Create("HMACSHA256");
    kha.Key = GetSigningKey(
"HMACSHA256", 
<SECRET_KEY>, 
dateString, 
"transcribe"
    );

    // generate the final signature for the request, place into the result
    var signature = kha.ComputeHash(Encoding.UTF8.GetBytes(stringToSign.ToString()));
    var signatureString = ToHex(signature, true);
    return signatureString;
}

// cache websocket for use throughout functions
private WebSocket websocket;

/// <summary>
/// Initialize and Connect WebSocket before sending any audio events
/// </summary>
/// <param name="url">websocket url to Amazon Transcribe</param>
private async void Connect(string url)
{
    websocket = new WebSocket(url);

    websocket.OnOpen += () =>
    {
        Debug.Log("WebSocket Connection Open");
    };

    websocket.OnError += (e) =>
    {
        Debug.LogError($"WebSocket Error: {e}");
    };

    websocket.OnClose += (e) =>
    {
        Debug.Log($"WebSocket Connection Closed: {e}");
    };
   
   // listens for received messages
    websocket.OnMessage += HandleMessage;

    // waiting for messages
    await websocket.Connect();
}

// <summary>
/// Parses the payload response from Amazon Transcribe
/// </summary>
/// <param name="bytes">byte array payload</param>
private void HandleMessage(byte[] bytes)
{
    Debug.Log($"WebSocket Message Received");

    //First 8 bytes are the prelude with info about header lengths and total length.
    byte[] totalByteLengthBytes = new byte[4];
    Array.Copy(bytes, totalByteLengthBytes, 4);
    if (BitConverter.IsLittleEndian)
    {
        Array.Reverse(totalByteLengthBytes);
    }

    //an int32 is 4 bytes
    int totalByteLength = BitConverter.ToInt32(totalByteLengthBytes, 0);

    byte[] headersByteLengthBytes = new byte[4];
    Array.Copy(bytes, 4, headersByteLengthBytes, 0, 4);
    if (BitConverter.IsLittleEndian)
    {
        Array.Reverse(headersByteLengthBytes);
    }

    int headersByteLength = BitConverter.ToInt32(headersByteLengthBytes, 0);

    //Use the prelude to get the offset of the message.
    int offset = headersByteLength + 12;

    //Message length is everything but the headers, CRCs, and prelude.
    int payloadLength = totalByteLength - (headersByteLength + 16);
    byte[] payload = new byte[payloadLength];
    Array.Copy(bytes, offset, payload, 0, payloadLength);
   
   // convert byte payload to readable string
    string message = ParseWebsocketPayload(payload);
    if (!string.IsNullOrEmpty(message))
    {
        // invoke Amazon Translate to send message to other resources
        Translate(message, sourceLanguage, targetLanguage);
    }
}

/// <summary>
/// Parses payload to readable string
/// </summary>
/// <param name="payload">byte payload from websocket response</param>
/// <returns>string message</returns>
private string ParseWebsocketPayload(byte[] payload)
{
    // convert byte payload to utf8 string
    string stringPayload = System.Text.Encoding.UTF8.GetString(payload);
    if (string.IsNullOrEmpty(stringPayload))
    {
        return null;
    }

    // get transcribed message
    TranslateWebsocketMessage jsonMessage = JsonUtility.FromJson<TranslateWebsocketMessage>(stringPayload);
    if (jsonMessage != null)
    {
        var results = jsonMessage.Transcript.Results;
        if (results != null && results.Count > 0)
        {
            // only return Transcript if it is a full response and not partial
            if (!results[0].IsPartial)
            {
                return results[0].Alternatives[0].Transcript;
            }
        }
    }
    return null;
}

microphoneAudioSource.clip = Microphone.Start(null, false, 300, sampleRate);

/// <summary>
/// Captures Microphone Audi and converts to Websocket Payload for Amazon Transcribe
/// </summary>
/// <param name="microphoneSource">Microphone Audio Source component</param>
/// <param name="websocket">Websocket reference</param>
/// <param name="delayTime">Incremental wait time</param>
private IEnumerator IESendAudioEvent(AudioSource microphoneSource, WebSocket websocket, float delayTime = 0.2f)
{
    int offset = 0;
    while (websocket != null && websocket.State == WebSocketState.Open)
    {
       // delay to capture new audio
        yield return new WaitForSeconds(delayTime);
        int position = Microphone.GetPosition(null);
        int diff = position - offset;
        
        if (diff > 0)
        {
           // get audio sample data
            float[] currentSample = new float[diff * microphoneSource.clip.channels];
            _microphoneAudioSource.clip.GetData(currentSample, offset);
           
           // create audio clip and set data
            AudioClip newClip = AudioClip.Create("", currentSample.Length, microphoneSource.clip.channels, microphoneSource.clip.frequency, false);
            newClip.SetData(currentSample, 0);
          
           // convert clip to 16-bit pcm encoded byte payload & send via websocket
            byte[] payload = AudioUtils.CreateAudioEvent(AudioUtils.To16BitPCM(newClip));
            websocket.Send(payload);
        }
        offset = position;
    }
}

/// <summary>
/// Calls Amazon Translate to translate provided text
/// </summary>
/// <param name="message">text to translate</param>
/// <param name="sourceLanguage">speakers language</param>
/// <param name="targetLanguage">foreign language</param>
private async void Translate(string message, string sourceLanguage, string targetLanguage)
{
    try
    {
        using (var client = new AmazonTranslateClient(<ACCESS_KEY_ID>, <SECRET_KEY>, <SESSION_TOKEN>, <REGION>))
        {
           // call translate text passing in the translate request object
            var response = await client.TranslateTextAsync(
                new TranslateTextRequest()
                {
                    Text = message,
                    SourceLanguageCode = sourceLanguage,
                    TargetLanguageCode = targetLanguage
                }
            );

            if (response.HttpStatusCode != System.Net.HttpStatusCode.OK)
            {
                Debug.LogError($"Failed to translate text. Status Code: {response.HttpStatusCode}");
            }
            
           // passes the translated text string to Amazon Polly for speech synthesis
           SynthesizeSpeech(response.TranslatedText);
        }
    }
    catch (Exception e)
    {
        throw new Exception($"Translation Failed. {e}");
    }
}

/// <summary>
/// Calls Amazon Polly to convert text to speech
/// </summary>
/// <param name="message">text to convert</param>
/// <param name="targetVoice">Polly Voice ID</param>
/// <param name="sampleRate">desired rate for generated audio stream</param>
private async void SynthesizeSpeech(string message, string targetVoice = "Emma", int sampleRate = 16000)
{
    try
    {
        using (var client = new AmazonPollyClient(<ACCESS_KEY_ID>, <SECRET_KEY>, <SESSION_TOKEN>, <REGION>))
        {
            // convert targetVoice string to Amazon.Polly.VoiceId
            VoiceId voiceId = VoiceId.FindValue(targetVoice);

            // call Polly passing in the translated text message
            var response = await client.SynthesizeSpeechAsync(
                new SynthesizeSpeechRequest()
                {
                    Text = message,
                    VoiceId = voiceId,
                    OutputFormat = OutputFormat.Mp3,
                    SampleRate = sampleRate.ToString()
                }
            );

            if (response.HttpStatusCode != System.Net.HttpStatusCode.OK)
            {
                throw new Exception($"Failed to get audio clip. Status code: {response.HttpStatusCode}");
            }

            // save response audio stream to a local file, overwrite existing
            if (response.AudioStream != null)
            {
                CreateAudioFile(response.AudioStream);
            }
        }
    }
    catch (Exception e)
    {
        throw new Exception($"Speech Synthesis Failed. {e}");
    }
}

/// <summary>
/// Writes Audio Stream to local file
/// </summary>
/// <param name="audioStream">stream to write to file</param>
private void CreateAudioFile(Stream audioStream)
{
    string path = $"{Application.persistentDataPath}/polly-audio.mp3";
    using (FileStream fs = File.Create(path))
    {
        await response.AudioStream.CopyToAsync(fs);
    }
    // call request coroutine to read in unity audio clip content data
    StartCoroutine(IERequestAudioClip($"file://{path}"));
}

/// <summary>
/// Request Coroutine reads in Unity AudioClip content data so audio can be played
/// </summary>
/// <param name="path">path to audio file</param>
/// <param name="audioType">unity audio type (default MPEG)</param>
private IEnumerator IERequestAudioClip(string path, AudioType audioType = AudioType.MPEG)
{
    // unity web request multimedia loads local audio file as unity audio clip
    using (UnityWebRequest request = UnityWebRequestMultimedia.GetAudioClip(path, audioType))
    {
        yield return request.SendWebRequest();
        switch (request.result)
        {
            case UnityWebRequest.Result.ConnectionError:
            case UnityWebRequest.Result.DataProcessingError:
            case UnityWebRequest.Result.ProtocolError:
                Debug.LogError($"{request.result}: {request.error}");
                break;
            case UnityWebRequest.Result.Success:
                // call PlayAudio passing downloadHandler payload 
                PlayAudio(DownloadHandlerAudioClip.GetContent(request));
                break;
        }
    }
}

/// <summary>
/// Plays loaded AudioClip
/// </summary>
/// <param name="clip">unity audio clip to play</param>
private void PlayAudio(AudioClip clip)
{
    if (clip != null)
    {
        pollyAudioSource.PlayOneShot(clip);
    }
}