diff --git a/software/source/clients/ios/react-native/src/screens/Main.tsx b/software/source/clients/ios/react-native/src/screens/Main.tsx index 4bbde42..d6a69cb 100644 --- a/software/source/clients/ios/react-native/src/screens/Main.tsx +++ b/software/source/clients/ios/react-native/src/screens/Main.tsx @@ -19,6 +19,37 @@ const Main: React.FC = ({ route }) => { const [audioQueue, setAudioQueue] = useState([]); const [sound, setSound] = useState(); const audioDir = FileSystem.documentDirectory + '01/audio/'; + const Buffer = require('buffer').Buffer; + + const toBuffer = async (blob: Blob) => { + + const uri = await toDataURI(blob); + const base64 = uri.replace(/^.*,/g, ""); + return Buffer.from(base64, "base64"); + }; + + const toDataURI = (blob: Blob) => + new Promise((resolve) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = () => { + const uri = reader.result?.toString(); + resolve(uri); + }; + }); + + const constructTempFilePath = async (buffer: Buffer) => { + const tempFilePath = `${audioDir}${Date.now()}.wav`; + await FileSystem.writeAsStringAsync( + tempFilePath, + buffer.toString(), + { + encoding: FileSystem.EncodingType.Base64, + } + ); + + return tempFilePath; + }; async function dirExists() { @@ -33,7 +64,6 @@ const Main: React.FC = ({ route }) => { } const playNextAudio = async () => { - await dirExists(); console.log("in playNextAudio audioQueue is", audioQueue.length); if (audioQueue.length > 0) { @@ -65,6 +95,12 @@ const Main: React.FC = ({ route }) => { : undefined; }, [sound]); + useEffect(() => { + console.log("audioQueue has been updated:", audioQueue.length); + if (audioQueue.length == 1) { + playNextAudio(); + } + }, [audioQueue]); useEffect(() => { let websocket: WebSocket; @@ -79,31 +115,38 @@ const Main: React.FC = ({ route }) => { }; websocket.onmessage = async (e) => { + console.log("Received message from WebSocket", e.data); + + const blob = await e.data; + const buffer = await toBuffer(blob); + const filePath = await constructTempFilePath(buffer); + setAudioQueue((prevQueue) => [...prevQueue, filePath]); + console.log("audio file written to", filePath); + + if (e.data.format === "bytes.raw" && e.data.end && audioQueue.length > 1) { + console.log("calling playNextAudio"); + playNextAudio(); + } + + /** const message = JSON.parse(e.data); if (message.content) { const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, ""); console.log("parsedMessage", parsedMessage.slice(0, 30)); - const filePath = `${audioDir}${Date.now()}.mp3`; - await FileSystem.writeAsStringAsync( - filePath, - parsedMessage, - { - encoding: FileSystem.EncodingType.Base64, - } - ); - - console.log("audio file written to", filePath); - + const filePath = await constructFilePath(parsedMessage); setAudioQueue((prevQueue) => [...prevQueue, filePath]); + console.log("audio file written to", filePath); } - if (message.format === "bytes.raw" && message.end) { + if (message.format === "bytes.raw" && message.end && audioQueue.length > 1) { console.log("calling playNextAudio"); playNextAudio(); } + */ + }; websocket.onerror = (error) => { diff --git a/software/source/server/server.py b/software/source/server/server.py index 444298d..2328f52 100644 --- a/software/source/server/server.py +++ b/software/source/server/server.py @@ -21,6 +21,7 @@ from ..utils.accumulator import Accumulator from .utils.logs import setup_logging from .utils.logs import logger import base64 +from google.cloud import storage from ..utils.print_markdown import print_markdown @@ -202,6 +203,10 @@ async def send_messages(websocket: WebSocket): await websocket.send_json(message) elif isinstance(message, bytes): message = base64.b64encode(message) + print(f"Sending to the device: {type(message)} {str(message)[:100]}") + await websocket.send_bytes(message) + + """ str_bytes = str(message) json_bytes = { "role": "assistant", @@ -213,6 +218,7 @@ async def send_messages(websocket: WebSocket): f"Sending to the device: {type(json_bytes)} {str(json_bytes)[:100]}" ) await websocket.send_json(json_bytes) + """ else: raise TypeError("Message must be a dict or bytes") except: @@ -254,6 +260,7 @@ async def listener(): # Format will be bytes.wav or bytes.opus mime_type = "audio/" + message["format"].split(".")[1] audio_file_path = bytes_to_wav(message["content"], mime_type) + print("Audio file path:", audio_file_path) # For microphone debugging: if False: @@ -387,6 +394,19 @@ def stream_tts(sentence): with open(audio_file, "rb") as f: audio_bytes = f.read() + + storage_client = storage.Client(project="react-native-421323") + bucket = storage_client.bucket("01-audio") + blob = bucket.blob(f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav") + generation_match_precondition = 0 + + blob.upload_from_filename( + audio_file, if_generation_match=generation_match_precondition + ) + print( + f"Audio file {audio_file} uploaded to {datetime.datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav" + ) + os.remove(audio_file) file_type = "bytes.raw"