MirrorMirror
/
01
mirror of https://github.com/OpenInterpreter/01.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add onPlayBackStatusUpdate

This commit is contained in:
Ben Xu 2024-04-25 08:24:02 -04:00
parent d80bdbafe4
commit cd22f22300
3 changed files with 93 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
software/source/clients/ios/react-native/package-lock.json generated
View File

@ -21,7 +21,8 @@
"react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0",
"text-encoding": "^0.7.0"
"text-encoding": "^0.7.0",
"zustand": "^4.5.2"
},
"devDependencies": {
"@babel/core": "^7.20.0",
@ -6093,13 +6094,13 @@
"version": "15.7.11",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",
"integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==",
"dev": true
"devOptional": true
},
"node_modules/@types/react": {
"version": "18.2.63",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.63.tgz",
"integrity": "sha512-ppaqODhs15PYL2nGUOaOu2RSCCB4Difu4UFrP4I3NHLloXC/ESQzQMi9nvjfT1+rudd0d2L3fQPJxRSey+rGlQ==",
"dev": true,
"devOptional": true,
"dependencies": {
"@types/prop-types": "*",
"@types/scheduler": "*",
@ -6116,7 +6117,7 @@
"version": "0.16.8",
"resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz",
"integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==",
"dev": true
"devOptional": true
},
"node_modules/@types/stack-utils": {
"version": "2.0.3",
@ -7243,7 +7244,7 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
"integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
"dev": true
"devOptional": true
},
"node_modules/dag-map": {
"version": "1.0.2",
@ -12956,6 +12957,14 @@
"react": ">=16.8"
}
},
"node_modules/use-sync-external-store": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz",
"integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==",
"peerDependencies": {
"react": "^16.8.0 || ^17.0.0 || ^18.0.0"
}
},
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@ -13301,6 +13310,33 @@
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/zustand": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.2.tgz",
"integrity": "sha512-2cN1tPkDVkwCy5ickKrI7vijSjPksFRfqS6237NzT0vqSsztTNnQdHw9mmN7uBdk3gceVXU0a+21jFzFzAc9+g==",
"dependencies": {
"use-sync-external-store": "1.2.0"
},
"engines": {
"node": ">=12.7.0"
},
"peerDependencies": {
"@types/react": ">=16.8",
"immer": ">=9.0.6",
"react": ">=16.8"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"immer": {
"optional": true
},
"react": {
"optional": true
}
}
}
}
}

3
software/source/clients/ios/react-native/package.json
View File

@ -23,7 +23,8 @@
"react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0",
"text-encoding": "^0.7.0"
"text-encoding": "^0.7.0",
"zustand": "^4.5.2"
},
"devDependencies": {
"@babel/core": "^7.20.0",

74
software/source/clients/ios/react-native/src/screens/Main.tsx
View File

@ -3,6 +3,7 @@ import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
import * as FileSystem from 'expo-file-system';
import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av";
import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding';
import { create } from 'zustand';
interface MainProps {
route: {
@ -12,23 +13,43 @@ interface MainProps {
};
}
interface AudioQueueState {
audioQueue: string[]; // Define the audio queue type
addToQueue: (uri: string) => void; // Function to set audio queue
}
const useAudioQueueStore = create<AudioQueueState>((set) => ({
audioQueue: [], // initial state
addToQueue: (uri) => set((state) => ({ audioQueue: [...state.audioQueue, uri] })), // action to set audio queue
}));
interface SoundState {
sound: Audio.Sound | null; // Define the sound type
setSound: (newSound: Audio.Sound | null) => void; // Function to set sound
}
const useSoundStore = create<SoundState>((set) => ({
sound: null, // initial state
setSound: (newSound) => set({ sound: newSound }), // action to set sound
}));
const Main: React.FC<MainProps> = ({ route }) => {
const { scannedData } = route.params;
const [connectionStatus, setConnectionStatus] = useState<string>("Connecting...");
const [ws, setWs] = useState<WebSocket | null>(null);
const [recording, setRecording] = useState<Audio.Recording | null>(null);
const [audioQueue, setAudioQueue] = useState<string[]>([]);
const [sound, setSound] = useState<Audio.Sound | null>();
const addToQueue = useAudioQueueStore((state) => state.addToQueue);
const audioQueue = useAudioQueueStore((state) => state.audioQueue);
const setSound = useSoundStore((state) => state.setSound);
const sound = useSoundStore((state) => state.sound);
const audioDir = FileSystem.documentDirectory + '01/audio/';
const [permissionResponse, requestPermission] = Audio.usePermissions();
polyfillEncoding();
const reader = new FileReader();
const constructTempFilePath = async (buffer: string) => {
await dirExists();
const tempFilePath = `${audioDir}${Date.now()}.wav`;
await FileSystem.writeAsStringAsync(
tempFilePath,
buffer,
@ -37,7 +58,6 @@ const Main: React.FC<MainProps> = ({ route }) => {
}
);
return tempFilePath;
};
@ -54,15 +74,10 @@ const Main: React.FC<MainProps> = ({ route }) => {
}
const playNextAudio = async () => {
console.log("in playNextAudio audioQueue is", audioQueue.length);
console.log(`in playNextAudio audioQueue is ${audioQueue.length} and sound is ${sound}`);
if (sound != null){
console.log('Unloading Sound');
await sound.unloadAsync();
setSound(null);
}
if (audioQueue.length > 0) {
if (audioQueue.length > 0 && sound == null) {
const uri = audioQueue.shift() as string;
console.log("load audio from", uri);
@ -80,21 +95,32 @@ const Main: React.FC<MainProps> = ({ route }) => {
playNextAudio();
}
} else {
console.log("audioQueue is empty or sound is not null");
return;
}
};
const _onPlayBackStatusUpdate = async (status: AVPlaybackStatus) => {
if (isAVPlaybackStatusSuccess(status) && status.didJustFinish === true){
console.log("on playback status update sound is ", sound);
if (sound != null){
console.log('Unloading Sound');
await sound.unloadAsync();
}
setSound(null);
console.log("audio has finished playing, playing next audio");
console.log(audioQueue);
playNextAudio();
}
}
const isAVPlaybackStatusSuccess = (
status: AVPlaybackStatus
): status is AVPlaybackStatusSuccess => {
return (status as AVPlaybackStatusSuccess).isLoaded !== undefined;
};
const _onPlayBackStatusUpdate = (status: AVPlaybackStatus) => {
if (isAVPlaybackStatusSuccess(status) && status.didJustFinish){
playNextAudio();
}
}
useEffect(() => {
console.log("audioQueue has been updated:", audioQueue.length);
if (audioQueue.length == 1) {
@ -102,6 +128,10 @@ const Main: React.FC<MainProps> = ({ route }) => {
}
}, [audioQueue]);
useEffect(() => {
console.log("sound has been updated:", sound);
}, [sound]);
useEffect(() => {
let websocket: WebSocket;
try {
@ -121,13 +151,8 @@ const Main: React.FC<MainProps> = ({ route }) => {
const buffer = await message.content as string;
const filePath = await constructTempFilePath(buffer);
setAudioQueue((prevQueue) => [...prevQueue, filePath]);
addToQueue(filePath);
console.log("audio file written to", filePath);
if (message.format === "bytes.raw" && message.end && audioQueue.length >= 1) {
console.log("calling playNextAudio");
playNextAudio();
}
};
websocket.onerror = (error) => {
@ -209,6 +234,7 @@ const Main: React.FC<MainProps> = ({ route }) => {
console.log("fetched audio file", response);
const blob = await response.blob();
const reader = new FileReader();
reader.readAsArrayBuffer(blob);
reader.onloadend = () => {
const audioBytes = reader.result;