MirrorMirror
/
01
mirror of https://github.com/OpenInterpreter/01.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

reset accumulator on server to process multiple requests

This commit is contained in:
Ben Xu 2024-04-25 06:15:15 -04:00
parent 1e1db0836c
commit d80bdbafe4
4 changed files with 142 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
software/source/clients/ios/react-native/package-lock.json generated
View File

@ -18,8 +18,10 @@
"react": "18.2.0",
"react-native": "0.73.4",
"react-native-base64": "^0.2.1",
"react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0"
"react-native-screens": "~3.29.0",
"text-encoding": "^0.7.0"
},
"devDependencies": {
"@babel/core": "^7.20.0",
@ -6492,6 +6494,12 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
},
"node_modules/base-64": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz",
"integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==",
"peer": true
},
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@ -7852,6 +7860,12 @@
"resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-1.11.1.tgz",
"integrity": "sha512-ddQEtCOgYHTLlFUe/yH67dDBIoct5VIULthyT3LRJbEwdpzAgueKsX2FYK02ldh440V87PWKCamh7R9evk1rrg=="
},
"node_modules/fast-base64-decode": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fast-base64-decode/-/fast-base64-decode-1.0.0.tgz",
"integrity": "sha512-qwaScUgUGBYeDNRnbc/KyllVU88Jk1pRHPStuF/lO7B0/RTRLj7U0lkdTAutlBblY08rwZDff6tNU9cjv6j//Q==",
"peer": true
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@ -10782,6 +10796,15 @@
"os-tmpdir": "^1.0.0"
}
},
"node_modules/p-defer": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz",
"integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==",
"peer": true,
"engines": {
"node": ">=8"
}
},
"node_modules/p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
@ -11505,6 +11528,40 @@
"resolved": "https://registry.npmjs.org/react-native-base64/-/react-native-base64-0.2.1.tgz",
"integrity": "sha512-eHgt/MA8y5ZF0aHfZ1aTPcIkDWxza9AaEk4GcpIX+ZYfZ04RcaNahO+527KR7J44/mD3efYfM23O2C1N44ByWA=="
},
"node_modules/react-native-fetch-api": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/react-native-fetch-api/-/react-native-fetch-api-3.0.0.tgz",
"integrity": "sha512-g2rtqPjdroaboDKTsJCTlcmtw54E25OjyaunUP0anOZn4Fuo2IKs8BVfe02zVggA/UysbmfSnRJIqtNkAgggNA==",
"peer": true,
"dependencies": {
"p-defer": "^3.0.0"
}
},
"node_modules/react-native-get-random-values": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/react-native-get-random-values/-/react-native-get-random-values-1.11.0.tgz",
"integrity": "sha512-4BTbDbRmS7iPdhYLRcz3PGFIpFJBwNZg9g42iwa2P6FOv9vZj/xJc678RZXnLNZzd0qd7Q3CCF6Yd+CU2eoXKQ==",
"peer": true,
"dependencies": {
"fast-base64-decode": "^1.0.0"
},
"peerDependencies": {
"react-native": ">=0.56"
}
},
"node_modules/react-native-polyfill-globals": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/react-native-polyfill-globals/-/react-native-polyfill-globals-3.1.0.tgz",
"integrity": "sha512-6ACmV1SjXvZP2LN6J2yK58yNACKddcvoiKLrSQdISx32IdYStfdmGXrbAfpd+TANrTlIaZ2SLoFXohNwhnqm/w==",
"peerDependencies": {
"base-64": "*",
"react-native-fetch-api": "*",
"react-native-get-random-values": "*",
"react-native-url-polyfill": "*",
"text-encoding": "*",
"web-streams-polyfill": "*"
}
},
"node_modules/react-native-safe-area-context": {
"version": "4.8.2",
"resolved": "https://registry.npmjs.org/react-native-safe-area-context/-/react-native-safe-area-context-4.8.2.tgz",
@ -11527,6 +11584,18 @@
"react-native": "*"
}
},
"node_modules/react-native-url-polyfill": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/react-native-url-polyfill/-/react-native-url-polyfill-2.0.0.tgz",
"integrity": "sha512-My330Do7/DvKnEvwQc0WdcBnFPploYKp9CYlefDXzIdEaA+PAhDYllkvGeEroEzvc4Kzzj2O4yVdz8v6fjRvhA==",
"peer": true,
"dependencies": {
"whatwg-url-without-unicode": "8.0.0-3"
},
"peerDependencies": {
"react-native": "*"
}
},
"node_modules/react-native/node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
@ -12589,6 +12658,12 @@
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
},
"node_modules/text-encoding": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.7.0.tgz",
"integrity": "sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA==",
"deprecated": "no longer maintained"
},
"node_modules/text-table": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@ -12949,6 +13024,15 @@
"defaults": "^1.0.3"
}
},
"node_modules/web-streams-polyfill": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0.tgz",
"integrity": "sha512-0zJXHRAYEjM2tUfZ2DiSOHAa2aw1tisnnhU3ufD57R8iefL+DcdJyRBRyJpG+NUimDgbTI/lH+gAE1PAvV3Cgw==",
"peer": true,
"engines": {
"node": ">= 8"
}
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",

4
software/source/clients/ios/react-native/package.json
View File

@ -20,8 +20,10 @@
"react": "18.2.0",
"react-native": "0.73.4",
"react-native-base64": "^0.2.1",
"react-native-polyfill-globals": "^3.1.0",
"react-native-safe-area-context": "4.8.2",
"react-native-screens": "~3.29.0"
"react-native-screens": "~3.29.0",
"text-encoding": "^0.7.0"
},
"devDependencies": {
"@babel/core": "^7.20.0",

96
software/source/clients/ios/react-native/src/screens/Main.tsx
View File

@ -1,7 +1,8 @@
import React, { useState, useEffect } from "react";
import React, { useState, useEffect, useCallback } from "react";
import { View, Text, TouchableOpacity, StyleSheet } from "react-native";
import * as FileSystem from 'expo-file-system';
import { AVPlaybackStatus, AVPlaybackStatusSuccess, Audio } from "expo-av";
import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding';
interface MainProps {
route: {
@ -19,18 +20,23 @@ const Main: React.FC<MainProps> = ({ route }) => {
const [audioQueue, setAudioQueue] = useState<string[]>([]);
const [sound, setSound] = useState<Audio.Sound | null>();
const audioDir = FileSystem.documentDirectory + '01/audio/';
const [permissionResponse, requestPermission] = Audio.usePermissions();
polyfillEncoding();
const reader = new FileReader();
const constructTempFilePath = async (buffer: string) => {
await dirExists();
const tempFilePath = `${audioDir}${Date.now()}.wav`;
await FileSystem.writeAsStringAsync(
tempFilePath,
buffer,
{
encoding: FileSystem.EncodingType.Base64,
}
);
await FileSystem.writeAsStringAsync(
tempFilePath,
buffer,
{
encoding: FileSystem.EncodingType.Base64,
}
);
return tempFilePath;
};
@ -111,9 +117,9 @@ const Main: React.FC<MainProps> = ({ route }) => {
websocket.onmessage = async (e) => {
const message = JSON.parse(e.data);
console.log(message.content);
console.log(message.content.slice(0, 50));
const buffer = await message.content;
const buffer = await message.content as string;
const filePath = await constructTempFilePath(buffer);
setAudioQueue((prevQueue) => [...prevQueue, filePath]);
console.log("audio file written to", filePath);
@ -122,26 +128,6 @@ const Main: React.FC<MainProps> = ({ route }) => {
console.log("calling playNextAudio");
playNextAudio();
}
/**
const message = JSON.parse(e.data);
if (message.content) {
const parsedMessage = message.content.replace(/^b'|['"]|['"]$/g, "");
console.log("parsedMessage", parsedMessage.slice(0, 30));
const filePath = await constructFilePath(parsedMessage);
setAudioQueue((prevQueue) => [...prevQueue, filePath]);
console.log("audio file written to", filePath);
}
if (message.format === "bytes.raw" && message.end && audioQueue.length > 1) {
console.log("calling playNextAudio");
playNextAudio();
}
*/
};
websocket.onerror = (error) => {
@ -167,56 +153,76 @@ const Main: React.FC<MainProps> = ({ route }) => {
};
}, [scannedData]);
const startRecording = async () => {
const startRecording = useCallback(async () => {
if (recording) {
console.log("A recording is already in progress.");
return;
}
try {
console.log("Requesting permissions..");
await Audio.requestPermissionsAsync();
if (permissionResponse !== null && permissionResponse.status !== `granted`) {
console.log("Requesting permission..");
await requestPermission();
}
await Audio.setAudioModeAsync({
allowsRecordingIOS: true,
playsInSilentModeIOS: true,
});
console.log("Starting recording..");
const { recording: newRecording } = await Audio.Recording.createAsync(
Audio.RecordingOptionsPresets.HIGH_QUALITY
);
const newRecording = new Audio.Recording();
await newRecording.prepareToRecordAsync(Audio.RecordingOptionsPresets.HIGH_QUALITY);
await newRecording.startAsync();
setRecording(newRecording);
console.log("Recording started");
} catch (err) {
console.error("Failed to start recording", err);
}
};
}, []);
const stopRecording = async () => {
const stopRecording = useCallback(async () => {
console.log("Stopping recording..");
setRecording(null);
if (recording) {
await recording.stopAndUnloadAsync();
await Audio.setAudioModeAsync({
allowsRecordingIOS: false,
});
const uri = recording.getURI();
console.log("Recording stopped and stored at", uri);
console.log("recording uri at ", uri);
setRecording(null);
// sanity check play the audio recording locally
// recording is working fine; is the server caching the audio file somewhere?
/**
if (uri) {
const { sound } = await Audio.Sound.createAsync({ uri });
sound.playAsync();
console.log("playing audio recording from", uri);
}
*/
if (ws && uri) {
const response = await fetch(uri);
console.log("fetched audio file", response);
const blob = await response.blob();
const reader = new FileReader();
reader.readAsArrayBuffer(blob);
reader.onloadend = () => {
const audioBytes = reader.result;
if (audioBytes) {
ws.send(audioBytes);
console.log("sent audio bytes to WebSocket");
const audioArray = new Uint8Array(audioBytes as ArrayBuffer);
const decoder = new TextDecoder("utf-8");
console.log("sent audio bytes to WebSocket", decoder.decode(audioArray).slice(0, 50));
}
};
}
}
};
}, [recording]);
return (
<View style={styles.container}>

5
software/source/server/server.py
View File

@ -39,8 +39,6 @@ print("")
setup_logging()
accumulator = Accumulator()
app = FastAPI()
app_dir = user_data_dir("01")
@ -229,6 +227,8 @@ async def send_messages(websocket: WebSocket):
async def listener():
while True:
try:
accumulator = Accumulator()
while True:
if not from_user.empty():
chunk = await from_user.get()
@ -258,6 +258,7 @@ async def listener():
# Convert bytes to audio file
# Format will be bytes.wav or bytes.opus
mime_type = "audio/" + message["format"].split(".")[1]
print("input audio file content", message["content"][:100])
audio_file_path = bytes_to_wav(message["content"], mime_type)
print("Audio file path:", audio_file_path)