MirrorMirror
/
01
mirror of https://github.com/OpenInterpreter/01.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Updated OI, added try

This commit is contained in:
killian 2024-03-19 20:18:53 -07:00
parent a82b4b264f
commit 83b776bd1a
4 changed files with 128 additions and 2989 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

237
01OS/01OS/server/server.py
View File

@ -1,6 +1,7 @@
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() # take environment variables from .env. load_dotenv() # take environment variables from .env.
import traceback
from platformdirs import user_data_dir from platformdirs import user_data_dir
import ast import ast
import json import json
@ -183,136 +184,138 @@ async def send_messages(websocket: WebSocket):
async def listener(): async def listener():
while True: while True:
while True: try:
if not from_user.empty(): while True:
chunk = await from_user.get() if not from_user.empty():
break chunk = await from_user.get()
elif not from_computer.empty(): break
chunk = from_computer.get() elif not from_computer.empty():
break chunk = from_computer.get()
await asyncio.sleep(1) break
await asyncio.sleep(1)
message = accumulator.accumulate(chunk) message = accumulator.accumulate(chunk)
if message == None: if message == None:
# Will be None until we have a full message ready # Will be None until we have a full message ready
continue
# print(str(message)[:1000])
# At this point, we have our message
if message["type"] == "audio" and message["format"].startswith("bytes"):
if not message["content"]: # If it was nothing / silence
continue continue
# Convert bytes to audio file # print(str(message)[:1000])
# Format will be bytes.wav or bytes.opus
mime_type = "audio/" + message["format"].split(".")[1]
audio_file_path = bytes_to_wav(message["content"], mime_type)
# For microphone debugging: # At this point, we have our message
if False:
os.system(f"open {audio_file_path}")
import time
time.sleep(15)
text = stt(audio_file_path) if message["type"] == "audio" and message["format"].startswith("bytes"):
print("> ", text)
message = {"role": "user", "type": "message", "content": text}
# At this point, we have only text messages if "content" not in message or message["content"] == None: # If it was nothing / silence
if type(message["content"]) != str:
print("This should be a string, but it's not:", message["content"])
message["content"] = message["content"].decode()
# Custom stop message will halt us
if message["content"].lower().strip(".,! ") == "stop":
continue
# Load, append, and save conversation history
with open(conversation_history_path, 'r') as file:
messages = json.load(file)
messages.append(message)
with open(conversation_history_path, 'w') as file:
json.dump(messages, file, indent=4)
accumulated_text = ""
if any([m["type"] == "image" for m in messages]) and interpreter.llm.model.startswith("gpt-"):
interpreter.llm.model = "gpt-4-vision-preview"
interpreter.llm.supports_vision = True
for chunk in interpreter.chat(messages, stream=True, display=True):
if any([m["type"] == "image" for m in interpreter.messages]):
interpreter.llm.model = "gpt-4-vision-preview"
logger.debug("Got chunk:", chunk)
# Send it to the user
await to_device.put(chunk)
# Yield to the event loop, so you actually send it out
await asyncio.sleep(0.01)
if os.getenv('TTS_RUNNER') == "server":
# Speak full sentences out loud
if chunk["role"] == "assistant" and "content" in chunk and chunk["type"] == "message":
accumulated_text += chunk["content"]
sentences = split_into_sentences(accumulated_text)
# If we're going to speak, say we're going to stop sending text.
# This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "end": True})
if is_full_sentence(sentences[-1]):
for sentence in sentences:
await stream_tts_to_device(sentence)
accumulated_text = ""
else:
for sentence in sentences[:-1]:
await stream_tts_to_device(sentence)
accumulated_text = sentences[-1]
# If we're going to speak, say we're going to stop sending text.
# This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "start": True})
# If we have a new message, save our progress and go back to the top
if not from_user.empty():
# Check if it's just an end flag. We ignore those.
temp_message = await from_user.get()
if type(temp_message) is dict and temp_message.get("role") == "user" and temp_message.get("end"):
# Yup. False alarm.
continue continue
else:
# Whoops! Put that back
await from_user.put(temp_message)
with open(conversation_history_path, 'w') as file: # Convert bytes to audio file
json.dump(interpreter.messages, file, indent=4) # Format will be bytes.wav or bytes.opus
mime_type = "audio/" + message["format"].split(".")[1]
audio_file_path = bytes_to_wav(message["content"], mime_type)
# TODO: is triggering seemingly randomly # For microphone debugging:
#logger.info("New user message recieved. Breaking.") if False:
#break os.system(f"open {audio_file_path}")
import time
time.sleep(15)
# Also check if there's any new computer messages text = stt(audio_file_path)
if not from_computer.empty(): print("> ", text)
message = {"role": "user", "type": "message", "content": text}
with open(conversation_history_path, 'w') as file:
json.dump(interpreter.messages, file, indent=4)
logger.info("New computer message recieved. Breaking.") # At this point, we have only text messages
break
if type(message["content"]) != str:
print("This should be a string, but it's not:", message["content"])
message["content"] = message["content"].decode()
# Custom stop message will halt us
if message["content"].lower().strip(".,! ") == "stop":
continue
# Load, append, and save conversation history
with open(conversation_history_path, 'r') as file:
messages = json.load(file)
messages.append(message)
with open(conversation_history_path, 'w') as file:
json.dump(messages, file, indent=4)
accumulated_text = ""
if any([m["type"] == "image" for m in messages]) and interpreter.llm.model.startswith("gpt-"):
interpreter.llm.model = "gpt-4-vision-preview"
interpreter.llm.supports_vision = True
for chunk in interpreter.chat(messages, stream=True, display=True):
if any([m["type"] == "image" for m in interpreter.messages]):
interpreter.llm.model = "gpt-4-vision-preview"
logger.debug("Got chunk:", chunk)
# Send it to the user
await to_device.put(chunk)
# Yield to the event loop, so you actually send it out
await asyncio.sleep(0.01)
if os.getenv('TTS_RUNNER') == "server":
# Speak full sentences out loud
if chunk["role"] == "assistant" and "content" in chunk and chunk["type"] == "message":
accumulated_text += chunk["content"]
sentences = split_into_sentences(accumulated_text)
# If we're going to speak, say we're going to stop sending text.
# This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "end": True})
if is_full_sentence(sentences[-1]):
for sentence in sentences:
await stream_tts_to_device(sentence)
accumulated_text = ""
else:
for sentence in sentences[:-1]:
await stream_tts_to_device(sentence)
accumulated_text = sentences[-1]
# If we're going to speak, say we're going to stop sending text.
# This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "start": True})
# If we have a new message, save our progress and go back to the top
if not from_user.empty():
# Check if it's just an end flag. We ignore those.
temp_message = await from_user.get()
if type(temp_message) is dict and temp_message.get("role") == "user" and temp_message.get("end"):
# Yup. False alarm.
continue
else:
# Whoops! Put that back
await from_user.put(temp_message)
with open(conversation_history_path, 'w') as file:
json.dump(interpreter.messages, file, indent=4)
# TODO: is triggering seemingly randomly
#logger.info("New user message recieved. Breaking.")
#break
# Also check if there's any new computer messages
if not from_computer.empty():
with open(conversation_history_path, 'w') as file:
json.dump(interpreter.messages, file, indent=4)
logger.info("New computer message recieved. Breaking.")
break
except:
traceback.print_exc()
async def stream_tts_to_device(sentence): async def stream_tts_to_device(sentence):
force_task_completion_responses = [ force_task_completion_responses = [

2
01OS/01OS/server/tunnel.py
View File

@ -71,7 +71,7 @@ def create_tunnel(tunnel_method='ngrok', server_host='localhost', server_port=80
# If ngrok is installed, start it on the specified port # If ngrok is installed, start it on the specified port
# process = subprocess.Popen(f'ngrok http {server_port} --log=stdout', shell=True, stdout=subprocess.PIPE) # process = subprocess.Popen(f'ngrok http {server_port} --log=stdout', shell=True, stdout=subprocess.PIPE)
process = subprocess.Popen(f'ngrok http {server_port} --scheme http,https --log=stdout', shell=True, stdout=subprocess.PIPE) process = subprocess.Popen(f'ngrok http {server_port} --scheme http,https --log=stdout', shell=True, stdout=subprocess.PIPE)
# Initially, no URL is found # Initially, no URL is found
found_url = False found_url = False

2876
01OS/poetry.lock generated
View File

File diff suppressed because one or more lines are too long

2
01OS/pyproject.toml
View File

@ -24,11 +24,11 @@ pydub = "^0.25.1"
ngrok = "^1.0.0" ngrok = "^1.0.0"
simpleaudio = "^1.0.4" simpleaudio = "^1.0.4"
opencv-python = "^4.9.0.80" opencv-python = "^4.9.0.80"
open-interpreter = { git = "https://github.com/KillianLucas/open-interpreter.git", branch = "development" }
psutil = "^5.9.8" psutil = "^5.9.8"
typer = "^0.9.0" typer = "^0.9.0"
platformdirs = "^4.2.0" platformdirs = "^4.2.0"
rich = "^13.7.1" rich = "^13.7.1"
open-interpreter = "^0.2.2"
[build-system] [build-system]
requires = ["poetry-core"] requires = ["poetry-core"]