MirrorMirror
/
01
mirror of https://github.com/OpenInterpreter/01.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix missing Device for Windows

This commit is contained in:
Davy Peter Braun 2024-03-29 16:46:47 +01:00
parent 7d1a4d9777
commit 621c5cd048
6 changed files with 212 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
software/source/clients/base_device.py
View File

@ -1,5 +1,8 @@
from dotenv import load_dotenv from dotenv import load_dotenv
from source.server.utils.logs import setup_logging, logger
load_dotenv() # take environment variables from .env. load_dotenv() # take environment variables from .env.
setup_logging()
import os import os
import asyncio import asyncio
@ -11,7 +14,7 @@ from queue import Queue
from pynput import keyboard from pynput import keyboard
import json import json
import traceback import traceback
import websockets import websocket as wsc
import queue import queue
import pydub import pydub
import ast import ast
@ -26,7 +29,7 @@ import cv2
import base64 import base64
from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run? from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run?
# In the future, I guess kernel watching code should be elsewhere? Somewhere server / client agnostic? # In the future, I guess kernel watching code should be elsewhere? Somewhere server / client agnostic?
from ..server.utils.kernel import put_kernel_messages_into_queue from ..server.utils.kernel import KernelChecker, put_kernel_messages_into_queue
from ..server.utils.get_system_info import get_system_info from ..server.utils.get_system_info import get_system_info
from ..server.utils.process_utils import kill_process_tree from ..server.utils.process_utils import kill_process_tree
@ -41,13 +44,11 @@ from ..utils.accumulator import Accumulator
accumulator = Accumulator() accumulator = Accumulator()
# Configuration for Audio Recording # AudioSegment configuration
CHUNK = 1024 # Record in chunks of 1024 samples AUDIO_SAMPLE_WIDTH = 2
FORMAT = pyaudio.paInt16 # 16 bits per sample AUDIO_FRAME_RATE = 16000
CHANNELS = 1 # Mono AUDIO_MONO_CHANNEL = 1
RATE = 44100 # Sample rate AUDIO_CODE_RUNNER_CLIENT = "client"
RECORDING = False # Flag to control recording state
SPACEBAR_PRESSED = False # Flag to track spacebar press state
# Camera configuration # Camera configuration
CAMERA_ENABLED = os.getenv('CAMERA_ENABLED', False) CAMERA_ENABLED = os.getenv('CAMERA_ENABLED', False)
@ -71,6 +72,14 @@ class Device:
self.audiosegments = [] self.audiosegments = []
self.server_url = "" self.server_url = ""
# Configuration for Audio Recording
self.CHUNK = 1024 # Record in chunks of 1024 samples
self.FORMAT = pyaudio.paInt16 # 16 bits per sample
self.CHANNELS = 1 # Mono
self.RATE = 44100 # Sample rate
self.RECORDING = False # Flag to control recording state
self.SPACEBAR_PRESSED = False # Flag to track spacebar press state
def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX): def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX):
"""Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list.""" """Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list."""
image_path = None image_path = None
@ -98,7 +107,7 @@ class Device:
cap.release() cap.release()
return image_path return image_path
def encode_image_to_base64(self, image_path): def encode_image_to_base64(self, image_path):
"""Encodes an image file to a base64 string.""" """Encodes an image file to a base64 string."""
@ -124,7 +133,7 @@ class Device:
self.add_image_to_send_queue(image_path) self.add_image_to_send_queue(image_path)
self.captured_images.clear() # Clear the list after sending self.captured_images.clear() # Clear the list after sending
async def play_audiosegments(self): async def play_audiosegments(self):
"""Plays them sequentially.""" """Plays them sequentially."""
while True: while True:
@ -141,7 +150,6 @@ class Device:
def record_audio(self): def record_audio(self):
if os.getenv('STT_RUNNER') == "server": if os.getenv('STT_RUNNER') == "server":
# STT will happen on the server. we're sending audio. # STT will happen on the server. we're sending audio.
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "start": True}) send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "start": True})
@ -152,20 +160,19 @@ class Device:
raise Exception("STT_RUNNER must be set to either 'client' or 'server'.") raise Exception("STT_RUNNER must be set to either 'client' or 'server'.")
"""Record audio from the microphone and add it to the queue.""" """Record audio from the microphone and add it to the queue."""
stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK) stream = p.open(format=self.FORMAT, channels=self.CHANNELS, rate=self.RATE, input=True, frames_per_buffer=self.CHUNK)
print("Recording started...") print("Recording started...")
global RECORDING
# Create a temporary WAV file to store the audio data # Create a temporary WAV file to store the audio data
temp_dir = tempfile.gettempdir() temp_dir = tempfile.gettempdir()
wav_path = os.path.join(temp_dir, f"audio_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav") wav_path = os.path.join(temp_dir, f"audio_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
wav_file = wave.open(wav_path, 'wb') wav_file = wave.open(wav_path, 'wb')
wav_file.setnchannels(CHANNELS) wav_file.setnchannels(self.CHANNELS)
wav_file.setsampwidth(p.get_sample_size(FORMAT)) wav_file.setsampwidth(p.get_sample_size(self.FORMAT))
wav_file.setframerate(RATE) wav_file.setframerate(self.RATE)
while RECORDING: while self.RECORDING:
data = stream.read(CHUNK, exception_on_overflow=False) data = stream.read(self.CHUNK, exception_on_overflow=False)
wav_file.writeframes(data) wav_file.writeframes(data)
wav_file.close() wav_file.close()
@ -173,7 +180,7 @@ class Device:
stream.close() stream.close()
print("Recording stopped.") print("Recording stopped.")
duration = wav_file.getnframes() / RATE duration = wav_file.getnframes() / self.RATE
if duration < 0.3: if duration < 0.3:
# Just pressed it. Send stop message # Just pressed it. Send stop message
if os.getenv('STT_RUNNER') == "client": if os.getenv('STT_RUNNER') == "client":
@ -198,10 +205,10 @@ class Device:
else: else:
# Stream audio # Stream audio
with open(wav_path, 'rb') as audio_file: with open(wav_path, 'rb') as audio_file:
byte_data = audio_file.read(CHUNK) byte_data = audio_file.read(self.CHUNK)
while byte_data: while byte_data:
send_queue.put(byte_data) send_queue.put(byte_data)
byte_data = audio_file.read(CHUNK) byte_data = audio_file.read(self.CHUNK)
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "end": True}) send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "end": True})
if os.path.exists(wav_path): if os.path.exists(wav_path):
@ -209,15 +216,14 @@ class Device:
def toggle_recording(self, state): def toggle_recording(self, state):
"""Toggle the recording state.""" """Toggle the recording state."""
global RECORDING, SPACEBAR_PRESSED if state and not self.SPACEBAR_PRESSED:
if state and not SPACEBAR_PRESSED: self.SPACEBAR_PRESSED = True
SPACEBAR_PRESSED = True if not self.RECORDING:
if not RECORDING: self.RECORDING = True
RECORDING = True
threading.Thread(target=self.record_audio).start() threading.Thread(target=self.record_audio).start()
elif not state and SPACEBAR_PRESSED: elif not state and self.SPACEBAR_PRESSED:
SPACEBAR_PRESSED = False self.SPACEBAR_PRESSED = False
RECORDING = False self.RECORDING = False
def on_press(self, key): def on_press(self, key):
"""Detect spacebar press and Ctrl+C combination.""" """Detect spacebar press and Ctrl+C combination."""
@ -239,7 +245,7 @@ class Device:
elif CAMERA_ENABLED and key == keyboard.KeyCode.from_char('c'): elif CAMERA_ENABLED and key == keyboard.KeyCode.from_char('c'):
self.fetch_image_from_camera() self.fetch_image_from_camera()
async def message_sender(self, websocket): async def message_sender(self, websocket):
while True: while True:
message = await asyncio.get_event_loop().run_in_executor(None, send_queue.get) message = await asyncio.get_event_loop().run_in_executor(None, send_queue.get)
@ -254,62 +260,74 @@ class Device:
show_connection_log = True show_connection_log = True
while True: while True:
try: try:
async with websockets.connect(WS_URL) as websocket: print("Attempting to connect to the WS server...")
if CAMERA_ENABLED:
print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
else:
print("\nHold the spacebar to start recording. Press CTRL-C to exit.")
asyncio.create_task(self.message_sender(websocket))
while True: try:
await asyncio.sleep(0.01) ws = wsc.create_connection(WS_URL, timeout=5)
chunk = await websocket.recv() except wsc.WebSocketTimeoutException:
print("Timeout while trying to connect to the WebSocket server.")
continue
print("Connected to the WS server.")
logger.debug(f"Got this message from the server: {type(chunk)} {chunk}") if CAMERA_ENABLED:
print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
else:
print("\nHold the spacebar to start recording. Press CTRL-C to exit.")
if type(chunk) == str: asyncio.create_task(self.message_sender(ws))
chunk = json.loads(chunk)
message = accumulator.accumulate(chunk) while True:
if message == None: await asyncio.sleep(0.01)
# Will be None until we have a full message ready chunk = await ws.recv()
continue
# At this point, we have our message logger.debug(f"Got this message from the server: {type(chunk)} {chunk}")
if message["type"] == "audio" and message["format"].startswith("bytes"): if isinstance(chunk, str):
chunk = json.loads(chunk)
# Convert bytes to audio file message = accumulator.accumulate(chunk)
if message is None:
# Will be None until we have a full message ready
continue
audio_bytes = message["content"] # At this point, we have our message
# Create an AudioSegment instance with the raw data if message["type"] == "audio" and message["format"].startswith("bytes"):
audio = AudioSegment( # Convert bytes to audio file
# raw audio data (bytes) audio_bytes = message["content"]
data=audio_bytes,
# signed 16-bit little-endian format
sample_width=2,
# 16,000 Hz frame rate
frame_rate=16000,
# mono sound
channels=1
)
self.audiosegments.append(audio) # Create an AudioSegment instance with the raw data
audio = AudioSegment(
data=audio_bytes,
sample_width=AUDIO_SAMPLE_WIDTH,
frame_rate=AUDIO_FRAME_RATE,
channels=AUDIO_MONO_CHANNEL
)
# Run the code if that's the client's job self.audiosegments.append(audio)
if os.getenv('CODE_RUNNER') == "client":
if message["type"] == "code" and "end" in message: # Run the code if that's the client's job
language = message["format"] if os.getenv('CODE_RUNNER') == AUDIO_CODE_RUNNER_CLIENT:
code = message["content"] if message["type"] == "code" and "end" in message:
result = interpreter.computer.run(language, code) language = message["format"]
send_queue.put(result) code = message["content"]
except: result = interpreter.computer.run(language, code)
send_queue.put(result)
except wsc.WebSocketConnectionClosedException:
print("WebSocket connection closed unexpectedly.")
if show_connection_log:
logger.info(f"Reconnecting to `{WS_URL}`...")
show_connection_log = False
await asyncio.sleep(2)
except wsc.WebSocketAddressException:
print(f"Invalid WebSocket URI: `{WS_URL}`. Please check the URI and try again.")
break # Exit the loop as the URI is invalid and retrying won't help
except Exception as e:
logger.debug(traceback.format_exc()) logger.debug(traceback.format_exc())
if show_connection_log: if show_connection_log:
logger.info(f"Connecting to `{WS_URL}`...") logger.info(f"Connecting to `{WS_URL}`...")
show_connection_log = False show_connection_log = False
await asyncio.sleep(2) await asyncio.sleep(2)
async def start_async(self): async def start_async(self):
@ -320,13 +338,14 @@ class Device:
# Start watching the kernel if it's your job to do that # Start watching the kernel if it's your job to do that
if os.getenv('CODE_RUNNER') == "client": if os.getenv('CODE_RUNNER') == "client":
asyncio.create_task(put_kernel_messages_into_queue(send_queue)) kernel_checker = KernelChecker()
asyncio.create_task(put_kernel_messages_into_queue(kernel_checker, send_queue))
asyncio.create_task(self.play_audiosegments()) asyncio.create_task(self.play_audiosegments())
# If Raspberry Pi, add the button listener, otherwise use the spacebar # If Raspberry Pi, add the button listener, otherwise use the spacebar
if current_platform.startswith("raspberry-pi"): if current_platform.startswith("raspberry-pi"):
logger.info("Raspberry Pi detected, using button on GPIO pin 15") print("Raspberry Pi detected, using button on GPIO pin 15")
# Use GPIO pin 15 # Use GPIO pin 15
pindef = ["gpiochip4", "15"] # gpiofind PIN15 pindef = ["gpiochip4", "15"] # gpiofind PIN15
print("PINDEF", pindef) print("PINDEF", pindef)

0
software/source/clients/windows/__init__.py Normal file
View File

10
software/source/clients/windows/device.py Normal file
View File

@ -0,0 +1,10 @@
from ..base_device import Device
device = Device()
def main(server_url):
device.server_url = server_url
device.start()
if __name__ == "__main__":
main()

39
software/source/server/server.py
View File

@ -16,7 +16,7 @@ from starlette.websockets import WebSocket, WebSocketDisconnect
from pathlib import Path from pathlib import Path
import asyncio import asyncio
import urllib.parse import urllib.parse
from .utils.kernel import put_kernel_messages_into_queue from .utils.kernel import KernelChecker, put_kernel_messages_into_queue
from .i import configure_interpreter from .i import configure_interpreter
from interpreter import interpreter from interpreter import interpreter
from ..utils.accumulator import Accumulator from ..utils.accumulator import Accumulator
@ -86,7 +86,7 @@ if os.getenv('CODE_RUNNER') == "device":
to_device.put({"role": "assistant", "type": "code", "format": "python", "start": True}) to_device.put({"role": "assistant", "type": "code", "format": "python", "start": True})
to_device.put(message) to_device.put(message)
to_device.put({"role": "assistant", "type": "code", "format": "python", "end": True}) to_device.put({"role": "assistant", "type": "code", "format": "python", "end": True})
# Stream the response # Stream the response
logger.info("Waiting for the device to respond...") logger.info("Waiting for the device to respond...")
while True: while True:
@ -165,13 +165,13 @@ async def receive_messages(websocket: WebSocket):
return return
else: else:
raise raise
async def send_messages(websocket: WebSocket): async def send_messages(websocket: WebSocket):
while True: while True:
message = await to_device.get() message = await to_device.get()
#print(f"Sending to the device: {type(message)} {str(message)[:100]}") #print(f"Sending to the device: {type(message)} {str(message)[:100]}")
try: try:
if isinstance(message, dict): if isinstance(message, dict):
await websocket.send_json(message) await websocket.send_json(message)
@ -197,7 +197,7 @@ async def listener():
break break
await asyncio.sleep(1) await asyncio.sleep(1)
message = accumulator.accumulate(chunk) message = accumulator.accumulate(chunk)
if message == None: if message == None:
@ -251,7 +251,7 @@ async def listener():
if any([m["type"] == "image" for m in messages]) and interpreter.llm.model.startswith("gpt-"): if any([m["type"] == "image" for m in messages]) and interpreter.llm.model.startswith("gpt-"):
interpreter.llm.model = "gpt-4-vision-preview" interpreter.llm.model = "gpt-4-vision-preview"
interpreter.llm.supports_vision = True interpreter.llm.supports_vision = True
for chunk in interpreter.chat(messages, stream=True, display=True): for chunk in interpreter.chat(messages, stream=True, display=True):
if any([m["type"] == "image" for m in interpreter.messages]): if any([m["type"] == "image" for m in interpreter.messages]):
@ -263,18 +263,18 @@ async def listener():
await to_device.put(chunk) await to_device.put(chunk)
# Yield to the event loop, so you actually send it out # Yield to the event loop, so you actually send it out
await asyncio.sleep(0.01) await asyncio.sleep(0.01)
if os.getenv('TTS_RUNNER') == "server": if os.getenv('TTS_RUNNER') == "server":
# Speak full sentences out loud # Speak full sentences out loud
if chunk["role"] == "assistant" and "content" in chunk and chunk["type"] == "message": if chunk["role"] == "assistant" and "content" in chunk and chunk["type"] == "message":
accumulated_text += chunk["content"] accumulated_text += chunk["content"]
sentences = split_into_sentences(accumulated_text) sentences = split_into_sentences(accumulated_text)
# If we're going to speak, say we're going to stop sending text. # If we're going to speak, say we're going to stop sending text.
# This should be fixed probably, we should be able to do both in parallel, or only one. # This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences): if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "end": True}) await to_device.put({"role": "assistant", "type": "message", "end": True})
if is_full_sentence(sentences[-1]): if is_full_sentence(sentences[-1]):
for sentence in sentences: for sentence in sentences:
await stream_tts_to_device(sentence) await stream_tts_to_device(sentence)
@ -288,13 +288,13 @@ async def listener():
# This should be fixed probably, we should be able to do both in parallel, or only one. # This should be fixed probably, we should be able to do both in parallel, or only one.
if any(is_full_sentence(sentence) for sentence in sentences): if any(is_full_sentence(sentence) for sentence in sentences):
await to_device.put({"role": "assistant", "type": "message", "start": True}) await to_device.put({"role": "assistant", "type": "message", "start": True})
# If we have a new message, save our progress and go back to the top # If we have a new message, save our progress and go back to the top
if not from_user.empty(): if not from_user.empty():
# Check if it's just an end flag. We ignore those. # Check if it's just an end flag. We ignore those.
temp_message = await from_user.get() temp_message = await from_user.get()
if type(temp_message) is dict and temp_message.get("role") == "user" and temp_message.get("end"): if type(temp_message) is dict and temp_message.get("role") == "user" and temp_message.get("end"):
# Yup. False alarm. # Yup. False alarm.
continue continue
@ -311,7 +311,7 @@ async def listener():
# Also check if there's any new computer messages # Also check if there's any new computer messages
if not from_computer.empty(): if not from_computer.empty():
with open(conversation_history_path, 'w') as file: with open(conversation_history_path, 'w') as file:
json.dump(interpreter.messages, file, indent=4) json.dump(interpreter.messages, file, indent=4)
@ -333,7 +333,7 @@ async def stream_tts_to_device(sentence):
await to_device.put(chunk) await to_device.put(chunk)
def stream_tts(sentence): def stream_tts(sentence):
audio_file = tts(sentence) audio_file = tts(sentence)
with open(audio_file, "rb") as f: with open(audio_file, "rb") as f:
@ -382,13 +382,13 @@ async def main(server_host, server_port, llm_service, model, llm_supports_vision
services_directory = os.path.join(application_directory, 'services') services_directory = os.path.join(application_directory, 'services')
service_dict = {'llm': llm_service, 'tts': tts_service, 'stt': stt_service} service_dict = {'llm': llm_service, 'tts': tts_service, 'stt': stt_service}
# Create a temp file with the session number # Create a temp file with the session number
session_file_path = os.path.join(user_data_dir('01'), '01-session.txt') session_file_path = os.path.join(user_data_dir('01'), '01-session.txt')
with open(session_file_path, 'w') as session_file: with open(session_file_path, 'w') as session_file:
session_id = int(datetime.datetime.now().timestamp() * 1000) session_id = int(datetime.datetime.now().timestamp() * 1000)
session_file.write(str(session_id)) session_file.write(str(session_id))
for service in service_dict: for service in service_dict:
service_directory = os.path.join(services_directory, service, service_dict[service]) service_directory = os.path.join(services_directory, service, service_dict[service])
@ -408,20 +408,21 @@ async def main(server_host, server_port, llm_service, model, llm_supports_vision
}) })
module = import_module(f'.server.services.{service}.{service_dict[service]}.{service}', package='source') module = import_module(f'.server.services.{service}.{service_dict[service]}.{service}', package='source')
ServiceClass = getattr(module, service.capitalize()) ServiceClass = getattr(module, service.capitalize())
service_instance = ServiceClass(config) service_instance = ServiceClass(config)
globals()[service] = getattr(service_instance, service) globals()[service] = getattr(service_instance, service)
interpreter.llm.completions = llm interpreter.llm.completions = llm
# Start listening # Start listening
asyncio.create_task(listener()) asyncio.create_task(listener())
# Start watching the kernel if it's your job to do that # Start watching the kernel if it's your job to do that
if True: # in the future, code can run on device. for now, just server. if True: # in the future, code can run on device. for now, just server.
asyncio.create_task(put_kernel_messages_into_queue(from_computer)) kernel_checker = KernelChecker()
asyncio.create_task(put_kernel_messages_into_queue(kernel_checker, from_computer))
config = Config(app, host=server_host, port=int(server_port), lifespan='on') config = Config(app, host=server_host, port=int(server_port), lifespan='on')
server = Server(config) server = Server(config)
await server.serve() await server.serve()

115
software/source/server/utils/kernel.py
View File

@ -1,63 +1,74 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import asyncio import asyncio
import subprocess import subprocess
import platform import platform
from dotenv import load_dotenv
from .logs import setup_logging, logger
from .logs import setup_logging load_dotenv() # take environment variables from .env.
from .logs import logger
setup_logging() setup_logging()
def get_kernel_messages(): class KernelChecker:
""" def __init__(self):
Is this the way to do this? self._last_messages = ""
"""
current_platform = platform.system()
if current_platform == "Darwin":
process = subprocess.Popen(['syslog'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
return output.decode('utf-8')
elif current_platform == "Linux":
with open('/var/log/dmesg', 'r') as file:
return file.read()
else:
logger.info("Unsupported platform.")
def custom_filter(message): def get_kernel_messages(self):
# Check for {TO_INTERPRETER{ message here }TO_INTERPRETER} pattern """
if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message: Fetch system logs or kernel message from the operating system.
start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{')
end = message.find('}TO_INTERPRETER}', start)
return message[start:end]
# Check for USB mention
# elif 'USB' in message:
# return message
# # Check for network related keywords
# elif any(keyword in message for keyword in ['network', 'IP', 'internet', 'LAN', 'WAN', 'router', 'switch']) and "networkStatusForFlags" not in message:
# return message
else:
return None
last_messages = ""
def check_filtered_kernel(): - For MacOS, it uses syslog.
messages = get_kernel_messages() - For Linux, it uses dmesg.
messages.replace(last_messages, "") - For Windows, it uses wevtutil with the 'qe' (query events) from the 'System' log
messages = messages.split("\n") with the '/f:text' (format text) flag.
"""
filtered_messages = [] current_platform = platform.system().lower()
for message in messages:
if custom_filter(message):
filtered_messages.append(message)
return "\n".join(filtered_messages)
async def put_kernel_messages_into_queue(queue): if current_platform == "darwin":
process = subprocess.Popen(['syslog'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
return output.decode('utf-8')
elif current_platform == "linux":
with open('/var/log/dmesg', 'r') as file:
return file.read()
elif current_platform == "windows":
process = subprocess.Popen(['wevtutil', 'qe', 'System', '/f:text'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
try:
return output.decode('utf-8')
except UnicodeDecodeError:
try:
return output.decode('utf-16')
except UnicodeDecodeError:
return output.decode('cp1252')
else:
logger.info("Unsupported platform.")
return ""
def custom_filter(self, message):
if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message:
start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{')
end = message.find('}TO_INTERPRETER}', start)
return message[start:end]
else:
return None
def check_filtered_kernel(self):
try:
messages = self.get_kernel_messages()
messages = messages.replace(self._last_messages, "")
messages = messages.split("\n")
filtered_messages = [message for message in messages if self.custom_filter(message)]
self._last_messages = "\n".join(filtered_messages)
return self._last_messages
except Exception as e:
logger.error(f"Error while checking kernel messages: {e}")
return None
async def put_kernel_messages_into_queue(kernel_checker, queue):
while True: while True:
text = check_filtered_kernel() text = kernel_checker.check_filtered_kernel()
if text: if text:
if isinstance(queue, asyncio.Queue): if isinstance(queue, asyncio.Queue):
await queue.put({"role": "computer", "type": "console", "start": True}) await queue.put({"role": "computer", "type": "console", "start": True})
@ -67,5 +78,5 @@ async def put_kernel_messages_into_queue(queue):
queue.put({"role": "computer", "type": "console", "start": True}) queue.put({"role": "computer", "type": "console", "start": True})
queue.put({"role": "computer", "type": "console", "format": "output", "content": text}) queue.put({"role": "computer", "type": "console", "format": "output", "content": text})
queue.put({"role": "computer", "type": "console", "end": True}) queue.put({"role": "computer", "type": "console", "end": True})
await asyncio.sleep(5) await asyncio.sleep(5)

44
software/start.py
View File

@ -15,32 +15,32 @@ app = typer.Typer()
@app.command() @app.command()
def run( def run(
server: bool = typer.Option(False, "--server", help="Run server"), server: bool = typer.Option(False, "--server", help="Run server"),
server_host: str = typer.Option("0.0.0.0", "--server-host", help="Specify the server host where the server will deploy"), server_host: str = typer.Option("127.0.0.1", "--server-host", help="Specify the server host where the server will deploy"),
server_port: int = typer.Option(10001, "--server-port", help="Specify the server port where the server will deploy"), server_port: int = typer.Option(10001, "--server-port", help="Specify the server port where the server will deploy"),
tunnel_service: str = typer.Option("ngrok", "--tunnel-service", help="Specify the tunnel service"), tunnel_service: str = typer.Option("ngrok", "--tunnel-service", help="Specify the tunnel service"),
expose: bool = typer.Option(False, "--expose", help="Expose server to internet"), expose: bool = typer.Option(False, "--expose", help="Expose server to internet"),
client: bool = typer.Option(False, "--client", help="Run client"), client: bool = typer.Option(False, "--client", help="Run client"),
server_url: str = typer.Option(None, "--server-url", help="Specify the server URL that the client should expect. Defaults to server-host and server-port"), server_url: str = typer.Option(None, "--server-url", help="Specify the server URL that the client should expect. Defaults to server-host and server-port"),
client_type: str = typer.Option("auto", "--client-type", help="Specify the client type"), client_type: str = typer.Option("auto", "--client-type", help="Specify the client type"),
llm_service: str = typer.Option("litellm", "--llm-service", help="Specify the LLM service"), llm_service: str = typer.Option("litellm", "--llm-service", help="Specify the LLM service"),
model: str = typer.Option("gpt-4", "--model", help="Specify the model"), model: str = typer.Option("gpt-4", "--model", help="Specify the model"),
llm_supports_vision: bool = typer.Option(False, "--llm-supports-vision", help="Specify if the LLM service supports vision"), llm_supports_vision: bool = typer.Option(False, "--llm-supports-vision", help="Specify if the LLM service supports vision"),
llm_supports_functions: bool = typer.Option(False, "--llm-supports-functions", help="Specify if the LLM service supports functions"), llm_supports_functions: bool = typer.Option(False, "--llm-supports-functions", help="Specify if the LLM service supports functions"),
context_window: int = typer.Option(2048, "--context-window", help="Specify the context window size"), context_window: int = typer.Option(2048, "--context-window", help="Specify the context window size"),
max_tokens: int = typer.Option(4096, "--max-tokens", help="Specify the maximum number of tokens"), max_tokens: int = typer.Option(4096, "--max-tokens", help="Specify the maximum number of tokens"),
temperature: float = typer.Option(0.8, "--temperature", help="Specify the temperature for generation"), temperature: float = typer.Option(0.8, "--temperature", help="Specify the temperature for generation"),
tts_service: str = typer.Option("openai", "--tts-service", help="Specify the TTS service"), tts_service: str = typer.Option("openai", "--tts-service", help="Specify the TTS service"),
stt_service: str = typer.Option("openai", "--stt-service", help="Specify the STT service"), stt_service: str = typer.Option("openai", "--stt-service", help="Specify the STT service"),
local: bool = typer.Option(False, "--local", help="Use recommended local services for LLM, STT, and TTS"), local: bool = typer.Option(False, "--local", help="Use recommended local services for LLM, STT, and TTS"),
): ):
_run( _run(
server=server, server=server,
server_host=server_host, server_host=server_host,
@ -66,39 +66,39 @@ def _run(
server: bool = False, server: bool = False,
server_host: str = "0.0.0.0", server_host: str = "0.0.0.0",
server_port: int = 10001, server_port: int = 10001,
tunnel_service: str = "bore", tunnel_service: str = "bore",
expose: bool = False, expose: bool = False,
client: bool = False, client: bool = False,
server_url: str = None, server_url: str = None,
client_type: str = "auto", client_type: str = "auto",
llm_service: str = "litellm", llm_service: str = "litellm",
model: str = "gpt-4", model: str = "gpt-4",
llm_supports_vision: bool = False, llm_supports_vision: bool = False,
llm_supports_functions: bool = False, llm_supports_functions: bool = False,
context_window: int = 2048, context_window: int = 2048,
max_tokens: int = 4096, max_tokens: int = 4096,
temperature: float = 0.8, temperature: float = 0.8,
tts_service: str = "openai", tts_service: str = "openai",
stt_service: str = "openai", stt_service: str = "openai",
local: bool = False local: bool = False
): ):
if local: if local:
tts_service = "piper" tts_service = "piper"
# llm_service = "llamafile" # llm_service = "llamafile"
stt_service = "local-whisper" stt_service = "local-whisper"
select_local_model() select_local_model()
if not server_url: if not server_url:
server_url = f"{server_host}:{server_port}" server_url = f"{server_host}:{server_port}"
if not server and not client: if not server and not client:
server = True server = True
client = True client = True
@ -120,10 +120,12 @@ def _run(
if client: if client:
if client_type == "auto": if client_type == "auto":
system_type = platform.system() system_type = platform.system().lower()
if system_type == "Darwin": # Mac OS if system_type == "darwin": # Mac OS
client_type = "mac" client_type = "mac"
elif system_type == "Linux": # Linux System elif system_type == "windows":
client_type = "windows"
elif system_type == "linux": # Linux System
try: try:
with open('/proc/device-tree/model', 'r') as m: with open('/proc/device-tree/model', 'r') as m:
if 'raspberry pi' in m.read().lower(): if 'raspberry pi' in m.read().lower():
@ -145,4 +147,4 @@ def _run(
if client: if client:
client_thread.join() client_thread.join()
except KeyboardInterrupt: except KeyboardInterrupt:
os.kill(os.getpid(), signal.SIGINT) os.kill(os.getpid(), signal.SIGINT)