MirrorMirror
/
01
mirror of https://github.com/OpenInterpreter/01.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix missing Device for Windows

This commit is contained in:
Davy Peter Braun 2024-03-29 16:46:47 +01:00
parent 7d1a4d9777
commit 621c5cd048
6 changed files with 212 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
software/source/clients/base_device.py
View File

@ -1,5 +1,8 @@
from dotenv import load_dotenv
from source.server.utils.logs import setup_logging, logger
load_dotenv() # take environment variables from .env.
setup_logging()
import os
import asyncio
@ -11,7 +14,7 @@ from queue import Queue
from pynput import keyboard
import json
import traceback
import websockets
import websocket as wsc
import queue
import pydub
import ast
@ -26,7 +29,7 @@ import cv2
import base64
from interpreter import interpreter # Just for code execution. Maybe we should let people do from interpreter.computer import run?
# In the future, I guess kernel watching code should be elsewhere? Somewhere server / client agnostic?
from ..server.utils.kernel import put_kernel_messages_into_queue
from ..server.utils.kernel import KernelChecker, put_kernel_messages_into_queue
from ..server.utils.get_system_info import get_system_info
from ..server.utils.process_utils import kill_process_tree
@ -41,13 +44,11 @@ from ..utils.accumulator import Accumulator
accumulator = Accumulator()
# Configuration for Audio Recording
CHUNK = 1024 # Record in chunks of 1024 samples
FORMAT = pyaudio.paInt16 # 16 bits per sample
CHANNELS = 1 # Mono
RATE = 44100 # Sample rate
RECORDING = False # Flag to control recording state
SPACEBAR_PRESSED = False # Flag to track spacebar press state
# AudioSegment configuration
AUDIO_SAMPLE_WIDTH = 2
AUDIO_FRAME_RATE = 16000
AUDIO_MONO_CHANNEL = 1
AUDIO_CODE_RUNNER_CLIENT = "client"
# Camera configuration
CAMERA_ENABLED = os.getenv('CAMERA_ENABLED', False)
@ -71,6 +72,14 @@ class Device:
self.audiosegments = []
self.server_url = ""
# Configuration for Audio Recording
self.CHUNK = 1024 # Record in chunks of 1024 samples
self.FORMAT = pyaudio.paInt16 # 16 bits per sample
self.CHANNELS = 1 # Mono
self.RATE = 44100 # Sample rate
self.RECORDING = False # Flag to control recording state
self.SPACEBAR_PRESSED = False # Flag to track spacebar press state
def fetch_image_from_camera(self, camera_index=CAMERA_DEVICE_INDEX):
"""Captures an image from the specified camera device and saves it to a temporary file. Adds the image to the captured_images list."""
image_path = None
@ -141,7 +150,6 @@ class Device:
def record_audio(self):
if os.getenv('STT_RUNNER') == "server":
# STT will happen on the server. we're sending audio.
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "start": True})
@ -152,20 +160,19 @@ class Device:
raise Exception("STT_RUNNER must be set to either 'client' or 'server'.")
"""Record audio from the microphone and add it to the queue."""
stream = p.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK)
stream = p.open(format=self.FORMAT, channels=self.CHANNELS, rate=self.RATE, input=True, frames_per_buffer=self.CHUNK)
print("Recording started...")
global RECORDING
# Create a temporary WAV file to store the audio data
temp_dir = tempfile.gettempdir()
wav_path = os.path.join(temp_dir, f"audio_{datetime.now().strftime('%Y%m%d%H%M%S%f')}.wav")
wav_file = wave.open(wav_path, 'wb')
wav_file.setnchannels(CHANNELS)
wav_file.setsampwidth(p.get_sample_size(FORMAT))
wav_file.setframerate(RATE)
wav_file.setnchannels(self.CHANNELS)
wav_file.setsampwidth(p.get_sample_size(self.FORMAT))
wav_file.setframerate(self.RATE)
while RECORDING:
data = stream.read(CHUNK, exception_on_overflow=False)
while self.RECORDING:
data = stream.read(self.CHUNK, exception_on_overflow=False)
wav_file.writeframes(data)
wav_file.close()
@ -173,7 +180,7 @@ class Device:
stream.close()
print("Recording stopped.")
duration = wav_file.getnframes() / RATE
duration = wav_file.getnframes() / self.RATE
if duration < 0.3:
# Just pressed it. Send stop message
if os.getenv('STT_RUNNER') == "client":
@ -198,10 +205,10 @@ class Device:
else:
# Stream audio
with open(wav_path, 'rb') as audio_file:
byte_data = audio_file.read(CHUNK)
byte_data = audio_file.read(self.CHUNK)
while byte_data:
send_queue.put(byte_data)
byte_data = audio_file.read(CHUNK)
byte_data = audio_file.read(self.CHUNK)
send_queue.put({"role": "user", "type": "audio", "format": "bytes.wav", "end": True})
if os.path.exists(wav_path):
@ -209,15 +216,14 @@ class Device:
def toggle_recording(self, state):
"""Toggle the recording state."""
global RECORDING, SPACEBAR_PRESSED
if state and not SPACEBAR_PRESSED:
SPACEBAR_PRESSED = True
if not RECORDING:
RECORDING = True
if state and not self.SPACEBAR_PRESSED:
self.SPACEBAR_PRESSED = True
if not self.RECORDING:
self.RECORDING = True
threading.Thread(target=self.record_audio).start()
elif not state and SPACEBAR_PRESSED:
SPACEBAR_PRESSED = False
RECORDING = False
elif not state and self.SPACEBAR_PRESSED:
self.SPACEBAR_PRESSED = False
self.RECORDING = False
def on_press(self, key):
"""Detect spacebar press and Ctrl+C combination."""
@ -254,62 +260,74 @@ class Device:
show_connection_log = True
while True:
try:
async with websockets.connect(WS_URL) as websocket:
if CAMERA_ENABLED:
print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
else:
print("\nHold the spacebar to start recording. Press CTRL-C to exit.")
print("Attempting to connect to the WS server...")
asyncio.create_task(self.message_sender(websocket))
try:
ws = wsc.create_connection(WS_URL, timeout=5)
except wsc.WebSocketTimeoutException:
print("Timeout while trying to connect to the WebSocket server.")
continue
print("Connected to the WS server.")
while True:
await asyncio.sleep(0.01)
chunk = await websocket.recv()
if CAMERA_ENABLED:
print("\nHold the spacebar to start recording. Press 'c' to capture an image from the camera. Press CTRL-C to exit.")
else:
print("\nHold the spacebar to start recording. Press CTRL-C to exit.")
logger.debug(f"Got this message from the server: {type(chunk)} {chunk}")
asyncio.create_task(self.message_sender(ws))
if type(chunk) == str:
chunk = json.loads(chunk)
while True:
await asyncio.sleep(0.01)
chunk = await ws.recv()
message = accumulator.accumulate(chunk)
if message == None:
# Will be None until we have a full message ready
continue
logger.debug(f"Got this message from the server: {type(chunk)} {chunk}")
# At this point, we have our message
if isinstance(chunk, str):
chunk = json.loads(chunk)
if message["type"] == "audio" and message["format"].startswith("bytes"):
message = accumulator.accumulate(chunk)
if message is None:
# Will be None until we have a full message ready
continue
# Convert bytes to audio file
# At this point, we have our message
audio_bytes = message["content"]
if message["type"] == "audio" and message["format"].startswith("bytes"):
# Convert bytes to audio file
audio_bytes = message["content"]
# Create an AudioSegment instance with the raw data
audio = AudioSegment(
# raw audio data (bytes)
data=audio_bytes,
# signed 16-bit little-endian format
sample_width=2,
# 16,000 Hz frame rate
frame_rate=16000,
# mono sound
channels=1
)
# Create an AudioSegment instance with the raw data
audio = AudioSegment(
data=audio_bytes,
sample_width=AUDIO_SAMPLE_WIDTH,
frame_rate=AUDIO_FRAME_RATE,
channels=AUDIO_MONO_CHANNEL
)
self.audiosegments.append(audio)
self.audiosegments.append(audio)
# Run the code if that's the client's job
if os.getenv('CODE_RUNNER') == "client":
if message["type"] == "code" and "end" in message:
language = message["format"]
code = message["content"]
result = interpreter.computer.run(language, code)
send_queue.put(result)
except:
# Run the code if that's the client's job
if os.getenv('CODE_RUNNER') == AUDIO_CODE_RUNNER_CLIENT:
if message["type"] == "code" and "end" in message:
language = message["format"]
code = message["content"]
result = interpreter.computer.run(language, code)
send_queue.put(result)
except wsc.WebSocketConnectionClosedException:
print("WebSocket connection closed unexpectedly.")
if show_connection_log:
logger.info(f"Reconnecting to `{WS_URL}`...")
show_connection_log = False
await asyncio.sleep(2)
except wsc.WebSocketAddressException:
print(f"Invalid WebSocket URI: `{WS_URL}`. Please check the URI and try again.")
break # Exit the loop as the URI is invalid and retrying won't help
except Exception as e:
logger.debug(traceback.format_exc())
if show_connection_log:
logger.info(f"Connecting to `{WS_URL}`...")
show_connection_log = False
logger.info(f"Connecting to `{WS_URL}`...")
show_connection_log = False
await asyncio.sleep(2)
async def start_async(self):
@ -320,13 +338,14 @@ class Device:
# Start watching the kernel if it's your job to do that
if os.getenv('CODE_RUNNER') == "client":
asyncio.create_task(put_kernel_messages_into_queue(send_queue))
kernel_checker = KernelChecker()
asyncio.create_task(put_kernel_messages_into_queue(kernel_checker, send_queue))
asyncio.create_task(self.play_audiosegments())
# If Raspberry Pi, add the button listener, otherwise use the spacebar
if current_platform.startswith("raspberry-pi"):
logger.info("Raspberry Pi detected, using button on GPIO pin 15")
print("Raspberry Pi detected, using button on GPIO pin 15")
# Use GPIO pin 15
pindef = ["gpiochip4", "15"] # gpiofind PIN15
print("PINDEF", pindef)

0
software/source/clients/windows/__init__.py Normal file
View File

10
software/source/clients/windows/device.py Normal file
View File

@ -0,0 +1,10 @@
from ..base_device import Device
device = Device()
def main(server_url):
device.server_url = server_url
device.start()
if __name__ == "__main__":
main()

5
software/source/server/server.py
View File

@ -16,7 +16,7 @@ from starlette.websockets import WebSocket, WebSocketDisconnect
from pathlib import Path
import asyncio
import urllib.parse
from .utils.kernel import put_kernel_messages_into_queue
from .utils.kernel import KernelChecker, put_kernel_messages_into_queue
from .i import configure_interpreter
from interpreter import interpreter
from ..utils.accumulator import Accumulator
@ -420,7 +420,8 @@ async def main(server_host, server_port, llm_service, model, llm_supports_vision
# Start watching the kernel if it's your job to do that
if True: # in the future, code can run on device. for now, just server.
asyncio.create_task(put_kernel_messages_into_queue(from_computer))
kernel_checker = KernelChecker()
asyncio.create_task(put_kernel_messages_into_queue(kernel_checker, from_computer))
config = Config(app, host=server_host, port=int(server_port), lifespan='on')
server = Server(config)

101
software/source/server/utils/kernel.py
View File

@ -1,63 +1,74 @@
from dotenv import load_dotenv
load_dotenv() # take environment variables from .env.
import asyncio
import subprocess
import platform
from dotenv import load_dotenv
from .logs import setup_logging, logger
from .logs import setup_logging
from .logs import logger
load_dotenv() # take environment variables from .env.
setup_logging()
def get_kernel_messages():
"""
Is this the way to do this?
"""
current_platform = platform.system()
class KernelChecker:
def __init__(self):
self._last_messages = ""
if current_platform == "Darwin":
process = subprocess.Popen(['syslog'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
return output.decode('utf-8')
elif current_platform == "Linux":
with open('/var/log/dmesg', 'r') as file:
return file.read()
else:
logger.info("Unsupported platform.")
def get_kernel_messages(self):
"""
Fetch system logs or kernel message from the operating system.
def custom_filter(message):
# Check for {TO_INTERPRETER{ message here }TO_INTERPRETER} pattern
if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message:
start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{')
end = message.find('}TO_INTERPRETER}', start)
return message[start:end]
# Check for USB mention
# elif 'USB' in message:
# return message
# # Check for network related keywords
# elif any(keyword in message for keyword in ['network', 'IP', 'internet', 'LAN', 'WAN', 'router', 'switch']) and "networkStatusForFlags" not in message:
- For MacOS, it uses syslog.
- For Linux, it uses dmesg.
- For Windows, it uses wevtutil with the 'qe' (query events) from the 'System' log
with the '/f:text' (format text) flag.
"""
current_platform = platform.system().lower()
# return message
else:
return None
if current_platform == "darwin":
process = subprocess.Popen(['syslog'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
return output.decode('utf-8')
elif current_platform == "linux":
with open('/var/log/dmesg', 'r') as file:
return file.read()
elif current_platform == "windows":
process = subprocess.Popen(['wevtutil', 'qe', 'System', '/f:text'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
output, _ = process.communicate()
try:
return output.decode('utf-8')
except UnicodeDecodeError:
try:
return output.decode('utf-16')
except UnicodeDecodeError:
return output.decode('cp1252')
else:
logger.info("Unsupported platform.")
return ""
last_messages = ""
def custom_filter(self, message):
if '{TO_INTERPRETER{' in message and '}TO_INTERPRETER}' in message:
start = message.find('{TO_INTERPRETER{') + len('{TO_INTERPRETER{')
end = message.find('}TO_INTERPRETER}', start)
return message[start:end]
else:
return None
def check_filtered_kernel():
messages = get_kernel_messages()
messages.replace(last_messages, "")
messages = messages.split("\n")
def check_filtered_kernel(self):
try:
messages = self.get_kernel_messages()
messages = messages.replace(self._last_messages, "")
messages = messages.split("\n")
filtered_messages = []
for message in messages:
if custom_filter(message):
filtered_messages.append(message)
filtered_messages = [message for message in messages if self.custom_filter(message)]
return "\n".join(filtered_messages)
self._last_messages = "\n".join(filtered_messages)
return self._last_messages
except Exception as e:
logger.error(f"Error while checking kernel messages: {e}")
return None
async def put_kernel_messages_into_queue(queue):
async def put_kernel_messages_into_queue(kernel_checker, queue):
while True:
text = check_filtered_kernel()
text = kernel_checker.check_filtered_kernel()
if text:
if isinstance(queue, asyncio.Queue):
await queue.put({"role": "computer", "type": "console", "start": True})

10
software/start.py
View File

@ -15,7 +15,7 @@ app = typer.Typer()
@app.command()
def run(
server: bool = typer.Option(False, "--server", help="Run server"),
server_host: str = typer.Option("0.0.0.0", "--server-host", help="Specify the server host where the server will deploy"),
server_host: str = typer.Option("127.0.0.1", "--server-host", help="Specify the server host where the server will deploy"),
server_port: int = typer.Option(10001, "--server-port", help="Specify the server port where the server will deploy"),
tunnel_service: str = typer.Option("ngrok", "--tunnel-service", help="Specify the tunnel service"),
@ -120,10 +120,12 @@ def _run(
if client:
if client_type == "auto":
system_type = platform.system()
if system_type == "Darwin": # Mac OS
system_type = platform.system().lower()
if system_type == "darwin": # Mac OS
client_type = "mac"
elif system_type == "Linux": # Linux System
elif system_type == "windows":
client_type = "windows"
elif system_type == "linux": # Linux System
try:
with open('/proc/device-tree/model', 'r') as m:
if 'raspberry pi' in m.read().lower():