From b4dd20950bba235a8b3c167c9ac327b1ab6eb277 Mon Sep 17 00:00:00 2001 From: Lincoln Roth Date: Sat, 17 Feb 2024 18:17:49 -0800 Subject: [PATCH 1/3] websockets from esp32 to server work, very chunky in other direction --- 01OS/01OS/clients/esp32/playback.ino | 110 ++++++++++++++++++++-- 01OS/01OS/clients/esp32/websocket_test.py | 33 +++++++ 2 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 01OS/01OS/clients/esp32/websocket_test.py diff --git a/01OS/01OS/clients/esp32/playback.ino b/01OS/01OS/clients/esp32/playback.ino index 61c800c..3dc91c1 100644 --- a/01OS/01OS/clients/esp32/playback.ino +++ b/01OS/01OS/clients/esp32/playback.ino @@ -3,6 +3,14 @@ #include #include +#include + +#include +#include +#include + +#include + #define CONFIG_I2S_BCK_PIN 19 #define CONFIG_I2S_LRCK_PIN 33 #define CONFIG_I2S_DATA_PIN 22 @@ -17,6 +25,24 @@ uint8_t microphonedata0[1024 * 70]; int data_offset = 0; +WebSocketsClient webSocket; + + +void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) { + const uint8_t* src = (const uint8_t*) mem; + Serial.printf("\n[HEXDUMP] Address: 0x%08X len: 0x%X (%d)", (ptrdiff_t)src, len, len); + for (uint32_t i = 0; i < len; i++) { + if (i % cols == 0) { + Serial.printf("\n[0x%08X] 0x%08X: ", (ptrdiff_t)src, i); + } + Serial.printf("%02X ", *src); + src++; + } + Serial.printf("\n"); +} + + + void InitI2SSpeakerOrMic(int mode) { esp_err_t err = ESP_OK; @@ -25,11 +51,11 @@ void InitI2SSpeakerOrMic(int mode) { .mode = (i2s_mode_t)(I2S_MODE_MASTER), .sample_rate = 16000, .bits_per_sample = - I2S_BITS_PER_SAMPLE_16BIT, // is fixed at 12bit, stereo, MSB + I2S_BITS_PER_SAMPLE_16BIT, // is fixed at 12bit, stereo, MSB .channel_format = I2S_CHANNEL_FMT_ALL_RIGHT, #if ESP_IDF_VERSION > ESP_IDF_VERSION_VAL(4, 1, 0) .communication_format = - I2S_COMM_FORMAT_STAND_I2S, // Set the format of the communication. + I2S_COMM_FORMAT_STAND_I2S, // Set the format of the communication. #else // 设置通讯格式 .communication_format = I2S_COMM_FORMAT_I2S, #endif @@ -64,12 +90,71 @@ void InitI2SSpeakerOrMic(int mode) { I2S_CHANNEL_MONO); } +void speaker_play(uint8_t *payload, uint32_t len){ + size_t bytes_written; + InitI2SSpeakerOrMic(MODE_SPK); + i2s_write(SPEAKER_I2S_NUMBER, payload, len, + &bytes_written, portMAX_DELAY); +} + +void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { + + switch (type) { + case WStype_DISCONNECTED: + Serial.printf("[WSc] Disconnected!\n"); + break; + case WStype_CONNECTED: + Serial.printf("[WSc] Connected to url: %s\n", payload); + + // send message to server when Connected + webSocket.sendTXT("Connected"); + break; + case WStype_TEXT: + Serial.printf("[WSc] get text: %s\n", payload); + + // send message to server + // webSocket.sendTXT("message here"); + break; + case WStype_BIN: + Serial.printf("[WSc] get binary length: %u\n", length); + // hexdump(payload, length); + speaker_play(payload, length); + + // send data to server + // webSocket.sendBIN(payload, length); + break; + case WStype_ERROR: + case WStype_FRAGMENT_TEXT_START: + case WStype_FRAGMENT_BIN_START: + case WStype_FRAGMENT: + case WStype_FRAGMENT_FIN: + break; + } + +} + +void websocket_setup() { + Serial.begin(115200); + WiFi.begin("Soundview_Guest", ""); + while (WiFi.status() != WL_CONNECTED){ + delay(500); + Serial.println("connecting to WiFi"); + } + Serial.println("connected to WiFi"); + webSocket.begin("192.168.68.71", 9001, "/"); + webSocket.onEvent(webSocketEvent); + // webSocket.setAuthorization("user", "Password"); + webSocket.setReconnectInterval(5000); + +} void setup() { M5.begin(true, false, true); M5.dis.drawpix(0, CRGB(128, 128, 0)); + websocket_setup(); delay(2000); } + void loop() { if (M5.Btn.isPressed()) { data_offset = 0; @@ -86,10 +171,21 @@ void loop() { if (M5.Btn.isReleased() || data_offset >= 71679) break; // delay(60); } - size_t bytes_written; - InitI2SSpeakerOrMic(MODE_SPK); - i2s_write(SPEAKER_I2S_NUMBER, microphonedata0, data_offset, - &bytes_written, portMAX_DELAY); + + Serial.println(data_offset); + + int chunk_size = 1004; + chunk_size = 4096; + + webSocket.sendBIN(microphonedata0, data_offset); + + // size_t bytes_written; + // InitI2SSpeakerOrMic(MODE_SPK); + // i2s_write(SPEAKER_I2S_NUMBER, microphonedata0, data_offset, + // &bytes_written, portMAX_DELAY); + } M5.update(); -} \ No newline at end of file + webSocket.loop(); + +} diff --git a/01OS/01OS/clients/esp32/websocket_test.py b/01OS/01OS/clients/esp32/websocket_test.py new file mode 100644 index 0000000..71c6813 --- /dev/null +++ b/01OS/01OS/clients/esp32/websocket_test.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import asyncio +import simpleaudio as sa +from websockets.server import serve + + +def divide_chunks(l, n): + # looping till length l + for i in range(0, len(l), n): + yield l[i:i + n] + + +async def echo(websocket): + async for message in websocket: + try: + play_obj = sa.play_buffer(bytearray(message), 1, 2, 16000) + play_obj.wait_done() + + x = list(divide_chunks(bytearray(message), 1000)) + for i in x: + await websocket.send(i) + + except Exception as e: + print('fail') + + + +async def main(): + async with serve(echo, "0.0.0.0", 9001): + await asyncio.Future() # run forever + +asyncio.run(main()) \ No newline at end of file From 1ca19d396a1be361f1211abaddf96569b0f924a2 Mon Sep 17 00:00:00 2001 From: Lincoln Roth Date: Sat, 17 Feb 2024 19:42:26 -0800 Subject: [PATCH 2/3] works on debugging websocket receive --- 01OS/01OS/clients/esp32/playback.ino | 31 +++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/01OS/01OS/clients/esp32/playback.ino b/01OS/01OS/clients/esp32/playback.ino index 3dc91c1..228639e 100644 --- a/01OS/01OS/clients/esp32/playback.ino +++ b/01OS/01OS/clients/esp32/playback.ino @@ -22,7 +22,9 @@ #define MODE_SPK 1 #define DATA_SIZE 1024 -uint8_t microphonedata0[1024 * 70]; +uint8_t microphonedata0[1024 * 30]; +uint8_t speakerdata0[1024 * 30]; +int speaker_offset = 0; int data_offset = 0; WebSocketsClient webSocket; @@ -91,6 +93,7 @@ void InitI2SSpeakerOrMic(int mode) { } void speaker_play(uint8_t *payload, uint32_t len){ + Serial.printf("received %lu bytes", len); size_t bytes_written; InitI2SSpeakerOrMic(MODE_SPK); i2s_write(SPEAKER_I2S_NUMBER, payload, len, @@ -111,14 +114,23 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { break; case WStype_TEXT: Serial.printf("[WSc] get text: %s\n", payload); + if ((char)payload[0] == 's'){ + Serial.println("start"); + speaker_offset = 0; + } + if ((char)payload[0] == 'e'){ + Serial.println("end"); + speaker_play(payload, speaker_offset); + } // send message to server // webSocket.sendTXT("message here"); break; case WStype_BIN: Serial.printf("[WSc] get binary length: %u\n", length); - // hexdump(payload, length); - speaker_play(payload, length); + memcpy(&(speakerdata0[speaker_offset]),&payload,length); // this line is likely the issue, the payloads here don't match the data that speaker_play contains + speaker_offset += length; + // send data to server // webSocket.sendBIN(payload, length); @@ -151,6 +163,8 @@ void setup() { M5.begin(true, false, true); M5.dis.drawpix(0, CRGB(128, 128, 0)); websocket_setup(); + InitI2SSpeakerOrMic(MODE_SPK); + delay(2000); } @@ -171,19 +185,8 @@ void loop() { if (M5.Btn.isReleased() || data_offset >= 71679) break; // delay(60); } - - Serial.println(data_offset); - - int chunk_size = 1004; - chunk_size = 4096; - webSocket.sendBIN(microphonedata0, data_offset); - // size_t bytes_written; - // InitI2SSpeakerOrMic(MODE_SPK); - // i2s_write(SPEAKER_I2S_NUMBER, microphonedata0, data_offset, - // &bytes_written, portMAX_DELAY); - } M5.update(); webSocket.loop(); From 22761d0344ded27e0de1d94b3ed071f8dc086572 Mon Sep 17 00:00:00 2001 From: Theia Vogel Date: Sat, 17 Feb 2024 22:45:22 -0800 Subject: [PATCH 3/3] streaming audio / playback --- .../clients/esp32/{ => playback}/playback.ino | 94 ++++++++++++++----- 01OS/01OS/clients/esp32/websocket_test.py | 48 ++++++---- 2 files changed, 100 insertions(+), 42 deletions(-) rename 01OS/01OS/clients/esp32/{ => playback}/playback.ino (72%) diff --git a/01OS/01OS/clients/esp32/playback.ino b/01OS/01OS/clients/esp32/playback/playback.ino similarity index 72% rename from 01OS/01OS/clients/esp32/playback.ino rename to 01OS/01OS/clients/esp32/playback/playback.ino index 228639e..7f2e2a3 100644 --- a/01OS/01OS/clients/esp32/playback.ino +++ b/01OS/01OS/clients/esp32/playback/playback.ino @@ -11,6 +11,9 @@ #include + +#define COMPUTER_IP "192.168.68.87" + #define CONFIG_I2S_BCK_PIN 19 #define CONFIG_I2S_LRCK_PIN 33 #define CONFIG_I2S_DATA_PIN 22 @@ -22,13 +25,36 @@ #define MODE_SPK 1 #define DATA_SIZE 1024 -uint8_t microphonedata0[1024 * 30]; -uint8_t speakerdata0[1024 * 30]; +uint8_t microphonedata0[1024 * 10]; +uint8_t speakerdata0[1024 * 1]; int speaker_offset = 0; int data_offset = 0; WebSocketsClient webSocket; +class ButtonChecker { + public: + void loop() { + lastTickState = thisTickState; + thisTickState = M5.Btn.isPressed() != 0; + } + + bool justPressed() { + return thisTickState && !lastTickState; + } + + bool justReleased() { + return !thisTickState && lastTickState; + } + + private: + bool lastTickState = false; + bool thisTickState = false; +}; + +ButtonChecker button = ButtonChecker(); + + void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) { const uint8_t* src = (const uint8_t*) mem; @@ -43,8 +69,6 @@ void hexdump(const void *mem, uint32_t len, uint8_t cols = 16) { Serial.printf("\n"); } - - void InitI2SSpeakerOrMic(int mode) { esp_err_t err = ESP_OK; @@ -101,7 +125,6 @@ void speaker_play(uint8_t *payload, uint32_t len){ } void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { - switch (type) { case WStype_DISCONNECTED: Serial.printf("[WSc] Disconnected!\n"); @@ -110,17 +133,18 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { Serial.printf("[WSc] Connected to url: %s\n", payload); // send message to server when Connected - webSocket.sendTXT("Connected"); break; case WStype_TEXT: Serial.printf("[WSc] get text: %s\n", payload); if ((char)payload[0] == 's'){ Serial.println("start"); speaker_offset = 0; + InitI2SSpeakerOrMic(MODE_SPK); } if ((char)payload[0] == 'e'){ Serial.println("end"); - speaker_play(payload, speaker_offset); + // speaker_play(speakerdata0, speaker_offset); + // speaker_offset = 0; } // send message to server @@ -128,8 +152,11 @@ void webSocketEvent(WStype_t type, uint8_t * payload, size_t length) { break; case WStype_BIN: Serial.printf("[WSc] get binary length: %u\n", length); - memcpy(&(speakerdata0[speaker_offset]),&payload,length); // this line is likely the issue, the payloads here don't match the data that speaker_play contains + memcpy(speakerdata0 + speaker_offset, payload, length); speaker_offset += length; + size_t bytes_written; + i2s_write(SPEAKER_I2S_NUMBER, speakerdata0, speaker_offset, &bytes_written, portMAX_DELAY); + speaker_offset = 0; // send data to server @@ -153,7 +180,7 @@ void websocket_setup() { Serial.println("connecting to WiFi"); } Serial.println("connected to WiFi"); - webSocket.begin("192.168.68.71", 9001, "/"); + webSocket.begin(COMPUTER_IP, 9001, "/"); webSocket.onEvent(webSocketEvent); // webSocket.setAuthorization("user", "Password"); webSocket.setReconnectInterval(5000); @@ -168,26 +195,43 @@ void setup() { delay(2000); } +bool recording = false; + +void flush_microphone() { + Serial.printf("[microphone] flushing %d bytes of data\n", data_offset); + if (data_offset == 0) return; + webSocket.sendBIN(microphonedata0, data_offset); + data_offset = 0; +} void loop() { - if (M5.Btn.isPressed()) { - data_offset = 0; + button.loop(); + if (button.justPressed()) { + Serial.println("Recording..."); + webSocket.sendTXT("s"); InitI2SSpeakerOrMic(MODE_MIC); - M5.dis.drawpix(0, CRGB(128, 128, 0)); - size_t byte_read; - - while (1) { - i2s_read(SPEAKER_I2S_NUMBER, - (char *)(microphonedata0 + data_offset), DATA_SIZE, - &byte_read, (100 / portTICK_RATE_MS)); - data_offset += 1024; - M5.update(); - if (M5.Btn.isReleased() || data_offset >= 71679) break; - // delay(60); - } - webSocket.sendBIN(microphonedata0, data_offset); - + recording = true; + } else if (button.justReleased()) { + Serial.println("Stopped recording."); + webSocket.sendTXT("e"); + flush_microphone(); + recording = false; } + + if (recording) { + size_t bytes_read; + i2s_read( + SPEAKER_I2S_NUMBER, + (char *)(microphonedata0 + data_offset), + DATA_SIZE, &bytes_read, (100 / portTICK_RATE_MS) + ); + data_offset += bytes_read; + + if (data_offset > 1024*10) { + flush_microphone(); + } + } + M5.update(); webSocket.loop(); diff --git a/01OS/01OS/clients/esp32/websocket_test.py b/01OS/01OS/clients/esp32/websocket_test.py index 71c6813..96afd54 100644 --- a/01OS/01OS/clients/esp32/websocket_test.py +++ b/01OS/01OS/clients/esp32/websocket_test.py @@ -1,33 +1,47 @@ #!/usr/bin/env python +"""A basic echo server for testing the device.""" + import asyncio -import simpleaudio as sa +import uuid +import websockets from websockets.server import serve +import traceback -def divide_chunks(l, n): - # looping till length l - for i in range(0, len(l), n): - yield l[i:i + n] +def divide_chunks(l, n): + # looping till length l + for i in range(0, len(l), n): + yield l[i : i + n] -async def echo(websocket): +buffers: dict[uuid.UUID, bytearray] = {} + + +async def echo(websocket: websockets.WebSocketServerProtocol): async for message in websocket: try: - play_obj = sa.play_buffer(bytearray(message), 1, 2, 16000) - play_obj.wait_done() - - x = list(divide_chunks(bytearray(message), 1000)) - for i in x: - await websocket.send(i) - - except Exception as e: - print('fail') - + if message == "s": + print("starting stream for", websocket.id) + buffers[websocket.id] = bytearray() + elif message == "e": + print("end, echoing stream for", websocket.id) + await websocket.send("s") + for chunk in divide_chunks(buffers[websocket.id], 1000): + await websocket.send(chunk) + await websocket.send("e") + elif type(message) is bytes: + print("recvd", len(message), "bytes from", websocket.id) + buffers[websocket.id].extend(message) + else: + print("ERR: recvd unknown message", message[:10], "from", websocket.id) + except Exception as _e: + traceback.print_exc() async def main(): async with serve(echo, "0.0.0.0", 9001): await asyncio.Future() # run forever -asyncio.run(main()) \ No newline at end of file + +asyncio.run(main())