From 928471b50bdb472bb1bb4fd94aeac3ac3f41aebe Mon Sep 17 00:00:00 2001 From: M Date: Sun, 12 Nov 2023 18:28:47 +0100 Subject: [PATCH] Push to talk, basic UI and ollama response stream --- assistant.png | Bin 0 -> 822 bytes assistant.py | 190 ++++++++++++++++++++++++++++++++++++++--------- assistant.yaml | 13 +++- requirements.txt | 3 +- 4 files changed, 165 insertions(+), 41 deletions(-) create mode 100644 assistant.png diff --git a/assistant.png b/assistant.png new file mode 100644 index 0000000000000000000000000000000000000000..ecf0baac967109126632ead3524160656871af52 GIT binary patch literal 822 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdz#^NA%Cx&(BWL^R}Ea{HEjtmSN z`?>!lvI6;>1s;*b3=CYaK$tP$%=>)|3{0mzT^vIq+>cK8_7+W)X+J+lIV!~My1#(O zk`TR|i{c%60&WtjoUSacv+W#GbPp8>#x4j=EK$_x+|VlUllP9>?rDLu%Uox_tjzd# zS$dBlXESI2|Hq&I@BJJnW1DVOGxh(>H8H-MHciakuyobVe_SU{*++0^CnyxUKX3>= zeA7JL?vDM0siv$vWnD$gYnnf}SJn!wdH80w@%tSgcvDh)#XqeuKYZ-eNs-BO7_u#X zKRdI1SHoT(BPQpQDftzD_pAabz5j~It;=txaGAr8**-<0VC_J$Z-yr|c+@A~)!g-z zw{Bb8=KOOt+1xLhz2Cf$@e$cC{-M{?F4S~RZSgZ%@qLYJ*1SCEX~FjIW^4V$m(%!X z-jO=D&+>C0_kD-3YhP|$t(?*H@d-;@dG*uezWc~h#dAJiv7F)CH#=YE|9WJJjg#36 zq}QpK-!WQ0{Q#0gg<^TB__E2}`;xb>hlsD$O=#e*+`hm4=#xe9JAc|WAvx-!Q~Zsm z>=mGV%P99$Sg%5Uox1qhzg%Fy+kWVXz1RHm3Cpyfm7Ea2-(vc8^DNU> z$va+Ur?-Eq28Z*#!iUl)PD%Y=7}K)3z~<&vr;FP2HwA-(w0v?7b7bV@WcB5VOnj>1 zbD!^cUU01dXkB^C54KI4j+swA{U>}k!{n1pbp{8YuWFmkQv58s+E{Bwf_=}ycPU2s z+ax!Df_Km3L*~G^+_(OHqWfV3mWSzCC`FYKbza){zEiru4YS*~^YqCHw zX{}cSwDz&zw!>jJLM$y0r!0qt$J_?-PnX{*tXV&&$M|C+*l1wP3CNo~dZ-ff?x2dC vS5&RRPwBdU-%qRsC#kC{+TopkKZPwCQ-A)SwO|!6tuuJK`njxgN@xNAYm13^ literal 0 HcmV?d00001 diff --git a/assistant.py b/assistant.py index 339922e..1c4eaa3 100644 --- a/assistant.py +++ b/assistant.py @@ -8,9 +8,18 @@ import requests import json import yaml from yaml import Loader +import pygame, sys +import pygame.locals + +BACK_COLOR = (0,0,0) +REC_COLOR = (255,0,0) +TEXT_COLOR = (255,255,255) +REC_SIZE = 80 +FONT_SIZE = 24 +WIDTH = 320 +HEIGHT = 240 + -if sys.version_info[0:3] != (3, 9, 13): - print('Warning, it was only tested with python 3.9.13, it may fail') INPUT_DEFAULT_DURATION_SECONDS = 5 INPUT_FORMAT = pyaudio.paInt16 @@ -25,18 +34,60 @@ class Assistant: def __init__(self): + self.config = self.initConfig() - print("Loading Whisper model...") + + programIcon = pygame.image.load('assistant.png') + + self.clock = pygame.time.Clock() + pygame.display.set_icon(programIcon) + pygame.display.set_caption("Assistant") + + self.windowSurface = pygame.display.set_mode((WIDTH, HEIGHT), 0, 32) + self.font = pygame.font.SysFont(None, FONT_SIZE) + + self.audio = pyaudio.PyAudio() + try: + self.audio.open(format=INPUT_FORMAT, + channels=INPUT_CHANNELS, + rate=INPUT_RATE, + input=True, + frames_per_buffer=INPUT_CHUNK).close() + except : + self.wait_exit() + + self.display_message(self.config.messages.loadingModel) self.model = whisper.load_model(self.config.whisperRecognition.modelPath) self.tts = pyttsx3.init() - self.audio = pyaudio.PyAudio() - self.conversation_history = [self.config.conversation.context+self.config.conversation.greeting+"\n"] + self.conversation_history = [self.config.conversation.context, + self.config.conversation.greeting] + self.context = [] + self.display_ready() + + self.text_to_speech(self.config.conversation.greeting) + + def wait_exit(self): + while True: + self.display_message(self.config.messages.noAudioInput) + self.clock.tick(60) + for event in pygame.event.get(): + if event.type == pygame.locals.QUIT: + self.shutdown() + + def shutdown(self): + self.audio.terminate() + pygame.quit() + sys.exit() def initConfig(self): class Inst: pass config=Inst(); + config.messages = Inst() + config.messages.pressSpace = "Pressez sur espace pour parler puis relachez." + config.messages.loadingModel = "Loading model..." + config.messages.noAudioInput = "Erreur: Pas d'entrée son" config.whisperRecognition = Inst() config.whisperRecognition.modelPath = "whisper/large-v3.pt" config.whisperRecognition.lang = "fr" @@ -54,72 +105,139 @@ class Assistant: #dic depth 2: map values to attributes def dic2Object(dic, object): for key in dic: - setattr(object, key, dic[key]) + if hasattr(object, key): + setattr(object, key, dic[key]) + else: + print("Ignoring unknow setting ", key) #dic depth 1: fill depth 2 attributes for key in dic: - dic2Object(dic[key], getattr(config, key)) + if hasattr(config, key): + dic2Object(dic[key], getattr(config, key)) + else: + print("Ignoring unknow setting ", key) + return config - def waveform_from_mic(self, duration=INPUT_DEFAULT_DURATION_SECONDS) -> np.ndarray: + def display_rec_start(self): + self.windowSurface.fill(BACK_COLOR) + pygame.draw.circle(self.windowSurface, REC_COLOR, (WIDTH/2, HEIGHT/2), REC_SIZE) + pygame.display.flip() - stream = self.audio.open(format=INPUT_FORMAT, channels=INPUT_CHANNELS, - rate=INPUT_RATE, input=True, + def display_message(self, text): + self.windowSurface.fill(BACK_COLOR) + + label = self.font.render(text, 1, TEXT_COLOR) + size = label.get_rect()[2:4] + self.windowSurface.blit(label, (WIDTH/2 - size[0]/2, HEIGHT/2 - size[1]/2)) + + pygame.display.flip() + + def display_ready(self): + self.display_message(self.config.messages.pressSpace) + + def waveform_from_mic(self, key = pygame.K_SPACE) -> np.ndarray: + + self.display_rec_start() + + stream = self.audio.open(format=INPUT_FORMAT, + channels=INPUT_CHANNELS, + rate=INPUT_RATE, + input=True, frames_per_buffer=INPUT_CHUNK) frames = [] - for _ in range(0, int(INPUT_RATE / INPUT_CHUNK * duration)): - data = stream.read(INPUT_CHUNK) - frames.append(data) + while True: + pygame.event.pump() # process event queue + pressed = pygame.key.get_pressed() + if pressed[key]: + data = stream.read(INPUT_CHUNK) + frames.append(data) + else: + break stream.stop_stream() stream.close() - self.audio.terminate() + + self.display_ready() return np.frombuffer(b''.join(frames), np.int16).astype(np.float32) * (1 / 32768.0) def speech_to_text(self, waveform): - print("Finished recording, converting to text...") self.text_to_speech(self.config.conversation.recognitionWaitMsg) - transcript = self.model.transcribe(waveform, language = self.config.whisperRecognition.lang, fp16=torch.cuda.is_available()) - return transcript["text"] + transcript = self.model.transcribe(waveform, + language = self.config.whisperRecognition.lang, + fp16=torch.cuda.is_available()) + text = transcript["text"] + self.text_to_speech(text) + return text - def ask_ollama(self, prompt): - print("Sending: ", prompt) - self.text_to_speech(prompt+self.config.conversation.llmWaitMsg) + def ask_ollama(self, prompt, responseCallback): + self.text_to_speech(self.config.conversation.llmWaitMsg) self.conversation_history.append(prompt) full_prompt = "\n".join(self.conversation_history) - response = requests.post(self.config.ollama.url, json= {"model": self.config.ollama.model,"stream":False,"prompt":full_prompt}, headers=OLLAMA_REST_HEADERS) - if response.status_code == 200: - data = json.loads(response.text) - response_text = data["response"] - self.conversation_history.append(response_text) - print("Received: ", response_text) - return response_text - else: - return "Erreur: " + response.text + jsonParam= {"model": self.config.ollama.model, + "stream":True, + "context":self.context, + "prompt":full_prompt} + print(jsonParam) + response = requests.post(self.config.ollama.url, + json=jsonParam, + headers=OLLAMA_REST_HEADERS, + stream=True) + response.raise_for_status() + + tokens = [] + for line in response.iter_lines(): + print(line) + body = json.loads(line) + token = body.get('response', '') + tokens.append(token) + # the response streams one token at a time, print that as we receive it + if token == "." or token == ":": + responseCallback("".join(tokens)) + tokens = [] + + if 'error' in body: + responseCallback("Erreur: " + body['error']) + + if body.get('done', False): + self.context = body['context'] def text_to_speech(self, text): + print(text) self.tts.say(text) self.tts.runAndWait() def main(): + if sys.version_info[0:3] != (3, 9, 13): + print('Warning, it was only tested with python 3.9.13, it may fail') + + pygame.init() + ass = Assistant() - ass.text_to_speech(ass.config.conversation.greeting) - print("Recording...") + push_to_talk_key = pygame.K_SPACE; + + while True: + ass.clock.tick(60) + for event in pygame.event.get(): + if event.type == pygame.KEYDOWN and event.key == push_to_talk_key: + print('Talk to me!') + speech = ass.waveform_from_mic(push_to_talk_key) - speech = ass.waveform_from_mic() + transcription = ass.speech_to_text(waveform=speech) + + ass.ask_ollama(transcription, ass.text_to_speech) + print('Done') - transcription = ass.speech_to_text(waveform=speech) - - response = ass.ask_ollama(transcription) + if event.type == pygame.locals.QUIT: + ass.shutdown() - ass.text_to_speech(text=response) if __name__ == "__main__": main() diff --git a/assistant.yaml b/assistant.yaml index 2f8e976..707ef01 100644 --- a/assistant.yaml +++ b/assistant.yaml @@ -1,3 +1,8 @@ +messages: + pressSpace: "Pressez sur espace pour parler puis relachez." + loadingModel: "Chargement du modèle..." + noAudioInput: "Erreur: Pas d'entrée son" + whisperRecognition: modelPath: "whisper/large-v3.pt" lang: "fr" @@ -7,7 +12,7 @@ ollama: model: "mistral" conversation: - context: "This is a discussion in french.\\n" - greeting: " Je vous écoute." - recognitionWaitMsg: " J'interprète votre demande." - llmWaitMsg: " Laissez moi réfléchir." + context: "This is a discussion in french." + greeting: "Je vous écoute." + recognitionWaitMsg: "Oui." + llmWaitMsg: "Laissez moi réfléchir." diff --git a/requirements.txt b/requirements.txt index e52fa06..4abe28c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ openai==1.2.3 Wave==0.0.2 openai-whisper @ git+https://github.com/openai/whisper.git@fcfeaf1b61994c071bba62da47d7846933576ac9 PyAudio==0.2.14 -pyyaml==6.0.1 \ No newline at end of file +pyyaml==6.0.1 +pygame==2.5.2 \ No newline at end of file