RoboHand_Sign/main.py at main · somebody241/RoboHand_Sign · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
from time import sleep

import whisper
import sounddevice as sd
import numpy as np
import map
from pyfirmata import Arduino

from sign_letters import *

board = Arduino('/dev/cu.usbmodem1101')
# board = Arduino('COM7')

right_pinky = board.get_pin("d:2:s")
right_ring = board.get_pin("d:4:s")
right_middle = board.get_pin("d:3:s")
right_index = board.get_pin("d:5:s")
right_thumb = board.get_pin("d:6:s")

left_pinky = board.get_pin("d:11:s")
left_ring = board.get_pin("d:8:s")
left_middle = board.get_pin("d:9:s")
left_index = board.get_pin("d:12:s")
left_thumb = board.get_pin("d:10:s")

right_hand = Hand([right_pinky, right_ring, right_middle, right_index, right_thumb])

left_hand = Hand([left_pinky, left_ring, left_middle, left_index, left_thumb])

alphabet = ['а', 'б', 'в', 'г', 'е', 'и', 'л', 'м', 'н', 'о', 'п', 'р', 'т', 'у', 'х', 'ц', 'ч', 'ш', 'ю']

left_hand_letters = ['л', 'м', 'п', 'т']

rh = right_hand
lh = left_hand

map.table['stop'](rh)
map.table['stop_left'](lh)

model = whisper.load_model("base")
sample_rate = 16000
dur = 3
running = True

print("SPACE to start/stop")
print("ESC to exit")

audio = sd.rec(int(dur * sample_rate), samplerate=sample_rate, channels=1, dtype=np.float32)
sd.wait()
result = model.transcribe(audio.flatten(), language="bg")['text']
arr = list(result)
for i in arr:
    if i == ' ':
        arr.remove(i)
print("result: " + str(arr))

for i in arr:
    if i in alphabet:
        if i in left_hand_letters:
            map.table[i](lh)
            map.table['stop'](rh)
        else:
            map.table[i](rh)
            map.table['stop_left'](lh)
    else:
        map.table['stop'](rh)
        map.table['stop_left'](lh)
    sleep(1)

map.table['stop'](rh)
map.table['stop_left'](lh)

board.exit()