SiLT-Project/video2text_joint.py at main · AkshTi/SiLT-Project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
import cv2
import numpy as np
import torch
from sign_recogn_joint import RecogJoint
from hand_cropping import crop_hand_joint
from data_processing import normalize_joints
import time
import mediapipe as mp
from signtotext import sign_to_text
from signtotext import filter_text

def videototext():
    model = RecogJoint()
    model.load_state_dict(torch.load('sign_recogn_joint_new',map_location=torch.device('cpu')))
    model.eval()
    st = time.time()
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    with torch.no_grad():
        cap = cv2.VideoCapture(0)
        texts = []
        pred_scores = []
        hands = mp.solutions.hands.Hands(static_image_mode=False,
                        max_num_hands=1,
                        min_detection_confidence=0.5,
                        min_tracking_confidence=0.45)
        while True:
            def detect(texts,pred_scores,cap,model):
                suc, img = cap.read()
                if suc is None:
                    time.sleep(0.01)
                    return -1
                results = crop_hand_joint(img,hands)
                if results is None:
                    texts.append(' ')
                    pred_scores.append(0)
                    time.sleep(0.01)
                    return -1
                # print(crops.shape)
                results = normalize_joints(results[0])
                preds = model(torch.tensor(results).to('cpu')).detach().numpy()[0]
                # N, 26
                pred_scores.append(np.max(preds))
                text = alphabet[np.argmax(preds)]
                texts.append(text)
                print(text)
                cv2.imshow('image',img)
                #print(normalize(crops)[0])
            if detect(texts,pred_scores,cap,model) == -1:
                continue
            if cv2.waitKey(1) & 0xFF == ord('q'):
                cv2.destroyAllWindows()
                break
        pred_scores = np.stack(pred_scores,axis=0)
        # print(pred_scores)
        # print(pred_scores.shape)
        text = sign_to_text(texts, pred_scores)
        filtered = filter_text(text)
        print(text)
        return filtered