-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathhighlight_filter_agent.py
More file actions
1459 lines (1222 loc) · 52.4 KB
/
Copy pathhighlight_filter_agent.py
File metadata and controls
1459 lines (1222 loc) · 52.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
highlight_filter_agent.py —— HighlightFilter Agent
项目:《面向无字幕赛事录像的体育视频自动字幕与高光剪辑多智能体系统》
职责:
1. 读取 MediaParseOutput(音频特征 + 字幕),执行双层融合高光筛选。
2. 第一层:音频响度突增候选框 + 字幕关键词匹配候选框,窗口并集融合。
3. 第二层:多模态评分(audio_score / text_score / combined_score)。
4. 第三层:轻量 LLM 语义二次校验,过滤广告/采访/无效片段。
5. 输出标准化 HighlightFilterOutput,供 EditPlanner 读取。
设计原则:
- 零跨模块对象传递:只读写本地 JSON 文件。
- 关键词兼容模糊匹配:应对 Whisper 错别字(如 "三芬"→"三分")。
- LLM 兜底但不强依赖:若 API 不可用,保留全部候选并告警。
- 允许纯文本触发:客队进球时观众安静,关键词必须能独立触发高光。
"""
from __future__ import annotations
import json
import logging
import os
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple
import requests
from config import settings
from pydantic import BaseModel, Field, ConfigDict
from schema import (
AudioFeatureSegment,
HighlightCandidate,
HighlightFilterOutput,
HighlightTrigger,
MediaParseOutput,
SubtitleSegment,
save_to_json,
)
# ---------------------------------------------------------------------------
# 日志配置
# ---------------------------------------------------------------------------
logging.basicConfig(
level=getattr(logging, settings.log_level, logging.INFO),
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("HighlightFilterAgent")
# ---------------------------------------------------------------------------
# 异常类
# ---------------------------------------------------------------------------
class HighlightFilterError(Exception):
"""HighlightFilter Agent 通用异常基类。"""
pass
class LLMValidationError(HighlightFilterError):
"""LLM 语义校验阶段异常(网络超时、返回格式错误等)。"""
pass
# ---------------------------------------------------------------------------
# 常量:进球触发词(用于语义化精剪)
# ---------------------------------------------------------------------------
_GOAL_TRIGGER_PATTERNS = [
# 英文篮球(注意:避免 \b 与标点冲突,使用 (^|\s) 和 (\s|$|!|\?|\.) 作为边界)
r"(?:^|\s)(it's good|it's going|bang!|downtown!|from downtown|three point|3-pointer|three)(?:\s|$|!|\?|\.)",
r"(?:^|\s)(goal!|scores?|makes? it|what a finish|equaliser|equalizer|penalty)(?:\s|$|!|\?|\.)",
# 中文篮球
r"(?:^|\s)(好球!|进了!|进球!|破门!|点球!|三分!|绝杀!)(?:\s|$|!|\?|\.)",
]
# These terms are direct scoring/play evidence. A language model may judge a
# short caption too vague, but it must not discard an explicit "three" or
# "dunk" call and leave the montage with a single clip.
_STRONG_EVENT_KEYWORDS = {
"three", "downtown", "3-pointer", "three point", "dunk", "layup", "jumper", "floater", "hook shot",
"two point", "up and good", "gets it to go", "goal", "penalty", "red card", "yellow card",
"offside", "corner", "save", "miss", "missed", "no good", "brick", "airball", "comes up short",
"scores", "score", "makes", "bang", "buzzer",
"三分", "扣篮", "进球", "点球", "红牌", "黄牌", "越位", "扑救", "绝杀",
}
# A scored two-pointer and a miss are both meaningful requests in this
# application, even when a general-purpose LLM considers the one-line call
# too mundane for a generic "highlight" reel. Preserve these calls so the
# later structured editor can accurately honour requests such as "只剪两分"
# and "只剪打铁".
_PROTECTED_BASKETBALL_PLAY = re.compile(
r"\b(three|3[- ]?pointer|dunk|layup|jumper|floater|hook shot|two[- ]?point|"
r"miss(?:es|ed)?|no good|comes up short|won'?t go|off the rim|rimmed out|"
r"brick|air ?ball|up and good|gets? it to go|lays? it in|finishes)\b|"
r"三分|两分|上篮|跳投|抛投|打铁|不中|没进|命中",
re.I,
)
# ---------------------------------------------------------------------------
# 语义化精剪动态参数
# ---------------------------------------------------------------------------
# 基础保留时长(秒)
_BASE_PRE_GOAL_SEC = 3.0 # 触发词前基础保留
_BASE_POST_GOAL_SEC = 7.0 # 触发词后基础保留:保留落网、欢呼与运动员反应
# 响度动态调整系数
# 原理:响度突增越大(crowd noise 越大),保留越多内容(欢呼是氛围的一部分)
# peak_dbfs 范围约 [-30, -10],映射到 [0, 1] 的响度强度
_LOUDNESS_DYNAMIC_MIN_DBFS = -35.0
_LOUDNESS_DYNAMIC_MAX_DBFS = -15.0
# 动态调整范围(在基础时长上增减)
_DYNAMIC_RANGE_PRE = 2.0 # 前最多 +/- 2s
_DYNAMIC_RANGE_POST = 3.0 # 后最多 +/- 3s
# 精剪后时长硬边界
_TRIM_MIN_DURATION = 3.0
_TRIM_MAX_DURATION = 14.0
# ---------------------------------------------------------------------------
# 内部数据结构:原始候选框(融合前)
# ---------------------------------------------------------------------------
@dataclass
class _RawCandidate:
"""融合前的原始候选框,仅用于内部计算。"""
start_sec: float
end_sec: float
source: str # "audio" | "text"
# 音频相关
surge_segments: List[AudioFeatureSegment] = None
# 文本相关
matched_subtitles: List[Tuple[SubtitleSegment, List[str]]] = None
def __post_init__(self):
if self.surge_segments is None:
self.surge_segments = []
if self.matched_subtitles is None:
self.matched_subtitles = []
# ---------------------------------------------------------------------------
# 工具函数:关键词匹配
# ---------------------------------------------------------------------------
def _normalize_text(text: str) -> str:
"""
文本归一化:转小写、去除多余空白、去除常见标点。
Args:
text: 原始字幕文本。
Returns:
str: 归一化后的文本。
"""
text = text.lower().strip()
# 保留字母、数字、中文、空格,其余替换为空格
text = re.sub(r"[^\w\s\u4e00-\u9fff]", " ", text)
text = re.sub(r"\s+", " ", text).strip()
return text
def _fuzzy_match_score(query: str, text: str) -> float:
"""
计算 query 与 text 的模糊匹配得分(基于最长公共子串比例)。
Args:
query: 关键词(已归一化)。
text: 待匹配文本(已归一化)。
Returns:
float: 相似度 [0.0, 1.0]。
"""
if not query or not text:
return 0.0
if query in text:
return 1.0
# 最长公共子串(LCS)长度
m, n = len(query), len(text)
# 使用一维 DP 节省内存
prev = [0] * (n + 1)
max_len = 0
for i in range(1, m + 1):
curr = [0] * (n + 1)
for j in range(1, n + 1):
if query[i - 1] == text[j - 1]:
curr[j] = prev[j - 1] + 1
max_len = max(max_len, curr[j])
else:
curr[j] = 0
prev = curr
return max_len / max(m, n)
def _match_keywords_in_text(
text: str,
keywords: List[str],
fuzzy_threshold: float = settings.keyword_fuzzy_threshold,
) -> List[str]:
"""
在单条字幕文本中匹配关键词列表,支持模糊匹配。
Args:
text: 字幕文本。
keywords: 关键词列表。
fuzzy_threshold: 模糊匹配阈值。
Returns:
List[str]: 命中的关键词列表(去重)。
"""
normalized = _normalize_text(text)
if not normalized:
return []
matched: List[str] = []
for kw in keywords:
kw_norm = _normalize_text(kw)
if not kw_norm:
continue
# 精确匹配或模糊匹配
if kw_norm in normalized or _fuzzy_match_score(kw_norm, normalized) >= fuzzy_threshold:
matched.append(kw)
# 去重并保持顺序
seen: Set[str] = set()
result: List[str] = []
for kw in matched:
if kw not in seen:
seen.add(kw)
result.append(kw)
return result
# ---------------------------------------------------------------------------
# 第一层:候选框提取
# ---------------------------------------------------------------------------
def _extract_audio_candidates(
segments: List[AudioFeatureSegment],
total_duration_sec: Optional[float] = None,
) -> List[_RawCandidate]:
"""
从音频特征片段中提取响度突增候选框。
规则:
- 遍历所有 segment,收集 loudness_surge_flag=True 的片段。
- 相邻(间隔 <= 2 秒)的突增片段合并为一个候选框。
Args:
segments: AudioFeatureSegment 列表。
Returns:
List[_RawCandidate]: 音频候选框列表。
"""
# 先过滤出突增片段
surge_segs = [seg for seg in segments if seg.loudness_surge_flag]
if not surge_segs:
return []
# 按时间排序
surge_segs.sort(key=lambda s: s.start_sec)
candidates: List[_RawCandidate] = []
current = _RawCandidate(
start_sec=surge_segs[0].start_sec,
end_sec=surge_segs[0].end_sec,
source="audio",
surge_segments=[surge_segs[0]],
)
GAP_THRESHOLD = 2.0 # 相邻合并阈值(秒)
for seg in surge_segs[1:]:
if seg.start_sec - current.end_sec <= GAP_THRESHOLD:
# 合并
current.end_sec = max(current.end_sec, seg.end_sec)
current.surge_segments.append(seg)
else:
candidates.append(current)
current = _RawCandidate(
start_sec=seg.start_sec,
end_sec=seg.end_sec,
source="audio",
surge_segments=[seg],
)
candidates.append(current)
# A crowd peak often happens at the basket, but the replay-worthy reaction
# happens immediately afterwards. Expand both sides before planning so a
# clip cannot end at the exact moment the ball goes in.
for candidate in candidates:
candidate.start_sec = max(0.0, candidate.start_sec - 3.0)
candidate.end_sec += 8.0
if total_duration_sec is not None:
candidate.end_sec = min(candidate.end_sec, total_duration_sec)
logger.info(f"音频突增候选框: {len(candidates)} 个")
for c in candidates:
logger.debug(
f" 音频框 [{c.start_sec:.1f}s-{c.end_sec:.1f}s] "
f"包含 {len(c.surge_segments)} 个突增段"
)
return candidates
def _extract_text_candidates(
subtitles: List[SubtitleSegment],
keywords: List[str],
expand_sec: float = 8.0,
) -> List[_RawCandidate]:
"""
从字幕中提取关键词命中候选框。
规则:
- 遍历字幕,匹配关键词。
- 命中的字幕时间区间前后各外扩 expand_sec 秒。
- 相邻(间隔 <= 2 秒)的候选框合并。
Args:
subtitles: SubtitleSegment 列表。
keywords: 关键词列表。
expand_sec: 前后外扩时长(秒)。
Returns:
List[_RawCandidate]: 文本候选框列表。
"""
hits: List[Tuple[float, float, SubtitleSegment, List[str]]] = []
for sub in subtitles:
matched = _match_keywords_in_text(sub.text, keywords)
if matched:
# 前后外扩
start = max(0.0, sub.start_sec - expand_sec)
end = sub.end_sec + expand_sec
hits.append((start, end, sub, matched))
if not hits:
return []
# 按开始时间排序
hits.sort(key=lambda x: x[0])
candidates: List[_RawCandidate] = []
current_start, current_end = hits[0][0], hits[0][1]
current_subs: List[Tuple[SubtitleSegment, List[str]]] = [(hits[0][2], hits[0][3])]
GAP_THRESHOLD = 2.0
for start, end, sub, matched in hits[1:]:
if start - current_end <= GAP_THRESHOLD:
# 合并
current_end = max(current_end, end)
current_subs.append((sub, matched))
else:
candidates.append(_RawCandidate(
start_sec=current_start,
end_sec=current_end,
source="text",
matched_subtitles=current_subs,
))
current_start, current_end = start, end
current_subs = [(sub, matched)]
candidates.append(_RawCandidate(
start_sec=current_start,
end_sec=current_end,
source="text",
matched_subtitles=current_subs,
))
logger.info(f"文本关键词候选框: {len(candidates)} 个")
for c in candidates:
kws = set()
for _, m in c.matched_subtitles:
kws.update(m)
logger.debug(
f" 文本框 [{c.start_sec:.1f}s-{c.end_sec:.1f}s] "
f"关键词: {', '.join(kws)}"
)
return candidates
# ---------------------------------------------------------------------------
# 窗口融合算法(Window Fusion)
# ---------------------------------------------------------------------------
def _fuse_candidates(
audio_candidates: List[_RawCandidate],
text_candidates: List[_RawCandidate],
gap_threshold: float = 2.0,
) -> List[_RawCandidate]:
"""
将音频与文本候选框进行并集融合。
规则:
- 所有候选框按开始时间排序。
- 重叠或相邻(间隔 <= gap_threshold)的框合并。
- 合并后的框同时继承音频和文本证据。
Args:
audio_candidates: 音频候选框列表。
text_candidates: 文本候选框列表。
gap_threshold: 相邻合并阈值(秒)。
Returns:
List[_RawCandidate]: 融合后的候选框列表。
"""
all_boxes = audio_candidates + text_candidates
if not all_boxes:
return []
# 按开始时间排序
all_boxes.sort(key=lambda b: b.start_sec)
fused: List[_RawCandidate] = []
current = _RawCandidate(
start_sec=all_boxes[0].start_sec,
end_sec=all_boxes[0].end_sec,
source=all_boxes[0].source,
surge_segments=list(all_boxes[0].surge_segments),
matched_subtitles=list(all_boxes[0].matched_subtitles),
)
for box in all_boxes[1:]:
if box.start_sec - current.end_sec <= gap_threshold:
# 合并
current.end_sec = max(current.end_sec, box.end_sec)
current.source = "fused" # 混合来源
current.surge_segments.extend(box.surge_segments)
current.matched_subtitles.extend(box.matched_subtitles)
else:
fused.append(current)
current = _RawCandidate(
start_sec=box.start_sec,
end_sec=box.end_sec,
source=box.source,
surge_segments=list(box.surge_segments),
matched_subtitles=list(box.matched_subtitles),
)
fused.append(current)
logger.info(f"融合后候选框: {len(fused)} 个")
for f in fused:
logger.debug(
f" 融合框 [{f.start_sec:.1f}s-{f.end_sec:.1f}s] "
f"source={f.source} audio={len(f.surge_segments)} text={len(f.matched_subtitles)}"
)
return fused
# ---------------------------------------------------------------------------
# 第二层:多模态评分
# ---------------------------------------------------------------------------
def _dbfs_to_energy(dbfs: float) -> float:
"""
将 dBFS 对数刻度转换为线性能量域。
公式:energy = 10^(dbfs / 20)
0 dBFS → 1.0(满幅),-60 dBFS → 0.001
Args:
dbfs: dBFS 值。
Returns:
float: 线性能量值(> 0)。
"""
return 10.0 ** (dbfs / 20.0)
def _compute_audio_score(
candidate: _RawCandidate,
global_mean_loudness: float,
min_dbfs: float = settings.loudness_min_dbfs,
) -> float:
"""
计算音频维度得分(能量域归一化,避免 dBFS 对数尺度线性运算误差)。
算法:
- 若无突增段,返回 0.0。
- 取候选框内所有突增段的 peak_loudness_dbfs 最大值。
- 绝对阈值拦截:若 peak < min_dbfs(默认 -35 dBFS),直接返回 0.0,
避免静音视频里正常语音被误判为高分。
- 能量域归一化:将对数 dBFS 转换为线性能量后计算相对位置。
score = (energy_peak - energy_min) / (energy_0dbfs - energy_min)
其中 energy_0dbfs = 1.0(满幅能量)。
Args:
candidate: 融合候选框。
global_mean_loudness: 全片平均响度(dBFS),保留用于日志/调试,不参与计算。
min_dbfs: 响度绝对下限(dBFS),低于此值视为无效突增。
Returns:
float: [0.0, 1.0]
"""
if not candidate.surge_segments:
return 0.0
peak = max(seg.peak_loudness_dbfs for seg in candidate.surge_segments)
# 绝对阈值拦截:低于 min_dbfs 的突增视为无效(可能是静音基线上的噪声)
if peak < min_dbfs:
logger.debug(
f" 音频得分=0.0(peak={peak:.1f} dBFS < 阈值={min_dbfs:.1f} dBFS)"
)
return 0.0
# 能量域归一化:避免在 quiet 视频中 -25 dBFS 相对 -85 dBFS 产生 2.0+ 的异常高分
energy_peak = _dbfs_to_energy(peak)
energy_min = _dbfs_to_energy(min_dbfs)
energy_max = 1.0 # 0 dBFS 对应的线性能量
score = (energy_peak - energy_min) / (energy_max - energy_min)
return round(max(min(score, 1.0), 0.0), 3)
def _compute_text_score(
candidate: _RawCandidate,
) -> float:
"""
计算文本维度得分。
算法:
- 若无关键词命中,返回 0.0。
- 统计命中关键词数量(去重)。
- 考虑匹配质量(精确匹配 > 模糊匹配)。
- 归一化:score = min(count / 3.0, 1.0)。
(假设 3 个不同关键词为满分)
Args:
candidate: 融合候选框。
Returns:
float: [0.0, 1.0]
"""
if not candidate.matched_subtitles:
return 0.0
unique_keywords: Set[str] = set()
exact_match_count = 0
for sub, matched_kws in candidate.matched_subtitles:
normalized_sub = _normalize_text(sub.text)
for kw in matched_kws:
unique_keywords.add(kw)
kw_norm = _normalize_text(kw)
if kw_norm in normalized_sub:
exact_match_count += 1
# 基础分:关键词数量
base_score = len(unique_keywords) / 3.0
# 精确匹配加成
bonus = exact_match_count * 0.1
score = min(base_score + bonus, 1.0)
return round(max(score, 0.0), 3)
def _determine_trigger(candidate: _RawCandidate) -> HighlightTrigger:
"""
根据候选框的证据来源确定触发类型。
Args:
candidate: 融合候选框。
Returns:
HighlightTrigger: AUDIO_SURGE / KEYWORD_MATCH / FUSED
"""
has_audio = len(candidate.surge_segments) > 0
has_text = len(candidate.matched_subtitles) > 0
if has_audio and has_text:
return HighlightTrigger.FUSED
elif has_audio:
return HighlightTrigger.AUDIO_SURGE
else:
return HighlightTrigger.KEYWORD_MATCH
# ---------------------------------------------------------------------------
# 第三层:LLM 语义二次校验
# ---------------------------------------------------------------------------
def _build_llm_prompt(subtitle_texts: List[str]) -> str:
"""
构建 LLM 校验 Prompt。
将高光时间窗口内【所有连续字幕文本】完整拼接为解说词上下文,
避免仅投喂离散关键词句导致 LLM 语义断层误判。
Args:
subtitle_texts: 该高光窗口内按时间顺序排列的全部字幕文本列表。
Returns:
str: 完整的 Prompt 字符串。
"""
combined_text = "\n".join(f"[{i+1}] {t}" for i, t in enumerate(subtitle_texts))
prompt = (
"你是一个资深篮球/足球球评。请阅读以下完整的解说词片段,判断其中是否包含"
"进球、绝杀、精彩过人、强烈身体对抗等核心赛事高光。"
"注意:客队进球时观众可能很安静,请完全依据解说文本的兴奋度和客观事实判断。"
"请直接返回 JSON 格式,不要有任何其他内容:\n\n"
f"{combined_text}\n\n"
'返回格式:{"is_highlight": true/false, "reason": "简短的中文过滤或保留理由"}'
)
return prompt
class _LLMResponse(BaseModel):
"""
LLM 语义校验返回的 JSON 结构约束。
使用 Pydantic v2 强类型校验,杜绝 "yes"/"no" 字符串被 bool() 误转为 True 的问题。
"""
is_highlight: bool = Field(
description="是否为有效高光,必须是布尔值 true/false,不接受字符串"
)
reason: str = Field(
default="无说明",
description="校验理由,若 is_highlight=false 则说明过滤原因"
)
model_config = ConfigDict(strict=True)
def _parse_llm_json(content: str) -> _LLMResponse:
"""
将 LLM 返回的 JSON 字符串解析为强类型 _LLMResponse。
处理常见 LLM 输出陷阱:
- "yes"/"no" 字符串 → 显式映射为 bool
- 数字 1/0 → 映射为 bool
- 字段缺失 → 使用默认值并告警
Args:
content: LLM 返回的原始 JSON 字符串。
Returns:
_LLMResponse: 校验后的结构化对象。
Raises:
LLMValidationError: 当 JSON 格式严重损坏无法挽救时抛出。
"""
# 先尝试原生 Pydantic 严格解析
try:
return _LLMResponse.model_validate_json(content)
except Exception:
pass # 降级到容错逻辑
# 容错:先解析为 dict,再手动清洗
try:
data = json.loads(content)
except json.JSONDecodeError as exc:
raise LLMValidationError(f"LLM 返回非合法 JSON: {exc}") from exc
if not isinstance(data, dict):
raise LLMValidationError(f"LLM 返回 JSON 根类型为 {type(data).__name__},期望 dict")
raw_flag = data.get("is_highlight")
# 清洗常见 LLM 错误输出
if isinstance(raw_flag, str):
raw_flag_lower = raw_flag.strip().lower()
if raw_flag_lower in ("yes", "true", "1", "是", "正确"):
raw_flag = True
elif raw_flag_lower in ("no", "false", "0", "否", "错误"):
raw_flag = False
else:
raw_flag = bool(raw_flag) # 兜底:非空字符串为 True
elif isinstance(raw_flag, (int, float)):
raw_flag = bool(raw_flag)
elif raw_flag is None:
raw_flag = True # 默认保留,避免误杀
reason = str(data.get("reason", "无说明"))
return _LLMResponse(is_highlight=raw_flag, reason=reason)
def _call_llm_single(
prompt: str,
api_base: Optional[str] = None,
api_key: Optional[str] = None,
model: Optional[str] = None,
timeout: Optional[int] = None,
max_tokens: Optional[int] = None,
temperature: Optional[float] = None,
) -> Tuple[bool, str]:
"""
单次调用 LLM API 进行高光校验。
采用 OpenAI 原生结构化输出(response_format=json_object)+ Pydantic 强类型校验,
彻底避免模型吐出多余自然语言或弱类型字符串导致解析失败。
Args:
prompt: 输入 Prompt。
api_base, api_key, model, timeout, max_tokens, temperature: API 配置。
Returns:
Tuple[bool, str]: (is_highlight, reason)
若调用失败,返回 (True, "LLM 调用失败,默认保留") 避免误杀。
"""
api_base = api_base or settings.llm_api_base
api_key = api_key or settings.llm_api_key or os.getenv("LLM_API_KEY", "")
model = model or settings.llm_model_name
timeout = timeout or settings.llm_timeout_sec
max_tokens = max_tokens or settings.llm_max_tokens
temperature = temperature if temperature is not None else settings.llm_temperature
if not api_key:
# 无 API Key,跳过校验,默认保留
return True, "LLM_API_KEY 未设置,默认保留"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
payload = {
"model": model,
"messages": [
{"role": "system", "content": "你是一个体育赛事内容审核助手,只返回 JSON。"},
{"role": "user", "content": prompt},
],
"max_tokens": max_tokens,
"temperature": temperature,
# 强制模型返回合法 JSON,杜绝多余自然语言
"response_format": {"type": "json_object"},
}
raw_content = ""
try:
response = requests.post(
f"{api_base}/chat/completions",
headers=headers,
json=payload,
timeout=timeout,
)
response.raise_for_status()
data = response.json()
raw_content = data["choices"][0]["message"]["content"].strip()
parsed = _parse_llm_json(raw_content)
return parsed.is_highlight, parsed.reason
except requests.Timeout:
logger.warning(f"LLM 请求超时 ({timeout}s),默认保留")
return True, f"LLM 请求超时 ({timeout}s),默认保留"
except requests.RequestException as exc:
logger.warning(f"LLM 请求失败: {exc},默认保留")
return True, f"LLM 请求失败: {exc},默认保留"
except LLMValidationError as exc:
snippet = raw_content[:300] if raw_content else "<空>"
logger.warning(f"LLM 返回解析失败: {exc},默认保留。原始返回: {snippet}")
return True, f"LLM 返回解析失败,默认保留"
except (json.JSONDecodeError, KeyError, IndexError, TypeError) as exc:
# 兜底:捕获所有残余解析异常
snippet = raw_content[:300] if raw_content else "<空>"
logger.warning(f"LLM 返回解析失败: {exc},默认保留。原始返回: {snippet}")
return True, f"LLM 返回解析失败,默认保留"
def _llm_validate_candidates(
candidates: List[HighlightCandidate],
subtitles: List[SubtitleSegment],
max_workers: int = min(4, settings.num_workers),
) -> Tuple[List[HighlightCandidate], int]:
"""
使用 LLM 对候选列表进行批量语义校验。
采用线程池并发请求,每个候选独立调用 LLM。
若 API 未配置,直接返回原列表并记录警告。
Args:
candidates: 待校验的 HighlightCandidate 列表。
subtitles: 全片字幕列表,用于提取关联文本。
max_workers: 并发线程数。
Returns:
Tuple[List[HighlightCandidate], int]: (有效候选列表, 被过滤数量)
"""
if not (settings.llm_api_key or os.getenv("LLM_API_KEY")):
logger.warning(
"LLM_API_KEY 未设置,跳过语义校验。"
"所有候选默认保留。如需启用,请设置环境变量 LLM_API_KEY。"
)
for c in candidates:
c.llm_validated = True
c.llm_reason = "LLM 未启用,默认保留"
return candidates, 0
# 构建字幕索引,便于快速查找
subtitle_by_index = {sub.index: sub for sub in subtitles}
def _validate_one(candidate: HighlightCandidate) -> HighlightCandidate:
# 根据候选框最终确定的时间轴区间 [start_sec, end_sec],
# 提取全片字幕中所有落在该区间内的连续字幕文本,
# 形成完整的解说词上下文,避免仅投喂离散关键词句导致语义断层。
window_texts: List[str] = []
for sub in subtitles:
if sub.end_sec >= candidate.start_sec and sub.start_sec <= candidate.end_sec:
window_texts.append(sub.text)
if not window_texts:
# 无关联字幕(纯音频触发),默认保留
candidate.llm_validated = True
candidate.llm_reason = "纯音频触发,无解说文本,默认保留"
return candidate
# Do not make a structured sports-edit request depend on whether a
# general LLM thinks an ordinary layup or missed shot is "exciting".
# Event classification and the user's later instruction determine
# whether to include it in the final reel.
if _PROTECTED_BASKETBALL_PLAY.search(" ".join(window_texts)):
candidate.llm_validated = True
candidate.llm_reason = "规则保留:字幕含明确两分或未命中回合。"
return candidate
prompt = _build_llm_prompt(window_texts)
is_highlight, reason = _call_llm_single(prompt)
candidate.llm_validated = is_highlight
candidate.llm_reason = reason
return candidate
logger.info(f"开始 LLM 语义校验: {len(candidates)} 个候选,并发数 {max_workers}")
validated: List[HighlightCandidate] = []
filtered_count = 0
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_candidate = {
executor.submit(_validate_one, c): c for c in candidates
}
for future in as_completed(future_to_candidate):
candidate = future_to_candidate[future]
try:
result = future.result()
strong_event = any(
keyword.lower() in _STRONG_EVENT_KEYWORDS
for keyword in result.matched_keywords
)
if not result.llm_validated and strong_event:
result.llm_validated = True
result.llm_reason = (
"规则保留:字幕含明确得分事件词,"
"不因 LLM 不确定而误删。"
)
if result.llm_validated:
validated.append(result)
else:
filtered_count += 1
logger.info(
f"LLM 过滤候选 {result.candidate_id}: {result.llm_reason}"
)
except Exception as exc:
logger.error(f"LLM 校验异常: {exc},默认保留")
candidate.llm_validated = True
candidate.llm_reason = f"校验异常: {exc},默认保留"
validated.append(candidate)
logger.info(
f"LLM 校验完成: 保留 {len(validated)} 个,过滤 {filtered_count} 个"
)
return validated, filtered_count
def _supplement_with_audio_coverage(
candidates: List[HighlightCandidate],
segments: List[AudioFeatureSegment],
subtitles: List[SubtitleSegment],
total_duration: float,
) -> List[HighlightCandidate]:
"""Add separated, high-energy windows when captions under-detect fast plays.
Sports broadcasts often keep crowd volume almost constant, and Whisper can
miss a rapid sequence of scores. Rather than returning a one-clip montage,
keep a small set of non-overlapping audio-peak windows as reviewable
candidates. Keyword/LLM candidates remain ranked above these fallbacks.
"""
desired_count = min(4, max(2, int(total_duration // 12)))
if len(candidates) >= desired_count:
return candidates
existing = [(c.start_sec, c.end_sec) for c in candidates]
peak_max = max((seg.peak_loudness_dbfs for seg in segments), default=-20.0)
ranked = sorted(segments, key=lambda seg: (seg.peak_loudness_dbfs, seg.mean_loudness_dbfs), reverse=True)
for seg in ranked:
center = (seg.start_sec + seg.end_sec) / 2
start = round(max(0.0, center - 5.0), 3)
end = round(min(total_duration, center + 8.0), 3)
# A montage must never reuse source frames. Even a one-second
# overlap reads as an obvious replay when clips are concatenated.
if any(max(0.0, min(end, old_end) - max(start, old_start)) > 0.0 for old_start, old_end in existing):
continue
related = [sub.index for sub in subtitles if sub.end_sec >= start and sub.start_sec <= end]
relative_peak = max(0.0, min(1.0, (seg.peak_loudness_dbfs + 35.0) / 35.0))
candidates.append(HighlightCandidate(
candidate_id=f"HL_{len(candidates) + 1:03d}",
start_sec=start,
end_sec=end,
duration_sec=round(end - start, 3),
trigger=HighlightTrigger.AUDIO_SURGE,
audio_score=round(max(0.25, relative_peak), 3),
text_score=0.0,
# Crowd volume is only a recall fallback. It deliberately has a
# small score so it cannot outrank a subtitle or verified-score
# candidate later in the edit planner.
combined_score=round(max(0.25, relative_peak) * settings.candidate_audio_weight, 3),
matched_keywords=[],
related_subtitle_indices=related,
llm_validated=True,
llm_reason="字幕召回不足时的音频峰值补充候选",
))
existing.append((start, end))
if len(candidates) >= desired_count:
break
return candidates
# ---------------------------------------------------------------------------
# 主流程:从 _RawCandidate 构建 HighlightCandidate
# ---------------------------------------------------------------------------
def _enforce_min_duration(
raw: _RawCandidate,
min_duration: float = settings.min_highlight_duration_sec,
video_end_sec: Optional[float] = None,
) -> Optional[_RawCandidate]:
"""
强制保证候选框时长不低于 min_duration。
策略:
- 若时长已 >= min_duration,直接返回原框。
- 否则尝试向两侧对称扩展,每次扩展 0.5 秒,直到满足 min_duration
或触及原始视频边界。
- 若扩展后仍不足 min_duration,返回 None(丢弃该候选)。
Args:
raw: 原始候选框。
min_duration: 最小允许时长(秒)。
video_end_sec: 视频总时长(秒),作为右边界;None 则不限制右边界。
Returns:
Optional[_RawCandidate]: 扩展后的候选框,或 None(若无法达标则丢弃)。
"""
duration = raw.end_sec - raw.start_sec
if duration >= min_duration:
return raw
# 需要扩展的总量
deficit = min_duration - duration
expand_each_side = deficit / 2.0
new_start = max(0.0, raw.start_sec - expand_each_side)
new_end = raw.end_sec + expand_each_side
if video_end_sec is not None:
new_end = min(new_end, video_end_sec)
# 如果单侧顶到边界,尝试把 deficit 全加到另一侧
if new_start == 0.0 and new_end < raw.end_sec + deficit:
new_end = min(raw.end_sec + deficit, video_end_sec if video_end_sec is not None else float("inf"))
if video_end_sec is not None and new_end == video_end_sec and new_start > max(0.0, raw.start_sec - deficit):
new_start = max(0.0, raw.start_sec - deficit)
expanded_duration = new_end - new_start
if expanded_duration < min_duration:
logger.debug(
f" 丢弃过短候选框 [{raw.start_sec:.1f}s-{raw.end_sec:.1f}s] "
f"扩展后仍仅 {expanded_duration:.1f}s < {min_duration:.1f}s"
)
return None
# 重新收集扩展区间内的音频和文本证据
expanded_surges = [
seg for seg in raw.surge_segments
if seg.end_sec >= new_start and seg.start_sec <= new_end
]
expanded_subs = [
(sub, kws) for sub, kws in raw.matched_subtitles
if sub.end_sec >= new_start and sub.start_sec <= new_end
]
return _RawCandidate(
start_sec=new_start,
end_sec=new_end,
source=raw.source,
surge_segments=expanded_surges,
matched_subtitles=expanded_subs,
)
def _split_oversized_candidate(
raw: _RawCandidate,
max_duration: float = settings.max_highlight_duration_sec,
expand_sec: float = 8.0,