Showing
13 changed files
with
994 additions
and
0 deletions
src/football_replay_match/cfg/config.yaml
0 → 100644
src/football_replay_match/config.py
0 → 100644
| 1 | +from aabd.base.cfg_loader import load_yaml_by_file_with_env | ||
| 2 | +from aabd.base.enhance_dict import EnhanceDict, read_prefixed_env_vars | ||
| 3 | +from pathlib import Path | ||
| 4 | +from logging import getLogger | ||
| 5 | +logger = getLogger(__name__) | ||
| 6 | +cfg_dir = Path(__file__).parent.absolute() / 'cfg' | ||
| 7 | +settings = EnhanceDict(load_yaml_by_file_with_env((cfg_dir / 'config.yaml').as_posix())) | ||
| 8 | +settings.update(read_prefixed_env_vars('FRM_')) | ||
| 9 | + | ||
| 10 | +logger.info(f'Settings: {settings}') |
src/football_replay_match/core/api.py
0 → 100644
src/football_replay_match/main.py
0 → 100644
| 1 | +import json | ||
| 2 | +import time | ||
| 3 | + | ||
| 4 | +from aabd.base.patched_logging import init_logging, get_logger | ||
| 5 | + | ||
| 6 | +init_logging() | ||
| 7 | +logger = get_logger(__name__) | ||
| 8 | +from threading import Thread | ||
| 9 | +from .core.api import replay_match_event | ||
| 10 | + | ||
| 11 | +from aabd.mq.kafka_client import KafkaMessageIterator, KafkaProducer | ||
| 12 | + | ||
| 13 | + | ||
| 14 | +def get_message_iterator(kafka_config): | ||
| 15 | + return KafkaMessageIterator(bootstrap_servers=kafka_config.get('servers', None), | ||
| 16 | + group_id=kafka_config.get('group_id', None), | ||
| 17 | + topic=kafka_config.get('topic', None), | ||
| 18 | + sasl_plain_username=kafka_config.get('username', None), | ||
| 19 | + sasl_plain_password=kafka_config.get('password', None), | ||
| 20 | + value_deserializer="str") | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +def get_message_producer(kafka_config): | ||
| 24 | + return KafkaProducer(bootstrap_servers=kafka_config.get('servers', None), | ||
| 25 | + sasl_plain_username=kafka_config.get('username', None), | ||
| 26 | + sasl_plain_password=kafka_config.get('password', None)) | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +def get_callback_func(producer, topic): | ||
| 30 | + def callback_func(task_id, call_data): | ||
| 31 | + try: | ||
| 32 | + st = time.time() | ||
| 33 | + producer.send_message_async(topic, call_data, key=task_id, timeout=5) | ||
| 34 | + logger.info(f"sent_message[{task_id}][{time.time() - st:.2f}s]: {call_data}") | ||
| 35 | + except: | ||
| 36 | + logger.exception(f"Error sending message: {call_data}") | ||
| 37 | + | ||
| 38 | + return callback_func | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +def kafka_message_iterator_thread(message_iterator, config, callback_func): | ||
| 42 | + with message_iterator: | ||
| 43 | + for message in message_iterator: | ||
| 44 | + try: | ||
| 45 | + json_message = json.loads(message) | ||
| 46 | + replay_match_event(json_message) | ||
| 47 | + logger.info(message) | ||
| 48 | + except: | ||
| 49 | + logger.exception(f"Error processing message: {message}") | ||
| 50 | + | ||
| 51 | + | ||
| 52 | +if __name__ == '__main__': | ||
| 53 | + from config import settings | ||
| 54 | + | ||
| 55 | + message_iter = get_message_iterator(settings.input_kafka) | ||
| 56 | + kafka_producer = get_message_producer(settings.output_kafka) | ||
| 57 | + try: | ||
| 58 | + | ||
| 59 | + Thread(target=kafka_message_iterator_thread, | ||
| 60 | + args=(message_iter, settings, | ||
| 61 | + get_callback_func(kafka_producer, settings.output_kafka.get('topic', None)))).start() | ||
| 62 | + | ||
| 63 | + except KeyboardInterrupt: | ||
| 64 | + message_iter.running = False | ||
| 65 | + kafka_producer.close() |
| 1 | +import json | ||
| 2 | +import os.path | ||
| 3 | +from pathlib import Path | ||
| 4 | + | ||
| 5 | +from langchain_core.messages import SystemMessage, HumanMessage | ||
| 6 | +from langchain_ollama import ChatOllama | ||
| 7 | + | ||
| 8 | +try: | ||
| 9 | + from .llm_video_content import contents as video_contents | ||
| 10 | +except: | ||
| 11 | + from llm_video_content import contents as video_contents | ||
| 12 | + | ||
| 13 | +req_prompt = """ | ||
| 14 | +# Role | ||
| 15 | +你是一名拥有20年经验的足球视频技术分析师,擅长结合**视觉画面**与**解说音频(ASR)**进行跨镜头的事件匹配。你能通过解说员的描述锁定时间背景,并通过视觉特征确认物理细节。 | ||
| 16 | + | ||
| 17 | +# Task | ||
| 18 | +输入包含: | ||
| 19 | +1. 【回放片段】(Replay):包含视频帧 + **对应的解说文本**。 | ||
| 20 | +2. 【直播进球片段列表】(Live Candidates):包含视频帧 + **对应的解说文本**。 | ||
| 21 | + | ||
| 22 | +目标:在【直播片段】中找到与【回放片段】属于**同一个进球事件**的片段。如果没有任何片段匹配,返回 null。 | ||
| 23 | + | ||
| 24 | +# Analysis Workflow (多模态分析流程) | ||
| 25 | + | ||
| 26 | +请按照以下步骤进行推理: | ||
| 27 | + | ||
| 28 | +### 第一步:解说词元数据提取 (听觉线索) | ||
| 29 | +分析【回放片段】的解说文本,寻找以下关键信息: | ||
| 30 | +- **时间指代**:解说员是否提到了具体时间?(如“上半场”、“第10分钟”、“开场不久”)。 | ||
| 31 | +- **事件描述**:解说员如何描述这个进球?(如“世界波”、“点球”、“补射”、“乌龙球”)。 | ||
| 32 | +- **球员提及**:解说员念出了谁的名字? | ||
| 33 | + | ||
| 34 | +### 第二步:视觉物理指纹提取 (视觉线索) | ||
| 35 | +忽略镜头语言(慢放、特写),提取核心物理特征: | ||
| 36 | +- **进攻与射门**:进攻方式(传中/直塞)、射门部位(头/脚)、射门位置。 | ||
| 37 | +- **球路与防守**:球的轨迹(高/低/折射)、门将扑救动作(侧扑/倒地/未动)。 | ||
| 38 | +- **庆祝**:进球后的庆祝动作(仅作为辅助验证)。 | ||
| 39 | + | ||
| 40 | +### 第三步:跨模态匹配与验证 | ||
| 41 | +遍历【直播片段】,结合视觉和听觉进行判断: | ||
| 42 | +- **视觉一致性**:直播片段的动作、轨迹、门将反应是否与回放完全吻合? | ||
| 43 | +- **听觉一致性**: | ||
| 44 | + - 如果回放解说提到“这是第5分钟的进球”,而直播片段的时间戳是90分钟,**不要直接排除**,而是检查直播片段的解说是否也提到了“回顾第5分钟”或者直播片段的视觉内容确实是第5分钟的动作。 | ||
| 45 | + - **核心原则**:视觉物理特征 > 解说词描述 > 时间戳元数据。 | ||
| 46 | + - **特殊情况**:如果视觉特征高度相似(如同一个球员、同一个角度),但解说词明确说“这是另一个进球(例如:这是他的第二个进球,而回放说是第一个)”,则判定为不匹配。 | ||
| 47 | + | ||
| 48 | +# Logic Constraints (逻辑约束 - 必须严格遵守) | ||
| 49 | + | ||
| 50 | +- **时间单向性原则(铁律)**: | ||
| 51 | + - **回看中的比赛时间一定在比赛片段画面时间之后**。 | ||
| 52 | + - 逻辑:回放是对过去发生事件的回顾。如果【回放片段】中解说提到的比赛时间(或画面显示的比赛时钟)**早于**【直播片段】中解说提到的比赛时间(或画面时钟),则**绝对不可能**是同一个事件。 | ||
| 53 | + - *示例*:回放解说在描述“第10分钟的进球”,而直播片段明确发生在“第5分钟”,则该直播片段**一定不是**目标。 | ||
| 54 | +- **时间陷阱**:回放可能是赛后集锦。如果解说员说“让我们看看**刚才**那个球”或者“**上半场**那个球”,即使当前比赛时间是90分钟,也要去匹配对应时间段的直播片段(或视觉特征)。 | ||
| 55 | +- **同名陷阱**:如果解说提到“又是**凯恩**进球了”,不能只看凯恩,必须看**怎么进的**(头球还是点球)。 | ||
| 56 | +- **无匹配处理**:如果所有候选片段在视觉动作(如射门方式、进球位置)或关键事件逻辑上与回放明显不符,必须判定为无匹配,将 `video_id` 设为 `null`。 | ||
| 57 | + | ||
| 58 | +### 输出要求 | ||
| 59 | + | ||
| 60 | +请仅输出一个JSON格式的结果,不要输出任何分析过程。不要包含 markdown 标记(如 ```json ... ```),不要包含任何解释或额外文本。 | ||
| 61 | +格式如下: | ||
| 62 | +{ | ||
| 63 | + "replay_summary": { | ||
| 64 | + "audio_cues": "解说提到的关键信息(如:'第15分钟', '远射', '德布劳内')", | ||
| 65 | + "visual_cues": "视觉关键特征(如:'禁区外右脚', '球挂死角', '门将飞身扑救')" | ||
| 66 | + }, | ||
| 67 | + "reasoning": "综合分析:回放解说提到是'上半场的远射',视觉显示'17号球员禁区外起脚'。Candidate_1 视觉是'近距离推射',排除。Candidate_2 视觉是'禁区外远射',且门将动作一致,虽然直播时间显示是下半场(可能是集锦回顾),但解说也提到了'回顾上半场',确认为同一事件。", | ||
| 68 | + "video_id": "Candidate_2" | ||
| 69 | +}""" | ||
| 70 | + | ||
| 71 | + | ||
| 72 | +class FootballReplayMatchLive: | ||
| 73 | + def __init__(self, base_url: str, model: str, temperature: float = 0.0, api_key: str = 'no_key'): | ||
| 74 | + self.base_url = base_url | ||
| 75 | + self.model = model | ||
| 76 | + self.temperature = temperature | ||
| 77 | + self.api_key = api_key | ||
| 78 | + # self.model = ChatOllama(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0", temperature=0.7, | ||
| 79 | + # keep_alive=-1, reasoning=False) | ||
| 80 | + # self.model = ChatOpenAI(base_url="http://192.168.1.59:11434/v1", model="qwen3.6:35b-a3b-q8_0", temperature=0.7, | ||
| 81 | + # api_key='no_key') | ||
| 82 | + self.model = ChatOllama(base_url=base_url, model=model, temperature=temperature, keep_alive=-1, reasoning=False) | ||
| 83 | + | ||
| 84 | + def _match_once(self, replay_video_contents: list, live_videos: list[dict], cache_path=None, record: list = None): | ||
| 85 | + | ||
| 86 | + if len(live_videos) == 0: | ||
| 87 | + return None | ||
| 88 | + elif len(live_videos) == 1: | ||
| 89 | + live_video_path = live_videos[0].get("video_path", None) | ||
| 90 | + live_video_id = live_videos[0].get("video_id", os.path.basename(live_video_path)) | ||
| 91 | + asr_text = live_videos[0].get("asr_text", '') | ||
| 92 | + live = { | ||
| 93 | + "video_id": live_video_id, | ||
| 94 | + "video_path": live_video_path, | ||
| 95 | + "asr_text": asr_text, | ||
| 96 | + } | ||
| 97 | + if record is not None: | ||
| 98 | + record.append({"live": live, "llm_result": None, 'live_list': [live]}) | ||
| 99 | + return live | ||
| 100 | + user_contents = [] | ||
| 101 | + # replay_video_contents = video_contents(replay_video["video_path"], "\n【回放片段信息】\n", | ||
| 102 | + # prompt_end=f"\n回放解说内容:{replay_video['asr_text']}\n", | ||
| 103 | + # video_name=os.path.basename(replay_video["video_path"]), | ||
| 104 | + # fps=2, max_frames=999, sampling_mode="head", max_short_edge=480) | ||
| 105 | + live_videos_contents = [] | ||
| 106 | + live_map = {} | ||
| 107 | + live_records = {} | ||
| 108 | + for live_video in live_videos: | ||
| 109 | + live_video_path = live_video.get("video_path", None) | ||
| 110 | + live_video_id = live_video.get("video_id", os.path.basename(live_video_path)) | ||
| 111 | + asr_text = live_video.get("asr_text", '') | ||
| 112 | + live_video_contents = live_video.get("llm_contents", None) | ||
| 113 | + if live_video_contents is None: | ||
| 114 | + live_video_contents = video_contents(live_video_path, | ||
| 115 | + prompt_start=f"### 候选片段 video_id: {live_video_id} ###", | ||
| 116 | + prompt_end=f"\n该片段解说内容: {asr_text}\n", | ||
| 117 | + video_name=os.path.basename(live_video_path), | ||
| 118 | + fps=2, max_frames=999, sampling_mode="head", max_short_edge=480) | ||
| 119 | + live_map[live_video_id] = { | ||
| 120 | + "video_id": live_video_id, | ||
| 121 | + "video_path": live_video_path, | ||
| 122 | + "asr_text": asr_text, | ||
| 123 | + "contents": live_video_contents, | ||
| 124 | + | ||
| 125 | + } | ||
| 126 | + live_records[live_video_id] = { | ||
| 127 | + "video_id": live_video_id, | ||
| 128 | + "video_path": live_video_path, | ||
| 129 | + "asr_text": asr_text, | ||
| 130 | + } | ||
| 131 | + live_videos_contents.extend(live_video_contents) | ||
| 132 | + | ||
| 133 | + user_contents.extend(replay_video_contents) | ||
| 134 | + user_contents.append({'type': "text", "text": "\n【候选直播片段列表】\n"}) | ||
| 135 | + user_contents.extend(live_videos_contents) | ||
| 136 | + user_contents.append({'type': "text", "text": "\n请根据上述片段进行匹配并按要求输出结果\n"}) | ||
| 137 | + system_message = SystemMessage(content=req_prompt) | ||
| 138 | + user_message = HumanMessage(content=user_contents) | ||
| 139 | + result = self.model.invoke([system_message, user_message]).content | ||
| 140 | + try: | ||
| 141 | + result_json = json.loads(result) | ||
| 142 | + except json.JSONDecodeError: | ||
| 143 | + try: | ||
| 144 | + result_json = json.loads(result.replace("```json", "").replace("```", "")) | ||
| 145 | + except Exception as e: | ||
| 146 | + print("JSON解析失败:", result) | ||
| 147 | + raise e | ||
| 148 | + | ||
| 149 | + video_id = result_json.get("video_id", None) | ||
| 150 | + result_live = live_map.get(video_id, None) | ||
| 151 | + if record is not None: | ||
| 152 | + record.append( | ||
| 153 | + {"live": live_records.get(video_id,None), "llm_result": result_json, 'live_list': list(live_records.values())}) | ||
| 154 | + return result_live | ||
| 155 | + | ||
| 156 | + def _match_batch(self, replay_video_contents: list, live_videos: list[dict], max_parallel: int = 3, cache_path=None, | ||
| 157 | + record: list = None): | ||
| 158 | + """ | ||
| 159 | + Match a replay video with live videos to find the most likely match. | ||
| 160 | + :param max_parallel: | ||
| 161 | + :param replay_video: Path to the replay video.(video_path,asr_text) | ||
| 162 | + :param live_videos: [(video_id,video_path,asr_text)] | ||
| 163 | + :param cache_path: Path to cache the result. | ||
| 164 | + :return: JSON object containing the match result. | ||
| 165 | + """ | ||
| 166 | + if cache_path is not None and os.path.exists(cache_path): | ||
| 167 | + with open(cache_path, 'r', encoding='utf-8') as f: | ||
| 168 | + return json.loads(f.read()) | ||
| 169 | + | ||
| 170 | + # 按照max_parallel对live_videos进行分组 | ||
| 171 | + live_videos_groups = [live_videos[i::max_parallel] for i in range(max_parallel)] | ||
| 172 | + # 过滤掉空的分组 | ||
| 173 | + live_videos_groups = [g for g in live_videos_groups if g] | ||
| 174 | + # 如果group 大于1 并且最后一个group 只有一个元素,将其唯一元素放入倒数第二个group,并删除最后一个group | ||
| 175 | + if len(live_videos_groups) > 1 and len(live_videos_groups[-1]) == 1: | ||
| 176 | + live_videos_groups[-2].append(live_videos_groups[-1][0]) | ||
| 177 | + live_videos_groups.pop() | ||
| 178 | + | ||
| 179 | + if len(live_videos_groups) > 1: | ||
| 180 | + match_result = [] | ||
| 181 | + for live_videos_group in live_videos_groups: | ||
| 182 | + g_live = self._match_once(replay_video_contents, live_videos_group, cache_path, record) | ||
| 183 | + if g_live is None: | ||
| 184 | + continue | ||
| 185 | + match_result.append(g_live) | ||
| 186 | + if len(match_result) == 0: | ||
| 187 | + return None | ||
| 188 | + else: | ||
| 189 | + return self._match_batch(replay_video_contents, match_result, max_parallel, cache_path, record) | ||
| 190 | + elif len(live_videos_groups) == 1: | ||
| 191 | + return self._match_once(replay_video_contents, live_videos_groups[0], cache_path, record) | ||
| 192 | + else: | ||
| 193 | + return None | ||
| 194 | + | ||
| 195 | + def match_batch(self, replay_video: dict, live_videos: list[dict], max_parallel: int = 3, cache_path=None): | ||
| 196 | + if cache_path is not None and os.path.exists(cache_path): | ||
| 197 | + try: | ||
| 198 | + with open(cache_path, 'r', encoding='utf-8') as f: | ||
| 199 | + return json.loads(f.read()).get("result", None) | ||
| 200 | + except: | ||
| 201 | + os.remove(cache_path) | ||
| 202 | + replay_video_contents = video_contents(replay_video["video_path"], "\n【回放片段信息】\n", | ||
| 203 | + prompt_end=f"\n回放解说内容:{replay_video['asr_text']}\n", | ||
| 204 | + video_name=os.path.basename(replay_video["video_path"]), | ||
| 205 | + fps=2, max_frames=999, sampling_mode="head", max_short_edge=480) | ||
| 206 | + live_record = [] | ||
| 207 | + result = self._match_batch(replay_video_contents, live_videos, max_parallel, cache_path, live_record) | ||
| 208 | + if result is not None: | ||
| 209 | + result_no_content = { | ||
| 210 | + "video_id": result.get("video_id", None), | ||
| 211 | + "video_path": result.get("video_path", None), | ||
| 212 | + "asr_text": result.get("asr_text", None), | ||
| 213 | + } | ||
| 214 | + else: | ||
| 215 | + result_no_content = None | ||
| 216 | + record = { | ||
| 217 | + "request": { | ||
| 218 | + "replay_video": replay_video, | ||
| 219 | + "live_videos": live_videos, | ||
| 220 | + "max_parallel": max_parallel, | ||
| 221 | + "cache_path": cache_path | ||
| 222 | + }, | ||
| 223 | + "result": result_no_content, | ||
| 224 | + "live_record": live_record | ||
| 225 | + } | ||
| 226 | + if cache_path is not None: | ||
| 227 | + os.makedirs(Path(cache_path).parent, exist_ok=True) | ||
| 228 | + with open(cache_path, 'w', encoding='utf-8') as f: | ||
| 229 | + f.write(json.dumps(record, ensure_ascii=False, indent=4)) | ||
| 230 | + return result_no_content |
| 1 | +import json | ||
| 2 | +import os.path | ||
| 3 | +from pathlib import Path | ||
| 4 | + | ||
| 5 | +from langchain_core.messages import SystemMessage, HumanMessage | ||
| 6 | +from langchain_ollama import ChatOllama | ||
| 7 | + | ||
| 8 | +try: | ||
| 9 | + from .llm_video_content import contents as video_contents | ||
| 10 | +except: | ||
| 11 | + from llm_video_content import contents as video_contents | ||
| 12 | + | ||
| 13 | +req_prompt = """ | ||
| 14 | +### 角色设定 | ||
| 15 | +你是一位拥有20年经验的足球赛事视频分析专家。你的任务是结合**视频画面**和**解说音频文本**,精准判断视频片段中是否发生了"有效进球"。 | ||
| 16 | + | ||
| 17 | +### 输入内容 | ||
| 18 | + | ||
| 19 | +1. **视频片段**:可能包含赛场、观众、教练、回放、演播室、宣传片、广告等多种画面。 | ||
| 20 | +2. **解说文本**:该片段对应的实时解说内容(ASR)。 | ||
| 21 | + | ||
| 22 | +### 分析逻辑(思维链) | ||
| 23 | + | ||
| 24 | +请综合以下三个维度的信息进行推理: | ||
| 25 | + | ||
| 26 | +#### 1. 场景维度(非比赛画面过滤 - 最高优先级) | ||
| 27 | + | ||
| 28 | +- **非比赛内容识别**:检查画面是否为**宣传片**、**商业广告**、**纯演播室解说**、**集锦混剪**(无连续比赛画面)或* | ||
| 29 | + *静态图文**。 | ||
| 30 | +- **处理规则**:如果视频内容主要是上述非比赛画面,且没有包含明确的实时进球片段,**直接判定为“无进球”** | ||
| 31 | + ,无需进行后续的进球逻辑分析。 | ||
| 32 | + | ||
| 33 | +#### 2. 视觉维度(寻找关键证据) | ||
| 34 | + | ||
| 35 | +- **核心画面**:足球入网、球在网内静止、裁判指中圈。 | ||
| 36 | +- **行为线索**:进攻方疯狂庆祝、防守方抱头懊恼、全场观众起立欢呼。 | ||
| 37 | +- **画面容忍度**:即使画面主要是观众或教练特写,只要上下文(如切入的进球回放)或行为暗示了进球,也应视为进球。 | ||
| 38 | + | ||
| 39 | +#### 3. 听觉维度(解说语义分析) | ||
| 40 | + | ||
| 41 | +- **进球关键词**:寻找如“球进啦”、“Goal”、“得分”、“世界波”、“破门”、“1比0”等肯定性词汇。 | ||
| 42 | +- **情绪语调**:解说员音量突然升高、语速加快、情绪激动(通常伴随进球发生)。 | ||
| 43 | +- **否定排除**:如果解说提到“越位在先”、“进球无效”、“击中横梁”、“偏出”,则视为未进球。 | ||
| 44 | + | ||
| 45 | +### 判定规则 | ||
| 46 | + | ||
| 47 | +- **判定为“无进球”**: | ||
| 48 | + - **画面为宣传片、广告、纯演播室或其他非比赛实时内容;** | ||
| 49 | + - 视觉显示未进(偏出/被扑/中柱); | ||
| 50 | + - 解说明确表示未进或进球无效; | ||
| 51 | + - 画面与解说均无进球迹象(如普通传球、界外球)。 | ||
| 52 | +- **判定为“进球”**: | ||
| 53 | + - 画面为比赛内容,且视觉清晰显示进球; | ||
| 54 | + - **或** 画面为比赛内容,视觉模糊但解说员明确喊出“球进了”且情绪激动; | ||
| 55 | + - **或** 画面显示庆祝/回放,配合解说确认进球。 | ||
| 56 | + | ||
| 57 | +### 输出要求 | ||
| 58 | + | ||
| 59 | +请仅输出一个JSON格式的结果,不要输出任何分析过程。不要包含 markdown 标记(如 ```json ... ```),不要包含任何解释或额外文本。 | ||
| 60 | +格式如下: | ||
| 61 | +{ | ||
| 62 | +"event_name": "进球" 或 "无进球", | ||
| 63 | +"description": "判定理由,若是宣传片请直接注明" | ||
| 64 | +}""" | ||
| 65 | + | ||
| 66 | + | ||
| 67 | +class FootballReplayVideoEvent: | ||
| 68 | + def __init__(self, base_url: str, model: str, temperature: float = 0.0, api_key: str = 'no_key'): | ||
| 69 | + self.base_url = base_url | ||
| 70 | + self.model = model | ||
| 71 | + self.temperature = temperature | ||
| 72 | + self.api_key = api_key | ||
| 73 | + # self.model = ChatOllama(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0", temperature=0.7, | ||
| 74 | + # keep_alive=-1, reasoning=False) | ||
| 75 | + # self.model = ChatOpenAI(base_url="http://192.168.1.59:11434/v1", model="qwen3.6:35b-a3b-q8_0", temperature=0.7, | ||
| 76 | + # api_key='no_key') | ||
| 77 | + self.model = ChatOllama(base_url=base_url, model=model, temperature=temperature, keep_alive=-1, reasoning=False) | ||
| 78 | + | ||
| 79 | + def video_event(self, video_path: str, asr_text: str = '无', cache_path=None): | ||
| 80 | + if cache_path is not None and os.path.exists(cache_path): | ||
| 81 | + with open(cache_path, 'r', encoding='utf-8') as f: | ||
| 82 | + return json.loads(f.read()) | ||
| 83 | + | ||
| 84 | + contents = video_contents(video_path, None, video_name=os.path.basename(video_path), | ||
| 85 | + fps=2, max_frames=999, sampling_mode="head", max_short_edge=480) | ||
| 86 | + system_message = SystemMessage(content=req_prompt) | ||
| 87 | + video_message = HumanMessage(content=contents) | ||
| 88 | + asr_message = HumanMessage(content=f"解说内容:{asr_text}") | ||
| 89 | + result = self.model.invoke([system_message, video_message, asr_message]).content | ||
| 90 | + try: | ||
| 91 | + result_json = json.loads(result) | ||
| 92 | + except json.JSONDecodeError: | ||
| 93 | + result_json = json.loads(result.replace("```json", "").replace("```", "")) | ||
| 94 | + if cache_path is not None: | ||
| 95 | + os.makedirs(Path(cache_path).parent, exist_ok=True) | ||
| 96 | + with open(cache_path, 'w', encoding='utf-8') as f: | ||
| 97 | + f.write(result) | ||
| 98 | + return result_json |
| 1 | +import base64 | ||
| 2 | +import tempfile | ||
| 3 | +from pathlib import Path | ||
| 4 | +from typing import Union | ||
| 5 | +import cv2 | ||
| 6 | +import httpx | ||
| 7 | + | ||
| 8 | + | ||
| 9 | +def _resize_frame(frame, max_short_edge: int): | ||
| 10 | + """按比例缩放帧,确保最短边不超过指定值""" | ||
| 11 | + h, w = frame.shape[:2] | ||
| 12 | + short_edge = min(h, w) | ||
| 13 | + if short_edge > max_short_edge: | ||
| 14 | + scale = max_short_edge / short_edge | ||
| 15 | + new_w = int(w * scale) | ||
| 16 | + new_h = int(h * scale) | ||
| 17 | + frame = cv2.resize(frame, (new_w, new_h), interpolation=cv2.INTER_AREA) | ||
| 18 | + return frame | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +def contents(video_source: Union[str, Path], prompt_start: str = None, prompt_end=None, video_name: str = None, | ||
| 22 | + fps: float = 1.0, | ||
| 23 | + max_frames: int = 10, | ||
| 24 | + max_short_edge: int = 768, | ||
| 25 | + sampling_mode: str = "uniform") -> list[str]: | ||
| 26 | + """ | ||
| 27 | + 从视频中提取帧并构建 LLM 消息内容。 | ||
| 28 | + | ||
| 29 | + Args: | ||
| 30 | + video_source: 视频源,可以是文件路径或 URL。 | ||
| 31 | + prompt_start: 提示词的开始部分。 | ||
| 32 | + prompt_end: 提示词的结束部分。 | ||
| 33 | + video_name: 视频名称。 | ||
| 34 | + fps: 提取帧的帧率。 | ||
| 35 | + max_frames: 最大帧数。 | ||
| 36 | + max_short_edge: 最大短边长度。 | ||
| 37 | + sampling_mode: 当帧数超过 max_frames 时的采样策略。 | ||
| 38 | + - "uniform": 均匀采样(默认) | ||
| 39 | + - "head": 保留前面的帧,抛弃后面的帧 | ||
| 40 | + """ | ||
| 41 | + source_str = str(video_source) | ||
| 42 | + temp_file = None | ||
| 43 | + | ||
| 44 | + try: | ||
| 45 | + # 如果是 URL,先下载到临时文件 | ||
| 46 | + if source_str.startswith(("http://", "https://")): | ||
| 47 | + resp = httpx.get(source_str, timeout=60.0) | ||
| 48 | + resp.raise_for_status() | ||
| 49 | + temp_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) | ||
| 50 | + temp_file.write(resp.content) | ||
| 51 | + temp_file.close() | ||
| 52 | + video_path = temp_file.name | ||
| 53 | + else: | ||
| 54 | + video_path = str(video_source) | ||
| 55 | + if not Path(video_path).exists(): | ||
| 56 | + raise FileNotFoundError(f"视频文件不存在: {video_path}") | ||
| 57 | + | ||
| 58 | + # 打开视频 | ||
| 59 | + cap = cv2.VideoCapture(video_path) | ||
| 60 | + if not cap.isOpened(): | ||
| 61 | + raise ValueError(f"无法打开视频: {video_source}") | ||
| 62 | + | ||
| 63 | + # 获取视频原始帧率 | ||
| 64 | + video_fps = cap.get(cv2.CAP_PROP_FPS) | ||
| 65 | + | ||
| 66 | + # 计算帧间隔 | ||
| 67 | + frame_interval = int(video_fps / fps) if fps > 0 else int(video_fps) | ||
| 68 | + if frame_interval < 1: | ||
| 69 | + frame_interval = 1 | ||
| 70 | + | ||
| 71 | + frames_base64 = [] | ||
| 72 | + frame_count = 0 | ||
| 73 | + | ||
| 74 | + while True: | ||
| 75 | + ret, frame = cap.read() | ||
| 76 | + if not ret: | ||
| 77 | + break | ||
| 78 | + | ||
| 79 | + # 按指定间隔提取帧 | ||
| 80 | + if frame_count % frame_interval == 0: | ||
| 81 | + # 缩放帧以控制最短边长度 | ||
| 82 | + frame = _resize_frame(frame, max_short_edge) | ||
| 83 | + # 编码为 JPEG 并转 base64 | ||
| 84 | + _, buffer = cv2.imencode('.jpg', frame) | ||
| 85 | + frame_base64 = base64.b64encode(buffer).decode('utf-8') | ||
| 86 | + frames_base64.append(frame_base64) | ||
| 87 | + | ||
| 88 | + frame_count += 1 | ||
| 89 | + | ||
| 90 | + cap.release() | ||
| 91 | + | ||
| 92 | + if len(frames_base64) > max_frames: | ||
| 93 | + import math | ||
| 94 | + step = len(frames_base64) / max_frames | ||
| 95 | + sampled = [frames_base64[min(int(i * step), len(frames_base64) - 1)] for i in range(max_frames)] | ||
| 96 | + frames_base64 = sampled | ||
| 97 | + | ||
| 98 | + # 构建消息内容:提示词 + 所有帧图片 | ||
| 99 | + video_prompt = ( | ||
| 100 | + f"以下是从视频({video_name})中按时间顺序提取的 {len(frames_base64)} 帧画面,视频原始帧率为 {video_fps:.2f} fps," | ||
| 101 | + f"抽帧间隔为 {frame_interval} 帧(约每 {frame_interval / video_fps:.2f} 秒一帧),请将它们视为一个连续的视频进行分析。" | ||
| 102 | + ) | ||
| 103 | + content = [] | ||
| 104 | + if prompt_start is not None: | ||
| 105 | + content.append({"type": "text", "text": prompt_start}) | ||
| 106 | + content.append({"type": "text", "text": video_prompt}) | ||
| 107 | + | ||
| 108 | + for frame_base64 in frames_base64: | ||
| 109 | + content.append({ | ||
| 110 | + "type": "image_url", | ||
| 111 | + "image_url": { | ||
| 112 | + "url": f"data:image/jpeg;base64,{frame_base64}" | ||
| 113 | + } | ||
| 114 | + }) | ||
| 115 | + if prompt_end is not None: | ||
| 116 | + content.append({"type": "text", "text": prompt_end}) | ||
| 117 | + return content | ||
| 118 | + | ||
| 119 | + finally: | ||
| 120 | + # 清理临时文件 | ||
| 121 | + if temp_file and Path(temp_file.name).exists(): | ||
| 122 | + Path(temp_file.name).unlink() |
| 1 | +from football_replay_match_live import FootballReplayMatchLive | ||
| 2 | +from qwen_asr_util import QwenAsr | ||
| 3 | +from football_replay_video_event_by_llm import FootballReplayVideoEvent | ||
| 4 | +import os | ||
| 5 | +import json | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +def batch_match(): | ||
| 9 | + replay_match_live = FootballReplayMatchLive(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0") | ||
| 10 | + qwen_asr = QwenAsr(base_url="http://192.168.1.59:8101/v1", model="Qwen/Qwen3-ASR-1.7B") | ||
| 11 | + fbrv = FootballReplayVideoEvent(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0", temperature=0.7) | ||
| 12 | + | ||
| 13 | + videos_dir = rf"R:\wdx\202604\20260420_download_football_video\finished" | ||
| 14 | + for video_name in sorted(os.listdir(videos_dir)): | ||
| 15 | + | ||
| 16 | + live_dir = os.path.join(videos_dir, video_name, 'live') | ||
| 17 | + live_video_dir = os.path.join(live_dir, 'videos') | ||
| 18 | + live_asr_dir = os.path.join(live_dir, 'asr') | ||
| 19 | + | ||
| 20 | + live_packs = [] | ||
| 21 | + for live_video_name in sorted(os.listdir(live_video_dir)): | ||
| 22 | + live_video_path = os.path.join(live_video_dir, live_video_name) | ||
| 23 | + live_asr_path = os.path.join(live_asr_dir, f"{live_video_name}.txt") | ||
| 24 | + # live_asr_text = qwen_asr.asr(live_video_path, cache_path=live_asr_path) | ||
| 25 | + live_asr_text = '' | ||
| 26 | + live_packs.append({ | ||
| 27 | + "video_id": os.path.basename(live_video_path), | ||
| 28 | + "video_path": live_video_path, | ||
| 29 | + "asr_text": live_asr_text | ||
| 30 | + }) | ||
| 31 | + | ||
| 32 | + replays_dir = os.path.join(videos_dir, video_name, 'replays') | ||
| 33 | + replay_videos_dir = os.path.join(replays_dir, 'videos') | ||
| 34 | + replay_goals_dir = os.path.join(replays_dir, 'goals') | ||
| 35 | + replay_asr_dir = os.path.join(replays_dir, 'asr') | ||
| 36 | + replay_matches_dir = os.path.join(replays_dir, 'matches') | ||
| 37 | + | ||
| 38 | + for replay_video_name in sorted(os.listdir(replay_videos_dir)): | ||
| 39 | + replay_video_path = os.path.join(replay_videos_dir, replay_video_name) | ||
| 40 | + replay_goals_path = os.path.join(replay_goals_dir, f"{replay_video_name}.json") | ||
| 41 | + | ||
| 42 | + replay_asr_path = os.path.join(replay_asr_dir, f"{replay_video_name}.txt") | ||
| 43 | + replay_match_path = os.path.join(replay_matches_dir, f"{replay_video_name}.json") | ||
| 44 | + | ||
| 45 | + # replay_asr_text = qwen_asr.asr(replay_video_path, cache_path=replay_asr_path) | ||
| 46 | + replay_asr_text = '' | ||
| 47 | + | ||
| 48 | + replay_goals = fbrv.video_event(replay_video_path, cache_path=replay_goals_path) | ||
| 49 | + if replay_goals['event_name'] == '无进球': | ||
| 50 | + continue | ||
| 51 | + replay_pack = {"video_path": replay_video_path, "asr_text": replay_asr_text} | ||
| 52 | + | ||
| 53 | + try: | ||
| 54 | + result = replay_match_live.match_batch(replay_pack, live_packs, max_parallel=2, | ||
| 55 | + cache_path=replay_match_path) | ||
| 56 | + print(replay_video_path) | ||
| 57 | + print(result) | ||
| 58 | + | ||
| 59 | + except Exception as e: | ||
| 60 | + print(f"Error processing {replay_video_path}: {e}") | ||
| 61 | + | ||
| 62 | + print('-' * 20) | ||
| 63 | + | ||
| 64 | + | ||
| 65 | +if __name__ == '__main__': | ||
| 66 | + batch_match() |
| 1 | +import os | ||
| 2 | +from io import BytesIO | ||
| 3 | +import re | ||
| 4 | +import requests | ||
| 5 | +import base64 | ||
| 6 | +import tempfile | ||
| 7 | +from pathlib import Path | ||
| 8 | +from typing import Union | ||
| 9 | +import httpx | ||
| 10 | +import subprocess | ||
| 11 | + | ||
| 12 | +from langchain_core.messages import HumanMessage | ||
| 13 | +from langchain_openai import ChatOpenAI | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +class QwenAsr: | ||
| 17 | + def __init__(self, base_url: str, model: str, temperature: float = 0.0, api_key: str = 'no_key'): | ||
| 18 | + self.base_url = base_url | ||
| 19 | + self.model = model | ||
| 20 | + self.temperature = temperature | ||
| 21 | + self.api_key = api_key | ||
| 22 | + # self.asr_model = ChatOpenAI(base_url="http://192.168.1.59:8101/v1", model="Qwen/Qwen3-ASR-1.7B", temperature=0.7, | ||
| 23 | + # api_key='no_key') | ||
| 24 | + self.asr_model = ChatOpenAI(base_url=self.base_url, model=self.model, temperature=self.temperature, | ||
| 25 | + api_key=self.api_key) | ||
| 26 | + | ||
| 27 | + @staticmethod | ||
| 28 | + def _audio_content(source: Union[str, Path]) -> list: | ||
| 29 | + """ | ||
| 30 | + 读取音频/视频文件并构建 LLM 消息内容。 | ||
| 31 | + 如果 source 是视频,会先提取音频;如果是 HTTP 音频 URL,直接引用 URL。 | ||
| 32 | + | ||
| 33 | + Args: | ||
| 34 | + source: 音频/视频文件路径或 URL | ||
| 35 | + Returns: | ||
| 36 | + list: LLM 消息内容列表 | ||
| 37 | + """ | ||
| 38 | + source_str = str(source) | ||
| 39 | + audio_exts = {"wav", "mp3", "m4a", "flac", "ogg", "webm", "aac", "wma"} | ||
| 40 | + video_exts = {"mp4", "avi", "mkv", "mov", "wmv", "flv", "webm", "mpeg", "mpg", "ts", "3gp", "m4v"} | ||
| 41 | + temp_files = [] | ||
| 42 | + | ||
| 43 | + try: | ||
| 44 | + content = [] | ||
| 45 | + is_url = source_str.startswith(("http://", "https://")) | ||
| 46 | + ext = Path(source_str).suffix.lstrip(".").lower() | ||
| 47 | + | ||
| 48 | + # HTTP 音频 URL:直接引用,无需下载 | ||
| 49 | + if is_url and ext in audio_exts: | ||
| 50 | + # audio_prompt = f"以下是音频文件({audio_name})的内容,请进行分析。" | ||
| 51 | + # content = [{"type": "text", "text": audio_prompt}] | ||
| 52 | + # if prompt is not None: | ||
| 53 | + # content.append({"type": "text", "text": prompt}) | ||
| 54 | + content.append({ | ||
| 55 | + "type": "audio_url", | ||
| 56 | + "audio_url": { | ||
| 57 | + "url": source_str | ||
| 58 | + } | ||
| 59 | + }) | ||
| 60 | + return content | ||
| 61 | + | ||
| 62 | + # 本地文件或需要下载的 URL(视频或其他媒体) | ||
| 63 | + if is_url: | ||
| 64 | + resp = httpx.get(source_str, timeout=60.0) | ||
| 65 | + resp.raise_for_status() | ||
| 66 | + suffix = Path(source_str).suffix or ".mp4" | ||
| 67 | + temp_file = tempfile.NamedTemporaryFile(suffix=suffix, delete=False) | ||
| 68 | + temp_file.write(resp.content) | ||
| 69 | + temp_file.close() | ||
| 70 | + temp_files.append(temp_file.name) | ||
| 71 | + media_path = temp_file.name | ||
| 72 | + else: | ||
| 73 | + media_path = str(source) | ||
| 74 | + if not Path(media_path).exists(): | ||
| 75 | + raise FileNotFoundError(f"文件不存在: {media_path}") | ||
| 76 | + | ||
| 77 | + media_ext = Path(media_path).suffix.lstrip(".").lower() | ||
| 78 | + | ||
| 79 | + # 如果是视频,提取音频为 wav | ||
| 80 | + if media_ext in video_exts or (is_url and ext not in audio_exts): | ||
| 81 | + temp_audio = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) | ||
| 82 | + temp_audio.close() | ||
| 83 | + temp_files.append(temp_audio.name) | ||
| 84 | + cmd = [ | ||
| 85 | + "ffmpeg", | ||
| 86 | + "-y", | ||
| 87 | + "-i", media_path, | ||
| 88 | + "-vn", | ||
| 89 | + "-acodec", "pcm_s16le", | ||
| 90 | + "-ar", "16000", | ||
| 91 | + "-ac", "1", | ||
| 92 | + "-hide_banner", | ||
| 93 | + "-loglevel", "error", | ||
| 94 | + temp_audio.name | ||
| 95 | + ] | ||
| 96 | + result = subprocess.run(cmd, capture_output=True, text=True) | ||
| 97 | + if result.returncode != 0: | ||
| 98 | + raise RuntimeError(f"ffmpeg 提取音频失败: {result.stderr.strip()}") | ||
| 99 | + audio_path = temp_audio.name | ||
| 100 | + media_ext = "wav" | ||
| 101 | + else: | ||
| 102 | + audio_path = media_path | ||
| 103 | + if media_ext not in audio_exts: | ||
| 104 | + media_ext = "wav" | ||
| 105 | + | ||
| 106 | + # 读取音频并 base64 编码 | ||
| 107 | + with open(audio_path, "rb") as f: | ||
| 108 | + audio_bytes = f.read() | ||
| 109 | + audio_base64 = base64.b64encode(audio_bytes).decode("utf-8") | ||
| 110 | + | ||
| 111 | + # 构建消息内容 | ||
| 112 | + # audio_prompt = f"以下是音频文件({audio_name})的内容,请进行分析。" | ||
| 113 | + # content = [{"type": "text", "text": audio_prompt}] | ||
| 114 | + # if prompt is not None: | ||
| 115 | + # content.append({"type": "text", "text": prompt}) | ||
| 116 | + | ||
| 117 | + content.append({ | ||
| 118 | + "type": "audio_url", | ||
| 119 | + "audio_url": { | ||
| 120 | + "url": f"data:audio/{media_ext};base64,{audio_base64}" | ||
| 121 | + } | ||
| 122 | + }) | ||
| 123 | + | ||
| 124 | + return content | ||
| 125 | + finally: | ||
| 126 | + # 清理所有临时文件 | ||
| 127 | + for tf in temp_files: | ||
| 128 | + p = Path(tf) | ||
| 129 | + if p.exists(): | ||
| 130 | + p.unlink() | ||
| 131 | + | ||
| 132 | + @staticmethod | ||
| 133 | + def _extract_asr_text(text: str) -> str: | ||
| 134 | + """ | ||
| 135 | + 从ASR响应文本中提取<asr_text>标签内容 | ||
| 136 | + | ||
| 137 | + Args: | ||
| 138 | + text: 原始响应文本,如 'language Chinese<asr_text>放大一点一倍。' | ||
| 139 | + | ||
| 140 | + Returns: | ||
| 141 | + str: 提取的转录文本,如 '放大一点一倍。' | ||
| 142 | + """ | ||
| 143 | + # 尝试匹配 <asr_text>...</asr_text> | ||
| 144 | + match = re.search(r'<asr_text>(.*?)(?:</asr_text>|$)', text, re.DOTALL) | ||
| 145 | + if match: | ||
| 146 | + return match.group(1).strip() | ||
| 147 | + return text | ||
| 148 | + | ||
| 149 | + def asr(self, source, cache_path:str=None): | ||
| 150 | + if cache_path is not None and os.path.exists(cache_path): | ||
| 151 | + with open(cache_path, 'r', encoding='utf-8') as f: | ||
| 152 | + return f.read() | ||
| 153 | + contents = self._audio_content(source) | ||
| 154 | + message = HumanMessage(content=contents) | ||
| 155 | + result = self.asr_model.invoke([message]) | ||
| 156 | + asr_text = self._extract_asr_text(result.content) | ||
| 157 | + if cache_path is not None: | ||
| 158 | + os.makedirs(Path(cache_path).parent, exist_ok=True) | ||
| 159 | + with open(cache_path, 'w', encoding='utf-8') as f: | ||
| 160 | + f.write(asr_text) | ||
| 161 | + return asr_text |
| 1 | +import json | ||
| 2 | +import os | ||
| 3 | + | ||
| 4 | +from football_replay_video_event_by_llm import FootballReplayVideoEvent | ||
| 5 | +from qwen_asr_util import QwenAsr | ||
| 6 | +def batch_with_asr(): | ||
| 7 | + fbrv = FootballReplayVideoEvent(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0", temperature=0.7) | ||
| 8 | + qwen_asr = QwenAsr(base_url="http://192.168.1.59:8101/v1", model="Qwen/Qwen3-ASR-1.7B",temperature=0.7, | ||
| 9 | + api_key='no_key') | ||
| 10 | + | ||
| 11 | + videos_dir = rf"R:\wdx\202604\20260420_download_football_video\finished" | ||
| 12 | + for video_name in sorted(os.listdir(videos_dir)): | ||
| 13 | + replays_dir = os.path.join(videos_dir, video_name, 'replays') | ||
| 14 | + replay_videos_dir = os.path.join(replays_dir, 'videos') | ||
| 15 | + replay_goals_dir = os.path.join(replays_dir, 'goals') | ||
| 16 | + replay_asr_dir = os.path.join(replays_dir, 'asr') | ||
| 17 | + | ||
| 18 | + for replay_video_name in sorted(os.listdir(replay_videos_dir)): | ||
| 19 | + replay_video_path = os.path.join(replay_videos_dir, replay_video_name) | ||
| 20 | + replay_goals_path = os.path.join(replay_goals_dir, f"{replay_video_name}.json") | ||
| 21 | + replay_asr_path = os.path.join(replay_asr_dir, f"{replay_video_name}.json") | ||
| 22 | + if os.path.exists(replay_goals_path): | ||
| 23 | + print("skip", replay_video_path) | ||
| 24 | + continue | ||
| 25 | + asr_text = qwen_asr.asr(replay_video_path, cache_path=replay_asr_path) | ||
| 26 | + event_json = fbrv.video_event(replay_video_name, asr_text) | ||
| 27 | + print(event_json) | ||
| 28 | + | ||
| 29 | +def batch_without_asr(): | ||
| 30 | + fbrv = FootballReplayVideoEvent(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0", temperature=0.7) | ||
| 31 | + # qwen_asr = QwenAsr(base_url="http://192.168.1.59:8101/v1", model="Qwen/Qwen3-ASR-1.7B", temperature=0.7, | ||
| 32 | + # api_key='no_key') | ||
| 33 | + | ||
| 34 | + videos_dir = rf"R:\wdx\202604\20260420_download_football_video\finished" | ||
| 35 | + for video_name in sorted(os.listdir(videos_dir)): | ||
| 36 | + replays_dir = os.path.join(videos_dir, video_name, 'replays') | ||
| 37 | + replay_videos_dir = os.path.join(replays_dir, 'videos') | ||
| 38 | + replay_goals_dir = os.path.join(replays_dir, 'goals') | ||
| 39 | + replay_asr_dir = os.path.join(replays_dir, 'asr') | ||
| 40 | + | ||
| 41 | + for replay_video_name in sorted(os.listdir(replay_videos_dir)): | ||
| 42 | + replay_video_path = os.path.join(replay_videos_dir, replay_video_name) | ||
| 43 | + replay_goals_path = os.path.join(replay_goals_dir, f"{replay_video_name}.json") | ||
| 44 | + replay_asr_path = os.path.join(replay_asr_dir, f"{replay_video_name}.json") | ||
| 45 | + if os.path.exists(replay_goals_path): | ||
| 46 | + print("skip", replay_video_path) | ||
| 47 | + continue | ||
| 48 | + # asr_text = qwen_asr.asr(replay_video_path, cache_path=replay_asr_path) | ||
| 49 | + event_json = fbrv.video_event(replay_video_path, None, cache_path=replay_goals_path) | ||
| 50 | + print(replay_video_name) | ||
| 51 | + print(event_json) | ||
| 52 | + | ||
| 53 | +def one_video_test(video_path): | ||
| 54 | + fbrv = FootballReplayVideoEvent(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0", temperature=0.7) | ||
| 55 | + event_json = fbrv.video_event(video_path, None) | ||
| 56 | + print(event_json) | ||
| 57 | + | ||
| 58 | +if __name__ == '__main__': | ||
| 59 | + # one_video_test(rf"D:\Code\py260417\src\llm_demo\lc_t\ftb\replay_1.mp4") | ||
| 60 | + batch_without_asr() |
| 1 | +import os | ||
| 2 | +import json | ||
| 3 | +from pathlib import Path | ||
| 4 | + | ||
| 5 | +from aabd.base.time_util import vms2str_auto | ||
| 6 | +from langchain_ollama import ChatOllama | ||
| 7 | + | ||
| 8 | + | ||
| 9 | +class ClipByEvent: | ||
| 10 | + def __init__(self, base_url, model, temperature=0.0, api_key='no_key'): | ||
| 11 | + self.base_url = base_url | ||
| 12 | + self.model = model | ||
| 13 | + self.temperature = temperature | ||
| 14 | + self.api_key = api_key | ||
| 15 | + self.clip_before = 10 * 1000 | ||
| 16 | + self.clip_after = 5 * 1000 | ||
| 17 | + | ||
| 18 | + # self.model = ChatOllama(base_url="http://192.168.1.59:11434", model="qwen3.6:35b-a3b-q8_0", temperature=0.7, | ||
| 19 | + # keep_alive=-1, reasoning=False) | ||
| 20 | + # self.model = ChatOpenAI(base_url="http://192.168.1.59:11434/v1", model="qwen3.6:35b-a3b-q8_0", temperature=0.7, | ||
| 21 | + # api_key='no_key') | ||
| 22 | + self.model = ChatOllama(base_url=base_url, model=model, temperature=temperature, keep_alive=-1, reasoning=False) | ||
| 23 | + | ||
| 24 | + def get_video_match_time(self, video_path, video_time): | ||
| 25 | + import cv2 | ||
| 26 | + import base64 | ||
| 27 | + from langchain_core.messages import HumanMessage | ||
| 28 | + | ||
| 29 | + # 从视频中精确提取指定时间的帧画面 | ||
| 30 | + cap = cv2.VideoCapture(video_path) | ||
| 31 | + if not cap.isOpened(): | ||
| 32 | + raise ValueError(f"无法打开视频文件: {video_path}") | ||
| 33 | + | ||
| 34 | + fps = cap.get(cv2.CAP_PROP_FPS) | ||
| 35 | + # video_time 单位为毫秒,转换为帧号 | ||
| 36 | + frame_number = int((video_time / 1000.0) * fps) | ||
| 37 | + cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number) | ||
| 38 | + ret, frame = cap.read() | ||
| 39 | + cap.release() | ||
| 40 | + | ||
| 41 | + if not ret: | ||
| 42 | + raise ValueError(f"无法从视频 {video_path} 中提取时间 {video_time}ms 的帧画面") | ||
| 43 | + | ||
| 44 | + # 将帧图像编码为base64 | ||
| 45 | + _, buffer = cv2.imencode('.jpg', frame) | ||
| 46 | + image_base64 = base64.b64encode(buffer).decode('utf-8') | ||
| 47 | + | ||
| 48 | + # 使用大模型分析比赛画面中的时间 | ||
| 49 | + message = HumanMessage( | ||
| 50 | + content=[ | ||
| 51 | + { | ||
| 52 | + "type": "text", | ||
| 53 | + "text": "请仔细观察这张比赛画面截图,找出画面中显示的比赛计时器或时间信息,并以'MM:SS'格式返回比赛时间。如果无法识别,请返回'unknown'。" | ||
| 54 | + }, | ||
| 55 | + { | ||
| 56 | + "type": "image_url", | ||
| 57 | + "image_url": { | ||
| 58 | + "url": f"data:image/jpeg;base64,{image_base64}" | ||
| 59 | + } | ||
| 60 | + } | ||
| 61 | + ] | ||
| 62 | + ) | ||
| 63 | + response = self.model.invoke([message]) | ||
| 64 | + match_time_text = response.content.strip() | ||
| 65 | + | ||
| 66 | + if match_time_text == 'unknown': | ||
| 67 | + raise ValueError(f"无法识别视频 {video_path} 中时间 {video_time}ms 的比赛时间") | ||
| 68 | + return match_time_text | ||
| 69 | + | ||
| 70 | + def clip_video(self, video_path, clip_time, out_path): | ||
| 71 | + # 精确截取视频指定视频段,从新编码 | ||
| 72 | + | ||
| 73 | + # 毫秒 | ||
| 74 | + start_time = clip_time - self.clip_before | ||
| 75 | + end_time = clip_time + self.clip_after | ||
| 76 | + | ||
| 77 | + start_time_s = start_time / 1000.0 | ||
| 78 | + duration_s = (end_time - start_time) / 1000.0 | ||
| 79 | + | ||
| 80 | + import subprocess | ||
| 81 | + import os | ||
| 82 | + | ||
| 83 | + os.makedirs(os.path.dirname(out_path), exist_ok=True) | ||
| 84 | + | ||
| 85 | + cmd = [ | ||
| 86 | + 'ffmpeg', | ||
| 87 | + '-i', video_path, | ||
| 88 | + '-ss', str(start_time_s), | ||
| 89 | + '-t', str(duration_s), | ||
| 90 | + '-c:v', 'libx264', | ||
| 91 | + '-c:a', 'aac', | ||
| 92 | + '-y', | ||
| 93 | + out_path | ||
| 94 | + ] | ||
| 95 | + | ||
| 96 | + subprocess.run(cmd, check=True) | ||
| 97 | + | ||
| 98 | + def clip_by_event(self, video_dir, json_dir, out_dir): | ||
| 99 | + | ||
| 100 | + for video_name in sorted(os.listdir(video_dir)): | ||
| 101 | + video_path = os.path.join(video_dir, video_name) | ||
| 102 | + f_name_0 = Path(video_name).stem | ||
| 103 | + json_path = os.path.join(json_dir, f"{f_name_0}.json") | ||
| 104 | + with open(json_path, 'r', encoding='utf-8') as f: | ||
| 105 | + data = json.load(f) | ||
| 106 | + | ||
| 107 | + events = data['dataObject']['data']['events'] | ||
| 108 | + | ||
| 109 | + event0 = data['dataObject']['data']['events'][0] | ||
| 110 | + event0_sei_utc = event0['seiUtc'] | ||
| 111 | + event0_text_time = event0['eventTimeText'] | ||
| 112 | + mm, ss = map(int, event0_text_time.split(':')) | ||
| 113 | + event0_time_seconds = mm * 60 + ss | ||
| 114 | + event0_time_ms = event0_time_seconds * 1000 | ||
| 115 | + match_start_sei_utc = event0_sei_utc - event0_time_ms | ||
| 116 | + | ||
| 117 | + take_frame_time = 30 * 60 * 1000 | ||
| 118 | + match_text_time = self.get_video_match_time(video_path, take_frame_time) | ||
| 119 | + mm, ss = map(int, match_text_time.split(':')) | ||
| 120 | + match_time_seconds = mm * 60 + ss | ||
| 121 | + match_time_ms = match_time_seconds * 1000 | ||
| 122 | + match_start_video_time = take_frame_time - match_time_ms | ||
| 123 | + | ||
| 124 | + video_start_sei_utc = match_start_sei_utc - match_start_video_time | ||
| 125 | + | ||
| 126 | + start_utc = video_start_sei_utc | ||
| 127 | + | ||
| 128 | + os.makedirs(os.path.join(out_dir, f_name_0), exist_ok=True) | ||
| 129 | + | ||
| 130 | + # 样例 | ||
| 131 | + # 41 传球 曼联,约罗 00:36 1776106869512 638160 00:10:38.160 | ||
| 132 | + # 41 传球 曼联,马兹拉维 00:39 1776106872512 641160 00:10:41.160 | ||
| 133 | + event_list = [] | ||
| 134 | + for event in events: | ||
| 135 | + event_enum = event['eventType'] | ||
| 136 | + eventTitle = event['eventTitle'] | ||
| 137 | + event_array = eventTitle.split(' ') | ||
| 138 | + event_2 = event_array[-1] | ||
| 139 | + event_1 = ','.join(event_array[:-1]) | ||
| 140 | + eventTimeText = event['eventTimeText'] | ||
| 141 | + seiUtc = event['seiUtc'] | ||
| 142 | + event_list.append([event_enum, event_2, event_1, eventTimeText, seiUtc, seiUtc - start_utc, | ||
| 143 | + vms2str_auto(seiUtc - start_utc)]) | ||
| 144 | + # out_f.write( | ||
| 145 | + # f'{event_enum}\t{event_2}\t{event_1}\t{eventTimeText}\t{seiUtc}\t{seiUtc - start_utc}\t{vms2str_auto(seiUtc - start_utc)}\n') | ||
| 146 | + | ||
| 147 | + with open(os.path.join(out_dir, f_name_0, f"sport_events.txt"), 'w', encoding='utf-8') as out_f: | ||
| 148 | + for event in event_list: | ||
| 149 | + out_f.write( | ||
| 150 | + f'{event[0]}\t{event[1]}\t{event[2]}\t{event[3]}\t{event[4]}\t{event[5]}\t{event[6]}\n') | ||
| 151 | + | ||
| 152 | + for event in event_list: | ||
| 153 | + if event[1] == '进球': | ||
| 154 | + out_path = os.path.join(out_dir, f_name_0, 'videos', | ||
| 155 | + f"{event[6].replace(':', '-').replace('.', '-')}.mp4") | ||
| 156 | + | ||
| 157 | + if not os.path.exists(out_path): | ||
| 158 | + print(f"Clipping {video_name} {event[6]} to {out_path}") | ||
| 159 | + self.clip_video(video_path, event[5], out_path) | ||
| 160 | + else: | ||
| 161 | + print(f"Clip skip {video_name} {event[6]} as {out_path} already exists") | ||
| 162 | + | ||
| 163 | + | ||
| 164 | +if __name__ == '__main__': | ||
| 165 | + cbe = ClipByEvent(base_url="http://192.168.1.215:11434", model="qwen3.5:9b-q8_0", temperature=0.7) | ||
| 166 | + cbe.clip_by_event(video_dir=rf"L:\wdx\202604\20260420_download_football_video\downloads", | ||
| 167 | + json_dir=rf"D:\Code\py260417\src\llm_demo\foot_event_clips\format_json", | ||
| 168 | + out_dir=rf"R:\wdx\202604\20260420_download_football_video\finished1") |
-
Please register or login to post a comment