Download processing_wemm.py from tencent/WeMM-Video-2B-AWQ: direct link, hf CLI and curl.
- Browser
- Download file 24.9 kB
-
https://huggingface.co/tencent/WeMM-Video-2B-AWQ/resolve/main/processing_wemm.py
- Command line
-
hf download hf://tencent/WeMM-Video-2B-AWQ/processing_wemm.py
-
curl -L -o processing_wemm.py https://huggingface.co/tencent/WeMM-Video-2B-AWQ/resolve/main/processing_wemm.py
24.9 kB
| """Unified HF processor for the WeMM Video 1.5 Qwen3.5 audio checkpoints.""" | |
| import copy | |
| import json | |
| import shutil | |
| from pathlib import Path | |
| import torch | |
| from transformers import BatchFeature, ProcessorMixin, Qwen3VLProcessor, WhisperFeatureExtractor | |
| from transformers.utils.hub import cached_file | |
| from transformers.video_utils import VideoMetadata | |
| from .wemm_media import AUDIO, IMAGE, VIDEO, audio_features, audio_ids, decode_audio | |
| from .wemm_media import decode_video, load_image, resolve_media, tokens_per_segment, video_ids | |
| from .wemm_media import audio_timestamp_groups, resolve_timestamp_interval, timestamp_interval_option | |
| from .wemm_video_policy import resolve_video_options, video_environment_defaults, video_resize_size | |
| # Transformers 5.2 must see these dependencies directly when caching remote | |
| # processor code; an import through wemm_media alone misses the nested file. | |
| from .wemm_zip import load_zip_video as _zip_decoder_dependency | |
| from .wemm_zip_policy import positive_frame_count | |
| class WeMMProcessor(ProcessorMixin): | |
| """Single-conversation media + question API, including ordered chat messages. | |
| ``processor(question='描述视频', videos='clip.mp4', use_audio_in_video=True)`` | |
| returns a BatchFeature directly accepted by ``model.generate(**inputs)``. | |
| Lists of media belong to the same conversation, not a batch of requests. | |
| """ | |
| image_processor_class = 'AutoImageProcessor' | |
| video_processor_class = 'AutoVideoProcessor' | |
| tokenizer_class = 'AutoTokenizer' | |
| feature_extractor_class = 'WhisperFeatureExtractor' | |
| def get_attributes(cls): | |
| return ['image_processor', 'tokenizer', 'video_processor', 'feature_extractor'] | |
| def __init__(self, image_processor, tokenizer, video_processor, feature_extractor, | |
| audio_config, fps=2.0, min_frames=4, max_frames=128, total_pixels=None, | |
| min_pixels=4096, system_prompt='You are a helpful assistant.', chat_template=None, | |
| timestamp_interval_sec=1.0, max_allowed_frames=None, max_allowed_total_pixels=None): | |
| super().__init__(image_processor=image_processor, tokenizer=tokenizer, | |
| video_processor=video_processor, feature_extractor=feature_extractor, | |
| chat_template=chat_template or tokenizer.chat_template) | |
| self.audio_config = copy.deepcopy(audio_config) | |
| self.fps, self.min_frames, self.max_frames = fps, min_frames, max_frames | |
| self.total_pixels = total_pixels or video_processor.size['longest_edge'] | |
| self.min_pixels, self.system_prompt = min_pixels, system_prompt | |
| self.timestamp_interval_sec = resolve_timestamp_interval(timestamp_interval_sec) | |
| self.max_allowed_frames = max_allowed_frames | |
| self.max_allowed_total_pixels = max_allowed_total_pixels | |
| self._native = Qwen3VLProcessor(image_processor=image_processor, | |
| tokenizer=tokenizer, video_processor=video_processor) | |
| def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): | |
| names = ('fps', 'min_frames', 'max_frames', 'total_pixels', 'min_pixels', 'system_prompt', | |
| 'timestamp_interval_sec', 'max_allowed_frames', 'max_allowed_total_pixels') | |
| overrides = {key: kwargs.pop(key) for key in names if key in kwargs} | |
| if 'TIMESTAMP_INTERVAL_SEC' in kwargs: | |
| raise ValueError('Use lowercase timestamp_interval_sec in processor arguments') | |
| trust_remote_code = kwargs.pop('trust_remote_code', True) | |
| kwargs.pop('_from_auto', None) | |
| hub = {key: kwargs[key] for key in ('cache_dir', 'force_download', 'local_files_only', | |
| 'token', 'revision', 'subfolder') if key in kwargs} | |
| proc_path = cached_file(pretrained_model_name_or_path, 'processor_config.json', | |
| _raise_exceptions_for_missing_entries=False, **hub) | |
| processor_config = json.loads(Path(proc_path).read_text()) if proc_path else {} | |
| config_path = cached_file(pretrained_model_name_or_path, 'config.json', | |
| _raise_exceptions_for_missing_entries=False, **hub) | |
| config = json.loads(Path(config_path).read_text()) if config_path else processor_config | |
| defaults = processor_config.get('wemm_defaults', {}) | |
| native = Qwen3VLProcessor.from_pretrained( | |
| pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs) | |
| audio = config['audio_config'] | |
| extractor = WhisperFeatureExtractor(feature_size=audio['num_mel_bins'], sampling_rate=16000, | |
| hop_length=160, chunk_length=30) | |
| environment = video_environment_defaults() | |
| effective = resolve_video_options({"total_pixels": native.video_processor.size['longest_edge'], | |
| **defaults, **environment}, overrides) | |
| parameters = {**defaults, **overrides, **{key: effective[key] for key in names if key in effective}} | |
| return cls(native.image_processor, native.tokenizer, native.video_processor, | |
| extractor, audio, chat_template=native.chat_template, **parameters) | |
| def save_pretrained(self, save_directory, **kwargs): | |
| """Save processor metadata/code beside a config containing audio_config.""" | |
| destination = Path(save_directory) | |
| destination.mkdir(parents=True, exist_ok=True) | |
| self._native.save_pretrained(destination, **kwargs) | |
| # Transformers 5.2 stores nested components in processor_config.json. | |
| # Also write standalone component configs before our custom metadata. | |
| self.image_processor.save_pretrained(destination) | |
| self.video_processor.save_pretrained(destination) | |
| defaults = {key: getattr(self, key) for key in | |
| ('fps', 'min_frames', 'max_frames', 'total_pixels', 'min_pixels', 'system_prompt', | |
| 'timestamp_interval_sec', 'max_allowed_frames', 'max_allowed_total_pixels')} | |
| mapping = {'AutoProcessor': 'processing_wemm.WeMMProcessor'} | |
| (destination / 'processor_config.json').write_text(json.dumps({ | |
| 'processor_class': 'WeMMProcessor', 'auto_map': mapping, | |
| 'wemm_defaults': defaults, 'audio_config': self.audio_config}, indent=2) + '\n') | |
| for name in ('processing_wemm.py', 'wemm_media.py', 'wemm_video_policy.py', | |
| 'wemm_zip.py', 'wemm_zip_policy.py'): | |
| source = Path(__file__).parent / name | |
| if source.resolve() != (destination / name).resolve(): | |
| shutil.copy2(source, destination / name) | |
| config_path = destination / 'config.json' | |
| if config_path.exists(): | |
| config = json.loads(config_path.read_text()) | |
| config.setdefault('auto_map', {}).update(mapping) | |
| config_path.write_text(json.dumps(config, indent=2) + '\n') | |
| return [str(destination / 'processor_config.json')] | |
| def model_input_names(self): | |
| return ['input_ids', 'attention_mask', 'pixel_values', 'image_grid_thw', | |
| 'pixel_values_videos', 'video_grid_thw', 'audio_feature'] | |
| def decode(self, *args, **kwargs): | |
| return self.tokenizer.decode(*args, **kwargs) | |
| def batch_decode(self, *args, **kwargs): | |
| return self.tokenizer.batch_decode(*args, **kwargs) | |
| def _calculate_timestamps(self): | |
| # Existing project helpers override native timestamp arithmetic for AV. | |
| return self._native._calculate_timestamps | |
| def _calculate_timestamps(self, function): | |
| self._native._calculate_timestamps = function | |
| def _items(value, modality): | |
| if value is None: | |
| return [] | |
| if modality == 'audio' and isinstance(value, tuple) and len(value) == 2: | |
| return [value] | |
| return value if isinstance(value, list) else [value] | |
| def _messages(self, question, media, images, videos, audios, video_audio, use_audio_in_video): | |
| items = [] | |
| for kind, value in (('image', images), ('video', videos), ('audio', audios), ('video_audio', video_audio)): | |
| for item in self._items(value, kind): | |
| items.append({'type': 'video_audio' if kind == 'video' and use_audio_in_video else kind, | |
| kind if kind != 'video_audio' else 'video': item}) | |
| for item in self._items(media, 'media'): | |
| if isinstance(item, dict): | |
| items.append(item) | |
| continue | |
| suffix = Path(str(item)).suffix.lower() | |
| if suffix in ('.jpg', '.jpeg', '.png', '.webp', '.bmp', '.tiff'): | |
| kind = 'image' | |
| elif suffix in ('.wav', '.flac', '.mp3', '.ogg', '.m4a', '.aac'): | |
| kind = 'audio' | |
| elif suffix in ('.mp4', '.mov', '.mkv', '.webm', '.avi', '.zip'): | |
| kind = 'video_audio' if use_audio_in_video else 'video' | |
| else: | |
| raise ValueError('Specify the media type explicitly for this input') | |
| items.append({'type': kind, 'video' if kind == 'video_audio' else kind: item}) | |
| content = [] | |
| for item in items: | |
| content.extend([item, {'type': 'text', 'text': '\n'}]) | |
| content.append({'type': 'text', 'text': question}) | |
| return [{'role': 'system', 'content': self.system_prompt}, {'role': 'user', 'content': content}] | |
| def _normalize_messages(self, messages, use_audio_in_video=False): | |
| normalized, routes = [], [] | |
| for message in messages: | |
| row = {key: value for key, value in message.items() if key != 'content'} | |
| if isinstance(message['content'], str): | |
| row['content'] = message['content'] | |
| else: | |
| parts = [] | |
| for item in message['content']: | |
| kind = item['type'] | |
| if kind in ('video_url', 'video_audio_url'): | |
| payload = item.get('video_url') | |
| nested = payload if isinstance(payload, dict) else {} | |
| video_source = nested.get('url') if nested else payload | |
| audio = item.get('audio_url') or nested.get('audio') | |
| audio_source = audio.get('url') if isinstance(audio, dict) else audio | |
| if video_source: | |
| kind = 'video_audio' if audio_source or nested.get('use_audio') else 'video' | |
| item = {**item, **nested, 'type': kind, 'video': video_source} | |
| if audio_source: | |
| item['audio'] = audio_source | |
| elif item['type'] == 'video_audio_url' and audio_source: | |
| kind, item = 'audio', {'type': 'audio', 'audio': audio_source} | |
| else: | |
| raise ValueError('Missing video URL in media item') | |
| if kind in ('audio_url', 'input_audio'): | |
| payload = item[kind] | |
| if kind == 'audio_url': | |
| source = payload['url'] if isinstance(payload, dict) else payload | |
| else: | |
| source = f"data:audio/{payload['format']};base64,{payload['data']}" | |
| nested = payload if isinstance(payload, dict) else {} | |
| interval = timestamp_interval_option(item, timestamp_interval_option(nested)) | |
| item = {'type': 'audio', 'audio': source, 'timestamp_interval_sec': interval} | |
| kind = 'audio' | |
| if kind == 'video' and item.get('use_audio_in_video', use_audio_in_video): | |
| kind = 'video_audio' | |
| if kind == 'text': | |
| parts.append(item['text']) | |
| elif kind in ('image', 'video', 'audio', 'video_audio'): | |
| key = 'video' if kind == 'video_audio' else kind | |
| if key not in item and 'path' not in item: | |
| raise ValueError(f'Missing {key} or path in media item') | |
| routes.append({**item, 'type': kind, 'source': item.get(key, item.get('path'))}) | |
| parts.append({'image': IMAGE, 'video': VIDEO, 'video_audio': VIDEO, 'audio': AUDIO}[kind]) | |
| else: | |
| raise ValueError(f'Unsupported content type: {kind}') | |
| row['content'] = ''.join(parts) | |
| normalized.append(row) | |
| return normalized, routes | |
| def apply_chat_template(self, conversation, *, tokenize=False, return_dict=False, | |
| return_tensors=None, add_generation_prompt=True, enable_thinking=False, **kwargs): | |
| if tokenize: | |
| result = self(messages=conversation, return_tensors=return_tensors or 'pt', | |
| add_generation_prompt=add_generation_prompt, enable_thinking=enable_thinking, **kwargs) | |
| return result if return_dict else result['input_ids'] | |
| messages, _ = self._normalize_messages(conversation) | |
| return self.tokenizer.apply_chat_template(messages, tokenize=False, | |
| add_generation_prompt=add_generation_prompt, enable_thinking=enable_thinking, **kwargs) | |
| def __call__(self, text=None, *, question=None, media=None, images=None, videos=None, | |
| audio=None, audios=None, video_audio=None, use_audio_in_video=False, messages=None, | |
| text_is_prompt=False, fps=None, num_frames=None, nframes=None, min_frames=None, max_frames=None, | |
| total_pixels=None, min_pixels=None, max_pixels=None, images_kwargs=None, | |
| videos_kwargs=None, return_tensors='pt', add_generation_prompt=True, | |
| enable_thinking=False, timestamp_interval_sec=None): | |
| timestamp_interval_sec = resolve_timestamp_interval( | |
| timestamp_interval_sec, default=self.timestamp_interval_sec) | |
| if return_tensors != 'pt': | |
| raise ValueError('WeMMProcessor currently returns PyTorch tensors; use return_tensors="pt"') | |
| if audio is not None and audios is not None: | |
| raise ValueError('Use either audio or audios') | |
| audios = audio if audio is not None else audios | |
| if messages is not None: | |
| if any(x is not None for x in (text, question, media, images, videos, audios, video_audio)): | |
| raise ValueError('Put media and text inside messages, or use the question/media interface') | |
| normalized, routes = self._normalize_messages(messages, use_audio_in_video=use_audio_in_video) | |
| prompt = self.tokenizer.apply_chat_template(normalized, tokenize=False, | |
| add_generation_prompt=add_generation_prompt, enable_thinking=enable_thinking) | |
| else: | |
| if text is not None and question is not None: | |
| raise ValueError('Use either text or question') | |
| question = question if question is not None else text | |
| if not isinstance(question, str): | |
| raise ValueError('Pass one question string or one messages conversation') | |
| normalized, routes = self._normalize_messages(self._messages( | |
| question, media, images, videos, audios, video_audio, use_audio_in_video), | |
| use_audio_in_video=use_audio_in_video) | |
| prompt = question if text_is_prompt else self.tokenizer.apply_chat_template( | |
| normalized, tokenize=False, add_generation_prompt=add_generation_prompt, | |
| enable_thinking=enable_thinking) | |
| image_pixels, image_grids, video_pixels, video_grids, features = [], [], [], [], [] | |
| blocks = {'image': [], 'video': [], 'audio': []} | |
| block_features = {kind: [] for kind in blocks} | |
| block_metadata = {kind: [] for kind in blocks} | |
| metadata = [] | |
| for route in routes: | |
| kind, source = route['type'], resolve_media(route['source']) | |
| item_info = {'type': kind} | |
| feature = None | |
| if kind == 'image': | |
| image = self.image_processor(images=[load_image(source)], return_tensors='pt', | |
| **(images_kwargs or {})) | |
| grid = image['image_grid_thw'][0] | |
| image_pixels.append(image['pixel_values']) | |
| image_grids.append(image['image_grid_thw']) | |
| blocks['image'].append(self.tokenizer.encode('<|vision_start|>', add_special_tokens=False) | |
| + [self.tokenizer.convert_tokens_to_ids('<|image_pad|>')] | |
| * (int(grid.prod()) // self.image_processor.merge_size**2) | |
| + self.tokenizer.encode('<|vision_end|>', add_special_tokens=False)) | |
| item_info['grid'] = grid.tolist() | |
| elif kind == 'audio': | |
| interval = resolve_timestamp_interval(timestamp_interval_option(route, timestamp_interval_sec)) | |
| samples = decode_audio(source) | |
| duration = len(samples) / 16000 | |
| feature = audio_features(samples, self.feature_extractor) | |
| count = feature.shape[0] * tokens_per_segment(self.audio_config) | |
| blocks['audio'].append(audio_ids(self.tokenizer, count, self.audio_config['second_per_window'], | |
| duration, interval)) | |
| groups, times, effective = audio_timestamp_groups( | |
| count, self.audio_config['second_per_window'], duration, interval) | |
| item_info.update(duration=duration, segments=len(feature), audio_tokens=count, | |
| timestamp_interval_sec=interval, audio_groups=groups, timestamps=times, | |
| effective_audio_tokens=effective, padding_audio_tokens=count-effective) | |
| else: | |
| options = resolve_video_options(dict( | |
| fps=self.fps if fps is None else fps, num_frames=num_frames, nframes=nframes, | |
| min_frames=self.min_frames if min_frames is None else min_frames, | |
| max_frames=self.max_frames if max_frames is None else max_frames, | |
| total_pixels=self.total_pixels if total_pixels is None else total_pixels, | |
| min_pixels=self.min_pixels if min_pixels is None else min_pixels, | |
| max_pixels=max_pixels, max_allowed_frames=self.max_allowed_frames, | |
| max_allowed_total_pixels=self.max_allowed_total_pixels), route) | |
| # SGLang supplies this internal cap from server-side defaults; | |
| # the HTTP parser never accepts it from a request. | |
| deployment_cap = positive_frame_count( | |
| route.get('wemm_zip_max_frames', self.max_frames), 'ZIP deployment max_frames') | |
| options['wemm_zip_max_frames'] = min( | |
| deployment_cap, int(options.get('max_allowed_frames') or deployment_cap)) | |
| decode_keys = ('fps', 'num_frames', 'min_frames', 'max_frames', 'min_pixels', | |
| 'max_pixels', 'total_pixels', 'max_allowed_frames', 'max_allowed_total_pixels', | |
| 'wemm_zip_max_frames', 'resized_height', 'resized_width') | |
| frames, info = decode_video(source, **{key: options[key] for key in decode_keys if key in options}) | |
| bundled_audio = info.pop('audio_bytes', None) | |
| pre_resized = bool(info.get('wemm_pre_resized', False)) | |
| area = int(frames.shape[1] * frames.shape[2] * len(frames)) | |
| size = ({'longest_edge': area, 'shortest_edge': area} if pre_resized else | |
| video_resize_size(len(frames), options)) | |
| kwargs = dict(videos_kwargs or {}) | |
| kwargs.setdefault('size', size) | |
| if pre_resized: | |
| kwargs.setdefault('do_resize', False) | |
| video_metadata = {key: info[key] for key in | |
| ('total_num_frames', 'fps', 'frames_indices', 'duration', 'width', 'height', 'video_backend') | |
| if key in info} | |
| kwargs.update(do_sample_frames=False, video_metadata=[VideoMetadata(**video_metadata)]) | |
| video = self.video_processor(videos=[frames], return_tensors='pt', **kwargs) | |
| grid = video['video_grid_thw'][0] | |
| video_pixels.append(video['pixel_values_videos']) | |
| video_grids.append(video['video_grid_thw']) | |
| count, duration = 0, None | |
| if kind == 'video_audio' or bundled_audio is not None: | |
| try: | |
| samples = decode_audio(route.get('audio', bundled_audio if bundled_audio is not None else source)) | |
| except (RuntimeError, ValueError) as exc: | |
| if 'audio' not in str(exc).lower() or not any(word in str(exc).lower() for word in ('stream', 'best')): | |
| raise | |
| item_info['audio_fallback'] = 'no audio stream; video only' | |
| else: | |
| duration = len(samples) / 16000 | |
| feature = audio_features(samples, self.feature_extractor, pcm16=False) | |
| count = len(feature) * tokens_per_segment(self.audio_config) | |
| item_info['segments'] = len(feature) | |
| ids, groups, times = video_ids(self.tokenizer, grid, info, | |
| self.video_processor.temporal_patch_size, self.video_processor.merge_size, | |
| count, duration, self.audio_config['second_per_window']) | |
| blocks['video'].append(ids) | |
| item_info.update(info, grid=grid.tolist(), size=kwargs['size'], | |
| audio_tokens=count, audio_duration=duration, audio_groups=groups, | |
| timestamps=times, sampled_frames=len(frames)) | |
| metadata.append(item_info) | |
| block_kind = 'video' if kind == 'video_audio' else kind | |
| block_features[block_kind].append(feature) | |
| block_metadata[block_kind].append(item_info) | |
| if blocks[block_kind][-1].count(self.tokenizer.convert_tokens_to_ids('<|audio_pad|>')) != item_info.get('audio_tokens', 0): | |
| raise ValueError('Audio features and placeholders differ for one media item') | |
| source_ids = self.tokenizer.encode(prompt, add_special_tokens=False) | |
| patterns = {kind: self.tokenizer.encode(pattern, add_special_tokens=False) | |
| for kind, pattern in (('image', IMAGE), ('video', VIDEO), ('audio', AUDIO))} | |
| offsets = dict.fromkeys(patterns, 0) | |
| ids, index, ordered_metadata = [], 0, [] | |
| while index < len(source_ids): | |
| for kind, pattern in patterns.items(): | |
| if source_ids[index:index + len(pattern)] == pattern: | |
| if offsets[kind] >= len(blocks[kind]): | |
| raise ValueError(f'Extra {kind} placeholder in the question/prompt') | |
| ids.extend(blocks[kind][offsets[kind]]) | |
| feature = block_features[kind][offsets[kind]] | |
| if feature is not None: | |
| features.append(feature) | |
| ordered_metadata.append(block_metadata[kind][offsets[kind]]) | |
| offsets[kind] += 1 | |
| index += len(pattern) | |
| break | |
| else: | |
| ids.append(source_ids[index]) | |
| index += 1 | |
| if any(offsets[kind] != len(blocks[kind]) for kind in blocks): | |
| raise ValueError('Media items and prompt placeholders differ') | |
| data = {'input_ids': torch.tensor([ids], dtype=torch.long), | |
| 'attention_mask': torch.ones(1, len(ids), dtype=torch.long)} | |
| for pixels, grids, pkey, gkey in ((image_pixels, image_grids, 'pixel_values', 'image_grid_thw'), | |
| (video_pixels, video_grids, 'pixel_values_videos', 'video_grid_thw')): | |
| if pixels: | |
| data[pkey], data[gkey] = torch.cat(pixels), torch.cat(grids) | |
| if features: | |
| data['audio_feature'] = torch.cat(features) | |
| expected = sum(item.get('audio_tokens', 0) for item in metadata) | |
| if ids.count(self.tokenizer.convert_tokens_to_ids('<|audio_pad|>')) != expected: | |
| raise ValueError('Audio features and placeholder counts differ') | |
| result = BatchFeature(data=data) | |
| # These attributes are diagnostic metadata, never forwarded to generate. | |
| result.media_metadata = ordered_metadata | |
| result.prompt_text = prompt | |
| return result | |