clean/video/pwtf_dvd/inference/model/video_encoder.py

import gc
import math

# Simplified config for inference
config_text = """
DATA:
  NUM_FRAMES: 8
  SAMPLING_RATE: 8
  TEST_CROP_SIZE: 256
  INPUT_CHANNEL_NUM: [3]
RESNET:
  ZERO_INIT_FINAL_BN: True
  WIDTH_PER_GROUP: 64
  NUM_GROUPS: 1
  DEPTH: 50
  TRANS_FUNC: bottleneck_transform
  STRIDE_1X1: False
  NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]]
NONLOCAL:
  LOCATION: [[[]], [[]], [[]], [[]]]
  GROUP: [[1], [1], [1], [1]]
  INSTANTIATION: softmax
BN:
  USE_PRECISE_STATS: True
  NUM_BATCHES_PRECISE: 200
MODEL:
  NUM_CLASSES: 1
  ARCH: i3d
  MODEL_NAME: ResNet
  DROPOUT_RATE: 0.1
  HEAD_ACT: sigmoid
TEST:
  ENABLE: True
  DATASET: kinetics
  BATCH_SIZE: 64
DATA_LOADER:
  NUM_WORKERS: 8
  PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
"""

# from .f3net import FAD_Head
from slowfast.models.video_model_builder import ResNet as ResNetOri
from slowfast.config.defaults import get_cfg
import torch
from torch import nn
from config_ftcn import config as my_cfg
from inspect import signature
# Removed TimeTransformer import - not used directly in this file
# Removed random import - not used in inference

my_cfg.init_with_yaml()
my_cfg.update_with_yaml("ftcn_tt.yaml")
my_cfg.freeze()



class CenterPatchPool(nn.Module):
    """Simplified patch pooling for inference - always use center patch"""
    def __init__(self):
        super().__init__()

    def forward(self, x):
        # batch,channel,16,7x7
        b, c, t, h, w = x.shape
        x = x.reshape(b, c, t, h * w)
        idx = h * w // 2  # Always use center patch for inference
        x = x[..., idx]
        return x


def valid_idx(idx, h):
    i = idx // h
    j = idx % h
    if j == 0 or i == h - 1 or j == h - 1:
        return False
    else:
        return True


class CenterAvgPool(nn.Module):
    """Simplified average pooling for inference - use all valid patches"""
    def __init__(self):
        super().__init__()

    def forward(self, x):
        # batch,channel,16,7x7
        b, c, t, h, w = x.shape
        x = x.reshape(b, c, t, h * w)
        candidates = list(range(h * w))
        candidates = [idx for idx in candidates if valid_idx(idx, h)]
        x = x[..., candidates].mean(-1)
        return x
    
        

# Removed duplicate TransformerHead class - using the one from temporal_transformer.py 


parameters = [parameter for parameter in signature(nn.Conv3d).parameters]

spatial_count = my_cfg.model.inco.spatial_count
keep_stride_count = my_cfg.model.inco.keep_stride_count


def temporal_only_conv(module, name, removed, stride_removed=0):
    """
    Recursively put desired batch norm in nn.module module.

    set module = net to start code.
    """
    # go through all attributes of module nn.module (e.g. network or layer) and put batch norms if present
    for attr_str in dir(module):
        sub_module = getattr(module, attr_str)
        if type(sub_module) == nn.Conv3d:
            target_spatial_size = 1
            predefine_padding = {1: 0, 3: 1, 5: 2, 7: 3}
            kernel_size = list(sub_module.kernel_size)
            assert kernel_size[1] == kernel_size[2]
            stride = sub_module.stride
            extra = None
            if stride[1] == stride[2] == 2:
                stride_removed += 1
                if stride_removed > keep_stride_count:
                    stride = [1, 1, 1]
                    extra = nn.MaxPool3d((1, 2, 2))

            if kernel_size[1] == 1 and extra is None:
                continue
            padding = list(sub_module.padding)

            kernel_size[1] = kernel_size[2] = target_spatial_size
            padding[1] = padding[2] = predefine_padding[target_spatial_size]
            if 'device' in parameters:
                parameters.remove('device')
            if 'dtype' in parameters:
                parameters.remove('dtype')
            param_dict = {key: getattr(sub_module, key) for key in parameters}

            param_dict.update(kernel_size=kernel_size, padding=padding, stride=stride)

            conv = nn.Conv3d(**param_dict)

            new_module = conv

            removed += 1
            if removed > spatial_count:
                setattr(module, attr_str, new_module)
                if extra is not None:
                    if attr_str == "conv":
                        bn_str = "bn"
                    else:
                        bn_str = f"{attr_str}_bn"
                    bn_module = getattr(module, bn_str)
                    assert isinstance(bn_module, nn.BatchNorm3d)
                    new_bn_module = nn.Sequential(bn_module, extra)
                    setattr(module, bn_str, new_bn_module)
            else:
                print("keep spatial")
        elif type(sub_module) == nn.Dropout:
            new_module = nn.Dropout(p=0.5)
            setattr(module, attr_str, new_module)
        if my_cfg.model.inco.no_time_pool:
            if type(sub_module) == nn.MaxPool3d:
                kernel_size = list(sub_module.kernel_size)
                if kernel_size[0] == 2:
                    kernel_size[0] = 1
                    setattr(module, attr_str, nn.MaxPool3d(kernel_size))
            elif type(sub_module) == nn.AvgPool3d:
                kernel_size = list(sub_module.kernel_size)
                kernel_size[0] = 2 * kernel_size[0]
                setattr(module, attr_str, nn.AvgPool3d(kernel_size))

    # iterate through immediate child modules. Note, the recursion is done by our code no need to use named_modules()
    old_name = name
    for name, immediate_child_module in module.named_children():
        removed, stride_removed = temporal_only_conv(
            immediate_child_module, old_name + "." + name, removed, stride_removed
        )
    return removed, stride_removed


class I3D8x8(nn.Module):
    def __init__(self) -> None:
        super(I3D8x8, self).__init__()
        cfg = get_cfg()
        cfg.merge_from_str(config_text)
        cfg.NUM_GPUS = 1
        cfg.TEST.BATCH_SIZE = 1
        cfg.TRAIN.BATCH_SIZE = 1

        cfg.DATA.NUM_FRAMES = my_cfg.clip_size
        SOLVER = my_cfg.model.inco.SOLVER
        if SOLVER is not None:
            for key, val in SOLVER.to_dict().items():
                old_val = getattr(cfg.SOLVER, key)
                val = type(old_val)(val)
                setattr(cfg.SOLVER, key, val)

        if my_cfg.model.inco.i3d_routine:
            self.cfg = cfg
        self.resnet = ResNetOri(cfg)
        temporal_only_conv(self.resnet, "model", 0)

        stop_point = my_cfg.model.transformer.stop_point
        
        for i in [5, 4, 3]:
            if stop_point <= i:
                setattr(self.resnet, f"s{i}", nn.Identity())
                if stop_point == 3:
                    setattr(self.resnet, f"pathway0_pool", nn.Identity())
        gc.collect()
        torch.cuda.empty_cache()

    def forward(
            self,
            images, ft_features=None,
            freeze_backbone=False
    ):
        assert not freeze_backbone

        inputs = [images]
        pred = self.resnet(inputs, ft_features)
        return pred