comfyui-solricks/taa.py at main · SOLRICKS/comfyui-solricks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# Video TAA + DLAA temporal state
# SOLRICKS


import torch
import torch.nn.functional as F

try:
    from .utils import rgb_luma
except ImportError:
    from utils import rgb_luma


class TAAState:
    """Keeps temporal history for the TAA pass."""

    def __init__(
        self,
        variance_gamma=1.5,
        edge_guard_strength=2.0,
        edge_guard_min=0.25,
        edge_guard_max=1.0,
    ):
        self.history = None
        self.frame_id = 0

        self.variance_gamma = variance_gamma
        self.edge_guard_strength = edge_guard_strength
        self.edge_guard_min = edge_guard_min
        self.edge_guard_max = edge_guard_max

    def reset(self):
        self.history = None
        self.frame_id = 0

    @staticmethod
    def _smoothstep(value: torch.Tensor) -> torch.Tensor:
        return value * value * (3.0 - 2.0 * value)

    def update(self, frame, alpha, sensitivity):
        if self.history is None or self.history.shape != frame.shape:
            self.history = frame.detach().clone().to(frame.device)
            return frame

        local_mean = F.avg_pool2d(frame, kernel_size=3, stride=1, padding=1)
        local_sq_mean = F.avg_pool2d(frame * frame, kernel_size=3, stride=1, padding=1)
        local_var = (local_sq_mean - local_mean * local_mean).clamp(min=0.0)
        local_std = torch.sqrt(local_var + 1e-6)

        local_min = local_mean - local_std * self.variance_gamma
        local_max = local_mean + local_std * self.variance_gamma

        history_clipped = torch.maximum(torch.minimum(self.history, local_max), local_min)

        diff = torch.abs(frame - history_clipped).mean(dim=1, keepdim=True)
        raw_diff = torch.abs(frame - self.history).mean(dim=1, keepdim=True)

        disocclusion = ((raw_diff - sensitivity * 2.0) / (sensitivity + 1e-6)).clamp(0.0, 1.0)
        disocclusion = self._smoothstep(disocclusion)

        gray = rgb_luma(frame)

        edge_x = torch.abs(gray[:, :, :, 1:] - gray[:, :, :, :-1])
        edge_y = torch.abs(gray[:, :, 1:, :] - gray[:, :, :-1, :])

        edge_x = F.pad(edge_x, (0, 1, 0, 0))
        edge_y = F.pad(edge_y, (0, 0, 0, 1))

        edge_strength = (edge_x + edge_y).clamp(0.0, 1.0)
        edge_guard = (
            self.edge_guard_max - edge_strength * self.edge_guard_strength
        ).clamp(self.edge_guard_min, self.edge_guard_max)

        motion_soft = ((diff - sensitivity) / (sensitivity + 1e-6)).clamp(0.0, 1.0)
        motion_soft = self._smoothstep(motion_soft)

        dynamic_alpha = alpha * (1.0 - motion_soft) * edge_guard

        confidence = torch.exp(-diff * 10.0)
        confidence = confidence * (1.0 - disocclusion)
        confidence = confidence.clamp(0.15, 1.0)

        dynamic_alpha = dynamic_alpha * confidence

        reject_strength = ((diff - sensitivity * 1.5) / (sensitivity + 1e-6)).clamp(0.0, 1.0)
        reject_strength = torch.maximum(reject_strength, disocclusion)
        reject_strength = self._smoothstep(reject_strength)

        history_clipped = torch.lerp(history_clipped, frame, reject_strength)

        out = torch.lerp(frame, history_clipped, dynamic_alpha)
        self.history = out.detach()

        return out