MadameMinty's picture
Add lighter CuNNy and ArtCNN sizes
d76a75b verified
Raw History Blame Contribute Delete
5.19 kB
"""Builds ncnn models of CuNNy DS (weights parsed from its mpv shaders) and ArtCNN DS (from its ONNX files), at every size given.
Forms:
- <name>-luma: 1 channel in (luma 0..1), 1 channel out (2x luma); the network alone, for checks.
- <name>: what the app's tile engine runs unchanged: RGB 0..1 in, RGB out at 2x. Luma is
taken by BT.709, run through the network; chroma comes from a bicubic 2x of the RGB input
(mpv uses Lanczos): out = bicubic(RGB) + (net luma - luma(bicubic(RGB))) in every channel,
which replaces the luma and keeps the chroma (both linear in RGB).
CuNNy also gets -rp forms: per-layer edge replication like the shader (exact at page borders; the
zero-padded forms differ in the outer ~8 output pixels of a page, and match inside).
<name>-gray: RGB in, the network's luma out in all channels; a timing proxy for a native luma path.
CuNNy-8x32-DS and ArtCNN C4F32 DS get every form (VARIANTS); every other size only the form the
app runs: <name>-rp for CuNNy, <name> for ArtCNN. Names come from the file names:
CuNNy-<size>-DS.glsl -> cunny-<size>-ds, ArtCNN_<size>_DS.onnx -> artcnn-<size>-ds (lowercase).
Usage: python convert.py <out dir> <CuNNy-*-DS.glsl or ArtCNN_*_DS.onnx> ...
"""
import re
import sys
from pathlib import Path
import glsl_net
from ncnn_graph import Graph
# The models built in every form; their variants are hosted beside the app's form.
VARIANTS = {"cunny-8x32-ds", "artcnn-c4f32-ds"}
def cunny_luma(g: Graph, y, passes, replicate=False):
"""CuNNy body on luma y [1, h, w] -> [1, 2h, 2w]: 3x3 convs (clamp 0..1 or ReLU between), depth-to-space, + bilinear y."""
x = y
for p in passes:
act = ("clip", 0.0, 1.0) if p.clamp01 else "relu" if p.relu else None
if replicate:
x = g.conv(g.replicate_pad(x), p.w, p.b, pad=0, act=act)
else:
x = g.conv(x, p.w, p.b, act=act)
return g.binary(g.pixel_shuffle(x), g.interp(y, "bilinear"), "add")
def artcnn_luma(g: Graph, y, onnx_path: Path):
"""ArtCNN C4 (DS) as its ONNX graph has it: conv0, 4x (conv, ReLU), conv5 + conv0, conv6, depth-to-space, clip 0..1."""
import onnx
from onnx import numpy_helper
m = onnx.load(str(onnx_path))
w = {i.name: numpy_helper.to_array(i) for i in m.graph.initializer}
convs = [n for n in m.graph.node if n.op_type == "Conv"]
ops = [n.op_type for n in m.graph.node]
assert ops == ["Conv", "Conv", "Relu", "Conv", "Relu", "Conv", "Relu", "Conv", "Relu", "Conv", "Add", "Conv", "DepthToSpace", "Clip"], ops
wb = [(w[n.input[1]], w[n.input[2]]) for n in convs]
c0 = g.conv(y, *wb[0])
x = c0
for k in range(1, 5):
x = g.conv(x, *wb[k], act="relu")
x = g.binary(g.conv(x, *wb[5]), c0, "add")
x = g.pixel_shuffle(g.conv(x, *wb[6]))
return g.clip(x)
def rgb_wrap(g: Graph, luma_net):
"""RGB in -> RGB out at 2x: the network's luma with bicubic chroma."""
rgb = g.input(3)
y2 = luma_net(g.luma(rgb))
up = g.interp(rgb, "bicubic")
return g.binary(up, g.spread(g.binary(y2, g.luma(up), "sub")), "add")
def gray_wrap(g: Graph, luma_net):
"""RGB in -> the network's 2x luma in all three channels: the cost of a native luma path (timing proxy)."""
return g.spread(luma_net(g.luma(g.input(3))))
def name_of(src: Path) -> str:
"""cunny-<size>-ds or artcnn-<size>-ds, from the upstream file name."""
m = re.fullmatch(r"CuNNy-(\w+)-DS\.glsl", src.name) or re.fullmatch(r"ArtCNN_(\w+)_DS\.onnx", src.name)
assert m, src.name
return f"{'cunny' if src.suffix == '.glsl' else 'artcnn'}-{m.group(1).lower()}-ds"
def build_cunny(shader: Path, save):
name = name_of(shader)
passes = glsl_net.parse(shader)
if name in VARIANTS:
g = Graph(); cunny_luma(g, g.input(1), passes); save(g, f"{name}-luma")
g = Graph(); cunny_luma(g, g.input(1), passes, replicate=True); save(g, f"{name}-luma-rp")
g = Graph(); rgb_wrap(g, lambda y: cunny_luma(g, y, passes)); save(g, name)
g = Graph(); rgb_wrap(g, lambda y: cunny_luma(g, y, passes, replicate=True)); save(g, f"{name}-rp")
if name in VARIANTS:
g = Graph(); gray_wrap(g, lambda y: cunny_luma(g, y, passes, replicate=True)); save(g, f"{name}-rp-gray")
def build_artcnn(onnx_path: Path, save):
name = name_of(onnx_path)
if name in VARIANTS:
g = Graph(); artcnn_luma(g, g.input(1), onnx_path); save(g, f"{name}-luma")
g = Graph(); gray_wrap(g, lambda y: artcnn_luma(g, y, onnx_path)); save(g, f"{name}-gray")
g = Graph(); rgb_wrap(g, lambda y: artcnn_luma(g, y, onnx_path)); save(g, name)
def build(out: Path, sources):
out.mkdir(parents=True, exist_ok=True)
made = {}
def save(g, name):
g.save(out / f"{name}.param", out / f"{name}.bin")
made[name] = g
for src in sources:
(build_cunny if src.suffix == ".glsl" else build_artcnn)(src, save)
return made
if __name__ == "__main__":
out = Path(sys.argv[1])
build(out, [Path(a) for a in sys.argv[2:]])
for f in sorted(out.glob("*.param")):
b = f.with_suffix(".bin")
print(f"{f.name:32} {f.stat().st_size:>7} B {b.name:30} {b.stat().st_size:>8} B")