ecg-rectifier / training /scene.py
pashaalex's picture
ECG photo rectifier: code, ONNX models, examples, training scripts
9cc92da verified
Raw History Blame Contribute Delete
12.2 kB
"""Photo-like render of an ECG sheet.
1. ecg_paper.make_sheet: flat texture;
2. sheet_geometry: developable bent / folded sheet -> 3-D mesh nodes (one per MESH_MM);
3. camera: distance, tilt, roll, aim point;
4. renderer: thin lens, radial distortion, disk light with soft shadows, background plane, optional
occluder -> rgb, sheet mask, uv (ground-truth paper coordinates per pixel);
5. depth of field and phone photometry (white balance, exposure, gamma, vignetting, noise,
sharpening, JPEG); uv and mask are never touched.
v5=True (detector dataset) adds sharp shadows on the flat texture and a local hue shift / colour cast.
"""
import math
import cv2
import numpy as np
import ecg_paper
import shadows
import sheet_geometry as geo
from renderer import BackgroundShadow, Distortion, Lighting, MeshRenderer, Optics, RectOccluder
MESH_MM = 1.0
HUE_PATCH_P = 0.5
HUE_SHIFT_DEG = (10.0, 25.0)
CAST_STRENGTH = (0.05, 0.25)
def _u(rng, lo, hi):
return float(rng.uniform(lo, hi))
def sample_shape(rng):
"""Profile components (+ accordion folds) and the ruling direction."""
style = str(rng.choice(["folded_strip", "bent", "flat_ish"], p=[0.45, 0.35, 0.20]))
comps = []
folds = []
if style == "folded_strip":
ruling_deg = _u(rng, -6, 6)
comps.append(dict(type="cylinder", s0=0.0, R=_u(rng, 300, 1500) * rng.choice([-1, 1])))
n = int(rng.integers(1, 4))
spacing = _u(rng, 45, 110)
start = _u(rng, -spacing, 0) - spacing * (n - 1) / 2
sign = rng.choice([-1.0, 1.0])
for k in range(n):
s0 = start + k * spacing
amp = _u(rng, 0.10, 0.45) * sign
sign = -sign
comps.append(dict(type="fold", s0=float(s0), width=_u(rng, 0.3, 2.0), amp=float(amp)))
folds.append(s0)
if rng.random() < 0.5:
comps.append(dict(type="wave", s0=0.0, wavelength=_u(rng, 60, 160), amp=_u(rng, 0.05, 0.25)))
elif style == "bent":
ruling_deg = _u(rng, 0, 180)
comps = geo.bent_components(rng, 0.0)
else:
ruling_deg = _u(rng, 0, 180)
comps.append(dict(type="cylinder", s0=0.0, R=_u(rng, 400, 3000) * rng.choice([-1, 1])))
return dict(style=style, comps=comps, ruling_deg=float(ruling_deg), folds_s=folds)
def hue_patch(x, rng):
"""Soft-edged region (ellipse or half-plane) with a hue rotation and a colour cast. x: BGR float32 0..1."""
h, w = x.shape[:2]
yy, xx = np.mgrid[0:h, 0:w].astype(np.float32)
s = float(max(h, w))
if rng.random() < 0.7:
cx, cy = _u(rng, 0, w), _u(rng, 0, h)
a, b = _u(rng, 0.15, 0.6) * s, _u(rng, 0.15, 0.6) * s
ang = _u(rng, 0, math.pi)
dx, dy = xx - cx, yy - cy
u = (dx * math.cos(ang) + dy * math.sin(ang)) / a
v = (-dx * math.sin(ang) + dy * math.cos(ang)) / b
dist = np.sqrt(u * u + v * v) - 1.0
soft = _u(rng, 0.1, 0.8)
region = np.clip(0.5 - dist / soft, 0.0, 1.0)
shape = "ellipse"
else:
ang = _u(rng, 0, 2 * math.pi)
d = ((xx - w / 2) * math.cos(ang) + (yy - h / 2) * math.sin(ang)) / s - _u(rng, -0.3, 0.3)
region = np.clip(0.5 + d / _u(rng, 0.05, 0.5), 0.0, 1.0)
shape = "half_plane"
shift = _u(rng, *HUE_SHIFT_DEG) * rng.choice([-1.0, 1.0])
sat = _u(rng, 0.9, 1.4)
hsv = cv2.cvtColor(np.clip(x, 0, 1), cv2.COLOR_BGR2HSV)
hsv[..., 0] = np.mod(hsv[..., 0] + shift * region, 360.0)
hsv[..., 1] = np.clip(hsv[..., 1] * (1 + (sat - 1) * region), 0, 1)
x = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cast_hue = _u(rng, 0, 180) if rng.random() < 0.5 else _u(rng, 8, 22)
tint = cv2.cvtColor(np.uint8([[[cast_hue, 255, 255]]]), cv2.COLOR_HSV2BGR)[0, 0].astype(np.float32) / 255
strength = _u(rng, *CAST_STRENGTH)
gain = 1.0 - strength * (1.0 - tint)
x = x * (1.0 - region[..., None] + region[..., None] * gain)
return x, dict(shape=shape, hue_shift_deg=shift, saturation=sat, cast_hue=cast_hue, cast=strength)
def photo_post(img, rng, v5):
x = img.astype(np.float32) / 255.0
hue_info = None
if v5 and rng.random() < HUE_PATCH_P:
x, hue_info = hue_patch(x, rng)
wb = np.array([_u(rng, 0.85, 1.20), _u(rng, 0.95, 1.05), _u(rng, 0.82, 1.12)], np.float32)
x = x * wb * _u(rng, 0.9, 1.15)
h, w = x.shape[:2]
yy, xx = np.mgrid[0:h, 0:w].astype(np.float32)
r2 = ((xx - w / 2) ** 2 + (yy - h / 2) ** 2) / (0.5 * max(h, w)) ** 2
x = x * (1 - _u(rng, 0.0, 0.25) * r2)[..., None]
x = np.clip(x, 0, 1) ** _u(rng, 0.8, 1.2)
x = x + rng.normal(0, 1, x.shape[:2])[..., None].astype(np.float32) * np.sqrt(x + 0.01) * _u(rng, 0.004, 0.025)
x = x + rng.normal(0, _u(rng, 0.0, 0.01), x.shape).astype(np.float32)
x = np.clip(x * 255, 0, 255)
if rng.random() < 0.7: # phone sharpening (halo)
blur = cv2.GaussianBlur(x, (0, 0), _u(rng, 0.8, 2.0))
x = np.clip(x + _u(rng, 0.3, 1.2) * (x - blur), 0, 255)
out = x.astype(np.uint8)
q = int(rng.integers(55, 96))
ok, buf = cv2.imencode(".jpg", out, [cv2.IMWRITE_JPEG_QUALITY, q])
return cv2.imdecode(buf, cv2.IMREAD_COLOR), dict(wb=wb.tolist(), jpeg=q, hue_patch=hue_info)
def depth_blur(img, sigma):
"""Spatially varying Gaussian blur: a blend of fixed-sigma layers."""
levels = [0.0, 0.5, 1.0, 1.6, 2.4, 3.4]
stack = [img.astype(np.float32) if lv == 0 else
cv2.GaussianBlur(img.astype(np.float32), (0, 0), lv) for lv in levels]
sig = np.clip(sigma, 0, levels[-1]).astype(np.float32)
out = np.zeros_like(stack[0])
for i in range(len(levels) - 1):
lo, hi = levels[i], levels[i + 1]
t = np.clip((sig - lo) / (hi - lo), 0, 1)
inside = ((sig >= lo) & (sig <= hi)).astype(np.float32)
if i == 0:
inside = (sig <= hi).astype(np.float32)
w = inside[..., None]
out += w * (stack[i] * (1 - t[..., None]) + stack[i + 1] * t[..., None])
sig = np.where(inside > 0, -1.0, sig)
return np.clip(out, 0, 255).astype(np.uint8)
def make_background(rng, w, h):
kind = str(rng.choice(["table_light", "table_grey", "wood", "dark", "white_sheet"]))
base = {"table_light": (205, 215, 220), "table_grey": (150, 155, 160), "wood": (90, 130, 170),
"dark": (45, 45, 50), "white_sheet": (235, 238, 240)}[kind]
bg = np.empty((h, w, 3), np.float32)
bg[:] = np.array(base, np.float32) * _u(rng, 0.85, 1.1)
yy, xx = np.mgrid[0:h, 0:w].astype(np.float32)
a = _u(rng, 0, 2 * math.pi)
bg *= (1 + _u(rng, 0, 0.2) * ((xx * math.cos(a) + yy * math.sin(a)) / max(w, h) - 0.5))[..., None]
if kind == "wood":
grain = np.sin(yy / _u(rng, 3, 9) + 3 * np.sin(xx / _u(rng, 40, 120)))
bg *= (1 + 0.06 * grain)[..., None]
bg += rng.normal(0, 2.0, (h, w, 1)).astype(np.float32)
return np.clip(bg, 0, 255).astype(np.uint8), kind
def render_scene(rng, out_size, px_per_mm, v5=False):
"""-> (photo BGR uint8, mask uint8 0/255, uv float32 texture px, texture, info)."""
probe_rng = np.random.default_rng(rng.integers(1 << 62))
tex, tex_info = ecg_paper.make_sheet(probe_rng, v5=v5)
W, H = tex_info["size_mm"]
shape = sample_shape(rng)
centre = (W / 2 + _u(rng, -0.25, 0.25) * W, H / 2 + _u(rng, -0.25, 0.25) * H)
if shape["folds_s"]:
# the texture is redrawn with crease marks along the fold lines (U - cx) cos + (V - cy) sin = s0
phi = math.radians(shape["ruling_deg"])
along0 = centre[0] * math.cos(phi) + centre[1] * math.sin(phi)
tex, tex_info = ecg_paper.make_sheet(np.random.default_rng(probe_rng.integers(1 << 62)),
folds_mm=[along0 + s for s in shape["folds_s"]],
fold_angle_deg=shape["ruling_deg"], v5=v5)
W, H = tex_info["size_mm"]
th, tw = tex.shape[:2]
shadow_info = None
if v5:
shadow_rng = np.random.default_rng(rng.integers(1 << 62))
fold_lines = [(along0 + s, shape["ruling_deg"]) for s in shape["folds_s"]] if shape["folds_s"] else None
shade, shadow_info = shadows.shadow_map(shadow_rng, th, tw, ecg_paper.PPM, fold_lines)
if shade is not None:
tex = shadows.apply(tex, shade, shadow_info["tint"])
ppm = px_per_mm
dist = _u(rng, 250, 600)
roll = _u(rng, -20, 20)
tilt_x, tilt_y = _u(rng, -25, 25), _u(rng, -20, 20)
ss = 2 # 2x supersampling
out_w = out_h = out_size * ss
L = ppm * ss * dist # sensor distance in output px
gw, gh = int(math.ceil(W / MESH_MM)), int(math.ceil(H / MESH_MM))
mr = MeshRenderer(tw, th, gw, gh)
U, V = mr.points2d[:, 0] / ecg_paper.PPM, mr.points2d[:, 1] / ecg_paper.PPM
Q = geo.sheet_points(shape, U, V, centre)
R_cw = geo.rot("z", math.radians(roll)) @ geo.rot("x", math.radians(tilt_x)) @ geo.rot("y", math.radians(tilt_y))
target = geo.sheet_points(shape, np.array([_u(rng, 0.25, 0.75) * W]), np.array([_u(rng, 0.25, 0.75) * H]),
centre)[0]
cam = target - dist * (R_cw @ np.array([0.0, 0.0, 1.0]))
Qc = (Q - cam) @ R_cw
mr.points3d[:] = Qc.astype(np.float32)
z = Qc[:, 2]
z_focus = float(np.median(z)) * _u(rng, 0.97, 1.03)
F = L * z_focus / (L + z_focus)
k1 = _u(rng, -0.08, 0.05)
light = dict(x=_u(rng, -0.8, 0.8) * dist, y=_u(rng, -0.8, 0.8) * dist, z=_u(rng, -0.2, 0.6) * dist)
lighting = Lighting(use=True, intensity=_u(rng, 0.85, 1.0), diameter=_u(rng, 40, 400),
light_mix_koef=_u(rng, 0.25, 0.7), **light)
bg_z = float(z.max() + _u(rng, 0.5, 4))
bg_shadow = BackgroundShadow(use=True, bg_z=bg_z, bottom_shadow_koef=_u(rng, 0.2, 0.6))
occ = None
if rng.random() < 0.3: # hand / phone between the light and the sheet
t = _u(rng, 0.3, 0.7)
occ = dict(cx=light["x"] * (1 - t) + _u(rng, -60, 60), cy=light["y"] * (1 - t) + _u(rng, -60, 60),
cz=light["z"] + (float(np.median(z)) - light["z"]) * t,
w=_u(rng, 60, 200), h=_u(rng, 60, 200), yaw=_u(rng, 0, 3.14))
occluder = RectOccluder(use=True, **occ)
else:
occluder = RectOccluder(use=False, cx=0, cy=0, cz=-1e6, w=0.0, h=0.0)
mr.configure(Optics(F=F, L=L, R=1.0), Distortion(use=abs(k1) > 0.01, k1=k1), lighting, bg_shadow, occluder)
background, bg_kind = make_background(rng, out_w, out_h)
rgb, mask, uv = mr.render(tex, (out_w, out_h), background)
mr.close()
rgb = cv2.resize(rgb, (out_size, out_size), interpolation=cv2.INTER_AREA)
# uv: mean of the 2 x 2 sub-pixels; a pixel is sheet only if all its sub-pixels are
blocks = (mask > 0).reshape(out_size, ss, out_size, ss)
uv = uv.reshape(out_size, ss, out_size, ss, 2).mean(axis=(1, 3))
mask = np.where(blocks.all(axis=(1, 3)), 255, 0).astype(np.uint8)
# depth of field: blur grows with |1/z_focus - 1/z|, z per pixel from the uv map
z_grid = z.reshape(gh + 1, gw + 1).astype(np.float32)
zx = (uv[..., 0] / ((tw - 1) / gw)).astype(np.float32)
zy = (uv[..., 1] / ((th - 1) / gh)).astype(np.float32)
z_pix = cv2.remap(z_grid, zx, zy, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE)
z_pix[mask == 0] = bg_z
defocus = np.abs(1.0 / z_focus - 1.0 / z_pix)
sharp = rng.random() < 0.5
max_sigma = 0.0 if sharp else _u(rng, 0.6, 2.2)
base_sigma = _u(rng, 0.3, 0.8)
scale = max_sigma / max(1e-9, float(np.percentile(defocus[mask > 0], 99)) if (mask > 0).any() else 1.0)
sigma = base_sigma + np.minimum(defocus * scale, max_sigma * 1.5)
rgb = depth_blur(rgb, sigma)
photo, photo_info = photo_post(rgb, rng, v5)
info = dict(texture=tex_info, shape=dict(style=shape["style"], ruling_deg=shape["ruling_deg"],
folds=len(shape["folds_s"])),
px_per_mm=ppm, dist_mm=dist, roll=roll, tilt=[tilt_x, tilt_y], defocus_px=max_sigma, k1=k1,
background=bg_kind, occluder=occ is not None, shadow=shadow_info, photo=photo_info)
return photo, mask, uv, tex, info