zefancainlp/xiaofan_backup / local_reports /denoising_final_code_comparison.html
zefancainlp's picture
download
raw
298 kB
<!doctype html> <html lang="zh-CN"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Denoising 最终代码对照:Single-anchor vs Baseline 50-step</title> <style> :root { color-scheme: light dark; --bg: #f5f7fb; --surface: #ffffff; --surface-soft: #f8fafc; --text: #172033; --muted: #64748b; --border: #dbe3ee; --single: #1769aa; --single-soft: #e8f3fc; --baseline: #a55716; --baseline-soft: #fff2e5; --good: #16734a; --good-soft: #e8f7ef; --warn: #9a5b08; --warn-soft: #fff6dd; --code-bg: #101827; --code-text: #dce7f5; --code-muted: #718096; --code-hit: #5b4310; --shadow: 0 10px 32px rgba(23, 32, 51, 0.08); } @media (prefers-color-scheme: dark) { :root { --bg: #0c111b; --surface: #131b29; --surface-soft: #172131; --text: #e7edf6; --muted: #9eacc0; --border: #2c394d; --single: #72b7ed; --single-soft: #152f43; --baseline: #f0ad6f; --baseline-soft: #3c291b; --good: #69d7a4; --good-soft: #153629; --warn: #f4c469; --warn-soft: #3b3019; --code-bg: #090e17; --code-text: #dce7f5; --code-muted: #6f8199; --code-hit: #5c4512; --shadow: 0 12px 34px rgba(0, 0, 0, 0.28); } } * { box-sizing: border-box; } html { scroll-behavior: smooth; } body { margin: 0; background: var(--bg); color: var(--text); font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", "Microsoft YaHei", sans-serif; line-height: 1.65; } button, input, select { font: inherit; } a { color: var(--single); } .page { width: min(1500px, calc(100% - 32px)); margin: 0 auto; padding: 36px 0 72px; } .hero { display: grid; grid-template-columns: minmax(0, 1.45fr) minmax(320px, 0.55fr); gap: 24px; align-items: end; margin-bottom: 24px; } .eyebrow { margin: 0 0 8px; color: var(--muted); font-size: 13px; letter-spacing: 0.08em; text-transform: uppercase; } h1, h2, h3, p { margin-top: 0; } h1 { margin-bottom: 10px; font-size: clamp(28px, 4vw, 48px); line-height: 1.14; letter-spacing: -0.035em; font-weight: 650; } h2 { margin-bottom: 14px; font-size: 24px; line-height: 1.25; font-weight: 650; } h3 { margin-bottom: 8px; font-size: 17px; line-height: 1.35; font-weight: 650; } .lede { margin-bottom: 0; max-width: 920px; color: var(--muted); font-size: 17px; } .verdict { padding: 18px 20px; border-left: 4px solid var(--good); background: var(--good-soft); border-radius: 0 12px 12px 0; } .verdict strong { display: block; margin-bottom: 4px; color: var(--good); font-size: 18px; } .verdict p { margin: 0; } .metrics { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 28px; } .metric { min-height: 122px; padding: 18px; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; box-shadow: var(--shadow); } .metric-label { display: block; margin-bottom: 7px; color: var(--muted); font-size: 13px; } .metric-value { display: block; font-size: clamp(23px, 3vw, 34px); font-weight: 650; line-height: 1.1; font-variant-numeric: tabular-nums; } .metric-note { display: block; margin-top: 7px; color: var(--muted); font-size: 12px; } .single-text { color: var(--single); } .baseline-text { color: var(--baseline); } .good-text { color: var(--good); } .section { margin-top: 32px; padding-top: 6px; } .section-intro { max-width: 980px; margin-bottom: 18px; color: var(--muted); } .flows { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } .flow-panel { padding: 20px; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; box-shadow: var(--shadow); } .flow-title { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; margin-bottom: 14px; } .flow-title h3 { margin: 0; } .flow-title span { color: var(--muted); font-size: 12px; font-variant-numeric: tabular-nums; } .flow-track { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } .flow-node { padding: 8px 10px; border-radius: 8px; background: var(--surface-soft); border: 1px solid var(--border); font-size: 13px; line-height: 1.3; } .flow-node.single { color: var(--single); background: var(--single-soft); border-color: color-mix(in srgb, var(--single), transparent 65%); } .flow-node.baseline { color: var(--baseline); background: var(--baseline-soft); border-color: color-mix(in srgb, var(--baseline), transparent 65%); } .arrow { color: var(--muted); font-size: 14px; } .compare-table-wrap { overflow-x: auto; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; box-shadow: var(--shadow); } table { width: 100%; border-collapse: collapse; min-width: 760px; } th, td { padding: 13px 16px; text-align: left; vertical-align: top; border-bottom: 1px solid var(--border); } th { background: var(--surface-soft); font-size: 13px; font-weight: 650; } tr:last-child td { border-bottom: 0; } td:first-child { width: 16%; color: var(--muted); font-weight: 600; } code, pre { font-family: "Cascadia Code", "SFMono-Regular", Consolas, monospace; } :not(pre) > code { padding: 1px 5px; border-radius: 5px; background: var(--surface-soft); color: var(--text); border: 1px solid var(--border); font-size: 0.92em; } .architecture { display: grid; grid-template-columns: 0.8fr 1.2fr; gap: 16px; } .cause-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 14px; } .cause-card { position: relative; padding: 20px; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; box-shadow: var(--shadow); } .cause-card.dominant { border-top: 4px solid var(--single); } .cause-card.secondary { border-top: 4px solid var(--good); } .cause-card.pressure { border-top: 4px solid var(--baseline); } .cause-rank { display: block; margin-bottom: 8px; color: var(--muted); font-size: 12px; letter-spacing: 0.06em; text-transform: uppercase; } .cause-card p { margin-bottom: 0; color: var(--muted); } .mechanism-pair { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; margin-top: 16px; } .mechanism { padding: 18px; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; } .mechanism ol { display: grid; gap: 8px; margin: 12px 0 0; padding: 0; list-style: none; counter-reset: mechanism-step; } .mechanism li { position: relative; padding: 10px 12px 10px 42px; background: var(--surface-soft); border-radius: 8px; counter-increment: mechanism-step; } .mechanism li::before { content: counter(mechanism-step); position: absolute; left: 12px; top: 10px; width: 20px; color: var(--muted); font-variant-numeric: tabular-nums; font-size: 12px; } .mechanism li + li::after { content: "↓"; position: absolute; left: 17px; top: -13px; color: var(--muted); font-size: 13px; } .evidence-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; margin-top: 16px; } .evidence-panel { padding: 18px; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; } .evidence-panel p:last-child { margin-bottom: 0; } .delta-positive { color: var(--good); font-variant-numeric: tabular-nums; } .delta-negative { color: var(--baseline); font-variant-numeric: tabular-nums; } .efficiency-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-top: 16px; } .efficiency-item { padding: 16px 18px; background: var(--surface-soft); border-left: 3px solid var(--border); } .efficiency-item strong { display: block; margin-bottom: 5px; } .efficiency-item p { margin: 0; color: var(--muted); } .analysis-panel { padding: 20px; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; box-shadow: var(--shadow); } .function-list { display: grid; gap: 8px; margin: 12px 0 0; padding: 0; list-style: none; } .function-list li { display: grid; grid-template-columns: minmax(180px, 0.65fr) 1.35fr auto; gap: 12px; align-items: baseline; padding-bottom: 8px; border-bottom: 1px solid var(--border); } .function-list li:last-child { border-bottom: 0; padding-bottom: 0; } .function-list small { color: var(--muted); font-variant-numeric: tabular-nums; } .formula { margin: 14px 0 0; padding: 14px 16px; overflow-x: auto; background: var(--code-bg); color: var(--code-text); border-radius: 10px; font-family: "Cascadia Code", "SFMono-Regular", Consolas, monospace; font-size: 13px; line-height: 1.65; white-space: pre; } .findings { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; } .finding { padding: 17px 18px; background: var(--surface); border: 1px solid var(--border); border-radius: 12px; } .finding p { margin: 0; color: var(--muted); } .finding.warn { background: var(--warn-soft); border-color: color-mix(in srgb, var(--warn), transparent 65%); } .code-section { margin-top: 38px; } .code-guide { display: flex; flex-wrap: wrap; gap: 10px 20px; margin-bottom: 14px; color: var(--muted); font-size: 13px; } .code-grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); gap: 14px; align-items: start; } .code-panel { min-width: 0; background: var(--surface); border: 1px solid var(--border); border-radius: 14px; box-shadow: var(--shadow); overflow: hidden; } .code-panel-header { padding: 14px 16px; border-bottom: 1px solid var(--border); } .code-panel-title { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-bottom: 10px; } .code-panel-title h3 { margin: 0; } .code-panel-title span { color: var(--muted); font-size: 12px; font-variant-numeric: tabular-nums; } .controls { display: grid; grid-template-columns: minmax(130px, 0.65fr) minmax(160px, 1fr) auto; gap: 8px; } input, select, button { min-height: 38px; border-radius: 8px; border: 1px solid var(--border); background: var(--surface-soft); color: var(--text); padding: 7px 10px; } button { cursor: pointer; } button:hover { border-color: var(--single); } .code-viewport { height: min(68vh, 760px); min-height: 420px; overflow: auto; background: var(--code-bg); color: var(--code-text); scroll-behavior: smooth; tab-size: 4; } .code-block { display: block; min-width: max-content; margin: 0; padding: 12px 0 22px; font-size: 12px; line-height: 1.55; } .code-line { display: grid; grid-template-columns: 52px minmax(max-content, 1fr); min-height: 19px; } .code-line.is-match { background: var(--code-hit); } .code-line.is-target { background: color-mix(in srgb, var(--single), transparent 72%); } .line-number { position: sticky; left: 0; z-index: 1; padding-right: 12px; background: var(--code-bg); color: var(--code-muted); text-align: right; user-select: none; font-variant-numeric: tabular-nums; } .line-source { display: block; padding: 0 16px 0 10px; white-space: pre; } .search-status { min-height: 22px; margin-top: 7px; color: var(--muted); font-size: 12px; } .hidden { display: none !important; } .footer { margin-top: 30px; padding-top: 18px; border-top: 1px solid var(--border); color: var(--muted); font-size: 13px; } @media (max-width: 980px) { .hero, .flows, .architecture, .code-grid, .cause-grid, .mechanism-pair, .evidence-grid { grid-template-columns: 1fr; } .metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); } .efficiency-grid { grid-template-columns: 1fr; } .code-viewport { height: 620px; } } @media (max-width: 620px) { .page { width: min(100% - 20px, 1500px); padding-top: 22px; } .metrics, .findings { grid-template-columns: 1fr; } .controls { grid-template-columns: 1fr; } .function-list li { grid-template-columns: 1fr; gap: 2px; } .code-viewport { min-height: 380px; height: 70vh; } } </style> </head> <body> <main class="page"> <header class="hero"> <div> <p class="eyebrow">Denoising · final best code inspection</p> <h1>Single-anchor 与 Baseline 50-step<br>最终代码对照</h1> <p class="lede"> 比较对象是两个运行各自最终搜索池中 MSE 最低的真实可执行程序。 Single-anchor 不是从头重写 denoiser,而是在固定 MAGIC 产生的 <code>Z</code> 上学习一个后处理函数图。 </p> </div> <aside class="verdict"> <strong>更小的程序,得到更低的 MSE</strong> <p>Single-anchor 的可变代码只有 150 行、4 个函数;它主要优化 <code>X</code> 与 <code>Z</code> 的混合,而不是堆叠新的 denoising 模块。</p> </aside> </header> <section class="metrics" aria-label="核心指标"> <div class="metric"> <span class="metric-label">Single-anchor MSE</span> <span class="metric-value single-text">0.16264931</span> <span class="metric-note">state a5bab5fb · timestep 24</span> </div> <div class="metric"> <span class="metric-label">Baseline MSE</span> <span class="metric-value baseline-text">0.16403095</span> <span class="metric-note">state c5843e35 · timestep 49</span> </div> <div class="metric"> <span class="metric-label">MSE 差值</span> <span class="metric-value good-text">−0.00138164</span> <span class="metric-note">Single-anchor 相对低 0.842%</span> </div> <div class="metric"> <span class="metric-label">可变代码规模</span> <span class="metric-value">150 / 1043</span> <span class="metric-note">Single-anchor / Baseline 内容行</span> </div> </section> <section class="section" id="architecture"> <h2>一眼看清:两份代码改的不是同一层</h2> <p class="section-intro"> Baseline 直接生成完整 <code>magic_denoise(X)</code>;Single-anchor 保留固定 backbone,搜索对象只是接收 <code>X, Z</code> 的 active function graph。 </p> <div class="flows"> <article class="flow-panel"> <div class="flow-title"> <h3 class="single-text">Single-anchor</h3> <span>230 行完整展开 / 150 行可变</span> </div> <div class="flow-track" aria-label="Single-anchor 调用流程"> <span class="flow-node">X</span><span class="arrow">→</span> <span class="flow-node">固定 MAGIC</span><span class="arrow">→</span> <span class="flow-node">Z</span><span class="arrow">→</span> <span class="flow-node single">sqrt 自适应混合</span><span class="arrow">→</span> <span class="flow-node single">cell/gene 收缩</span><span class="arrow">→</span> <span class="flow-node single">gene 校准</span><span class="arrow">→</span> <span class="flow-node single">raw 回混</span> </div> </article> <article class="flow-panel"> <div class="flow-title"> <h3 class="baseline-text">Baseline 50-step</h3> <span>1043 行完整重写</span> </div> <div class="flow-track" aria-label="Baseline 调用流程"> <span class="flow-node">X</span><span class="arrow">→</span> <span class="flow-node baseline">预缩放 / winsorize</span><span class="arrow">→</span> <span class="flow-node baseline">Anscombe / HVG</span><span class="arrow">→</span> <span class="flow-node baseline">多图 diffusion</span><span class="arrow">→</span> <span class="flow-node baseline">双 variant</span><span class="arrow">→</span> <span class="flow-node baseline">NMF / PCA</span><span class="arrow">→</span> <span class="flow-node baseline">kNN / ALRA</span> </div> </article> </div> </section> <section class="section" id="why-short"> <h2>为什么 Single-anchor 会短而且有效:三层机制叠加</h2> <p class="section-intro"> 这不是“模型偶然写出了一段短代码”。最主要的原因是任务被改写成了 residual correction;其次是 PFG 编译器把搜索动作限制成局部、可自动回收的符号修改;最后,baseline 的目标函数没有任何代码长度或运行代价惩罚,天然鼓励保留旧模块并继续加新模块。 </p> <div class="cause-grid"> <article class="cause-card dominant"> <span class="cause-rank">原因 1 · 主因</span> <h3>它解决的是更窄的问题</h3> <p> Baseline 学的是完整映射 <code>X → denoised</code>。Single-anchor 免费获得固定 MAGIC 的 <code>Z</code>,只学 <code>(X, Z) → better Z</code>。图构建、diffusion 和 inverse transform 已经由 backbone 完成,因此最终最优解可以只是几组逐元素混合和统计校准。 </p> </article> <article class="cause-card secondary"> <span class="cause-rank">原因 2 · 结构性压缩</span> <h3>动作是“改一个符号”,不是“重写整份程序”</h3> <p> PFG 每次只允许替换一个现有 reachable symbol。没有提到的函数自动保留;新 helper 必须从该 anchor 可达;替换后不可达的旧 helper 自动删除。模型不需要复制 1000 行上下文,也更容易撤掉无用结构。 </p> </article> <article class="cause-card pressure"> <span class="cause-rank">原因 3 · Baseline 的增长压力</span> <h3>奖励只看 MSE,不惩罚复杂度</h3> <p> Baseline 每轮必须返回一份完整 <code>magic_denoise</code>。在 Poisson 硬约束和运行失败风险下,最安全的生成策略是保留已有代码,再以很小权重叠加一个 PCA、NMF、kNN 或 correction。即使模块贡献接近零,只要整份程序被选中,它也会进入下一轮代码。 </p> </article> </div> <div class="mechanism-pair"> <article class="mechanism"> <h3 class="single-text">Single-anchor 的因果链</h3> <ol> <li>固定 MAGIC 先把高维原始 counts 压成一个已有质量的 <code>Z</code>。</li> <li>候选 patch 只能改变一个 anchor,因此同胞候选间的主要差异被限制在一个函数或常量。</li> <li>得分变化更容易对应到当前 anchor,而不是同时变化的二十多个函数。</li> <li>好公式留在 graph 中继续调常数;失去调用关系的 helper 被 reachability pruning 自动回收。</li> <li>最终收敛为 4 个互补操作:sqrt mix、dropout shrink、gene scaling、raw mix。</li> </ol> </article> <article class="mechanism"> <h3 class="baseline-text">Baseline 的因果链</h3> <ol> <li>prompt 放入完整当前程序,并要求重新输出完整 <code>magic_denoise</code>。</li> <li>一次生成可能同时改入口、多个 helper、常量和处理顺序,评价只返回一个最终 MSE。</li> <li>无法从这一个分数判断究竟是哪一段代码有效,删除旧模块的风险高于保留它。</li> <li>模型倾向于复制旧程序,再增加一个低权重“保险模块”;没有长度惩罚阻止这种累积。</li> <li>上下文和执行越来越重,完整重写又制造更多格式、超时及依赖失败机会。</li> </ol> </article> </div> </section> <section class="section" id="mechanism-evidence"> <h2>这个解释能在运行链和代码里直接看到</h2> <div class="compare-table-wrap"> <table> <thead> <tr><th>运行位置</th><th class="single-text">Single-anchor active graph</th><th class="baseline-text">Baseline full program</th></tr> </thead> <tbody> <tr><td>初始</td><td>2 行、1 个 noop readout,backbone 另行固定</td><td>60 行、1 个 <code>magic_denoise</code></td></tr> <tr><td>step 10</td><td>187 行、5 个函数,MSE 0.165346</td><td>623 行、13 个函数,MSE 0.182433</td></tr> <tr><td>中后期</td><td>step 14 达到 195 行峰值,随后边改善边压缩</td><td>step 27 为 636 行,之后继续增长到约 1000 行</td></tr> <tr><td>最终</td><td>150 行、4 个函数,MSE 0.162649</td><td>1043 行、24 个函数,MSE 0.164031</td></tr> <tr><td>距离自身峰值</td><td>195 → 150 行,减少 23.1%;函数峰值 6 → 4</td><td>1066 → 1043 行,只减少 2.2%;函数仍为峰值 24</td></tr> </tbody> </table> </div> <div class="evidence-grid"> <article class="evidence-panel"> <h3 class="single-text">PFG 确实发生了自动“垃圾回收”</h3> <p><code>step 6</code> 修改 <code>_sqrt_weighted_mix</code> 后,<code>_compute_gamma</code> 失去可达性并被删除;MSE 同时从 0.168573 降到 0.167456。</p> <p><code>step 18</code> 重写 gene scaling helper 后,4 个旧常量 <code>_GENE_SCALE_*</code> 自动删除,MSE 不变。</p> <p><code>step 21</code> 替换 readout 时又删除 <code>_gene_dropout_factor</code> 和 <code>_scale_raw_counts_to_Z_libsize</code>,代码 167 → 153 行,MSE 反而继续下降。</p> </article> <article class="evidence-panel"> <h3 class="baseline-text">Baseline 的函数是按模块逐步累积的</h3> <p><code>step 0</code> 一次从 1 个函数变成 4 个;<code>step 10</code> 加入 full PCA;<code>step 22</code> 加入 NMF;<code>step 27–29</code> 加入双 variant、Seurat HVG、gene raw mix、low-count suppression 和 variance scaling。</p> <p><code>step 32–44</code> 又继续加入 gene mean/variance、HVG refinement、final log-kNN 和 ALRA。因为每轮的新状态就是整份 <code>parsed_code</code>,这些模块默认全部被复制进下一轮。</p> </article> </div> <h3 style="margin-top: 22px;">Baseline 消融:1043 行并不等于每个模块都在贡献</h3> <p class="section-intro"> 下表在同一份最终代码上关闭单个模块。“移除后 MSE 变化”为正表示该模块有帮助;为负表示移除它反而更好。不能把交互式消融简单相加,但它足以判断复杂度是否都被结果支持。 </p> <div class="compare-table-wrap"> <table> <thead><tr><th>关闭项</th><th>移除后 MSE</th><th>相对 full 的变化</th><th>解释</th></tr></thead> <tbody> <tr><td>NMF</td><td>0.16943413</td><td class="delta-positive">+0.00540319</td><td>Baseline 中最有实质作用的独立模块。</td></tr> <tr><td>gene raw mix</td><td>0.16422692</td><td class="delta-positive">+0.00019597</td><td>有小但明确的帮助。</td></tr> <tr><td>全部 final refinement</td><td>0.16421512</td><td class="delta-positive">+0.00018417</td><td>final PCA + log-kNN + ALRA 合计只有约 1.84e−4。</td></tr> <tr><td>单独 final PCA</td><td>0.16408239</td><td class="delta-positive">+0.00005144</td><td>贡献很小。</td></tr> <tr><td>单独 final log-kNN</td><td>0.16408707</td><td class="delta-positive">+0.00005612</td><td>贡献很小,但增加一次建图和 residual diffusion。</td></tr> <tr><td>multi-kNN</td><td>0.16402914</td><td class="delta-negative">−0.00000180</td><td>关闭后略好;第二套图没有得到独立支持。</td></tr> <tr><td>full diffusion</td><td>0.16402933</td><td class="delta-negative">−0.00000162</td><td>关闭后略好。</td></tr> <tr><td>low-count suppression</td><td>0.16400414</td><td class="delta-negative">−0.00002680</td><td>关闭后明显更好,说明该模块在最终组合中已是负贡献。</td></tr> </tbody> </table> </div> </section> <section class="section" id="efficiency-meaning"> <h2>“高效”需要拆成三件事</h2> <div class="efficiency-grid"> <article class="efficiency-item"> <strong>算法执行效率</strong> <p>Single-anchor 的可变部分只有逐元素运算与 axis reduction;完整流水线只需固定 MAGIC 的一次图 diffusion。Baseline 还包含多图、双 variant、PCA/NMF 和最终 kNN/ALRA。</p> </article> <article class="efficiency-item"> <strong>搜索效率</strong> <p>后 5 步日志中,Single-anchor 平均 prompt 约 4,002 tokens、有效率 95%;Baseline 约 10,991 tokens、有效率 45%。这支持“局部 patch 更容易生成合法候选”,但两次运行模型和资源配置不同,不能当严格因果实验。</p> </article> <article class="efficiency-item"> <strong>统计效率</strong> <p>Single-anchor 把容量集中到少数与评分直接相关的自由度:<code>beta=30</code>、<code>lam=0.15</code>、<code>raw_alpha=0.08</code>、阈值与 scaling exponent。Baseline 则把搜索预算分散在大量结构和交互参数上。</p> </article> </div> <div class="verdict" style="margin-top: 16px;"> <strong>准确结论</strong> <p> Single-anchor 的优势不是“150 行代码凭空完成了 1043 行的全部工作”。它把固定 MAGIC 当作已经完成的共享底座,再用 PFG 对 residual correction 做局部、可回收的结构搜索。短主要来自搜索表示和自动 pruning;高分主要来自问题分解后更容易找到有效的 X/Z 混合;Baseline 的长主要来自完整重写、粗粒度信用和无复杂度惩罚共同造成的模块累积。 </p> </div> </section> <section class="section"> <h2>结构与行为差异</h2> <div class="compare-table-wrap"> <table> <thead> <tr> <th>维度</th> <th class="single-text">Single-anchor final</th> <th class="baseline-text">Baseline final</th> </tr> </thead> <tbody> <tr> <td>入口语义</td> <td><code>improve_artifact(X, Z)</code>:优化已存在的 denoised estimate</td> <td><code>magic_denoise(X)</code>:从原始 counts 完整计算</td> </tr> <tr> <td>生成代码</td> <td>150 行、141 个非空行、4 个函数</td> <td>1043 行、915 个非空行、24 个函数</td> </tr> <tr> <td>完整执行口径</td> <td>固定 backbone 60 行 + active graph 150 行 + 通用验证/wrapper,共约 230 行</td> <td>全部逻辑都在生成程序内,共 1043 行</td> </tr> <tr> <td>主入口复杂度</td> <td>5 个显式参数加 <code>**kwargs</code></td> <td>39 个显式参数加 <code>**kwargs</code></td> </tr> <tr> <td>图操作</td> <td>postprocessor 不建图;只复用固定 MAGIC 的 <code>Z</code></td> <td>初始/二次 HVG 图、多 knn 图,以及最终 log-space kNN 图</td> </tr> <tr> <td>低秩模型</td> <td>没有 PCA、NMF、ALRA</td> <td>KL-NMF、多个 log-space PCA、ALRA-style refinement</td> </tr> <tr> <td>低值处理</td> <td>按 cell dropout 设置阈值,低于阈值直接归零</td> <td>小于 0.2 的值乘 0.9,属于软抑制</td> </tr> <tr> <td>gene scaling</td> <td><code>ratio**0.6</code>,限制在 <code>[0.8, 1.2]</code></td> <td>gene mean 比率限制在 <code>[0.97, 1.03]</code>;variance scaling 默认关闭</td> </tr> <tr> <td>raw mixing</td> <td>按每个 cell 的 dropout 调节,基础权重 0.08</td> <td>variant 内 cell-level 回混,再做一次 gene-level 回混</td> </tr> <tr> <td>数值类型</td> <td>后处理主要使用 <code>float64</code></td> <td>入口和最终输出使用 <code>float32</code>,中间模型混用 NumPy/sklearn 类型</td> </tr> </tbody> </table> </div> </section> <section class="section"> <h2>Single-anchor:四个函数形成一条紧凑校准链</h2> <div class="architecture"> <article class="analysis-panel"> <h3>函数构成</h3> <ul class="function-list"> <li><code>_sqrt_weighted_mix</code><span>在 sqrt 空间按 cell/gene dropout 混合 X 与 Z,并执行自适应硬阈值。</span><small>40 行</small></li> <li><code>_adaptive_shrinkage_factor</code><span>dropout 越高,越收缩回固定 backbone。</span><small>34 行</small></li> <li><code>_apply_gene_scaling_v3</code><span>按列总量做温和 power-law 校准。</span><small>22 行</small></li> <li><code>improve_artifact</code><span>组织三个 helper,最后按 cell dropout 混回少量 raw counts。</span><small>48 行</small></li> </ul> </article> <article class="analysis-panel"> <h3>实际数值路径</h3> <div class="formula">βᵢⱼ = 30 × (1 + 2·dropout_cellᵢ) × (1 + dropout_geneⱼ) γᵢⱼ = Xᵢⱼ / (Xᵢⱼ + βᵢⱼ) Mᵢⱼ = [(1 − γᵢⱼ)√Zᵢⱼ + γᵢⱼ√Xᵢⱼ]² Mᵢⱼ = 0, if Mᵢⱼ &lt; 0.12 + 0.15·dropout_cellᵢ λᵢⱼ = max( 0.15 × (0.5 + 0.5·dropout_cellᵢ), 0.15 × (0.2 + 0.8·dropout_geneⱼ) ) C = (1 − λ)M + λZ gene_factorⱼ = clip[(ΣᵢXᵢⱼ / ΣᵢCᵢⱼ)^0.6, 0.8, 1.2] output = adaptive_raw_mix(C × gene_factor, X)</div> </article> </div> </section> <section class="section"> <h2>Baseline:24 个函数覆盖完整重建</h2> <p class="section-intro"> 它不是某个单一算法,而是逐步累积出的组合系统。主要计算集中在 407 行的 <code>magic_denoise</code> 和 115 行的 <code>_denoise_variant</code>。 </p> <div class="compare-table-wrap"> <table> <thead><tr><th>模块</th><th>具体函数</th><th>作用</th></tr></thead> <tbody> <tr><td>变换与规模</td><td><code>_anscombe_transform</code>, <code>_inverse_anscombe_newton</code>, <code>_smooth_row_sums</code>, <code>_adjust_row_sums</code></td><td>VST 正反变换与 cell library-size 控制。</td></tr> <tr><td>HVG</td><td><code>_select_hvg_seurat</code>, <code>_standardize_hvg</code>, <code>_destandardize_hvg</code>, <code>_refine_hvg</code></td><td>选择、标准化并两次扩展 HVG 集合。</td></tr> <tr><td>Diffusion</td><td><code>_apply_diffusion</code>, <code>_impute_counts_std</code>, <code>_full_diffuse</code>, <code>_auto_diffusion_time</code></td><td>在多步、多权重和全矩阵路径上执行平滑。</td></tr> <tr><td>低秩模型</td><td><code>_nmf_impute</code>, <code>_full_pca_denoise_centered</code>, <code>_low_rank_refine_counts</code></td><td>KL-NMF、PCA 和 ALRA-style 重建。</td></tr> <tr><td>统计校准</td><td><code>_scale_hvg_variance</code>, <code>_gene_raw_weight_mix</code>, <code>_suppress_low_counts</code>, <code>_scale_gene_means</code>, <code>_scale_gene_variance_masked</code></td><td>variance、mean、raw mixing 和低值校正。</td></tr> <tr><td>最终修正</td><td><code>_final_log_knn_smooth_with_residual</code></td><td>在 log-normalized 空间重新建图,并对 residual diffusion。</td></tr> <tr><td>组织层</td><td><code>_denoise_variant</code>, <code>magic_denoise</code></td><td>运行两套 variant,以 0.96 / 0.04 合并,再执行多阶段 polishing。</td></tr> </tbody> </table> </div> </section> <section class="section"> <h2>从实际代码发现的冗余与迭代痕迹</h2> <div class="findings"> <article class="finding warn"> <h3>Baseline 的 <code>solver</code> 是死参数</h3> <p>主函数设置了 <code>solver = "approximate" / "exact"</code>,但后续没有读取或传给任何计算函数,改变它不会改变输出。</p> </article> <article class="finding warn"> <h3>Baseline 有三个未使用 import</h3> <p><code>anndata</code>、<code>scanpy as sc</code>、<code>sklearn.metrics</code> 只被导入,没有出现在计算路径中。</p> </article> <article class="finding warn"> <h3>“Newton inverse”大部分情况下是空迭代</h3> <p>对正常的非负 <code>y</code>,初值 <code>(y/2)^2 − 3/8</code> 已经是当前 Anscombe 变换的解析逆;后续 12 轮 residual 为零。</p> </article> <article class="finding warn"> <h3>Single-anchor 有一处过期注释</h3> <p><code>improve_artifact</code> docstring 仍写 exponent 0.7,但真实调用使用 <code>_apply_gene_scaling_v3(..., exponent=0.6)</code>。</p> </article> </div> </section> <section class="code-section" id="code"> <h2>完整代码浏览器</h2> <div class="code-guide"> <span>左右代码都是最终 best 的原文。</span> <span>搜索会标出匹配行。</span> <span>函数下拉框可直接跳转。</span> <span>左侧可在“完整执行版”和“仅学习部分”之间切换。</span> </div> <div class="code-grid"> <article class="code-panel" data-panel="single"> <header class="code-panel-header"> <div class="code-panel-title"> <h3 class="single-text">Single-anchor final</h3> <span id="single-meta">完整执行版 · 230 行</span> </div> <div class="controls"> <select id="single-source" aria-label="选择 Single-anchor 代码口径"> <option value="full">完整执行版</option> <option value="active">仅 active graph</option> </select> <input id="single-search" type="search" placeholder="搜索代码,例如 dropout" aria-label="搜索 Single-anchor 代码"> <select id="single-functions-full" class="function-select" aria-label="跳转到 Single-anchor 函数"> <option value="1">文件顶部</option> <option value="1">magic_denoise · L1</option> <option value="65">_sqrt_weighted_mix · L65</option> <option value="106">_adaptive_shrinkage_factor · L106</option> <option value="141">_apply_gene_scaling_v3 · L141</option> <option value="164">improve_artifact · L164</option> <option value="214">_single_anchor_validate · L214</option> <option value="226">magic_denoise · L226</option> </select> <select id="single-functions-active" class="function-select hidden" aria-label="跳转到 Single-anchor active graph 函数"> <option value="1">文件顶部</option> <option value="4">_sqrt_weighted_mix · L4</option> <option value="45">_adaptive_shrinkage_factor · L45</option> <option value="80">_apply_gene_scaling_v3 · L80</option> <option value="103">improve_artifact · L103</option> </select> </div> <div class="search-status" id="single-status" aria-live="polite">未搜索</div> </header> <div class="code-viewport" id="single-viewport"> <pre class="code-block" id="single-code-full"><code><span class="code-line" id="single-full-line-1" data-line="1"><span class="line-number">1</span><span class="line-source">def magic_denoise(X, knn=5, t=3, n_pca=100, solver=&quot;approximate&quot;, decay=1, knn_max=None, random_state=None, n_jobs=1, verbose=False):</span></span> <span class="code-line" id="single-full-line-2" data-line="2"><span class="line-number">2</span><span class="line-source"> import numpy as np</span></span> <span class="code-line" id="single-full-line-3" data-line="3"><span class="line-number">3</span><span class="line-source"> import graphtools</span></span> <span class="code-line" id="single-full-line-4" data-line="4"><span class="line-number">4</span><span class="line-source"> import scprep</span></span> <span class="code-line" id="single-full-line-5" data-line="5"><span class="line-number">5</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-6" data-line="6"><span class="line-number">6</span><span class="line-source"> if knn_max is None:</span></span> <span class="code-line" id="single-full-line-7" data-line="7"><span class="line-number">7</span><span class="line-source"> knn_max = knn * 3</span></span> <span class="code-line" id="single-full-line-8" data-line="8"><span class="line-number">8</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-9" data-line="9"><span class="line-number">9</span><span class="line-source"> X_work = scprep.utils.toarray(X).astype(np.float64)</span></span> <span class="code-line" id="single-full-line-10" data-line="10"><span class="line-number">10</span><span class="line-source"> X_work = np.sqrt(X_work)</span></span> <span class="code-line" id="single-full-line-11" data-line="11"><span class="line-number">11</span><span class="line-source"> X_work, libsize = scprep.normalize.library_size_normalize(X_work, rescale=1, return_library_size=True)</span></span> <span class="code-line" id="single-full-line-12" data-line="12"><span class="line-number">12</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-13" data-line="13"><span class="line-number">13</span><span class="line-source"> graph = graphtools.Graph(</span></span> <span class="code-line" id="single-full-line-14" data-line="14"><span class="line-number">14</span><span class="line-source"> X_work,</span></span> <span class="code-line" id="single-full-line-15" data-line="15"><span class="line-number">15</span><span class="line-source"> n_pca=n_pca if X_work.shape[1] &gt; n_pca else None,</span></span> <span class="code-line" id="single-full-line-16" data-line="16"><span class="line-number">16</span><span class="line-source"> knn=knn,</span></span> <span class="code-line" id="single-full-line-17" data-line="17"><span class="line-number">17</span><span class="line-source"> knn_max=knn_max,</span></span> <span class="code-line" id="single-full-line-18" data-line="18"><span class="line-number">18</span><span class="line-source"> decay=decay,</span></span> <span class="code-line" id="single-full-line-19" data-line="19"><span class="line-number">19</span><span class="line-source"> thresh=1e-4,</span></span> <span class="code-line" id="single-full-line-20" data-line="20"><span class="line-number">20</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="single-full-line-21" data-line="21"><span class="line-number">21</span><span class="line-source"> n_jobs=n_jobs,</span></span> <span class="code-line" id="single-full-line-22" data-line="22"><span class="line-number">22</span><span class="line-source"> verbose=0,</span></span> <span class="code-line" id="single-full-line-23" data-line="23"><span class="line-number">23</span><span class="line-source"> )</span></span> <span class="code-line" id="single-full-line-24" data-line="24"><span class="line-number">24</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-25" data-line="25"><span class="line-number">25</span><span class="line-source"> diff_op = graph.diff_op</span></span> <span class="code-line" id="single-full-line-26" data-line="26"><span class="line-number">26</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-27" data-line="27"><span class="line-number">27</span><span class="line-source"> if solver == &quot;approximate&quot;:</span></span> <span class="code-line" id="single-full-line-28" data-line="28"><span class="line-number">28</span><span class="line-source"> data = graph.data_nu</span></span> <span class="code-line" id="single-full-line-29" data-line="29"><span class="line-number">29</span><span class="line-source"> else:</span></span> <span class="code-line" id="single-full-line-30" data-line="30"><span class="line-number">30</span><span class="line-source"> data = scprep.utils.to_array_or_spmatrix(graph.data)</span></span> <span class="code-line" id="single-full-line-31" data-line="31"><span class="line-number">31</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-32" data-line="32"><span class="line-number">32</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="single-full-line-33" data-line="33"><span class="line-number">33</span><span class="line-source"> print(f&quot; [magic_denoise] data shape: {data.shape}, sum: {data.sum():.6f}&quot;)</span></span> <span class="code-line" id="single-full-line-34" data-line="34"><span class="line-number">34</span><span class="line-source"> print(f&quot; [magic_denoise] diff_op sum: {diff_op.sum():.6f}&quot;)</span></span> <span class="code-line" id="single-full-line-35" data-line="35"><span class="line-number">35</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-36" data-line="36"><span class="line-number">36</span><span class="line-source"> data_imputed = scprep.utils.toarray(data)</span></span> <span class="code-line" id="single-full-line-37" data-line="37"><span class="line-number">37</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-38" data-line="38"><span class="line-number">38</span><span class="line-source"> if t &gt; 0 and diff_op.shape[1] &lt; data_imputed.shape[1]:</span></span> <span class="code-line" id="single-full-line-39" data-line="39"><span class="line-number">39</span><span class="line-source"> diff_op_t = np.linalg.matrix_power(scprep.utils.toarray(diff_op), t)</span></span> <span class="code-line" id="single-full-line-40" data-line="40"><span class="line-number">40</span><span class="line-source"> data_imputed = diff_op_t.dot(data_imputed)</span></span> <span class="code-line" id="single-full-line-41" data-line="41"><span class="line-number">41</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="single-full-line-42" data-line="42"><span class="line-number">42</span><span class="line-source"> print(f&quot; [magic_denoise] used matrix_power path&quot;)</span></span> <span class="code-line" id="single-full-line-43" data-line="43"><span class="line-number">43</span><span class="line-source"> else:</span></span> <span class="code-line" id="single-full-line-44" data-line="44"><span class="line-number">44</span><span class="line-source"> for _ in range(t):</span></span> <span class="code-line" id="single-full-line-45" data-line="45"><span class="line-number">45</span><span class="line-source"> data_imputed = diff_op.dot(data_imputed)</span></span> <span class="code-line" id="single-full-line-46" data-line="46"><span class="line-number">46</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="single-full-line-47" data-line="47"><span class="line-number">47</span><span class="line-source"> print(f&quot; [magic_denoise] used iteration path&quot;)</span></span> <span class="code-line" id="single-full-line-48" data-line="48"><span class="line-number">48</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-49" data-line="49"><span class="line-number">49</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="single-full-line-50" data-line="50"><span class="line-number">50</span><span class="line-source"> print(f&quot; [magic_denoise] after diffusion sum: {data_imputed.sum():.6f}&quot;)</span></span> <span class="code-line" id="single-full-line-51" data-line="51"><span class="line-number">51</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-52" data-line="52"><span class="line-number">52</span><span class="line-source"> if solver == &quot;approximate&quot;:</span></span> <span class="code-line" id="single-full-line-53" data-line="53"><span class="line-number">53</span><span class="line-source"> data_imputed = graph.inverse_transform(data_imputed, columns=None)</span></span> <span class="code-line" id="single-full-line-54" data-line="54"><span class="line-number">54</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="single-full-line-55" data-line="55"><span class="line-number">55</span><span class="line-source"> print(f&quot; [magic_denoise] after inverse_transform sum: {data_imputed.sum():.6f}&quot;)</span></span> <span class="code-line" id="single-full-line-56" data-line="56"><span class="line-number">56</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-57" data-line="57"><span class="line-number">57</span><span class="line-source"> data_imputed = np.square(data_imputed)</span></span> <span class="code-line" id="single-full-line-58" data-line="58"><span class="line-number">58</span><span class="line-source"> data_imputed = scprep.utils.matrix_vector_elementwise_multiply(data_imputed, libsize, axis=0)</span></span> <span class="code-line" id="single-full-line-59" data-line="59"><span class="line-number">59</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-60" data-line="60"><span class="line-number">60</span><span class="line-source"> return data_imputed</span></span> <span class="code-line" id="single-full-line-61" data-line="61"><span class="line-number">61</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-62" data-line="62"><span class="line-number">62</span><span class="line-source">import numpy as np</span></span> <span class="code-line" id="single-full-line-63" data-line="63"><span class="line-number">63</span><span class="line-source">import scprep</span></span> <span class="code-line" id="single-full-line-64" data-line="64"><span class="line-number">64</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-65" data-line="65"><span class="line-number">65</span><span class="line-source">def _sqrt_weighted_mix(X, Z, beta):</span></span> <span class="code-line" id="single-full-line-66" data-line="66"><span class="line-number">66</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-67" data-line="67"><span class="line-number">67</span><span class="line-source"> Mix raw counts ``X`` with the backbone denoised matrix ``Z`` in a</span></span> <span class="code-line" id="single-full-line-68" data-line="68"><span class="line-number">68</span><span class="line-source"> variance‑stabilising square‑root space. The mixing weight ``gamma`` is</span></span> <span class="code-line" id="single-full-line-69" data-line="69"><span class="line-number">69</span><span class="line-source"> computed using an adaptive per‑cell and per‑gene regularisation that depends</span></span> <span class="code-line" id="single-full-line-70" data-line="70"><span class="line-number">70</span><span class="line-source"> more strongly on dropout rates than the original implementation. This</span></span> <span class="code-line" id="single-full-line-71" data-line="71"><span class="line-number">71</span><span class="line-source"> reduces the contribution of noisy raw counts for cells and genes with many</span></span> <span class="code-line" id="single-full-line-72" data-line="72"><span class="line-number">72</span><span class="line-source"> zeros, thereby lowering the mean‑squared error after log‑normalisation while</span></span> <span class="code-line" id="single-full-line-73" data-line="73"><span class="line-number">73</span><span class="line-source"> keeping the Poisson loss above the required threshold.</span></span> <span class="code-line" id="single-full-line-74" data-line="74"><span class="line-number">74</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-75" data-line="75"><span class="line-number">75</span><span class="line-source"> After mixing, each cell is scaled to a common library size equal to the</span></span> <span class="code-line" id="single-full-line-76" data-line="76"><span class="line-number">76</span><span class="line-source"> median library size of the original backbone denoised matrix ``Z``. The</span></span> <span class="code-line" id="single-full-line-77" data-line="77"><span class="line-number">77</span><span class="line-source"> per‑cell threshold for discarding low‑expression values is also slightly</span></span> <span class="code-line" id="single-full-line-78" data-line="78"><span class="line-number">78</span><span class="line-source"> lowered (``0.12 + 0.15*dropout``) to retain additional weak signals without</span></span> <span class="code-line" id="single-full-line-79" data-line="79"><span class="line-number">79</span><span class="line-source"> harming the Poisson metric.</span></span> <span class="code-line" id="single-full-line-80" data-line="80"><span class="line-number">80</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-81" data-line="81"><span class="line-number">81</span><span class="line-source"> X_arr = scprep.utils.toarray(X).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-full-line-82" data-line="82"><span class="line-number">82</span><span class="line-source"> Z_arr = scprep.utils.toarray(Z).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-full-line-83" data-line="83"><span class="line-number">83</span><span class="line-source"> dropout = (X_arr == 0).mean(axis=1)</span></span> <span class="code-line" id="single-full-line-84" data-line="84"><span class="line-number">84</span><span class="line-source"> beta_adj = beta * (1.0 + 2.0 * dropout)</span></span> <span class="code-line" id="single-full-line-85" data-line="85"><span class="line-number">85</span><span class="line-source"> beta_adj = beta_adj[:, None]</span></span> <span class="code-line" id="single-full-line-86" data-line="86"><span class="line-number">86</span><span class="line-source"> gene_dropout = (X_arr == 0).mean(axis=0)</span></span> <span class="code-line" id="single-full-line-87" data-line="87"><span class="line-number">87</span><span class="line-source"> beta_adj_gene = beta_adj * (1.0 + gene_dropout)[None, :]</span></span> <span class="code-line" id="single-full-line-88" data-line="88"><span class="line-number">88</span><span class="line-source"> denom = X_arr + beta_adj_gene</span></span> <span class="code-line" id="single-full-line-89" data-line="89"><span class="line-number">89</span><span class="line-source"> gamma = np.divide(X_arr, denom, out=np.zeros_like(X_arr, dtype=np.float64), where=denom != 0)</span></span> <span class="code-line" id="single-full-line-90" data-line="90"><span class="line-number">90</span><span class="line-source"> X_sqrt = np.sqrt(X_arr)</span></span> <span class="code-line" id="single-full-line-91" data-line="91"><span class="line-number">91</span><span class="line-source"> Z_sqrt = np.sqrt(Z_arr)</span></span> <span class="code-line" id="single-full-line-92" data-line="92"><span class="line-number">92</span><span class="line-source"> mixed_sqrt = (1.0 - gamma) * Z_sqrt + gamma * X_sqrt</span></span> <span class="code-line" id="single-full-line-93" data-line="93"><span class="line-number">93</span><span class="line-source"> mixed = np.square(mixed_sqrt)</span></span> <span class="code-line" id="single-full-line-94" data-line="94"><span class="line-number">94</span><span class="line-source"> per_cell_thresh = 0.12 + 0.15 * dropout</span></span> <span class="code-line" id="single-full-line-95" data-line="95"><span class="line-number">95</span><span class="line-source"> mask = mixed &lt; per_cell_thresh[:, None]</span></span> <span class="code-line" id="single-full-line-96" data-line="96"><span class="line-number">96</span><span class="line-source"> mixed[mask] = 0.0</span></span> <span class="code-line" id="single-full-line-97" data-line="97"><span class="line-number">97</span><span class="line-source"> libsize = Z_arr.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-full-line-98" data-line="98"><span class="line-number">98</span><span class="line-source"> median_lib = np.median(libsize)</span></span> <span class="code-line" id="single-full-line-99" data-line="99"><span class="line-number">99</span><span class="line-source"> row_sum = mixed.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-full-line-100" data-line="100"><span class="line-number">100</span><span class="line-source"> scale = np.ones_like(row_sum)</span></span> <span class="code-line" id="single-full-line-101" data-line="101"><span class="line-number">101</span><span class="line-source"> nonzero = row_sum != 0</span></span> <span class="code-line" id="single-full-line-102" data-line="102"><span class="line-number">102</span><span class="line-source"> scale[nonzero] = median_lib / row_sum[nonzero]</span></span> <span class="code-line" id="single-full-line-103" data-line="103"><span class="line-number">103</span><span class="line-source"> improved = mixed * scale</span></span> <span class="code-line" id="single-full-line-104" data-line="104"><span class="line-number">104</span><span class="line-source"> return improved</span></span> <span class="code-line" id="single-full-line-105" data-line="105"><span class="line-number">105</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-106" data-line="106"><span class="line-number">106</span><span class="line-source">def _adaptive_shrinkage_factor(X, base_lam):</span></span> <span class="code-line" id="single-full-line-107" data-line="107"><span class="line-number">107</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-108" data-line="108"><span class="line-number">108</span><span class="line-source"> Compute a per‑cell shrinkage factor towards the backbone denoised matrix.</span></span> <span class="code-line" id="single-full-line-109" data-line="109"><span class="line-number">109</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-110" data-line="110"><span class="line-number">110</span><span class="line-source"> The factor is scaled by the cell–specific dropout rate</span></span> <span class="code-line" id="single-full-line-111" data-line="111"><span class="line-number">111</span><span class="line-source"> (fraction of zero counts). Cells with many zeros receive a larger</span></span> <span class="code-line" id="single-full-line-112" data-line="112"><span class="line-number">112</span><span class="line-source"> shrinkage (more reliance on the backbone) while cells with few zeros</span></span> <span class="code-line" id="single-full-line-113" data-line="113"><span class="line-number">113</span><span class="line-source"> rely more on the mixed estimate. The returned array has shape</span></span> <span class="code-line" id="single-full-line-114" data-line="114"><span class="line-number">114</span><span class="line-source"> (n_cells, 1) and can be broadcast against matrices of shape</span></span> <span class="code-line" id="single-full-line-115" data-line="115"><span class="line-number">115</span><span class="line-source"> (n_cells, n_genes).</span></span> <span class="code-line" id="single-full-line-116" data-line="116"><span class="line-number">116</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-117" data-line="117"><span class="line-number">117</span><span class="line-source"> Parameters</span></span> <span class="code-line" id="single-full-line-118" data-line="118"><span class="line-number">118</span><span class="line-source"> ----------</span></span> <span class="code-line" id="single-full-line-119" data-line="119"><span class="line-number">119</span><span class="line-source"> X : array‑like, shape (n_cells, n_genes)</span></span> <span class="code-line" id="single-full-line-120" data-line="120"><span class="line-number">120</span><span class="line-source"> Raw count matrix.</span></span> <span class="code-line" id="single-full-line-121" data-line="121"><span class="line-number">121</span><span class="line-source"> base_lam : float</span></span> <span class="code-line" id="single-full-line-122" data-line="122"><span class="line-number">122</span><span class="line-source"> Global shrinkage strength (default from ``improve_artifact``).</span></span> <span class="code-line" id="single-full-line-123" data-line="123"><span class="line-number">123</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-124" data-line="124"><span class="line-number">124</span><span class="line-source"> Returns</span></span> <span class="code-line" id="single-full-line-125" data-line="125"><span class="line-number">125</span><span class="line-source"> -------</span></span> <span class="code-line" id="single-full-line-126" data-line="126"><span class="line-number">126</span><span class="line-source"> np.ndarray, shape (n_cells, 1)</span></span> <span class="code-line" id="single-full-line-127" data-line="127"><span class="line-number">127</span><span class="line-source"> Column vector of adaptive shrinkage factors in [0, 1].</span></span> <span class="code-line" id="single-full-line-128" data-line="128"><span class="line-number">128</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-129" data-line="129"><span class="line-number">129</span><span class="line-source"> import scipy.sparse as sp</span></span> <span class="code-line" id="single-full-line-130" data-line="130"><span class="line-number">130</span><span class="line-source"> if sp.issparse(X):</span></span> <span class="code-line" id="single-full-line-131" data-line="131"><span class="line-number">131</span><span class="line-source"> n_genes = X.shape[1]</span></span> <span class="code-line" id="single-full-line-132" data-line="132"><span class="line-number">132</span><span class="line-source"> nnz_per_cell = np.ravel(X.getnnz(axis=1))</span></span> <span class="code-line" id="single-full-line-133" data-line="133"><span class="line-number">133</span><span class="line-source"> dropout = 1.0 - nnz_per_cell / float(n_genes)</span></span> <span class="code-line" id="single-full-line-134" data-line="134"><span class="line-number">134</span><span class="line-source"> else:</span></span> <span class="code-line" id="single-full-line-135" data-line="135"><span class="line-number">135</span><span class="line-source"> X_arr = np.asarray(X)</span></span> <span class="code-line" id="single-full-line-136" data-line="136"><span class="line-number">136</span><span class="line-source"> dropout = (X_arr == 0).mean(axis=1)</span></span> <span class="code-line" id="single-full-line-137" data-line="137"><span class="line-number">137</span><span class="line-source"> lam_vec = base_lam * (0.5 + 0.5 * dropout)</span></span> <span class="code-line" id="single-full-line-138" data-line="138"><span class="line-number">138</span><span class="line-source"> lam_vec = np.clip(lam_vec, 0.0, 1.0)</span></span> <span class="code-line" id="single-full-line-139" data-line="139"><span class="line-number">139</span><span class="line-source"> return lam_vec[:, None]</span></span> <span class="code-line" id="single-full-line-140" data-line="140"><span class="line-number">140</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-141" data-line="141"><span class="line-number">141</span><span class="line-source">def _apply_gene_scaling_v3(matrix, X_ref, exponent=0.6):</span></span> <span class="code-line" id="single-full-line-142" data-line="142"><span class="line-number">142</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-143" data-line="143"><span class="line-number">143</span><span class="line-source"> Scale columns of ``matrix`` toward the column sums of ``X_ref`` using a</span></span> <span class="code-line" id="single-full-line-144" data-line="144"><span class="line-number">144</span><span class="line-source"> moderated power‑law correction. The default exponent (0.6) is slightly</span></span> <span class="code-line" id="single-full-line-145" data-line="145"><span class="line-number">145</span><span class="line-source"> less aggressive than the previous 0.7, and scaling factors are constrained</span></span> <span class="code-line" id="single-full-line-146" data-line="146"><span class="line-number">146</span><span class="line-source"> to the interval [0.8, 1.2] to avoid over‑correction. This yields expression</span></span> <span class="code-line" id="single-full-line-147" data-line="147"><span class="line-number">147</span><span class="line-source"> profiles that better match the raw data after log‑normalisation, lowering</span></span> <span class="code-line" id="single-full-line-148" data-line="148"><span class="line-number">148</span><span class="line-source"> the mean‑squared error while preserving the Poisson loss constraint.</span></span> <span class="code-line" id="single-full-line-149" data-line="149"><span class="line-number">149</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-150" data-line="150"><span class="line-number">150</span><span class="line-source"> import numpy as np</span></span> <span class="code-line" id="single-full-line-151" data-line="151"><span class="line-number">151</span><span class="line-source"> import scprep</span></span> <span class="code-line" id="single-full-line-152" data-line="152"><span class="line-number">152</span><span class="line-source"> M = np.asarray(matrix, dtype=np.float64, copy=False)</span></span> <span class="code-line" id="single-full-line-153" data-line="153"><span class="line-number">153</span><span class="line-source"> X_arr = scprep.utils.toarray(X_ref).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-full-line-154" data-line="154"><span class="line-number">154</span><span class="line-source"> gene_sum_M = M.sum(axis=0, keepdims=True)</span></span> <span class="code-line" id="single-full-line-155" data-line="155"><span class="line-number">155</span><span class="line-source"> gene_sum_X = X_arr.sum(axis=0, keepdims=True)</span></span> <span class="code-line" id="single-full-line-156" data-line="156"><span class="line-number">156</span><span class="line-source"> eps = 1e-08</span></span> <span class="code-line" id="single-full-line-157" data-line="157"><span class="line-number">157</span><span class="line-source"> ratio = (gene_sum_X + eps) / (gene_sum_M + eps)</span></span> <span class="code-line" id="single-full-line-158" data-line="158"><span class="line-number">158</span><span class="line-source"> factor = np.power(ratio, exponent)</span></span> <span class="code-line" id="single-full-line-159" data-line="159"><span class="line-number">159</span><span class="line-source"> factor = np.clip(factor, 0.8, 1.2)</span></span> <span class="code-line" id="single-full-line-160" data-line="160"><span class="line-number">160</span><span class="line-source"> scaled = M * factor</span></span> <span class="code-line" id="single-full-line-161" data-line="161"><span class="line-number">161</span><span class="line-source"> np.clip(scaled, a_min=0.0, a_max=None, out=scaled)</span></span> <span class="code-line" id="single-full-line-162" data-line="162"><span class="line-number">162</span><span class="line-source"> return scaled</span></span> <span class="code-line" id="single-full-line-163" data-line="163"><span class="line-number">163</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-164" data-line="164"><span class="line-number">164</span><span class="line-source">def improve_artifact(X, Z, beta=30, lam=0.15, raw_alpha=0.08, **kwargs):</span></span> <span class="code-line" id="single-full-line-165" data-line="165"><span class="line-number">165</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-166" data-line="166"><span class="line-number">166</span><span class="line-source"> Refined post‑processing of the fixed‑backbone denoised matrix ``Z``.</span></span> <span class="code-line" id="single-full-line-167" data-line="167"><span class="line-number">167</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-168" data-line="168"><span class="line-number">168</span><span class="line-source"> Key changes over the previous version:</span></span> <span class="code-line" id="single-full-line-169" data-line="169"><span class="line-number">169</span><span class="line-source"> * ``beta`` increased to 30, giving stronger reliance on the backbone in the</span></span> <span class="code-line" id="single-full-line-170" data-line="170"><span class="line-number">170</span><span class="line-source"> square‑root mixing stage.</span></span> <span class="code-line" id="single-full-line-171" data-line="171"><span class="line-number">171</span><span class="line-source"> * Gene‑wise shrinkage is more aggressive: ``lam_gene = lam * (0.2 + 0.8 *</span></span> <span class="code-line" id="single-full-line-172" data-line="172"><span class="line-number">172</span><span class="line-source"> gene_dropout)``.</span></span> <span class="code-line" id="single-full-line-173" data-line="173"><span class="line-number">173</span><span class="line-source"> * Column scaling uses the more aggressive exponent 0.7 via</span></span> <span class="code-line" id="single-full-line-174" data-line="174"><span class="line-number">174</span><span class="line-source"> ``_apply_gene_scaling_v3``.</span></span> <span class="code-line" id="single-full-line-175" data-line="175"><span class="line-number">175</span><span class="line-source"> * The contribution of raw counts in the final blending is reduced</span></span> <span class="code-line" id="single-full-line-176" data-line="176"><span class="line-number">176</span><span class="line-source"> (``raw_alpha`` lowered to 0.08), which typically lowers MSE without</span></span> <span class="code-line" id="single-full-line-177" data-line="177"><span class="line-number">177</span><span class="line-source"> harming the Poisson loss.</span></span> <span class="code-line" id="single-full-line-178" data-line="178"><span class="line-number">178</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-full-line-179" data-line="179"><span class="line-number">179</span><span class="line-source"> import numpy as np</span></span> <span class="code-line" id="single-full-line-180" data-line="180"><span class="line-number">180</span><span class="line-source"> import scprep</span></span> <span class="code-line" id="single-full-line-181" data-line="181"><span class="line-number">181</span><span class="line-source"> mixed = _sqrt_weighted_mix(X, Z, beta)</span></span> <span class="code-line" id="single-full-line-182" data-line="182"><span class="line-number">182</span><span class="line-source"> lam_cell = _adaptive_shrinkage_factor(X, lam)</span></span> <span class="code-line" id="single-full-line-183" data-line="183"><span class="line-number">183</span><span class="line-source"> X_arr = scprep.utils.toarray(X).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-full-line-184" data-line="184"><span class="line-number">184</span><span class="line-source"> gene_dropout = (X_arr == 0).mean(axis=0)</span></span> <span class="code-line" id="single-full-line-185" data-line="185"><span class="line-number">185</span><span class="line-source"> lam_gene = lam * (0.2 + 0.8 * gene_dropout)</span></span> <span class="code-line" id="single-full-line-186" data-line="186"><span class="line-number">186</span><span class="line-source"> lam_gene = lam_gene[np.newaxis, :]</span></span> <span class="code-line" id="single-full-line-187" data-line="187"><span class="line-number">187</span><span class="line-source"> lam_mat = np.maximum(lam_cell, lam_gene)</span></span> <span class="code-line" id="single-full-line-188" data-line="188"><span class="line-number">188</span><span class="line-source"> np.clip(lam_mat, 0.0, 1.0, out=lam_mat)</span></span> <span class="code-line" id="single-full-line-189" data-line="189"><span class="line-number">189</span><span class="line-source"> Z_arr = scprep.utils.toarray(Z).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-full-line-190" data-line="190"><span class="line-number">190</span><span class="line-source"> combined = (1.0 - lam_mat) * mixed + lam_mat * Z_arr</span></span> <span class="code-line" id="single-full-line-191" data-line="191"><span class="line-number">191</span><span class="line-source"> np.clip(combined, a_min=0.0, a_max=None, out=combined)</span></span> <span class="code-line" id="single-full-line-192" data-line="192"><span class="line-number">192</span><span class="line-source"> gene_scaled = _apply_gene_scaling_v3(combined, X)</span></span> <span class="code-line" id="single-full-line-193" data-line="193"><span class="line-number">193</span><span class="line-source"> if raw_alpha &gt; 0:</span></span> <span class="code-line" id="single-full-line-194" data-line="194"><span class="line-number">194</span><span class="line-source"> raw_arr = X_arr</span></span> <span class="code-line" id="single-full-line-195" data-line="195"><span class="line-number">195</span><span class="line-source"> raw_sum = raw_arr.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-full-line-196" data-line="196"><span class="line-number">196</span><span class="line-source"> libsize_Z = Z_arr.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-full-line-197" data-line="197"><span class="line-number">197</span><span class="line-source"> median_lib = np.median(libsize_Z)</span></span> <span class="code-line" id="single-full-line-198" data-line="198"><span class="line-number">198</span><span class="line-source"> scale = np.ones_like(raw_sum)</span></span> <span class="code-line" id="single-full-line-199" data-line="199"><span class="line-number">199</span><span class="line-source"> nonzero = raw_sum != 0</span></span> <span class="code-line" id="single-full-line-200" data-line="200"><span class="line-number">200</span><span class="line-source"> scale[nonzero] = (0.5 * libsize_Z[nonzero] + 0.5 * median_lib) / raw_sum[nonzero]</span></span> <span class="code-line" id="single-full-line-201" data-line="201"><span class="line-number">201</span><span class="line-source"> raw_scaled = raw_arr * scale</span></span> <span class="code-line" id="single-full-line-202" data-line="202"><span class="line-number">202</span><span class="line-source"> dropout = (X_arr == 0).mean(axis=1)</span></span> <span class="code-line" id="single-full-line-203" data-line="203"><span class="line-number">203</span><span class="line-source"> mean_dropout = float(dropout.mean())</span></span> <span class="code-line" id="single-full-line-204" data-line="204"><span class="line-number">204</span><span class="line-source"> denom = max(1e-06, 1.0 - mean_dropout)</span></span> <span class="code-line" id="single-full-line-205" data-line="205"><span class="line-number">205</span><span class="line-source"> cell_alpha = raw_alpha * (1.0 - dropout) / denom</span></span> <span class="code-line" id="single-full-line-206" data-line="206"><span class="line-number">206</span><span class="line-source"> cell_alpha = np.clip(cell_alpha, 0.0, 1.0)[:, None]</span></span> <span class="code-line" id="single-full-line-207" data-line="207"><span class="line-number">207</span><span class="line-source"> final = (1.0 - cell_alpha) * gene_scaled + cell_alpha * raw_scaled</span></span> <span class="code-line" id="single-full-line-208" data-line="208"><span class="line-number">208</span><span class="line-source"> else:</span></span> <span class="code-line" id="single-full-line-209" data-line="209"><span class="line-number">209</span><span class="line-source"> final = gene_scaled</span></span> <span class="code-line" id="single-full-line-210" data-line="210"><span class="line-number">210</span><span class="line-source"> np.clip(final, a_min=0.0, a_max=None, out=final)</span></span> <span class="code-line" id="single-full-line-211" data-line="211"><span class="line-number">211</span><span class="line-source"> return final</span></span> <span class="code-line" id="single-full-line-212" data-line="212"><span class="line-number">212</span><span class="line-source">_SINGLE_ANCHOR_BACKBONE = magic_denoise</span></span> <span class="code-line" id="single-full-line-213" data-line="213"><span class="line-number">213</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-214" data-line="214"><span class="line-number">214</span><span class="line-source">def _single_anchor_validate(Z, X):</span></span> <span class="code-line" id="single-full-line-215" data-line="215"><span class="line-number">215</span><span class="line-source"> import numpy as np</span></span> <span class="code-line" id="single-full-line-216" data-line="216"><span class="line-number">216</span><span class="line-source"> import scprep</span></span> <span class="code-line" id="single-full-line-217" data-line="217"><span class="line-number">217</span><span class="line-source"> Z = scprep.utils.toarray(Z)</span></span> <span class="code-line" id="single-full-line-218" data-line="218"><span class="line-number">218</span><span class="line-source"> if Z.shape != X.shape:</span></span> <span class="code-line" id="single-full-line-219" data-line="219"><span class="line-number">219</span><span class="line-source"> raise ValueError(f&quot;artifact shape {Z.shape} != input shape {X.shape}&quot;)</span></span> <span class="code-line" id="single-full-line-220" data-line="220"><span class="line-number">220</span><span class="line-source"> if not np.isfinite(Z).all():</span></span> <span class="code-line" id="single-full-line-221" data-line="221"><span class="line-number">221</span><span class="line-source"> raise ValueError(&quot;artifact contains non-finite values&quot;)</span></span> <span class="code-line" id="single-full-line-222" data-line="222"><span class="line-number">222</span><span class="line-source"> if np.any(Z &lt; 0):</span></span> <span class="code-line" id="single-full-line-223" data-line="223"><span class="line-number">223</span><span class="line-source"> raise ValueError(&quot;artifact contains negative values&quot;)</span></span> <span class="code-line" id="single-full-line-224" data-line="224"><span class="line-number">224</span><span class="line-source"> return Z</span></span> <span class="code-line" id="single-full-line-225" data-line="225"><span class="line-number">225</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-full-line-226" data-line="226"><span class="line-number">226</span><span class="line-source">def magic_denoise(X, **kwargs):</span></span> <span class="code-line" id="single-full-line-227" data-line="227"><span class="line-number">227</span><span class="line-source"> Z = _SINGLE_ANCHOR_BACKBONE(X, **kwargs)</span></span> <span class="code-line" id="single-full-line-228" data-line="228"><span class="line-number">228</span><span class="line-source"> Z = _single_anchor_validate(Z, X)</span></span> <span class="code-line" id="single-full-line-229" data-line="229"><span class="line-number">229</span><span class="line-source"> Z = improve_artifact(X, Z, **kwargs)</span></span> <span class="code-line" id="single-full-line-230" data-line="230"><span class="line-number">230</span><span class="line-source"> return _single_anchor_validate(Z, X)</span></span></code></pre> <pre class="code-block hidden" id="single-code-active"><code><span class="code-line" id="single-active-line-1" data-line="1"><span class="line-number">1</span><span class="line-source">import numpy as np</span></span> <span class="code-line" id="single-active-line-2" data-line="2"><span class="line-number">2</span><span class="line-source">import scprep</span></span> <span class="code-line" id="single-active-line-3" data-line="3"><span class="line-number">3</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-4" data-line="4"><span class="line-number">4</span><span class="line-source">def _sqrt_weighted_mix(X, Z, beta):</span></span> <span class="code-line" id="single-active-line-5" data-line="5"><span class="line-number">5</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-6" data-line="6"><span class="line-number">6</span><span class="line-source"> Mix raw counts ``X`` with the backbone denoised matrix ``Z`` in a</span></span> <span class="code-line" id="single-active-line-7" data-line="7"><span class="line-number">7</span><span class="line-source"> variance‑stabilising square‑root space. The mixing weight ``gamma`` is</span></span> <span class="code-line" id="single-active-line-8" data-line="8"><span class="line-number">8</span><span class="line-source"> computed using an adaptive per‑cell and per‑gene regularisation that depends</span></span> <span class="code-line" id="single-active-line-9" data-line="9"><span class="line-number">9</span><span class="line-source"> more strongly on dropout rates than the original implementation. This</span></span> <span class="code-line" id="single-active-line-10" data-line="10"><span class="line-number">10</span><span class="line-source"> reduces the contribution of noisy raw counts for cells and genes with many</span></span> <span class="code-line" id="single-active-line-11" data-line="11"><span class="line-number">11</span><span class="line-source"> zeros, thereby lowering the mean‑squared error after log‑normalisation while</span></span> <span class="code-line" id="single-active-line-12" data-line="12"><span class="line-number">12</span><span class="line-source"> keeping the Poisson loss above the required threshold.</span></span> <span class="code-line" id="single-active-line-13" data-line="13"><span class="line-number">13</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-14" data-line="14"><span class="line-number">14</span><span class="line-source"> After mixing, each cell is scaled to a common library size equal to the</span></span> <span class="code-line" id="single-active-line-15" data-line="15"><span class="line-number">15</span><span class="line-source"> median library size of the original backbone denoised matrix ``Z``. The</span></span> <span class="code-line" id="single-active-line-16" data-line="16"><span class="line-number">16</span><span class="line-source"> per‑cell threshold for discarding low‑expression values is also slightly</span></span> <span class="code-line" id="single-active-line-17" data-line="17"><span class="line-number">17</span><span class="line-source"> lowered (``0.12 + 0.15*dropout``) to retain additional weak signals without</span></span> <span class="code-line" id="single-active-line-18" data-line="18"><span class="line-number">18</span><span class="line-source"> harming the Poisson metric.</span></span> <span class="code-line" id="single-active-line-19" data-line="19"><span class="line-number">19</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-20" data-line="20"><span class="line-number">20</span><span class="line-source"> X_arr = scprep.utils.toarray(X).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-active-line-21" data-line="21"><span class="line-number">21</span><span class="line-source"> Z_arr = scprep.utils.toarray(Z).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-active-line-22" data-line="22"><span class="line-number">22</span><span class="line-source"> dropout = (X_arr == 0).mean(axis=1)</span></span> <span class="code-line" id="single-active-line-23" data-line="23"><span class="line-number">23</span><span class="line-source"> beta_adj = beta * (1.0 + 2.0 * dropout)</span></span> <span class="code-line" id="single-active-line-24" data-line="24"><span class="line-number">24</span><span class="line-source"> beta_adj = beta_adj[:, None]</span></span> <span class="code-line" id="single-active-line-25" data-line="25"><span class="line-number">25</span><span class="line-source"> gene_dropout = (X_arr == 0).mean(axis=0)</span></span> <span class="code-line" id="single-active-line-26" data-line="26"><span class="line-number">26</span><span class="line-source"> beta_adj_gene = beta_adj * (1.0 + gene_dropout)[None, :]</span></span> <span class="code-line" id="single-active-line-27" data-line="27"><span class="line-number">27</span><span class="line-source"> denom = X_arr + beta_adj_gene</span></span> <span class="code-line" id="single-active-line-28" data-line="28"><span class="line-number">28</span><span class="line-source"> gamma = np.divide(X_arr, denom, out=np.zeros_like(X_arr, dtype=np.float64), where=denom != 0)</span></span> <span class="code-line" id="single-active-line-29" data-line="29"><span class="line-number">29</span><span class="line-source"> X_sqrt = np.sqrt(X_arr)</span></span> <span class="code-line" id="single-active-line-30" data-line="30"><span class="line-number">30</span><span class="line-source"> Z_sqrt = np.sqrt(Z_arr)</span></span> <span class="code-line" id="single-active-line-31" data-line="31"><span class="line-number">31</span><span class="line-source"> mixed_sqrt = (1.0 - gamma) * Z_sqrt + gamma * X_sqrt</span></span> <span class="code-line" id="single-active-line-32" data-line="32"><span class="line-number">32</span><span class="line-source"> mixed = np.square(mixed_sqrt)</span></span> <span class="code-line" id="single-active-line-33" data-line="33"><span class="line-number">33</span><span class="line-source"> per_cell_thresh = 0.12 + 0.15 * dropout</span></span> <span class="code-line" id="single-active-line-34" data-line="34"><span class="line-number">34</span><span class="line-source"> mask = mixed &lt; per_cell_thresh[:, None]</span></span> <span class="code-line" id="single-active-line-35" data-line="35"><span class="line-number">35</span><span class="line-source"> mixed[mask] = 0.0</span></span> <span class="code-line" id="single-active-line-36" data-line="36"><span class="line-number">36</span><span class="line-source"> libsize = Z_arr.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-active-line-37" data-line="37"><span class="line-number">37</span><span class="line-source"> median_lib = np.median(libsize)</span></span> <span class="code-line" id="single-active-line-38" data-line="38"><span class="line-number">38</span><span class="line-source"> row_sum = mixed.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-active-line-39" data-line="39"><span class="line-number">39</span><span class="line-source"> scale = np.ones_like(row_sum)</span></span> <span class="code-line" id="single-active-line-40" data-line="40"><span class="line-number">40</span><span class="line-source"> nonzero = row_sum != 0</span></span> <span class="code-line" id="single-active-line-41" data-line="41"><span class="line-number">41</span><span class="line-source"> scale[nonzero] = median_lib / row_sum[nonzero]</span></span> <span class="code-line" id="single-active-line-42" data-line="42"><span class="line-number">42</span><span class="line-source"> improved = mixed * scale</span></span> <span class="code-line" id="single-active-line-43" data-line="43"><span class="line-number">43</span><span class="line-source"> return improved</span></span> <span class="code-line" id="single-active-line-44" data-line="44"><span class="line-number">44</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-45" data-line="45"><span class="line-number">45</span><span class="line-source">def _adaptive_shrinkage_factor(X, base_lam):</span></span> <span class="code-line" id="single-active-line-46" data-line="46"><span class="line-number">46</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-47" data-line="47"><span class="line-number">47</span><span class="line-source"> Compute a per‑cell shrinkage factor towards the backbone denoised matrix.</span></span> <span class="code-line" id="single-active-line-48" data-line="48"><span class="line-number">48</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-49" data-line="49"><span class="line-number">49</span><span class="line-source"> The factor is scaled by the cell–specific dropout rate</span></span> <span class="code-line" id="single-active-line-50" data-line="50"><span class="line-number">50</span><span class="line-source"> (fraction of zero counts). Cells with many zeros receive a larger</span></span> <span class="code-line" id="single-active-line-51" data-line="51"><span class="line-number">51</span><span class="line-source"> shrinkage (more reliance on the backbone) while cells with few zeros</span></span> <span class="code-line" id="single-active-line-52" data-line="52"><span class="line-number">52</span><span class="line-source"> rely more on the mixed estimate. The returned array has shape</span></span> <span class="code-line" id="single-active-line-53" data-line="53"><span class="line-number">53</span><span class="line-source"> (n_cells, 1) and can be broadcast against matrices of shape</span></span> <span class="code-line" id="single-active-line-54" data-line="54"><span class="line-number">54</span><span class="line-source"> (n_cells, n_genes).</span></span> <span class="code-line" id="single-active-line-55" data-line="55"><span class="line-number">55</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-56" data-line="56"><span class="line-number">56</span><span class="line-source"> Parameters</span></span> <span class="code-line" id="single-active-line-57" data-line="57"><span class="line-number">57</span><span class="line-source"> ----------</span></span> <span class="code-line" id="single-active-line-58" data-line="58"><span class="line-number">58</span><span class="line-source"> X : array‑like, shape (n_cells, n_genes)</span></span> <span class="code-line" id="single-active-line-59" data-line="59"><span class="line-number">59</span><span class="line-source"> Raw count matrix.</span></span> <span class="code-line" id="single-active-line-60" data-line="60"><span class="line-number">60</span><span class="line-source"> base_lam : float</span></span> <span class="code-line" id="single-active-line-61" data-line="61"><span class="line-number">61</span><span class="line-source"> Global shrinkage strength (default from ``improve_artifact``).</span></span> <span class="code-line" id="single-active-line-62" data-line="62"><span class="line-number">62</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-63" data-line="63"><span class="line-number">63</span><span class="line-source"> Returns</span></span> <span class="code-line" id="single-active-line-64" data-line="64"><span class="line-number">64</span><span class="line-source"> -------</span></span> <span class="code-line" id="single-active-line-65" data-line="65"><span class="line-number">65</span><span class="line-source"> np.ndarray, shape (n_cells, 1)</span></span> <span class="code-line" id="single-active-line-66" data-line="66"><span class="line-number">66</span><span class="line-source"> Column vector of adaptive shrinkage factors in [0, 1].</span></span> <span class="code-line" id="single-active-line-67" data-line="67"><span class="line-number">67</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-68" data-line="68"><span class="line-number">68</span><span class="line-source"> import scipy.sparse as sp</span></span> <span class="code-line" id="single-active-line-69" data-line="69"><span class="line-number">69</span><span class="line-source"> if sp.issparse(X):</span></span> <span class="code-line" id="single-active-line-70" data-line="70"><span class="line-number">70</span><span class="line-source"> n_genes = X.shape[1]</span></span> <span class="code-line" id="single-active-line-71" data-line="71"><span class="line-number">71</span><span class="line-source"> nnz_per_cell = np.ravel(X.getnnz(axis=1))</span></span> <span class="code-line" id="single-active-line-72" data-line="72"><span class="line-number">72</span><span class="line-source"> dropout = 1.0 - nnz_per_cell / float(n_genes)</span></span> <span class="code-line" id="single-active-line-73" data-line="73"><span class="line-number">73</span><span class="line-source"> else:</span></span> <span class="code-line" id="single-active-line-74" data-line="74"><span class="line-number">74</span><span class="line-source"> X_arr = np.asarray(X)</span></span> <span class="code-line" id="single-active-line-75" data-line="75"><span class="line-number">75</span><span class="line-source"> dropout = (X_arr == 0).mean(axis=1)</span></span> <span class="code-line" id="single-active-line-76" data-line="76"><span class="line-number">76</span><span class="line-source"> lam_vec = base_lam * (0.5 + 0.5 * dropout)</span></span> <span class="code-line" id="single-active-line-77" data-line="77"><span class="line-number">77</span><span class="line-source"> lam_vec = np.clip(lam_vec, 0.0, 1.0)</span></span> <span class="code-line" id="single-active-line-78" data-line="78"><span class="line-number">78</span><span class="line-source"> return lam_vec[:, None]</span></span> <span class="code-line" id="single-active-line-79" data-line="79"><span class="line-number">79</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-80" data-line="80"><span class="line-number">80</span><span class="line-source">def _apply_gene_scaling_v3(matrix, X_ref, exponent=0.6):</span></span> <span class="code-line" id="single-active-line-81" data-line="81"><span class="line-number">81</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-82" data-line="82"><span class="line-number">82</span><span class="line-source"> Scale columns of ``matrix`` toward the column sums of ``X_ref`` using a</span></span> <span class="code-line" id="single-active-line-83" data-line="83"><span class="line-number">83</span><span class="line-source"> moderated power‑law correction. The default exponent (0.6) is slightly</span></span> <span class="code-line" id="single-active-line-84" data-line="84"><span class="line-number">84</span><span class="line-source"> less aggressive than the previous 0.7, and scaling factors are constrained</span></span> <span class="code-line" id="single-active-line-85" data-line="85"><span class="line-number">85</span><span class="line-source"> to the interval [0.8, 1.2] to avoid over‑correction. This yields expression</span></span> <span class="code-line" id="single-active-line-86" data-line="86"><span class="line-number">86</span><span class="line-source"> profiles that better match the raw data after log‑normalisation, lowering</span></span> <span class="code-line" id="single-active-line-87" data-line="87"><span class="line-number">87</span><span class="line-source"> the mean‑squared error while preserving the Poisson loss constraint.</span></span> <span class="code-line" id="single-active-line-88" data-line="88"><span class="line-number">88</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-89" data-line="89"><span class="line-number">89</span><span class="line-source"> import numpy as np</span></span> <span class="code-line" id="single-active-line-90" data-line="90"><span class="line-number">90</span><span class="line-source"> import scprep</span></span> <span class="code-line" id="single-active-line-91" data-line="91"><span class="line-number">91</span><span class="line-source"> M = np.asarray(matrix, dtype=np.float64, copy=False)</span></span> <span class="code-line" id="single-active-line-92" data-line="92"><span class="line-number">92</span><span class="line-source"> X_arr = scprep.utils.toarray(X_ref).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-active-line-93" data-line="93"><span class="line-number">93</span><span class="line-source"> gene_sum_M = M.sum(axis=0, keepdims=True)</span></span> <span class="code-line" id="single-active-line-94" data-line="94"><span class="line-number">94</span><span class="line-source"> gene_sum_X = X_arr.sum(axis=0, keepdims=True)</span></span> <span class="code-line" id="single-active-line-95" data-line="95"><span class="line-number">95</span><span class="line-source"> eps = 1e-08</span></span> <span class="code-line" id="single-active-line-96" data-line="96"><span class="line-number">96</span><span class="line-source"> ratio = (gene_sum_X + eps) / (gene_sum_M + eps)</span></span> <span class="code-line" id="single-active-line-97" data-line="97"><span class="line-number">97</span><span class="line-source"> factor = np.power(ratio, exponent)</span></span> <span class="code-line" id="single-active-line-98" data-line="98"><span class="line-number">98</span><span class="line-source"> factor = np.clip(factor, 0.8, 1.2)</span></span> <span class="code-line" id="single-active-line-99" data-line="99"><span class="line-number">99</span><span class="line-source"> scaled = M * factor</span></span> <span class="code-line" id="single-active-line-100" data-line="100"><span class="line-number">100</span><span class="line-source"> np.clip(scaled, a_min=0.0, a_max=None, out=scaled)</span></span> <span class="code-line" id="single-active-line-101" data-line="101"><span class="line-number">101</span><span class="line-source"> return scaled</span></span> <span class="code-line" id="single-active-line-102" data-line="102"><span class="line-number">102</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-103" data-line="103"><span class="line-number">103</span><span class="line-source">def improve_artifact(X, Z, beta=30, lam=0.15, raw_alpha=0.08, **kwargs):</span></span> <span class="code-line" id="single-active-line-104" data-line="104"><span class="line-number">104</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-105" data-line="105"><span class="line-number">105</span><span class="line-source"> Refined post‑processing of the fixed‑backbone denoised matrix ``Z``.</span></span> <span class="code-line" id="single-active-line-106" data-line="106"><span class="line-number">106</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="single-active-line-107" data-line="107"><span class="line-number">107</span><span class="line-source"> Key changes over the previous version:</span></span> <span class="code-line" id="single-active-line-108" data-line="108"><span class="line-number">108</span><span class="line-source"> * ``beta`` increased to 30, giving stronger reliance on the backbone in the</span></span> <span class="code-line" id="single-active-line-109" data-line="109"><span class="line-number">109</span><span class="line-source"> square‑root mixing stage.</span></span> <span class="code-line" id="single-active-line-110" data-line="110"><span class="line-number">110</span><span class="line-source"> * Gene‑wise shrinkage is more aggressive: ``lam_gene = lam * (0.2 + 0.8 *</span></span> <span class="code-line" id="single-active-line-111" data-line="111"><span class="line-number">111</span><span class="line-source"> gene_dropout)``.</span></span> <span class="code-line" id="single-active-line-112" data-line="112"><span class="line-number">112</span><span class="line-source"> * Column scaling uses the more aggressive exponent 0.7 via</span></span> <span class="code-line" id="single-active-line-113" data-line="113"><span class="line-number">113</span><span class="line-source"> ``_apply_gene_scaling_v3``.</span></span> <span class="code-line" id="single-active-line-114" data-line="114"><span class="line-number">114</span><span class="line-source"> * The contribution of raw counts in the final blending is reduced</span></span> <span class="code-line" id="single-active-line-115" data-line="115"><span class="line-number">115</span><span class="line-source"> (``raw_alpha`` lowered to 0.08), which typically lowers MSE without</span></span> <span class="code-line" id="single-active-line-116" data-line="116"><span class="line-number">116</span><span class="line-source"> harming the Poisson loss.</span></span> <span class="code-line" id="single-active-line-117" data-line="117"><span class="line-number">117</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="single-active-line-118" data-line="118"><span class="line-number">118</span><span class="line-source"> import numpy as np</span></span> <span class="code-line" id="single-active-line-119" data-line="119"><span class="line-number">119</span><span class="line-source"> import scprep</span></span> <span class="code-line" id="single-active-line-120" data-line="120"><span class="line-number">120</span><span class="line-source"> mixed = _sqrt_weighted_mix(X, Z, beta)</span></span> <span class="code-line" id="single-active-line-121" data-line="121"><span class="line-number">121</span><span class="line-source"> lam_cell = _adaptive_shrinkage_factor(X, lam)</span></span> <span class="code-line" id="single-active-line-122" data-line="122"><span class="line-number">122</span><span class="line-source"> X_arr = scprep.utils.toarray(X).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-active-line-123" data-line="123"><span class="line-number">123</span><span class="line-source"> gene_dropout = (X_arr == 0).mean(axis=0)</span></span> <span class="code-line" id="single-active-line-124" data-line="124"><span class="line-number">124</span><span class="line-source"> lam_gene = lam * (0.2 + 0.8 * gene_dropout)</span></span> <span class="code-line" id="single-active-line-125" data-line="125"><span class="line-number">125</span><span class="line-source"> lam_gene = lam_gene[np.newaxis, :]</span></span> <span class="code-line" id="single-active-line-126" data-line="126"><span class="line-number">126</span><span class="line-source"> lam_mat = np.maximum(lam_cell, lam_gene)</span></span> <span class="code-line" id="single-active-line-127" data-line="127"><span class="line-number">127</span><span class="line-source"> np.clip(lam_mat, 0.0, 1.0, out=lam_mat)</span></span> <span class="code-line" id="single-active-line-128" data-line="128"><span class="line-number">128</span><span class="line-source"> Z_arr = scprep.utils.toarray(Z).astype(np.float64, copy=False)</span></span> <span class="code-line" id="single-active-line-129" data-line="129"><span class="line-number">129</span><span class="line-source"> combined = (1.0 - lam_mat) * mixed + lam_mat * Z_arr</span></span> <span class="code-line" id="single-active-line-130" data-line="130"><span class="line-number">130</span><span class="line-source"> np.clip(combined, a_min=0.0, a_max=None, out=combined)</span></span> <span class="code-line" id="single-active-line-131" data-line="131"><span class="line-number">131</span><span class="line-source"> gene_scaled = _apply_gene_scaling_v3(combined, X)</span></span> <span class="code-line" id="single-active-line-132" data-line="132"><span class="line-number">132</span><span class="line-source"> if raw_alpha &gt; 0:</span></span> <span class="code-line" id="single-active-line-133" data-line="133"><span class="line-number">133</span><span class="line-source"> raw_arr = X_arr</span></span> <span class="code-line" id="single-active-line-134" data-line="134"><span class="line-number">134</span><span class="line-source"> raw_sum = raw_arr.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-active-line-135" data-line="135"><span class="line-number">135</span><span class="line-source"> libsize_Z = Z_arr.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="single-active-line-136" data-line="136"><span class="line-number">136</span><span class="line-source"> median_lib = np.median(libsize_Z)</span></span> <span class="code-line" id="single-active-line-137" data-line="137"><span class="line-number">137</span><span class="line-source"> scale = np.ones_like(raw_sum)</span></span> <span class="code-line" id="single-active-line-138" data-line="138"><span class="line-number">138</span><span class="line-source"> nonzero = raw_sum != 0</span></span> <span class="code-line" id="single-active-line-139" data-line="139"><span class="line-number">139</span><span class="line-source"> scale[nonzero] = (0.5 * libsize_Z[nonzero] + 0.5 * median_lib) / raw_sum[nonzero]</span></span> <span class="code-line" id="single-active-line-140" data-line="140"><span class="line-number">140</span><span class="line-source"> raw_scaled = raw_arr * scale</span></span> <span class="code-line" id="single-active-line-141" data-line="141"><span class="line-number">141</span><span class="line-source"> dropout = (X_arr == 0).mean(axis=1)</span></span> <span class="code-line" id="single-active-line-142" data-line="142"><span class="line-number">142</span><span class="line-source"> mean_dropout = float(dropout.mean())</span></span> <span class="code-line" id="single-active-line-143" data-line="143"><span class="line-number">143</span><span class="line-source"> denom = max(1e-06, 1.0 - mean_dropout)</span></span> <span class="code-line" id="single-active-line-144" data-line="144"><span class="line-number">144</span><span class="line-source"> cell_alpha = raw_alpha * (1.0 - dropout) / denom</span></span> <span class="code-line" id="single-active-line-145" data-line="145"><span class="line-number">145</span><span class="line-source"> cell_alpha = np.clip(cell_alpha, 0.0, 1.0)[:, None]</span></span> <span class="code-line" id="single-active-line-146" data-line="146"><span class="line-number">146</span><span class="line-source"> final = (1.0 - cell_alpha) * gene_scaled + cell_alpha * raw_scaled</span></span> <span class="code-line" id="single-active-line-147" data-line="147"><span class="line-number">147</span><span class="line-source"> else:</span></span> <span class="code-line" id="single-active-line-148" data-line="148"><span class="line-number">148</span><span class="line-source"> final = gene_scaled</span></span> <span class="code-line" id="single-active-line-149" data-line="149"><span class="line-number">149</span><span class="line-source"> np.clip(final, a_min=0.0, a_max=None, out=final)</span></span> <span class="code-line" id="single-active-line-150" data-line="150"><span class="line-number">150</span><span class="line-source"> return final</span></span></code></pre> </div> </article> <article class="code-panel" data-panel="baseline"> <header class="code-panel-header"> <div class="code-panel-title"> <h3 class="baseline-text">Baseline 50-step final</h3> <span>完整重写 · 1043 行</span> </div> <div class="controls"> <button type="button" id="baseline-top">回到顶部</button> <input id="baseline-search" type="search" placeholder="搜索代码,例如 _denoise_variant" aria-label="搜索 Baseline 代码"> <select id="baseline-functions" class="function-select" aria-label="跳转到 Baseline 函数"> <option value="1">文件顶部</option> <option value="29">_anscombe_transform · L29</option> <option value="34">_inverse_anscombe_newton · L34</option> <option value="49">_apply_diffusion · L49</option> <option value="57">_smooth_row_sums · L57</option> <option value="63">_adjust_row_sums · L63</option> <option value="70">_winsorize_counts · L70</option> <option value="77">_select_hvg_seurat · L77</option> <option value="110">_standardize_hvg · L110</option> <option value="125">_destandardize_hvg · L125</option> <option value="133">_impute_counts_std · L133</option> <option value="170">_full_diffuse · L170</option> <option value="178">_nmf_impute · L178</option> <option value="203">_full_pca_denoise_centered · L203</option> <option value="248">_auto_diffusion_time · L248</option> <option value="265">_scale_hvg_variance · L265</option> <option value="285">_gene_raw_weight_mix · L285</option> <option value="312">_suppress_low_counts · L312</option> <option value="328">_scale_gene_means · L328</option> <option value="351">_scale_gene_variance_masked · L351</option> <option value="377">_refine_hvg · L377</option> <option value="409">_final_log_knn_smooth_with_residual · L409</option> <option value="476">_low_rank_refine_counts · L476</option> <option value="517">_denoise_variant · L517</option> <option value="637">magic_denoise · L637</option> </select> </div> <div class="search-status" id="baseline-status" aria-live="polite">未搜索</div> </header> <div class="code-viewport" id="baseline-viewport"> <pre class="code-block" id="baseline-code"><code><span class="code-line" id="baseline-line-1" data-line="1"><span class="line-number">1</span><span class="line-source">#!/usr/bin/env python</span></span> <span class="code-line" id="baseline-line-2" data-line="2"><span class="line-number">2</span><span class="line-source"># -*- coding: utf-8 -*-</span></span> <span class="code-line" id="baseline-line-3" data-line="3"><span class="line-number">3</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-4" data-line="4"><span class="line-number">4</span><span class="line-source">&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-5" data-line="5"><span class="line-number">5</span><span class="line-source">magic_denoise – an enhanced MAGIC‑style denoiser for raw scRNA‑seq count matrices.</span></span> <span class="code-line" id="baseline-line-6" data-line="6"><span class="line-number">6</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-7" data-line="7"><span class="line-number">7</span><span class="line-source">The implementation keeps the “reversed‑normalisation’’ pipeline that guarantees</span></span> <span class="code-line" id="baseline-line-8" data-line="8"><span class="line-number">8</span><span class="line-source">Poisson‑norm ≥ 0.97 while applying a set of refinements that lower the MSE.</span></span> <span class="code-line" id="baseline-line-9" data-line="9"><span class="line-number">9</span><span class="line-source">&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-10" data-line="10"><span class="line-number">10</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-11" data-line="11"><span class="line-number">11</span><span class="line-source"># ----------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-12" data-line="12"><span class="line-number">12</span><span class="line-source"># Imports</span></span> <span class="code-line" id="baseline-line-13" data-line="13"><span class="line-number">13</span><span class="line-source"># ----------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-14" data-line="14"><span class="line-number">14</span><span class="line-source">import numpy as np</span></span> <span class="code-line" id="baseline-line-15" data-line="15"><span class="line-number">15</span><span class="line-source">import scipy.sparse as sp</span></span> <span class="code-line" id="baseline-line-16" data-line="16"><span class="line-number">16</span><span class="line-source">import scprep</span></span> <span class="code-line" id="baseline-line-17" data-line="17"><span class="line-number">17</span><span class="line-source">import graphtools</span></span> <span class="code-line" id="baseline-line-18" data-line="18"><span class="line-number">18</span><span class="line-source">import anndata</span></span> <span class="code-line" id="baseline-line-19" data-line="19"><span class="line-number">19</span><span class="line-source">import scanpy as sc</span></span> <span class="code-line" id="baseline-line-20" data-line="20"><span class="line-number">20</span><span class="line-source">import sklearn.metrics</span></span> <span class="code-line" id="baseline-line-21" data-line="21"><span class="line-number">21</span><span class="line-source">from sklearn.decomposition import PCA, NMF</span></span> <span class="code-line" id="baseline-line-22" data-line="22"><span class="line-number">22</span><span class="line-source">from typing import Optional, Tuple</span></span> <span class="code-line" id="baseline-line-23" data-line="23"><span class="line-number">23</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-24" data-line="24"><span class="line-number">24</span><span class="line-source"># ----------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-25" data-line="25"><span class="line-number">25</span><span class="line-source"># Helper utilities – must stay at top level</span></span> <span class="code-line" id="baseline-line-26" data-line="26"><span class="line-number">26</span><span class="line-source"># ----------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-27" data-line="27"><span class="line-number">27</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-28" data-line="28"><span class="line-number">28</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-29" data-line="29"><span class="line-number">29</span><span class="line-source">def _anscombe_transform(counts: np.ndarray) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-30" data-line="30"><span class="line-number">30</span><span class="line-source"> &quot;&quot;&quot;Anscombe variance‑stabilising transform for Poisson data.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-31" data-line="31"><span class="line-number">31</span><span class="line-source"> return 2.0 * np.sqrt(counts + 3.0 / 8.0)</span></span> <span class="code-line" id="baseline-line-32" data-line="32"><span class="line-number">32</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-33" data-line="33"><span class="line-number">33</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-34" data-line="34"><span class="line-number">34</span><span class="line-source">def _inverse_anscombe_newton(y: np.ndarray, n_iter: int = 12) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-35" data-line="35"><span class="line-number">35</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-36" data-line="36"><span class="line-number">36</span><span class="line-source"> Unbiased inverse of the Anscombe transform using Newton iterations.</span></span> <span class="code-line" id="baseline-line-37" data-line="37"><span class="line-number">37</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-38" data-line="38"><span class="line-number">38</span><span class="line-source"> x = (y / 2.0) ** 2 - 3.0 / 8.0</span></span> <span class="code-line" id="baseline-line-39" data-line="39"><span class="line-number">39</span><span class="line-source"> eps = 1e-12</span></span> <span class="code-line" id="baseline-line-40" data-line="40"><span class="line-number">40</span><span class="line-source"> for _ in range(n_iter):</span></span> <span class="code-line" id="baseline-line-41" data-line="41"><span class="line-number">41</span><span class="line-source"> sqrt_term = np.sqrt(np.maximum(x + 3.0 / 8.0, 0.0))</span></span> <span class="code-line" id="baseline-line-42" data-line="42"><span class="line-number">42</span><span class="line-source"> resid = 2.0 * sqrt_term - y</span></span> <span class="code-line" id="baseline-line-43" data-line="43"><span class="line-number">43</span><span class="line-source"> delta = resid * sqrt_term</span></span> <span class="code-line" id="baseline-line-44" data-line="44"><span class="line-number">44</span><span class="line-source"> x -= delta</span></span> <span class="code-line" id="baseline-line-45" data-line="45"><span class="line-number">45</span><span class="line-source"> x = np.maximum(x, 0.0)</span></span> <span class="code-line" id="baseline-line-46" data-line="46"><span class="line-number">46</span><span class="line-source"> return x</span></span> <span class="code-line" id="baseline-line-47" data-line="47"><span class="line-number">47</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-48" data-line="48"><span class="line-number">48</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-49" data-line="49"><span class="line-number">49</span><span class="line-source">def _apply_diffusion(diff_op, mat):</span></span> <span class="code-line" id="baseline-line-50" data-line="50"><span class="line-number">50</span><span class="line-source"> &quot;&quot;&quot;Multiply a diffusion operator with a dense matrix.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-51" data-line="51"><span class="line-number">51</span><span class="line-source"> if sp.issparse(diff_op):</span></span> <span class="code-line" id="baseline-line-52" data-line="52"><span class="line-number">52</span><span class="line-source"> return diff_op.dot(mat)</span></span> <span class="code-line" id="baseline-line-53" data-line="53"><span class="line-number">53</span><span class="line-source"> else:</span></span> <span class="code-line" id="baseline-line-54" data-line="54"><span class="line-number">54</span><span class="line-source"> return diff_op @ mat</span></span> <span class="code-line" id="baseline-line-55" data-line="55"><span class="line-number">55</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-56" data-line="56"><span class="line-number">56</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-57" data-line="57"><span class="line-number">57</span><span class="line-source">def _smooth_row_sums(diff_op, row_sums: np.ndarray, blend: float = 0.5) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-58" data-line="58"><span class="line-number">58</span><span class="line-source"> &quot;&quot;&quot;Smooth per‑cell library sizes.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-59" data-line="59"><span class="line-number">59</span><span class="line-source"> smooth = _apply_diffusion(diff_op, row_sums)</span></span> <span class="code-line" id="baseline-line-60" data-line="60"><span class="line-number">60</span><span class="line-source"> return (1.0 - blend) * row_sums + blend * smooth</span></span> <span class="code-line" id="baseline-line-61" data-line="61"><span class="line-number">61</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-62" data-line="62"><span class="line-number">62</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-63" data-line="63"><span class="line-number">63</span><span class="line-source">def _adjust_row_sums(matrix: np.ndarray, target_libsize: np.ndarray, eps: float = 1e-12) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-64" data-line="64"><span class="line-number">64</span><span class="line-source"> &quot;&quot;&quot;Rescale each cell so its library matches `target_libsize`.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-65" data-line="65"><span class="line-number">65</span><span class="line-source"> cur_sum = matrix.sum(axis=1, keepdims=True) + eps</span></span> <span class="code-line" id="baseline-line-66" data-line="66"><span class="line-number">66</span><span class="line-source"> scale = target_libsize / cur_sum</span></span> <span class="code-line" id="baseline-line-67" data-line="67"><span class="line-number">67</span><span class="line-source"> return matrix * scale</span></span> <span class="code-line" id="baseline-line-68" data-line="68"><span class="line-number">68</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-69" data-line="69"><span class="line-number">69</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-70" data-line="70"><span class="line-number">70</span><span class="line-source">def _winsorize_counts(counts: np.ndarray, q_low: float = 0.001, q_high: float = 0.999) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-71" data-line="71"><span class="line-number">71</span><span class="line-source"> &quot;&quot;&quot;Clip extreme per‑gene values to robust quantiles.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-72" data-line="72"><span class="line-number">72</span><span class="line-source"> low = np.quantile(counts, q_low, axis=0, keepdims=True)</span></span> <span class="code-line" id="baseline-line-73" data-line="73"><span class="line-number">73</span><span class="line-source"> high = np.quantile(counts, q_high, axis=0, keepdims=True)</span></span> <span class="code-line" id="baseline-line-74" data-line="74"><span class="line-number">74</span><span class="line-source"> return np.clip(counts, low, high)</span></span> <span class="code-line" id="baseline-line-75" data-line="75"><span class="line-number">75</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-76" data-line="76"><span class="line-number">76</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-77" data-line="77"><span class="line-number">77</span><span class="line-source">def _select_hvg_seurat(Y: np.ndarray, n_hvg: Optional[int] = None) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-78" data-line="78"><span class="line-number">78</span><span class="line-source"> &quot;&quot;&quot;Seurat‑style highly‑variable‑gene selection.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-79" data-line="79"><span class="line-number">79</span><span class="line-source"> n_genes = Y.shape[1]</span></span> <span class="code-line" id="baseline-line-80" data-line="80"><span class="line-number">80</span><span class="line-source"> if n_hvg is None or n_hvg &gt;= n_genes:</span></span> <span class="code-line" id="baseline-line-81" data-line="81"><span class="line-number">81</span><span class="line-source"> return np.ones(n_genes, dtype=bool)</span></span> <span class="code-line" id="baseline-line-82" data-line="82"><span class="line-number">82</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-83" data-line="83"><span class="line-number">83</span><span class="line-source"> eps = 1e-12</span></span> <span class="code-line" id="baseline-line-84" data-line="84"><span class="line-number">84</span><span class="line-source"> mean = Y.mean(axis=0)</span></span> <span class="code-line" id="baseline-line-85" data-line="85"><span class="line-number">85</span><span class="line-source"> var = Y.var(axis=0)</span></span> <span class="code-line" id="baseline-line-86" data-line="86"><span class="line-number">86</span><span class="line-source"> dispersion = var / (mean + eps)</span></span> <span class="code-line" id="baseline-line-87" data-line="87"><span class="line-number">87</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-88" data-line="88"><span class="line-number">88</span><span class="line-source"> log_mean = np.log1p(mean)</span></span> <span class="code-line" id="baseline-line-89" data-line="89"><span class="line-number">89</span><span class="line-source"> nbins = max(10, min(100, int(np.sqrt(n_genes))))</span></span> <span class="code-line" id="baseline-line-90" data-line="90"><span class="line-number">90</span><span class="line-source"> bins = np.linspace(log_mean.min(), log_mean.max(), nbins + 1)</span></span> <span class="code-line" id="baseline-line-91" data-line="91"><span class="line-number">91</span><span class="line-source"> bin_idx = np.digitize(log_mean, bins) - 1</span></span> <span class="code-line" id="baseline-line-92" data-line="92"><span class="line-number">92</span><span class="line-source"> bin_idx = np.clip(bin_idx, 0, nbins - 1)</span></span> <span class="code-line" id="baseline-line-93" data-line="93"><span class="line-number">93</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-94" data-line="94"><span class="line-number">94</span><span class="line-source"> norm_disp = np.zeros_like(dispersion)</span></span> <span class="code-line" id="baseline-line-95" data-line="95"><span class="line-number">95</span><span class="line-source"> for b in range(nbins):</span></span> <span class="code-line" id="baseline-line-96" data-line="96"><span class="line-number">96</span><span class="line-source"> mask = bin_idx == b</span></span> <span class="code-line" id="baseline-line-97" data-line="97"><span class="line-number">97</span><span class="line-source"> if not np.any(mask):</span></span> <span class="code-line" id="baseline-line-98" data-line="98"><span class="line-number">98</span><span class="line-source"> continue</span></span> <span class="code-line" id="baseline-line-99" data-line="99"><span class="line-number">99</span><span class="line-source"> disp_bin = dispersion[mask]</span></span> <span class="code-line" id="baseline-line-100" data-line="100"><span class="line-number">100</span><span class="line-source"> med = np.median(disp_bin)</span></span> <span class="code-line" id="baseline-line-101" data-line="101"><span class="line-number">101</span><span class="line-source"> mad = np.median(np.abs(disp_bin - med)) + eps</span></span> <span class="code-line" id="baseline-line-102" data-line="102"><span class="line-number">102</span><span class="line-source"> norm_disp[mask] = (disp_bin - med) / mad</span></span> <span class="code-line" id="baseline-line-103" data-line="103"><span class="line-number">103</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-104" data-line="104"><span class="line-number">104</span><span class="line-source"> idx = np.argpartition(norm_disp, -n_hvg)[-n_hvg:]</span></span> <span class="code-line" id="baseline-line-105" data-line="105"><span class="line-number">105</span><span class="line-source"> mask = np.zeros(n_genes, dtype=bool)</span></span> <span class="code-line" id="baseline-line-106" data-line="106"><span class="line-number">106</span><span class="line-source"> mask[idx] = True</span></span> <span class="code-line" id="baseline-line-107" data-line="107"><span class="line-number">107</span><span class="line-source"> return mask</span></span> <span class="code-line" id="baseline-line-108" data-line="108"><span class="line-number">108</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-109" data-line="109"><span class="line-number">109</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-110" data-line="110"><span class="line-number">110</span><span class="line-source">def _standardize_hvg(Y_vst: np.ndarray, hvgs_mask: np.ndarray, robust: bool = False,</span></span> <span class="code-line" id="baseline-line-111" data-line="111"><span class="line-number">111</span><span class="line-source"> eps: float = 1e-12):</span></span> <span class="code-line" id="baseline-line-112" data-line="112"><span class="line-number">112</span><span class="line-source"> &quot;&quot;&quot;Zero‑mean, unit‑variance standardisation of the HVG columns.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-113" data-line="113"><span class="line-number">113</span><span class="line-source"> if robust:</span></span> <span class="code-line" id="baseline-line-114" data-line="114"><span class="line-number">114</span><span class="line-source"> loc = np.median(Y_vst[:, hvgs_mask], axis=0)</span></span> <span class="code-line" id="baseline-line-115" data-line="115"><span class="line-number">115</span><span class="line-source"> scale = np.median(np.abs(Y_vst[:, hvgs_mask] - loc), axis=0) + eps</span></span> <span class="code-line" id="baseline-line-116" data-line="116"><span class="line-number">116</span><span class="line-source"> else:</span></span> <span class="code-line" id="baseline-line-117" data-line="117"><span class="line-number">117</span><span class="line-source"> loc = Y_vst[:, hvgs_mask].mean(axis=0)</span></span> <span class="code-line" id="baseline-line-118" data-line="118"><span class="line-number">118</span><span class="line-source"> scale = Y_vst[:, hvgs_mask].std(axis=0) + eps</span></span> <span class="code-line" id="baseline-line-119" data-line="119"><span class="line-number">119</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-120" data-line="120"><span class="line-number">120</span><span class="line-source"> Y_std = Y_vst.copy()</span></span> <span class="code-line" id="baseline-line-121" data-line="121"><span class="line-number">121</span><span class="line-source"> Y_std[:, hvgs_mask] = (Y_vst[:, hvgs_mask] - loc) / scale</span></span> <span class="code-line" id="baseline-line-122" data-line="122"><span class="line-number">122</span><span class="line-source"> return Y_std, loc, scale</span></span> <span class="code-line" id="baseline-line-123" data-line="123"><span class="line-number">123</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-124" data-line="124"><span class="line-number">124</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-125" data-line="125"><span class="line-number">125</span><span class="line-source">def _destandardize_hvg(Y_std: np.ndarray, hvgs_mask: np.ndarray,</span></span> <span class="code-line" id="baseline-line-126" data-line="126"><span class="line-number">126</span><span class="line-source"> loc_hvg: np.ndarray, scale_hvg: np.ndarray):</span></span> <span class="code-line" id="baseline-line-127" data-line="127"><span class="line-number">127</span><span class="line-source"> &quot;&quot;&quot;Inverse of `_standardize_hvg` (only HVG columns).&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-128" data-line="128"><span class="line-number">128</span><span class="line-source"> Y_vst = Y_std.copy()</span></span> <span class="code-line" id="baseline-line-129" data-line="129"><span class="line-number">129</span><span class="line-source"> Y_vst[:, hvgs_mask] = Y_std[:, hvgs_mask] * scale_hvg + loc_hvg</span></span> <span class="code-line" id="baseline-line-130" data-line="130"><span class="line-number">130</span><span class="line-source"> return Y_vst</span></span> <span class="code-line" id="baseline-line-131" data-line="131"><span class="line-number">131</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-132" data-line="132"><span class="line-number">132</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-133" data-line="133"><span class="line-number">133</span><span class="line-source">def _impute_counts_std(</span></span> <span class="code-line" id="baseline-line-134" data-line="134"><span class="line-number">134</span><span class="line-source"> Y_std: np.ndarray,</span></span> <span class="code-line" id="baseline-line-135" data-line="135"><span class="line-number">135</span><span class="line-source"> hvgs_mask: np.ndarray,</span></span> <span class="code-line" id="baseline-line-136" data-line="136"><span class="line-number">136</span><span class="line-source"> diff_op,</span></span> <span class="code-line" id="baseline-line-137" data-line="137"><span class="line-number">137</span><span class="line-source"> dropout: np.ndarray,</span></span> <span class="code-line" id="baseline-line-138" data-line="138"><span class="line-number">138</span><span class="line-source"> median_dropout: float,</span></span> <span class="code-line" id="baseline-line-139" data-line="139"><span class="line-number">139</span><span class="line-source"> max_t: int,</span></span> <span class="code-line" id="baseline-line-140" data-line="140"><span class="line-number">140</span><span class="line-source"> dropout_exponent: float = 1.0,</span></span> <span class="code-line" id="baseline-line-141" data-line="141"><span class="line-number">141</span><span class="line-source"> diffusion_weight_exponent: float = 1.0,</span></span> <span class="code-line" id="baseline-line-142" data-line="142"><span class="line-number">142</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-143" data-line="143"><span class="line-number">143</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-144" data-line="144"><span class="line-number">144</span><span class="line-source"> Classic MAGIC weighting performed in the standardised HVG space.</span></span> <span class="code-line" id="baseline-line-145" data-line="145"><span class="line-number">145</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-146" data-line="146"><span class="line-number">146</span><span class="line-source"> eps = 1e-12</span></span> <span class="code-line" id="baseline-line-147" data-line="147"><span class="line-number">147</span><span class="line-source"> dw = dropout / (dropout + median_dropout + eps) # confidence</span></span> <span class="code-line" id="baseline-line-148" data-line="148"><span class="line-number">148</span><span class="line-source"> dw_pow = np.power(dw, dropout_exponent)</span></span> <span class="code-line" id="baseline-line-149" data-line="149"><span class="line-number">149</span><span class="line-source"> dw_hvg = dw_pow[hvgs_mask]</span></span> <span class="code-line" id="baseline-line-150" data-line="150"><span class="line-number">150</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-151" data-line="151"><span class="line-number">151</span><span class="line-source"> orig_hvg = Y_std[:, hvgs_mask]</span></span> <span class="code-line" id="baseline-line-152" data-line="152"><span class="line-number">152</span><span class="line-source"> weight_orig = 1.0 - dw_hvg</span></span> <span class="code-line" id="baseline-line-153" data-line="153"><span class="line-number">153</span><span class="line-source"> imputed_hvg = orig_hvg * weight_orig[None, :]</span></span> <span class="code-line" id="baseline-line-154" data-line="154"><span class="line-number">154</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-155" data-line="155"><span class="line-number">155</span><span class="line-source"> # diffusion‑step weighting – earlier steps receive larger weight when exponent &lt; 1</span></span> <span class="code-line" id="baseline-line-156" data-line="156"><span class="line-number">156</span><span class="line-source"> t_idx = np.arange(1, max_t + 1, dtype=np.float64)</span></span> <span class="code-line" id="baseline-line-157" data-line="157"><span class="line-number">157</span><span class="line-source"> step_weight_raw = np.power(t_idx, diffusion_weight_exponent)</span></span> <span class="code-line" id="baseline-line-158" data-line="158"><span class="line-number">158</span><span class="line-source"> step_weight = step_weight_raw / step_weight_raw.sum()</span></span> <span class="code-line" id="baseline-line-159" data-line="159"><span class="line-number">159</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-160" data-line="160"><span class="line-number">160</span><span class="line-source"> diff_low = orig_hvg.copy()</span></span> <span class="code-line" id="baseline-line-161" data-line="161"><span class="line-number">161</span><span class="line-source"> for i in range(max_t):</span></span> <span class="code-line" id="baseline-line-162" data-line="162"><span class="line-number">162</span><span class="line-source"> diff_low = _apply_diffusion(diff_op, diff_low)</span></span> <span class="code-line" id="baseline-line-163" data-line="163"><span class="line-number">163</span><span class="line-source"> imputed_hvg += diff_low * (dw_hvg[None, :] * step_weight[i])</span></span> <span class="code-line" id="baseline-line-164" data-line="164"><span class="line-number">164</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-165" data-line="165"><span class="line-number">165</span><span class="line-source"> Y_std_imputed = Y_std.copy()</span></span> <span class="code-line" id="baseline-line-166" data-line="166"><span class="line-number">166</span><span class="line-source"> Y_std_imputed[:, hvgs_mask] = imputed_hvg</span></span> <span class="code-line" id="baseline-line-167" data-line="167"><span class="line-number">167</span><span class="line-source"> return Y_std_imputed</span></span> <span class="code-line" id="baseline-line-168" data-line="168"><span class="line-number">168</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-169" data-line="169"><span class="line-number">169</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-170" data-line="170"><span class="line-number">170</span><span class="line-source">def _full_diffuse(Y: np.ndarray, diff_op, weight: float = 0.25) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-171" data-line="171"><span class="line-number">171</span><span class="line-source"> &quot;&quot;&quot;Very light diffusion of the *full* gene matrix.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-172" data-line="172"><span class="line-number">172</span><span class="line-source"> if weight &lt;= 0.0:</span></span> <span class="code-line" id="baseline-line-173" data-line="173"><span class="line-number">173</span><span class="line-source"> return Y</span></span> <span class="code-line" id="baseline-line-174" data-line="174"><span class="line-number">174</span><span class="line-source"> diffed = _apply_diffusion(diff_op, Y)</span></span> <span class="code-line" id="baseline-line-175" data-line="175"><span class="line-number">175</span><span class="line-source"> return (1.0 - weight) * Y + weight * diffed</span></span> <span class="code-line" id="baseline-line-176" data-line="176"><span class="line-number">176</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-177" data-line="177"><span class="line-number">177</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-178" data-line="178"><span class="line-number">178</span><span class="line-source">def _nmf_impute(</span></span> <span class="code-line" id="baseline-line-179" data-line="179"><span class="line-number">179</span><span class="line-source"> counts: np.ndarray,</span></span> <span class="code-line" id="baseline-line-180" data-line="180"><span class="line-number">180</span><span class="line-source"> n_components: int = 20,</span></span> <span class="code-line" id="baseline-line-181" data-line="181"><span class="line-number">181</span><span class="line-source"> random_state: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-182" data-line="182"><span class="line-number">182</span><span class="line-source"> max_iter: int = 200,</span></span> <span class="code-line" id="baseline-line-183" data-line="183"><span class="line-number">183</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-184" data-line="184"><span class="line-number">184</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-185" data-line="185"><span class="line-number">185</span><span class="line-source"> &quot;&quot;&quot;KL‑NMF low‑rank imputation on a non‑negative count matrix.&quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-186" data-line="186"><span class="line-number">186</span><span class="line-source"> X = counts + eps</span></span> <span class="code-line" id="baseline-line-187" data-line="187"><span class="line-number">187</span><span class="line-source"> model = NMF(</span></span> <span class="code-line" id="baseline-line-188" data-line="188"><span class="line-number">188</span><span class="line-source"> n_components=n_components,</span></span> <span class="code-line" id="baseline-line-189" data-line="189"><span class="line-number">189</span><span class="line-source"> init=&quot;nndsvda&quot;,</span></span> <span class="code-line" id="baseline-line-190" data-line="190"><span class="line-number">190</span><span class="line-source"> solver=&quot;mu&quot;,</span></span> <span class="code-line" id="baseline-line-191" data-line="191"><span class="line-number">191</span><span class="line-source"> beta_loss=&quot;kullback-leibler&quot;,</span></span> <span class="code-line" id="baseline-line-192" data-line="192"><span class="line-number">192</span><span class="line-source"> max_iter=max_iter,</span></span> <span class="code-line" id="baseline-line-193" data-line="193"><span class="line-number">193</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-194" data-line="194"><span class="line-number">194</span><span class="line-source"> verbose=0,</span></span> <span class="code-line" id="baseline-line-195" data-line="195"><span class="line-number">195</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-196" data-line="196"><span class="line-number">196</span><span class="line-source"> W = model.fit_transform(X)</span></span> <span class="code-line" id="baseline-line-197" data-line="197"><span class="line-number">197</span><span class="line-source"> H = model.components_</span></span> <span class="code-line" id="baseline-line-198" data-line="198"><span class="line-number">198</span><span class="line-source"> imputed = W @ H</span></span> <span class="code-line" id="baseline-line-199" data-line="199"><span class="line-number">199</span><span class="line-source"> imputed = np.maximum(imputed - eps, 0.0)</span></span> <span class="code-line" id="baseline-line-200" data-line="200"><span class="line-number">200</span><span class="line-source"> return imputed</span></span> <span class="code-line" id="baseline-line-201" data-line="201"><span class="line-number">201</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-202" data-line="202"><span class="line-number">202</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-203" data-line="203"><span class="line-number">203</span><span class="line-source">def _full_pca_denoise_centered(</span></span> <span class="code-line" id="baseline-line-204" data-line="204"><span class="line-number">204</span><span class="line-source"> counts: np.ndarray,</span></span> <span class="code-line" id="baseline-line-205" data-line="205"><span class="line-number">205</span><span class="line-source"> target_sum: float = 10000.0,</span></span> <span class="code-line" id="baseline-line-206" data-line="206"><span class="line-number">206</span><span class="line-source"> random_state: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-207" data-line="207"><span class="line-number">207</span><span class="line-source"> n_components: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-208" data-line="208"><span class="line-number">208</span><span class="line-source"> variance: Optional[float] = None,</span></span> <span class="code-line" id="baseline-line-209" data-line="209"><span class="line-number">209</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-210" data-line="210"><span class="line-number">210</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-211" data-line="211"><span class="line-number">211</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-212" data-line="212"><span class="line-number">212</span><span class="line-source"> Centered‑PCA denoiser performed in log‑space (classic MAGIC step).</span></span> <span class="code-line" id="baseline-line-213" data-line="213"><span class="line-number">213</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-214" data-line="214"><span class="line-number">214</span><span class="line-source"> row_sum = counts.sum(axis=1, keepdims=True) + eps</span></span> <span class="code-line" id="baseline-line-215" data-line="215"><span class="line-number">215</span><span class="line-source"> norm_counts = counts * (target_sum / row_sum)</span></span> <span class="code-line" id="baseline-line-216" data-line="216"><span class="line-number">216</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-217" data-line="217"><span class="line-number">217</span><span class="line-source"> log_counts = np.log1p(norm_counts)</span></span> <span class="code-line" id="baseline-line-218" data-line="218"><span class="line-number">218</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-219" data-line="219"><span class="line-number">219</span><span class="line-source"> max_allowed = min(log_counts.shape[0] - 1, log_counts.shape[1])</span></span> <span class="code-line" id="baseline-line-220" data-line="220"><span class="line-number">220</span><span class="line-source"> if variance is not None and 0 &lt; variance &lt; 1:</span></span> <span class="code-line" id="baseline-line-221" data-line="221"><span class="line-number">221</span><span class="line-source"> prelim = min(max_allowed, 300)</span></span> <span class="code-line" id="baseline-line-222" data-line="222"><span class="line-number">222</span><span class="line-source"> pca_tmp = PCA(</span></span> <span class="code-line" id="baseline-line-223" data-line="223"><span class="line-number">223</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-224" data-line="224"><span class="line-number">224</span><span class="line-source"> svd_solver=&quot;randomized&quot;,</span></span> <span class="code-line" id="baseline-line-225" data-line="225"><span class="line-number">225</span><span class="line-source"> n_components=prelim,</span></span> <span class="code-line" id="baseline-line-226" data-line="226"><span class="line-number">226</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-227" data-line="227"><span class="line-number">227</span><span class="line-source"> pca_tmp.fit(log_counts)</span></span> <span class="code-line" id="baseline-line-228" data-line="228"><span class="line-number">228</span><span class="line-source"> cumvar = np.cumsum(pca_tmp.explained_variance_ratio_)</span></span> <span class="code-line" id="baseline-line-229" data-line="229"><span class="line-number">229</span><span class="line-source"> k = np.searchsorted(cumvar, variance) + 1</span></span> <span class="code-line" id="baseline-line-230" data-line="230"><span class="line-number">230</span><span class="line-source"> n_components_use = max(1, min(k, max_allowed))</span></span> <span class="code-line" id="baseline-line-231" data-line="231"><span class="line-number">231</span><span class="line-source"> else:</span></span> <span class="code-line" id="baseline-line-232" data-line="232"><span class="line-number">232</span><span class="line-source"> n_components_use = min(n_components or 30, max_allowed)</span></span> <span class="code-line" id="baseline-line-233" data-line="233"><span class="line-number">233</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-234" data-line="234"><span class="line-number">234</span><span class="line-source"> pca = PCA(</span></span> <span class="code-line" id="baseline-line-235" data-line="235"><span class="line-number">235</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-236" data-line="236"><span class="line-number">236</span><span class="line-source"> svd_solver=&quot;randomized&quot;,</span></span> <span class="code-line" id="baseline-line-237" data-line="237"><span class="line-number">237</span><span class="line-source"> n_components=n_components_use,</span></span> <span class="code-line" id="baseline-line-238" data-line="238"><span class="line-number">238</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-239" data-line="239"><span class="line-number">239</span><span class="line-source"> lowrank = pca.fit_transform(log_counts)</span></span> <span class="code-line" id="baseline-line-240" data-line="240"><span class="line-number">240</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-241" data-line="241"><span class="line-number">241</span><span class="line-source"> log_recon = pca.inverse_transform(lowrank)</span></span> <span class="code-line" id="baseline-line-242" data-line="242"><span class="line-number">242</span><span class="line-source"> norm_recon = np.expm1(log_recon)</span></span> <span class="code-line" id="baseline-line-243" data-line="243"><span class="line-number">243</span><span class="line-source"> recon_counts = norm_recon * (row_sum / target_sum)</span></span> <span class="code-line" id="baseline-line-244" data-line="244"><span class="line-number">244</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-245" data-line="245"><span class="line-number">245</span><span class="line-source"> return np.maximum(recon_counts, 0.0)</span></span> <span class="code-line" id="baseline-line-246" data-line="246"><span class="line-number">246</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-247" data-line="247"><span class="line-number">247</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-248" data-line="248"><span class="line-number">248</span><span class="line-source">def _auto_diffusion_time(graph, max_t: int = 12, target_decay: float = 0.1) -&gt; int:</span></span> <span class="code-line" id="baseline-line-249" data-line="249"><span class="line-number">249</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-250" data-line="250"><span class="line-number">250</span><span class="line-source"> Choose diffusion depth `t` such that the second eigenvalue λ₂ satisfies λ₂ᵗ ≤ target_decay.</span></span> <span class="code-line" id="baseline-line-251" data-line="251"><span class="line-number">251</span><span class="line-source"> Falls back to 2 if eigen‑decomposition fails.</span></span> <span class="code-line" id="baseline-line-252" data-line="252"><span class="line-number">252</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-253" data-line="253"><span class="line-number">253</span><span class="line-source"> try:</span></span> <span class="code-line" id="baseline-line-254" data-line="254"><span class="line-number">254</span><span class="line-source"> eigvals = graph.P.eigsh(k=5, which=&quot;LM&quot;, return_eigenvectors=False)</span></span> <span class="code-line" id="baseline-line-255" data-line="255"><span class="line-number">255</span><span class="line-source"> lambda2 = eigvals[1] # λ₂ (λ₁ == 1)</span></span> <span class="code-line" id="baseline-line-256" data-line="256"><span class="line-number">256</span><span class="line-source"> if lambda2 &lt;= 0.0 or lambda2 &gt;= 1.0:</span></span> <span class="code-line" id="baseline-line-257" data-line="257"><span class="line-number">257</span><span class="line-source"> return 2</span></span> <span class="code-line" id="baseline-line-258" data-line="258"><span class="line-number">258</span><span class="line-source"> t = int(np.ceil(np.log(target_decay) / np.log(lambda2)))</span></span> <span class="code-line" id="baseline-line-259" data-line="259"><span class="line-number">259</span><span class="line-source"> t = max(1, min(t, max_t))</span></span> <span class="code-line" id="baseline-line-260" data-line="260"><span class="line-number">260</span><span class="line-source"> return t</span></span> <span class="code-line" id="baseline-line-261" data-line="261"><span class="line-number">261</span><span class="line-source"> except Exception:</span></span> <span class="code-line" id="baseline-line-262" data-line="262"><span class="line-number">262</span><span class="line-source"> return 2</span></span> <span class="code-line" id="baseline-line-263" data-line="263"><span class="line-number">263</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-264" data-line="264"><span class="line-number">264</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-265" data-line="265"><span class="line-number">265</span><span class="line-source">def _scale_hvg_variance(</span></span> <span class="code-line" id="baseline-line-266" data-line="266"><span class="line-number">266</span><span class="line-source"> Y_vst_imputed: np.ndarray,</span></span> <span class="code-line" id="baseline-line-267" data-line="267"><span class="line-number">267</span><span class="line-source"> Y_vst_original: np.ndarray,</span></span> <span class="code-line" id="baseline-line-268" data-line="268"><span class="line-number">268</span><span class="line-source"> hvgs_mask: np.ndarray,</span></span> <span class="code-line" id="baseline-line-269" data-line="269"><span class="line-number">269</span><span class="line-source"> min_factor: float = 0.5,</span></span> <span class="code-line" id="baseline-line-270" data-line="270"><span class="line-number">270</span><span class="line-source"> max_factor: float = 2.0,</span></span> <span class="code-line" id="baseline-line-271" data-line="271"><span class="line-number">271</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-272" data-line="272"><span class="line-number">272</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-273" data-line="273"><span class="line-number">273</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-274" data-line="274"><span class="line-number">274</span><span class="line-source"> Rescale each HVG column so that its variance in the Anscombe‑stabilised</span></span> <span class="code-line" id="baseline-line-275" data-line="275"><span class="line-number">275</span><span class="line-source"> space matches the original variance.</span></span> <span class="code-line" id="baseline-line-276" data-line="276"><span class="line-number">276</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-277" data-line="277"><span class="line-number">277</span><span class="line-source"> orig_var = Y_vst_original[:, hvgs_mask].var(axis=0)</span></span> <span class="code-line" id="baseline-line-278" data-line="278"><span class="line-number">278</span><span class="line-source"> imp_var = Y_vst_imputed[:, hvgs_mask].var(axis=0) + eps</span></span> <span class="code-line" id="baseline-line-279" data-line="279"><span class="line-number">279</span><span class="line-source"> scale = np.sqrt((orig_var + eps) / imp_var)</span></span> <span class="code-line" id="baseline-line-280" data-line="280"><span class="line-number">280</span><span class="line-source"> scale = np.clip(scale, min_factor, max_factor)</span></span> <span class="code-line" id="baseline-line-281" data-line="281"><span class="line-number">281</span><span class="line-source"> Y_vst_imputed[:, hvgs_mask] = Y_vst_imputed[:, hvgs_mask] * scale[None, :]</span></span> <span class="code-line" id="baseline-line-282" data-line="282"><span class="line-number">282</span><span class="line-source"> return Y_vst_imputed</span></span> <span class="code-line" id="baseline-line-283" data-line="283"><span class="line-number">283</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-284" data-line="284"><span class="line-number">284</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-285" data-line="285"><span class="line-number">285</span><span class="line-source">def _gene_raw_weight_mix(</span></span> <span class="code-line" id="baseline-line-286" data-line="286"><span class="line-number">286</span><span class="line-source"> denoised: np.ndarray,</span></span> <span class="code-line" id="baseline-line-287" data-line="287"><span class="line-number">287</span><span class="line-source"> raw_counts: np.ndarray,</span></span> <span class="code-line" id="baseline-line-288" data-line="288"><span class="line-number">288</span><span class="line-source"> dropout: np.ndarray,</span></span> <span class="code-line" id="baseline-line-289" data-line="289"><span class="line-number">289</span><span class="line-source"> max_raw_weight: float = 0.02,</span></span> <span class="code-line" id="baseline-line-290" data-line="290"><span class="line-number">290</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-291" data-line="291"><span class="line-number">291</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-292" data-line="292"><span class="line-number">292</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-293" data-line="293"><span class="line-number">293</span><span class="line-source"> Adaptive gene‑wise mixing of original counts back into the denoised matrix.</span></span> <span class="code-line" id="baseline-line-294" data-line="294"><span class="line-number">294</span><span class="line-source"> Higher weight for genes that are well detected **and** strongly expressed.</span></span> <span class="code-line" id="baseline-line-295" data-line="295"><span class="line-number">295</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-296" data-line="296"><span class="line-number">296</span><span class="line-source"> w_det = 1.0 - dropout # (genes,)</span></span> <span class="code-line" id="baseline-line-297" data-line="297"><span class="line-number">297</span><span class="line-source"> gene_mean = raw_counts.mean(axis=0)</span></span> <span class="code-line" id="baseline-line-298" data-line="298"><span class="line-number">298</span><span class="line-source"> w_exp = gene_mean / (gene_mean + 1.0) # (genes,)</span></span> <span class="code-line" id="baseline-line-299" data-line="299"><span class="line-number">299</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-300" data-line="300"><span class="line-number">300</span><span class="line-source"> raw_weight = max_raw_weight * w_det * w_exp</span></span> <span class="code-line" id="baseline-line-301" data-line="301"><span class="line-number">301</span><span class="line-source"> raw_weight = np.clip(raw_weight, 0.0, max_raw_weight)</span></span> <span class="code-line" id="baseline-line-302" data-line="302"><span class="line-number">302</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-303" data-line="303"><span class="line-number">303</span><span class="line-source"> cur_lib = denoised.sum(axis=1, keepdims=True) + eps</span></span> <span class="code-line" id="baseline-line-304" data-line="304"><span class="line-number">304</span><span class="line-source"> raw_lib = raw_counts.sum(axis=1, keepdims=True) + eps</span></span> <span class="code-line" id="baseline-line-305" data-line="305"><span class="line-number">305</span><span class="line-source"> raw_scaled = raw_counts * (cur_lib / raw_lib)</span></span> <span class="code-line" id="baseline-line-306" data-line="306"><span class="line-number">306</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-307" data-line="307"><span class="line-number">307</span><span class="line-source"> blend = raw_weight[None, :] # (1, genes)</span></span> <span class="code-line" id="baseline-line-308" data-line="308"><span class="line-number">308</span><span class="line-source"> denoised_adj = denoised * (1.0 - blend) + raw_scaled * blend</span></span> <span class="code-line" id="baseline-line-309" data-line="309"><span class="line-number">309</span><span class="line-source"> return denoised_adj</span></span> <span class="code-line" id="baseline-line-310" data-line="310"><span class="line-number">310</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-311" data-line="311"><span class="line-number">311</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-312" data-line="312"><span class="line-number">312</span><span class="line-source">def _suppress_low_counts(</span></span> <span class="code-line" id="baseline-line-313" data-line="313"><span class="line-number">313</span><span class="line-source"> matrix: np.ndarray,</span></span> <span class="code-line" id="baseline-line-314" data-line="314"><span class="line-number">314</span><span class="line-source"> low_thresh: float = 0.5,</span></span> <span class="code-line" id="baseline-line-315" data-line="315"><span class="line-number">315</span><span class="line-source"> shrink: float = 0.95,</span></span> <span class="code-line" id="baseline-line-316" data-line="316"><span class="line-number">316</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-317" data-line="317"><span class="line-number">317</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-318" data-line="318"><span class="line-number">318</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-319" data-line="319"><span class="line-number">319</span><span class="line-source"> Shrink values that are below ``low_thresh`` towards zero.</span></span> <span class="code-line" id="baseline-line-320" data-line="320"><span class="line-number">320</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-321" data-line="321"><span class="line-number">321</span><span class="line-source"> mat = matrix.copy()</span></span> <span class="code-line" id="baseline-line-322" data-line="322"><span class="line-number">322</span><span class="line-source"> mask = mat &lt; low_thresh</span></span> <span class="code-line" id="baseline-line-323" data-line="323"><span class="line-number">323</span><span class="line-source"> mat[mask] *= shrink</span></span> <span class="code-line" id="baseline-line-324" data-line="324"><span class="line-number">324</span><span class="line-source"> mat[mat &lt; eps] = 0.0</span></span> <span class="code-line" id="baseline-line-325" data-line="325"><span class="line-number">325</span><span class="line-source"> return mat</span></span> <span class="code-line" id="baseline-line-326" data-line="326"><span class="line-number">326</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-327" data-line="327"><span class="line-number">327</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-328" data-line="328"><span class="line-number">328</span><span class="line-source">def _scale_gene_means(</span></span> <span class="code-line" id="baseline-line-329" data-line="329"><span class="line-number">329</span><span class="line-source"> denoised: np.ndarray,</span></span> <span class="code-line" id="baseline-line-330" data-line="330"><span class="line-number">330</span><span class="line-source"> raw: np.ndarray,</span></span> <span class="code-line" id="baseline-line-331" data-line="331"><span class="line-number">331</span><span class="line-source"> min_factor: float = 0.99,</span></span> <span class="code-line" id="baseline-line-332" data-line="332"><span class="line-number">332</span><span class="line-source"> max_factor: float = 1.01,</span></span> <span class="code-line" id="baseline-line-333" data-line="333"><span class="line-number">333</span><span class="line-source"> mean_thresh: float = 0.15,</span></span> <span class="code-line" id="baseline-line-334" data-line="334"><span class="line-number">334</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-335" data-line="335"><span class="line-number">335</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-336" data-line="336"><span class="line-number">336</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-337" data-line="337"><span class="line-number">337</span><span class="line-source"> Rescale each gene so that its mean matches the original raw‑count mean.</span></span> <span class="code-line" id="baseline-line-338" data-line="338"><span class="line-number">338</span><span class="line-source"> The scaling is limited to a narrow band to avoid over‑correction.</span></span> <span class="code-line" id="baseline-line-339" data-line="339"><span class="line-number">339</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-340" data-line="340"><span class="line-number">340</span><span class="line-source"> raw_means = raw.mean(axis=0)</span></span> <span class="code-line" id="baseline-line-341" data-line="341"><span class="line-number">341</span><span class="line-source"> den_means = denoised.mean(axis=0) + eps</span></span> <span class="code-line" id="baseline-line-342" data-line="342"><span class="line-number">342</span><span class="line-source"> scale = raw_means / den_means</span></span> <span class="code-line" id="baseline-line-343" data-line="343"><span class="line-number">343</span><span class="line-source"> scale = np.clip(scale, min_factor, max_factor)</span></span> <span class="code-line" id="baseline-line-344" data-line="344"><span class="line-number">344</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-345" data-line="345"><span class="line-number">345</span><span class="line-source"> low_mask = raw_means &lt; mean_thresh</span></span> <span class="code-line" id="baseline-line-346" data-line="346"><span class="line-number">346</span><span class="line-source"> scale[low_mask] = 1.0</span></span> <span class="code-line" id="baseline-line-347" data-line="347"><span class="line-number">347</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-348" data-line="348"><span class="line-number">348</span><span class="line-source"> return denoised * scale</span></span> <span class="code-line" id="baseline-line-349" data-line="349"><span class="line-number">349</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-350" data-line="350"><span class="line-number">350</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-351" data-line="351"><span class="line-number">351</span><span class="line-source">def _scale_gene_variance_masked(</span></span> <span class="code-line" id="baseline-line-352" data-line="352"><span class="line-number">352</span><span class="line-source"> denoised: np.ndarray,</span></span> <span class="code-line" id="baseline-line-353" data-line="353"><span class="line-number">353</span><span class="line-source"> raw: np.ndarray,</span></span> <span class="code-line" id="baseline-line-354" data-line="354"><span class="line-number">354</span><span class="line-source"> min_factor: float = 0.7,</span></span> <span class="code-line" id="baseline-line-355" data-line="355"><span class="line-number">355</span><span class="line-source"> max_factor: float = 1.3,</span></span> <span class="code-line" id="baseline-line-356" data-line="356"><span class="line-number">356</span><span class="line-source"> mean_thresh: float = 0.15,</span></span> <span class="code-line" id="baseline-line-357" data-line="357"><span class="line-number">357</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-358" data-line="358"><span class="line-number">358</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-359" data-line="359"><span class="line-number">359</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-360" data-line="360"><span class="line-number">360</span><span class="line-source"> Rescale genes with sufficiently high mean so that their variance matches</span></span> <span class="code-line" id="baseline-line-361" data-line="361"><span class="line-number">361</span><span class="line-source"> the original raw‑count variance. Low‑expressed genes are left untouched.</span></span> <span class="code-line" id="baseline-line-362" data-line="362"><span class="line-number">362</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-363" data-line="363"><span class="line-number">363</span><span class="line-source"> raw_means = raw.mean(axis=0)</span></span> <span class="code-line" id="baseline-line-364" data-line="364"><span class="line-number">364</span><span class="line-source"> mask = raw_means &gt; mean_thresh</span></span> <span class="code-line" id="baseline-line-365" data-line="365"><span class="line-number">365</span><span class="line-source"> if not np.any(mask):</span></span> <span class="code-line" id="baseline-line-366" data-line="366"><span class="line-number">366</span><span class="line-source"> return denoised</span></span> <span class="code-line" id="baseline-line-367" data-line="367"><span class="line-number">367</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-368" data-line="368"><span class="line-number">368</span><span class="line-source"> raw_var = raw.var(axis=0) + eps</span></span> <span class="code-line" id="baseline-line-369" data-line="369"><span class="line-number">369</span><span class="line-source"> den_var = denoised.var(axis=0) + eps</span></span> <span class="code-line" id="baseline-line-370" data-line="370"><span class="line-number">370</span><span class="line-source"> scale = np.sqrt(raw_var / den_var)</span></span> <span class="code-line" id="baseline-line-371" data-line="371"><span class="line-number">371</span><span class="line-source"> scale = np.clip(scale, min_factor, max_factor)</span></span> <span class="code-line" id="baseline-line-372" data-line="372"><span class="line-number">372</span><span class="line-source"> scale[~mask] = 1.0</span></span> <span class="code-line" id="baseline-line-373" data-line="373"><span class="line-number">373</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-374" data-line="374"><span class="line-number">374</span><span class="line-source"> return denoised * scale</span></span> <span class="code-line" id="baseline-line-375" data-line="375"><span class="line-number">375</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-376" data-line="376"><span class="line-number">376</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-377" data-line="377"><span class="line-number">377</span><span class="line-source">def _refine_hvg(</span></span> <span class="code-line" id="baseline-line-378" data-line="378"><span class="line-number">378</span><span class="line-source"> Y_vst: np.ndarray,</span></span> <span class="code-line" id="baseline-line-379" data-line="379"><span class="line-number">379</span><span class="line-source"> Y_std: np.ndarray,</span></span> <span class="code-line" id="baseline-line-380" data-line="380"><span class="line-number">380</span><span class="line-source"> hvgs_mask: np.ndarray,</span></span> <span class="code-line" id="baseline-line-381" data-line="381"><span class="line-number">381</span><span class="line-source"> loc_hvg: np.ndarray,</span></span> <span class="code-line" id="baseline-line-382" data-line="382"><span class="line-number">382</span><span class="line-source"> scale_hvg: np.ndarray,</span></span> <span class="code-line" id="baseline-line-383" data-line="383"><span class="line-number">383</span><span class="line-source"> diff_op,</span></span> <span class="code-line" id="baseline-line-384" data-line="384"><span class="line-number">384</span><span class="line-source"> dropout: np.ndarray,</span></span> <span class="code-line" id="baseline-line-385" data-line="385"><span class="line-number">385</span><span class="line-source"> median_dropout: float,</span></span> <span class="code-line" id="baseline-line-386" data-line="386"><span class="line-number">386</span><span class="line-source"> n_hvg_target: int,</span></span> <span class="code-line" id="baseline-line-387" data-line="387"><span class="line-number">387</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-388" data-line="388"><span class="line-number">388</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-389" data-line="389"><span class="line-number">389</span><span class="line-source"> One‑step diffusion, re‑select HVGs on the partially imputed matrix and</span></span> <span class="code-line" id="baseline-line-390" data-line="390"><span class="line-number">390</span><span class="line-source"> return the *union* of the original and refined masks.</span></span> <span class="code-line" id="baseline-line-391" data-line="391"><span class="line-number">391</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-392" data-line="392"><span class="line-number">392</span><span class="line-source"> Y_std_quick = _impute_counts_std(</span></span> <span class="code-line" id="baseline-line-393" data-line="393"><span class="line-number">393</span><span class="line-source"> Y_std=Y_std,</span></span> <span class="code-line" id="baseline-line-394" data-line="394"><span class="line-number">394</span><span class="line-source"> hvgs_mask=hvgs_mask,</span></span> <span class="code-line" id="baseline-line-395" data-line="395"><span class="line-number">395</span><span class="line-source"> diff_op=diff_op,</span></span> <span class="code-line" id="baseline-line-396" data-line="396"><span class="line-number">396</span><span class="line-source"> dropout=dropout,</span></span> <span class="code-line" id="baseline-line-397" data-line="397"><span class="line-number">397</span><span class="line-source"> median_dropout=median_dropout,</span></span> <span class="code-line" id="baseline-line-398" data-line="398"><span class="line-number">398</span><span class="line-source"> max_t=1,</span></span> <span class="code-line" id="baseline-line-399" data-line="399"><span class="line-number">399</span><span class="line-source"> dropout_exponent=1.0,</span></span> <span class="code-line" id="baseline-line-400" data-line="400"><span class="line-number">400</span><span class="line-source"> diffusion_weight_exponent=1.0,</span></span> <span class="code-line" id="baseline-line-401" data-line="401"><span class="line-number">401</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-402" data-line="402"><span class="line-number">402</span><span class="line-source"> Y_vst_quick = _destandardize_hvg(Y_std_quick, hvgs_mask, loc_hvg, scale_hvg)</span></span> <span class="code-line" id="baseline-line-403" data-line="403"><span class="line-number">403</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-404" data-line="404"><span class="line-number">404</span><span class="line-source"> refined_mask = _select_hvg_seurat(Y_vst_quick, n_hvg=n_hvg_target)</span></span> <span class="code-line" id="baseline-line-405" data-line="405"><span class="line-number">405</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-406" data-line="406"><span class="line-number">406</span><span class="line-source"> return hvgs_mask | refined_mask</span></span> <span class="code-line" id="baseline-line-407" data-line="407"><span class="line-number">407</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-408" data-line="408"><span class="line-number">408</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-409" data-line="409"><span class="line-number">409</span><span class="line-source">def _final_log_knn_smooth_with_residual(</span></span> <span class="code-line" id="baseline-line-410" data-line="410"><span class="line-number">410</span><span class="line-source"> counts: np.ndarray,</span></span> <span class="code-line" id="baseline-line-411" data-line="411"><span class="line-number">411</span><span class="line-source"> raw_counts: np.ndarray,</span></span> <span class="code-line" id="baseline-line-412" data-line="412"><span class="line-number">412</span><span class="line-source"> target_sum: float = 10000.0,</span></span> <span class="code-line" id="baseline-line-413" data-line="413"><span class="line-number">413</span><span class="line-source"> knn: int = 30,</span></span> <span class="code-line" id="baseline-line-414" data-line="414"><span class="line-number">414</span><span class="line-source"> weight: float = 0.01,</span></span> <span class="code-line" id="baseline-line-415" data-line="415"><span class="line-number">415</span><span class="line-source"> residual_weight: float = 0.02,</span></span> <span class="code-line" id="baseline-line-416" data-line="416"><span class="line-number">416</span><span class="line-source"> n_pca: int = 50,</span></span> <span class="code-line" id="baseline-line-417" data-line="417"><span class="line-number">417</span><span class="line-source"> random_state: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-418" data-line="418"><span class="line-number">418</span><span class="line-source"> n_jobs: int = 2,</span></span> <span class="code-line" id="baseline-line-419" data-line="419"><span class="line-number">419</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-420" data-line="420"><span class="line-number">420</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-421" data-line="421"><span class="line-number">421</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-422" data-line="422"><span class="line-number">422</span><span class="line-source"> One‑step neighbour‑average smoothing performed in log‑normalised space,</span></span> <span class="code-line" id="baseline-line-423" data-line="423"><span class="line-number">423</span><span class="line-source"> followed by a residual correction that nudges the result toward the original</span></span> <span class="code-line" id="baseline-line-424" data-line="424"><span class="line-number">424</span><span class="line-source"> raw counts.</span></span> <span class="code-line" id="baseline-line-425" data-line="425"><span class="line-number">425</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-426" data-line="426"><span class="line-number">426</span><span class="line-source"> # normalise both matrices to the target library size</span></span> <span class="code-line" id="baseline-line-427" data-line="427"><span class="line-number">427</span><span class="line-source"> lib_den = counts.sum(axis=1, keepdims=True) + eps</span></span> <span class="code-line" id="baseline-line-428" data-line="428"><span class="line-number">428</span><span class="line-source"> norm_den = counts * (target_sum / lib_den)</span></span> <span class="code-line" id="baseline-line-429" data-line="429"><span class="line-number">429</span><span class="line-source"> log_den = np.log1p(norm_den)</span></span> <span class="code-line" id="baseline-line-430" data-line="430"><span class="line-number">430</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-431" data-line="431"><span class="line-number">431</span><span class="line-source"> lib_raw = raw_counts.sum(axis=1, keepdims=True) + eps</span></span> <span class="code-line" id="baseline-line-432" data-line="432"><span class="line-number">432</span><span class="line-source"> norm_raw = raw_counts * (target_sum / lib_raw)</span></span> <span class="code-line" id="baseline-line-433" data-line="433"><span class="line-number">433</span><span class="line-source"> log_raw = np.log1p(norm_raw)</span></span> <span class="code-line" id="baseline-line-434" data-line="434"><span class="line-number">434</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-435" data-line="435"><span class="line-number">435</span><span class="line-source"> # low‑dimensional embedding for graph construction</span></span> <span class="code-line" id="baseline-line-436" data-line="436"><span class="line-number">436</span><span class="line-source"> n_comp = min(n_pca, log_den.shape[1], 100)</span></span> <span class="code-line" id="baseline-line-437" data-line="437"><span class="line-number">437</span><span class="line-source"> if n_comp &lt; log_den.shape[1]:</span></span> <span class="code-line" id="baseline-line-438" data-line="438"><span class="line-number">438</span><span class="line-source"> pca = PCA(</span></span> <span class="code-line" id="baseline-line-439" data-line="439"><span class="line-number">439</span><span class="line-source"> n_components=n_comp,</span></span> <span class="code-line" id="baseline-line-440" data-line="440"><span class="line-number">440</span><span class="line-source"> svd_solver=&quot;randomized&quot;,</span></span> <span class="code-line" id="baseline-line-441" data-line="441"><span class="line-number">441</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-442" data-line="442"><span class="line-number">442</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-443" data-line="443"><span class="line-number">443</span><span class="line-source"> embed = pca.fit_transform(log_den)</span></span> <span class="code-line" id="baseline-line-444" data-line="444"><span class="line-number">444</span><span class="line-source"> else:</span></span> <span class="code-line" id="baseline-line-445" data-line="445"><span class="line-number">445</span><span class="line-source"> embed = log_den.copy()</span></span> <span class="code-line" id="baseline-line-446" data-line="446"><span class="line-number">446</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-447" data-line="447"><span class="line-number">447</span><span class="line-source"> # k‑NN graph (tiny, fast)</span></span> <span class="code-line" id="baseline-line-448" data-line="448"><span class="line-number">448</span><span class="line-source"> graph = graphtools.Graph(</span></span> <span class="code-line" id="baseline-line-449" data-line="449"><span class="line-number">449</span><span class="line-source"> embed,</span></span> <span class="code-line" id="baseline-line-450" data-line="450"><span class="line-number">450</span><span class="line-source"> n_pca=None,</span></span> <span class="code-line" id="baseline-line-451" data-line="451"><span class="line-number">451</span><span class="line-source"> knn=knn,</span></span> <span class="code-line" id="baseline-line-452" data-line="452"><span class="line-number">452</span><span class="line-source"> knn_max=knn * 3,</span></span> <span class="code-line" id="baseline-line-453" data-line="453"><span class="line-number">453</span><span class="line-source"> decay=0.5,</span></span> <span class="code-line" id="baseline-line-454" data-line="454"><span class="line-number">454</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-455" data-line="455"><span class="line-number">455</span><span class="line-source"> n_jobs=n_jobs,</span></span> <span class="code-line" id="baseline-line-456" data-line="456"><span class="line-number">456</span><span class="line-source"> verbose=0,</span></span> <span class="code-line" id="baseline-line-457" data-line="457"><span class="line-number">457</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-458" data-line="458"><span class="line-number">458</span><span class="line-source"> diff_op = graph.diff_op</span></span> <span class="code-line" id="baseline-line-459" data-line="459"><span class="line-number">459</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-460" data-line="460"><span class="line-number">460</span><span class="line-source"> # diffusion in log‑space</span></span> <span class="code-line" id="baseline-line-461" data-line="461"><span class="line-number">461</span><span class="line-source"> smooth_log = _apply_diffusion(diff_op, log_den)</span></span> <span class="code-line" id="baseline-line-462" data-line="462"><span class="line-number">462</span><span class="line-source"> blended_log = (1.0 - weight) * log_den + weight * smooth_log</span></span> <span class="code-line" id="baseline-line-463" data-line="463"><span class="line-number">463</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-464" data-line="464"><span class="line-number">464</span><span class="line-source"> # residual correction</span></span> <span class="code-line" id="baseline-line-465" data-line="465"><span class="line-number">465</span><span class="line-source"> residual = log_raw - blended_log</span></span> <span class="code-line" id="baseline-line-466" data-line="466"><span class="line-number">466</span><span class="line-source"> smooth_res = _apply_diffusion(diff_op, residual)</span></span> <span class="code-line" id="baseline-line-467" data-line="467"><span class="line-number">467</span><span class="line-source"> corrected_log = blended_log + residual_weight * smooth_res</span></span> <span class="code-line" id="baseline-line-468" data-line="468"><span class="line-number">468</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-469" data-line="469"><span class="line-number">469</span><span class="line-source"> # back to raw counts</span></span> <span class="code-line" id="baseline-line-470" data-line="470"><span class="line-number">470</span><span class="line-source"> norm_corrected = np.expm1(corrected_log)</span></span> <span class="code-line" id="baseline-line-471" data-line="471"><span class="line-number">471</span><span class="line-source"> denoised_counts = norm_corrected * (lib_den / target_sum)</span></span> <span class="code-line" id="baseline-line-472" data-line="472"><span class="line-number">472</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-473" data-line="473"><span class="line-number">473</span><span class="line-source"> return np.maximum(denoised_counts, 0.0)</span></span> <span class="code-line" id="baseline-line-474" data-line="474"><span class="line-number">474</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-475" data-line="475"><span class="line-number">475</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-476" data-line="476"><span class="line-number">476</span><span class="line-source">def _low_rank_refine_counts(</span></span> <span class="code-line" id="baseline-line-477" data-line="477"><span class="line-number">477</span><span class="line-source"> counts: np.ndarray,</span></span> <span class="code-line" id="baseline-line-478" data-line="478"><span class="line-number">478</span><span class="line-source"> target_sum: float = 10000.0,</span></span> <span class="code-line" id="baseline-line-479" data-line="479"><span class="line-number">479</span><span class="line-source"> variance: float = 0.97,</span></span> <span class="code-line" id="baseline-line-480" data-line="480"><span class="line-number">480</span><span class="line-source"> blend: float = 0.04,</span></span> <span class="code-line" id="baseline-line-481" data-line="481"><span class="line-number">481</span><span class="line-source"> random_state: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-482" data-line="482"><span class="line-number">482</span><span class="line-source"> eps: float = 1e-12,</span></span> <span class="code-line" id="baseline-line-483" data-line="483"><span class="line-number">483</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-484" data-line="484"><span class="line-number">484</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-485" data-line="485"><span class="line-number">485</span><span class="line-source"> Light ALRA‑style low‑rank refinement performed **in log‑normalised space**.</span></span> <span class="code-line" id="baseline-line-486" data-line="486"><span class="line-number">486</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-487" data-line="487"><span class="line-number">487</span><span class="line-source"> lib = counts.sum(axis=1, keepdims=True) + eps</span></span> <span class="code-line" id="baseline-line-488" data-line="488"><span class="line-number">488</span><span class="line-source"> norm_counts = counts * (target_sum / lib)</span></span> <span class="code-line" id="baseline-line-489" data-line="489"><span class="line-number">489</span><span class="line-source"> log_counts = np.log1p(norm_counts)</span></span> <span class="code-line" id="baseline-line-490" data-line="490"><span class="line-number">490</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-491" data-line="491"><span class="line-number">491</span><span class="line-source"> max_pc = min(80, log_counts.shape[1])</span></span> <span class="code-line" id="baseline-line-492" data-line="492"><span class="line-number">492</span><span class="line-source"> pca_tmp = PCA(</span></span> <span class="code-line" id="baseline-line-493" data-line="493"><span class="line-number">493</span><span class="line-source"> n_components=max_pc,</span></span> <span class="code-line" id="baseline-line-494" data-line="494"><span class="line-number">494</span><span class="line-source"> svd_solver=&quot;randomized&quot;,</span></span> <span class="code-line" id="baseline-line-495" data-line="495"><span class="line-number">495</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-496" data-line="496"><span class="line-number">496</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-497" data-line="497"><span class="line-number">497</span><span class="line-source"> pca_tmp.fit(log_counts)</span></span> <span class="code-line" id="baseline-line-498" data-line="498"><span class="line-number">498</span><span class="line-source"> cumvar = np.cumsum(pca_tmp.explained_variance_ratio_)</span></span> <span class="code-line" id="baseline-line-499" data-line="499"><span class="line-number">499</span><span class="line-source"> n_pc = int(np.searchsorted(cumvar, variance) + 1)</span></span> <span class="code-line" id="baseline-line-500" data-line="500"><span class="line-number">500</span><span class="line-source"> n_pc = max(10, min(n_pc, max_pc))</span></span> <span class="code-line" id="baseline-line-501" data-line="501"><span class="line-number">501</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-502" data-line="502"><span class="line-number">502</span><span class="line-source"> pca = PCA(</span></span> <span class="code-line" id="baseline-line-503" data-line="503"><span class="line-number">503</span><span class="line-source"> n_components=n_pc,</span></span> <span class="code-line" id="baseline-line-504" data-line="504"><span class="line-number">504</span><span class="line-source"> svd_solver=&quot;randomized&quot;,</span></span> <span class="code-line" id="baseline-line-505" data-line="505"><span class="line-number">505</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-506" data-line="506"><span class="line-number">506</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-507" data-line="507"><span class="line-number">507</span><span class="line-source"> low = pca.fit_transform(log_counts)</span></span> <span class="code-line" id="baseline-line-508" data-line="508"><span class="line-number">508</span><span class="line-source"> log_recon = pca.inverse_transform(low)</span></span> <span class="code-line" id="baseline-line-509" data-line="509"><span class="line-number">509</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-510" data-line="510"><span class="line-number">510</span><span class="line-source"> norm_recon = np.expm1(log_recon)</span></span> <span class="code-line" id="baseline-line-511" data-line="511"><span class="line-number">511</span><span class="line-source"> recon_counts = norm_recon * (lib / target_sum)</span></span> <span class="code-line" id="baseline-line-512" data-line="512"><span class="line-number">512</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-513" data-line="513"><span class="line-number">513</span><span class="line-source"> refined = blend * recon_counts + (1.0 - blend) * counts</span></span> <span class="code-line" id="baseline-line-514" data-line="514"><span class="line-number">514</span><span class="line-source"> return np.maximum(refined, 0.0)</span></span> <span class="code-line" id="baseline-line-515" data-line="515"><span class="line-number">515</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-516" data-line="516"><span class="line-number">516</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-517" data-line="517"><span class="line-number">517</span><span class="line-source">def _denoise_variant(</span></span> <span class="code-line" id="baseline-line-518" data-line="518"><span class="line-number">518</span><span class="line-source"> X_raw: np.ndarray,</span></span> <span class="code-line" id="baseline-line-519" data-line="519"><span class="line-number">519</span><span class="line-source"> X_pre: np.ndarray,</span></span> <span class="code-line" id="baseline-line-520" data-line="520"><span class="line-number">520</span><span class="line-source"> Y_vst: np.ndarray,</span></span> <span class="code-line" id="baseline-line-521" data-line="521"><span class="line-number">521</span><span class="line-source"> Y_std: np.ndarray,</span></span> <span class="code-line" id="baseline-line-522" data-line="522"><span class="line-number">522</span><span class="line-source"> loc_hvg: np.ndarray,</span></span> <span class="code-line" id="baseline-line-523" data-line="523"><span class="line-number">523</span><span class="line-source"> scale_hvg: np.ndarray,</span></span> <span class="code-line" id="baseline-line-524" data-line="524"><span class="line-number">524</span><span class="line-source"> hvgs_mask: np.ndarray,</span></span> <span class="code-line" id="baseline-line-525" data-line="525"><span class="line-number">525</span><span class="line-source"> diff_op,</span></span> <span class="code-line" id="baseline-line-526" data-line="526"><span class="line-number">526</span><span class="line-source"> libsize: np.ndarray,</span></span> <span class="code-line" id="baseline-line-527" data-line="527"><span class="line-number">527</span><span class="line-source"> dropout: np.ndarray,</span></span> <span class="code-line" id="baseline-line-528" data-line="528"><span class="line-number">528</span><span class="line-source"> median_dropout: float,</span></span> <span class="code-line" id="baseline-line-529" data-line="529"><span class="line-number">529</span><span class="line-source"> row_smooth_blend: float,</span></span> <span class="code-line" id="baseline-line-530" data-line="530"><span class="line-number">530</span><span class="line-source"> target_sum: float,</span></span> <span class="code-line" id="baseline-line-531" data-line="531"><span class="line-number">531</span><span class="line-source"> random_state: Optional[int],</span></span> <span class="code-line" id="baseline-line-532" data-line="532"><span class="line-number">532</span><span class="line-source"> eps: float,</span></span> <span class="code-line" id="baseline-line-533" data-line="533"><span class="line-number">533</span><span class="line-source"> # hyper‑parameters (exposed via outer function)</span></span> <span class="code-line" id="baseline-line-534" data-line="534"><span class="line-number">534</span><span class="line-source"> dropout_exponent: float = 1.0,</span></span> <span class="code-line" id="baseline-line-535" data-line="535"><span class="line-number">535</span><span class="line-source"> diffusion_weight_exponent: float = 0.85,</span></span> <span class="code-line" id="baseline-line-536" data-line="536"><span class="line-number">536</span><span class="line-source"> max_t: int = 3,</span></span> <span class="code-line" id="baseline-line-537" data-line="537"><span class="line-number">537</span><span class="line-source"> nmf_weight: float = 0.12,</span></span> <span class="code-line" id="baseline-line-538" data-line="538"><span class="line-number">538</span><span class="line-source"> nmf_components: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-539" data-line="539"><span class="line-number">539</span><span class="line-source"> full_pca_variance: Optional[float] = 0.985,</span></span> <span class="code-line" id="baseline-line-540" data-line="540"><span class="line-number">540</span><span class="line-source"> full_pca_blend: float = 0.20,</span></span> <span class="code-line" id="baseline-line-541" data-line="541"><span class="line-number">541</span><span class="line-source"> full_diffuse_weight: float = 0.05,</span></span> <span class="code-line" id="baseline-line-542" data-line="542"><span class="line-number">542</span><span class="line-source"> raw_mix_weight: Optional[float] = None,</span></span> <span class="code-line" id="baseline-line-543" data-line="543"><span class="line-number">543</span><span class="line-source"> variance_scaling: bool = True,</span></span> <span class="code-line" id="baseline-line-544" data-line="544"><span class="line-number">544</span><span class="line-source"> inverse_anscombe_iters: int = 12,</span></span> <span class="code-line" id="baseline-line-545" data-line="545"><span class="line-number">545</span><span class="line-source"> verbose: bool = False,</span></span> <span class="code-line" id="baseline-line-546" data-line="546"><span class="line-number">546</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-547" data-line="547"><span class="line-number">547</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-548" data-line="548"><span class="line-number">548</span><span class="line-source"> A single MAGIC‑style denoising run (variant).</span></span> <span class="code-line" id="baseline-line-549" data-line="549"><span class="line-number">549</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-550" data-line="550"><span class="line-number">550</span><span class="line-source"> # 1️⃣ Impute in the standardised HVG space</span></span> <span class="code-line" id="baseline-line-551" data-line="551"><span class="line-number">551</span><span class="line-source"> Y_std_imputed = _impute_counts_std(</span></span> <span class="code-line" id="baseline-line-552" data-line="552"><span class="line-number">552</span><span class="line-source"> Y_std=Y_std,</span></span> <span class="code-line" id="baseline-line-553" data-line="553"><span class="line-number">553</span><span class="line-source"> hvgs_mask=hvgs_mask,</span></span> <span class="code-line" id="baseline-line-554" data-line="554"><span class="line-number">554</span><span class="line-source"> diff_op=diff_op,</span></span> <span class="code-line" id="baseline-line-555" data-line="555"><span class="line-number">555</span><span class="line-source"> dropout=dropout,</span></span> <span class="code-line" id="baseline-line-556" data-line="556"><span class="line-number">556</span><span class="line-source"> median_dropout=median_dropout,</span></span> <span class="code-line" id="baseline-line-557" data-line="557"><span class="line-number">557</span><span class="line-source"> max_t=max_t,</span></span> <span class="code-line" id="baseline-line-558" data-line="558"><span class="line-number">558</span><span class="line-source"> dropout_exponent=dropout_exponent,</span></span> <span class="code-line" id="baseline-line-559" data-line="559"><span class="line-number">559</span><span class="line-source"> diffusion_weight_exponent=diffusion_weight_exponent,</span></span> <span class="code-line" id="baseline-line-560" data-line="560"><span class="line-number">560</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-561" data-line="561"><span class="line-number">561</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-562" data-line="562"><span class="line-number">562</span><span class="line-source"> # 2️⃣ Destandardise back to the Anscombe scale</span></span> <span class="code-line" id="baseline-line-563" data-line="563"><span class="line-number">563</span><span class="line-source"> Y_vst_imputed = _destandardize_hvg(Y_std_imputed, hvgs_mask, loc_hvg, scale_hvg)</span></span> <span class="code-line" id="baseline-line-564" data-line="564"><span class="line-number">564</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-565" data-line="565"><span class="line-number">565</span><span class="line-source"> # 3️⃣ (optional) variance matching for HVGs</span></span> <span class="code-line" id="baseline-line-566" data-line="566"><span class="line-number">566</span><span class="line-source"> if variance_scaling:</span></span> <span class="code-line" id="baseline-line-567" data-line="567"><span class="line-number">567</span><span class="line-source"> Y_vst_imputed = _scale_hvg_variance(</span></span> <span class="code-line" id="baseline-line-568" data-line="568"><span class="line-number">568</span><span class="line-source"> Y_vst_imputed,</span></span> <span class="code-line" id="baseline-line-569" data-line="569"><span class="line-number">569</span><span class="line-source"> Y_vst_original=Y_vst,</span></span> <span class="code-line" id="baseline-line-570" data-line="570"><span class="line-number">570</span><span class="line-source"> hvgs_mask=hvgs_mask,</span></span> <span class="code-line" id="baseline-line-571" data-line="571"><span class="line-number">571</span><span class="line-source"> min_factor=0.5,</span></span> <span class="code-line" id="baseline-line-572" data-line="572"><span class="line-number">572</span><span class="line-source"> max_factor=2.0,</span></span> <span class="code-line" id="baseline-line-573" data-line="573"><span class="line-number">573</span><span class="line-source"> eps=eps,</span></span> <span class="code-line" id="baseline-line-574" data-line="574"><span class="line-number">574</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-575" data-line="575"><span class="line-number">575</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-576" data-line="576"><span class="line-number">576</span><span class="line-source"> # 4️⃣ Smooth per‑cell library sizes and adjust rows</span></span> <span class="code-line" id="baseline-line-577" data-line="577"><span class="line-number">577</span><span class="line-source"> smoothed_libsize = _smooth_row_sums(diff_op, libsize, blend=row_smooth_blend)</span></span> <span class="code-line" id="baseline-line-578" data-line="578"><span class="line-number">578</span><span class="line-source"> Y_vst_adjusted = _adjust_row_sums(Y_vst_imputed, smoothed_libsize)</span></span> <span class="code-line" id="baseline-line-579" data-line="579"><span class="line-number">579</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-580" data-line="580"><span class="line-number">580</span><span class="line-source"> # 5️⃣ Light full‑matrix diffusion</span></span> <span class="code-line" id="baseline-line-581" data-line="581"><span class="line-number">581</span><span class="line-source"> Y_vst_final = _full_diffuse(Y_vst_adjusted, diff_op, weight=full_diffuse_weight)</span></span> <span class="code-line" id="baseline-line-582" data-line="582"><span class="line-number">582</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-583" data-line="583"><span class="line-number">583</span><span class="line-source"> # 6️⃣ Inverse Anscombe → raw scale</span></span> <span class="code-line" id="baseline-line-584" data-line="584"><span class="line-number">584</span><span class="line-source"> counts_imputed = _inverse_anscombe_newton(Y_vst_final, n_iter=inverse_anscombe_iters)</span></span> <span class="code-line" id="baseline-line-585" data-line="585"><span class="line-number">585</span><span class="line-source"> counts_imputed = np.maximum(counts_imputed, 0.0)</span></span> <span class="code-line" id="baseline-line-586" data-line="586"><span class="line-number">586</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-587" data-line="587"><span class="line-number">587</span><span class="line-source"> # 7️⃣ KL‑NMF low‑rank blending (only for modest data)</span></span> <span class="code-line" id="baseline-line-588" data-line="588"><span class="line-number">588</span><span class="line-source"> if nmf_weight &gt; 0.0 and X_raw.shape[0] &lt;= 5000:</span></span> <span class="code-line" id="baseline-line-589" data-line="589"><span class="line-number">589</span><span class="line-source"> if nmf_components is None:</span></span> <span class="code-line" id="baseline-line-590" data-line="590"><span class="line-number">590</span><span class="line-source"> n_cells, n_genes = X_raw.shape</span></span> <span class="code-line" id="baseline-line-591" data-line="591"><span class="line-number">591</span><span class="line-source"> nmf_components = max(10, min(20, n_cells // 5, n_genes // 5))</span></span> <span class="code-line" id="baseline-line-592" data-line="592"><span class="line-number">592</span><span class="line-source"> counts_nmf = _nmf_impute(</span></span> <span class="code-line" id="baseline-line-593" data-line="593"><span class="line-number">593</span><span class="line-source"> X_pre,</span></span> <span class="code-line" id="baseline-line-594" data-line="594"><span class="line-number">594</span><span class="line-source"> n_components=nmf_components,</span></span> <span class="code-line" id="baseline-line-595" data-line="595"><span class="line-number">595</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-596" data-line="596"><span class="line-number">596</span><span class="line-source"> max_iter=200,</span></span> <span class="code-line" id="baseline-line-597" data-line="597"><span class="line-number">597</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-598" data-line="598"><span class="line-number">598</span><span class="line-source"> # match per‑cell library size before blending</span></span> <span class="code-line" id="baseline-line-599" data-line="599"><span class="line-number">599</span><span class="line-source"> target_lib = counts_imputed.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="baseline-line-600" data-line="600"><span class="line-number">600</span><span class="line-source"> counts_nmf = _adjust_row_sums(counts_nmf, target_lib)</span></span> <span class="code-line" id="baseline-line-601" data-line="601"><span class="line-number">601</span><span class="line-source"> counts_imputed = (1.0 - nmf_weight) * counts_imputed + nmf_weight * counts_nmf</span></span> <span class="code-line" id="baseline-line-602" data-line="602"><span class="line-number">602</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-603" data-line="603"><span class="line-number">603</span><span class="line-source"> # 8️⃣ Centered‑PCA denoising (log‑space)</span></span> <span class="code-line" id="baseline-line-604" data-line="604"><span class="line-number">604</span><span class="line-source"> if (full_pca_variance is not None and 0 &lt; full_pca_variance &lt; 1) or full_pca_blend &gt; 0:</span></span> <span class="code-line" id="baseline-line-605" data-line="605"><span class="line-number">605</span><span class="line-source"> pca_counts = _full_pca_denoise_centered(</span></span> <span class="code-line" id="baseline-line-606" data-line="606"><span class="line-number">606</span><span class="line-source"> counts_imputed,</span></span> <span class="code-line" id="baseline-line-607" data-line="607"><span class="line-number">607</span><span class="line-source"> target_sum=target_sum,</span></span> <span class="code-line" id="baseline-line-608" data-line="608"><span class="line-number">608</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-609" data-line="609"><span class="line-number">609</span><span class="line-source"> n_components=None,</span></span> <span class="code-line" id="baseline-line-610" data-line="610"><span class="line-number">610</span><span class="line-source"> variance=full_pca_variance,</span></span> <span class="code-line" id="baseline-line-611" data-line="611"><span class="line-number">611</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-612" data-line="612"><span class="line-number">612</span><span class="line-source"> counts_imputed = (</span></span> <span class="code-line" id="baseline-line-613" data-line="613"><span class="line-number">613</span><span class="line-source"> full_pca_blend * pca_counts + (1.0 - full_pca_blend) * counts_imputed</span></span> <span class="code-line" id="baseline-line-614" data-line="614"><span class="line-number">614</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-615" data-line="615"><span class="line-number">615</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-616" data-line="616"><span class="line-number">616</span><span class="line-source"> # 9️⃣ Adaptive cell‑level raw‑mix (helps Poisson loss)</span></span> <span class="code-line" id="baseline-line-617" data-line="617"><span class="line-number">617</span><span class="line-source"> if raw_mix_weight is None:</span></span> <span class="code-line" id="baseline-line-618" data-line="618"><span class="line-number">618</span><span class="line-source"> lib_cv = np.std(libsize) / (np.mean(libsize) + eps)</span></span> <span class="code-line" id="baseline-line-619" data-line="619"><span class="line-number">619</span><span class="line-source"> raw_mix_weight = float(np.clip(0.02 + 0.12 * lib_cv, 0.02, 0.07))</span></span> <span class="code-line" id="baseline-line-620" data-line="620"><span class="line-number">620</span><span class="line-source"> raw_mix_weight = min(max(raw_mix_weight, 0.0), 0.07)</span></span> <span class="code-line" id="baseline-line-621" data-line="621"><span class="line-number">621</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-622" data-line="622"><span class="line-number">622</span><span class="line-source"> if raw_mix_weight &gt; 0.0:</span></span> <span class="code-line" id="baseline-line-623" data-line="623"><span class="line-number">623</span><span class="line-source"> cur_lib = counts_imputed.sum(axis=1, keepdims=True)</span></span> <span class="code-line" id="baseline-line-624" data-line="624"><span class="line-number">624</span><span class="line-source"> raw_scaled = X_raw * (cur_lib / (libsize + eps))</span></span> <span class="code-line" id="baseline-line-625" data-line="625"><span class="line-number">625</span><span class="line-source"> counts_imputed = (</span></span> <span class="code-line" id="baseline-line-626" data-line="626"><span class="line-number">626</span><span class="line-source"> raw_mix_weight * raw_scaled + (1.0 - raw_mix_weight) * counts_imputed</span></span> <span class="code-line" id="baseline-line-627" data-line="627"><span class="line-number">627</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-628" data-line="628"><span class="line-number">628</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-629" data-line="629"><span class="line-number">629</span><span class="line-source"> # final sanity check</span></span> <span class="code-line" id="baseline-line-630" data-line="630"><span class="line-number">630</span><span class="line-source"> counts_imputed = np.maximum(counts_imputed, 0.0)</span></span> <span class="code-line" id="baseline-line-631" data-line="631"><span class="line-number">631</span><span class="line-source"> return counts_imputed</span></span> <span class="code-line" id="baseline-line-632" data-line="632"><span class="line-number">632</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-633" data-line="633"><span class="line-number">633</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-634" data-line="634"><span class="line-number">634</span><span class="line-source"># ----------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-635" data-line="635"><span class="line-number">635</span><span class="line-source"># Main denoising routine – tuned for lower MSE while preserving Poisson‑norm</span></span> <span class="code-line" id="baseline-line-636" data-line="636"><span class="line-number">636</span><span class="line-source"># ----------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-637" data-line="637"><span class="line-number">637</span><span class="line-source">def magic_denoise(</span></span> <span class="code-line" id="baseline-line-638" data-line="638"><span class="line-number">638</span><span class="line-source"> X,</span></span> <span class="code-line" id="baseline-line-639" data-line="639"><span class="line-number">639</span><span class="line-source"> # Graph / diffusion hyper‑parameters</span></span> <span class="code-line" id="baseline-line-640" data-line="640"><span class="line-number">640</span><span class="line-source"> knn: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-641" data-line="641"><span class="line-number">641</span><span class="line-source"> t: Optional[int] = None, # diffusion depth (None ⇒ auto)</span></span> <span class="code-line" id="baseline-line-642" data-line="642"><span class="line-number">642</span><span class="line-source"> n_pca: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-643" data-line="643"><span class="line-number">643</span><span class="line-source"> solver: Optional[str] = None, # &quot;approximate&quot; or &quot;exact&quot;</span></span> <span class="code-line" id="baseline-line-644" data-line="644"><span class="line-number">644</span><span class="line-source"> decay: Optional[float] = None,</span></span> <span class="code-line" id="baseline-line-645" data-line="645"><span class="line-number">645</span><span class="line-source"> knn_max: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-646" data-line="646"><span class="line-number">646</span><span class="line-source"> random_state: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-647" data-line="647"><span class="line-number">647</span><span class="line-source"> n_jobs: int = 2,</span></span> <span class="code-line" id="baseline-line-648" data-line="648"><span class="line-number">648</span><span class="line-source"> # HVG selector</span></span> <span class="code-line" id="baseline-line-649" data-line="649"><span class="line-number">649</span><span class="line-source"> n_hvg: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-650" data-line="650"><span class="line-number">650</span><span class="line-source"> # Row‑library‑size smoothing</span></span> <span class="code-line" id="baseline-line-651" data-line="651"><span class="line-number">651</span><span class="line-source"> row_smooth_blend: Optional[float] = None, # adaptive if None</span></span> <span class="code-line" id="baseline-line-652" data-line="652"><span class="line-number">652</span><span class="line-source"> # Primary MAGIC weighting</span></span> <span class="code-line" id="baseline-line-653" data-line="653"><span class="line-number">653</span><span class="line-source"> dropout_exponent: float = 1.0,</span></span> <span class="code-line" id="baseline-line-654" data-line="654"><span class="line-number">654</span><span class="line-source"> diffusion_weight_exponent: float = 0.85,</span></span> <span class="code-line" id="baseline-line-655" data-line="655"><span class="line-number">655</span><span class="line-source"> # NMF imputation</span></span> <span class="code-line" id="baseline-line-656" data-line="656"><span class="line-number">656</span><span class="line-source"> nmf_weight: float = 0.12,</span></span> <span class="code-line" id="baseline-line-657" data-line="657"><span class="line-number">657</span><span class="line-source"> nmf_components: Optional[int] = None,</span></span> <span class="code-line" id="baseline-line-658" data-line="658"><span class="line-number">658</span><span class="line-source"> # Global PCA‑denoise (inside each variant)</span></span> <span class="code-line" id="baseline-line-659" data-line="659"><span class="line-number">659</span><span class="line-source"> full_pca_variance: float = 0.985, # keep ~98.5 % variance</span></span> <span class="code-line" id="baseline-line-660" data-line="660"><span class="line-number">660</span><span class="line-source"> full_pca_blend: float = 0.20, # blend of PCA‑reconstruction</span></span> <span class="code-line" id="baseline-line-661" data-line="661"><span class="line-number">661</span><span class="line-source"> # Full‑matrix diffusion (new)</span></span> <span class="code-line" id="baseline-line-662" data-line="662"><span class="line-number">662</span><span class="line-source"> full_diffuse_weight: float = 0.05,</span></span> <span class="code-line" id="baseline-line-663" data-line="663"><span class="line-number">663</span><span class="line-source"> # Raw‑mix (helps Poisson loss) – cell‑level mixing</span></span> <span class="code-line" id="baseline-line-664" data-line="664"><span class="line-number">664</span><span class="line-source"> raw_mix_weight: Optional[float] = None,</span></span> <span class="code-line" id="baseline-line-665" data-line="665"><span class="line-number">665</span><span class="line-source"> # Additional gene‑level raw mixing (new)</span></span> <span class="code-line" id="baseline-line-666" data-line="666"><span class="line-number">666</span><span class="line-source"> gene_raw_weight: float = 0.03, # max weight per‑gene for raw counts</span></span> <span class="code-line" id="baseline-line-667" data-line="667"><span class="line-number">667</span><span class="line-source"> # Low‑count suppression (new)</span></span> <span class="code-line" id="baseline-line-668" data-line="668"><span class="line-number">668</span><span class="line-source"> low_thresh: float = 0.2,</span></span> <span class="code-line" id="baseline-line-669" data-line="669"><span class="line-number">669</span><span class="line-source"> low_shrink: float = 0.90,</span></span> <span class="code-line" id="baseline-line-670" data-line="670"><span class="line-number">670</span><span class="line-source"> # Gene‑mean matching (new)</span></span> <span class="code-line" id="baseline-line-671" data-line="671"><span class="line-number">671</span><span class="line-source"> gene_mean_scaling: bool = True,</span></span> <span class="code-line" id="baseline-line-672" data-line="672"><span class="line-number">672</span><span class="line-source"> gene_mean_scaling_range: Tuple[float, float] = (0.97, 1.03),</span></span> <span class="code-line" id="baseline-line-673" data-line="673"><span class="line-number">673</span><span class="line-source"> gene_mean_scaling_thresh: float = 0.15,</span></span> <span class="code-line" id="baseline-line-674" data-line="674"><span class="line-number">674</span><span class="line-source"> # Gene‑variance matching (optional – off by default; enable for extra MSE gain)</span></span> <span class="code-line" id="baseline-line-675" data-line="675"><span class="line-number">675</span><span class="line-source"> gene_variance_scaling: bool = False,</span></span> <span class="code-line" id="baseline-line-676" data-line="676"><span class="line-number">676</span><span class="line-source"> # Final polishing PCA (new)</span></span> <span class="code-line" id="baseline-line-677" data-line="677"><span class="line-number">677</span><span class="line-source"> final_pca_variance: float = 0.99,</span></span> <span class="code-line" id="baseline-line-678" data-line="678"><span class="line-number">678</span><span class="line-source"> final_pca_blend: float = 0.15,</span></span> <span class="code-line" id="baseline-line-679" data-line="679"><span class="line-number">679</span><span class="line-source"> # Final log‑space k‑NN smoothing (new) + residual correction</span></span> <span class="code-line" id="baseline-line-680" data-line="680"><span class="line-number">680</span><span class="line-source"> final_log_knn: bool = True,</span></span> <span class="code-line" id="baseline-line-681" data-line="681"><span class="line-number">681</span><span class="line-source"> final_log_knn_weight: float = 0.01,</span></span> <span class="code-line" id="baseline-line-682" data-line="682"><span class="line-number">682</span><span class="line-source"> final_log_knn_knn: int = 30,</span></span> <span class="code-line" id="baseline-line-683" data-line="683"><span class="line-number">683</span><span class="line-source"> residual_correction_weight: float = 0.02,</span></span> <span class="code-line" id="baseline-line-684" data-line="684"><span class="line-number">684</span><span class="line-source"> # Low‑rank (ALRA‑style) refinement (new)</span></span> <span class="code-line" id="baseline-line-685" data-line="685"><span class="line-number">685</span><span class="line-source"> final_alra: bool = True,</span></span> <span class="code-line" id="baseline-line-686" data-line="686"><span class="line-number">686</span><span class="line-source"> final_alra_blend: float = 0.07,</span></span> <span class="code-line" id="baseline-line-687" data-line="687"><span class="line-number">687</span><span class="line-source"> final_alra_variance: float = 0.99,</span></span> <span class="code-line" id="baseline-line-688" data-line="688"><span class="line-number">688</span><span class="line-source"> # Misc</span></span> <span class="code-line" id="baseline-line-689" data-line="689"><span class="line-number">689</span><span class="line-source"> target_sum: float = 10000.0,</span></span> <span class="code-line" id="baseline-line-690" data-line="690"><span class="line-number">690</span><span class="line-source"> inverse_anscombe_iters: int = 12,</span></span> <span class="code-line" id="baseline-line-691" data-line="691"><span class="line-number">691</span><span class="line-source"> multi_knn: bool = True, # average two graphs (if feasible)</span></span> <span class="code-line" id="baseline-line-692" data-line="692"><span class="line-number">692</span><span class="line-source"> verbose: bool = False,</span></span> <span class="code-line" id="baseline-line-693" data-line="693"><span class="line-number">693</span><span class="line-source"> **kwargs,</span></span> <span class="code-line" id="baseline-line-694" data-line="694"><span class="line-number">694</span><span class="line-source">) -&gt; np.ndarray:</span></span> <span class="code-line" id="baseline-line-695" data-line="695"><span class="line-number">695</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-696" data-line="696"><span class="line-number">696</span><span class="line-source"> Perform denoising of a raw count matrix `X` (cells &#215; genes) using a</span></span> <span class="code-line" id="baseline-line-697" data-line="697"><span class="line-number">697</span><span class="line-source"> MAGIC‑style diffusion, followed by several corrective post‑processing</span></span> <span class="code-line" id="baseline-line-698" data-line="698"><span class="line-number">698</span><span class="line-source"> steps. The routine follows the “reversed‑normalisation’’ strategy that</span></span> <span class="code-line" id="baseline-line-699" data-line="699"><span class="line-number">699</span><span class="line-source"> guarantees a Poisson‑norm ≥ 0.97 while striving for a lower MSE.</span></span> <span class="code-line" id="baseline-line-700" data-line="700"><span class="line-number">700</span><span class="line-source"> &quot;&quot;&quot;</span></span> <span class="code-line" id="baseline-line-701" data-line="701"><span class="line-number">701</span><span class="line-source"> eps = 1e-12</span></span> <span class="code-line" id="baseline-line-702" data-line="702"><span class="line-number">702</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-703" data-line="703"><span class="line-number">703</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-704" data-line="704"><span class="line-number">704</span><span class="line-source"> # 0️⃣ Input handling &amp; basic statistics</span></span> <span class="code-line" id="baseline-line-705" data-line="705"><span class="line-number">705</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-706" data-line="706"><span class="line-number">706</span><span class="line-source"> X = scprep.utils.toarray(X).astype(np.float32)</span></span> <span class="code-line" id="baseline-line-707" data-line="707"><span class="line-number">707</span><span class="line-source"> if X.ndim != 2:</span></span> <span class="code-line" id="baseline-line-708" data-line="708"><span class="line-number">708</span><span class="line-source"> raise ValueError(&quot;`X` must be a 2‑D array (cells &#215; genes).&quot;)</span></span> <span class="code-line" id="baseline-line-709" data-line="709"><span class="line-number">709</span><span class="line-source"> n_cells, n_genes = X.shape</span></span> <span class="code-line" id="baseline-line-710" data-line="710"><span class="line-number">710</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-711" data-line="711"><span class="line-number">711</span><span class="line-source"> libsize = X.sum(axis=1, keepdims=True) # (cells, 1)</span></span> <span class="code-line" id="baseline-line-712" data-line="712"><span class="line-number">712</span><span class="line-source"> libsize[libsize == 0] = 1.0 # avoid division by zero</span></span> <span class="code-line" id="baseline-line-713" data-line="713"><span class="line-number">713</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-714" data-line="714"><span class="line-number">714</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-715" data-line="715"><span class="line-number">715</span><span class="line-source"> # 0.5️⃣ Median‑library pre‑scale + winsorising (graph stabilisation)</span></span> <span class="code-line" id="baseline-line-716" data-line="716"><span class="line-number">716</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-717" data-line="717"><span class="line-number">717</span><span class="line-source"> median_lib = np.median(libsize)</span></span> <span class="code-line" id="baseline-line-718" data-line="718"><span class="line-number">718</span><span class="line-source"> pre_scale_factor = median_lib / libsize</span></span> <span class="code-line" id="baseline-line-719" data-line="719"><span class="line-number">719</span><span class="line-source"> X_pre = X * pre_scale_factor</span></span> <span class="code-line" id="baseline-line-720" data-line="720"><span class="line-number">720</span><span class="line-source"> X_pre = _winsorize_counts(X_pre, q_low=0.001, q_high=0.999)</span></span> <span class="code-line" id="baseline-line-721" data-line="721"><span class="line-number">721</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-722" data-line="722"><span class="line-number">722</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-723" data-line="723"><span class="line-number">723</span><span class="line-source"> # 1️⃣ Anscombe VST (variance stabilising)</span></span> <span class="code-line" id="baseline-line-724" data-line="724"><span class="line-number">724</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-725" data-line="725"><span class="line-number">725</span><span class="line-source"> Y_vst = _anscombe_transform(X_pre)</span></span> <span class="code-line" id="baseline-line-726" data-line="726"><span class="line-number">726</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-727" data-line="727"><span class="line-number">727</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-728" data-line="728"><span class="line-number">728</span><span class="line-source"> # 2️⃣ Initial HVG selection (Seurat style)</span></span> <span class="code-line" id="baseline-line-729" data-line="729"><span class="line-number">729</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-730" data-line="730"><span class="line-number">730</span><span class="line-source"> if n_hvg is None:</span></span> <span class="code-line" id="baseline-line-731" data-line="731"><span class="line-number">731</span><span class="line-source"> # generous default – up to half the genes but not less than 15 k</span></span> <span class="code-line" id="baseline-line-732" data-line="732"><span class="line-number">732</span><span class="line-source"> n_hvg = min(30000, max(15000, n_genes // 2))</span></span> <span class="code-line" id="baseline-line-733" data-line="733"><span class="line-number">733</span><span class="line-source"> hvgs_mask = _select_hvg_seurat(Y_vst, n_hvg=n_hvg)</span></span> <span class="code-line" id="baseline-line-734" data-line="734"><span class="line-number">734</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-735" data-line="735"><span class="line-number">735</span><span class="line-source"> # filter ultra‑sparse / low‑expressed genes before building the graph</span></span> <span class="code-line" id="baseline-line-736" data-line="736"><span class="line-number">736</span><span class="line-source"> dropout = (X == 0).mean(axis=0) # (genes,)</span></span> <span class="code-line" id="baseline-line-737" data-line="737"><span class="line-number">737</span><span class="line-source"> gene_mean = X.mean(axis=0) # (genes,)</span></span> <span class="code-line" id="baseline-line-738" data-line="738"><span class="line-number">738</span><span class="line-source"> hvgs_mask &amp;= (dropout &lt; 0.85) &amp; (gene_mean &gt; 0.08)</span></span> <span class="code-line" id="baseline-line-739" data-line="739"><span class="line-number">739</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-740" data-line="740"><span class="line-number">740</span><span class="line-source"> if hvgs_mask.sum() &lt; max(500, n_hvg // 4):</span></span> <span class="code-line" id="baseline-line-741" data-line="741"><span class="line-number">741</span><span class="line-source"> hvgs_mask = _select_hvg_seurat(Y_vst, n_hvg=n_hvg)</span></span> <span class="code-line" id="baseline-line-742" data-line="742"><span class="line-number">742</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-743" data-line="743"><span class="line-number">743</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="baseline-line-744" data-line="744"><span class="line-number">744</span><span class="line-source"> print(f&quot;[magic_denoise] Initial HVGs: {hvgs_mask.sum()} / {n_genes}&quot;)</span></span> <span class="code-line" id="baseline-line-745" data-line="745"><span class="line-number">745</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-746" data-line="746"><span class="line-number">746</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-747" data-line="747"><span class="line-number">747</span><span class="line-source"> # 3️⃣ Build the primary k‑NN graph (used also for cheap HVG refinement)</span></span> <span class="code-line" id="baseline-line-748" data-line="748"><span class="line-number">748</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-749" data-line="749"><span class="line-number">749</span><span class="line-source"> Y_graph = Y_vst[:, hvgs_mask] # graph built on VST‑transformed HVGs</span></span> <span class="code-line" id="baseline-line-750" data-line="750"><span class="line-number">750</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-751" data-line="751"><span class="line-number">751</span><span class="line-source"> # sensible defaults for the graph</span></span> <span class="code-line" id="baseline-line-752" data-line="752"><span class="line-number">752</span><span class="line-source"> if knn is None:</span></span> <span class="code-line" id="baseline-line-753" data-line="753"><span class="line-number">753</span><span class="line-source"> knn = max(30, min(80, int(np.sqrt(n_cells) * 2)))</span></span> <span class="code-line" id="baseline-line-754" data-line="754"><span class="line-number">754</span><span class="line-source"> if n_pca is None:</span></span> <span class="code-line" id="baseline-line-755" data-line="755"><span class="line-number">755</span><span class="line-source"> n_pca = min(100, max(15, int(np.log2(n_genes))))</span></span> <span class="code-line" id="baseline-line-756" data-line="756"><span class="line-number">756</span><span class="line-source"> if knn_max is None:</span></span> <span class="code-line" id="baseline-line-757" data-line="757"><span class="line-number">757</span><span class="line-source"> knn_max = knn * 3</span></span> <span class="code-line" id="baseline-line-758" data-line="758"><span class="line-number">758</span><span class="line-source"> if decay is None:</span></span> <span class="code-line" id="baseline-line-759" data-line="759"><span class="line-number">759</span><span class="line-source"> decay = 0.5</span></span> <span class="code-line" id="baseline-line-760" data-line="760"><span class="line-number">760</span><span class="line-source"> if solver is None:</span></span> <span class="code-line" id="baseline-line-761" data-line="761"><span class="line-number">761</span><span class="line-source"> solver = &quot;approximate&quot; if n_cells &gt; 2000 else &quot;exact&quot;</span></span> <span class="code-line" id="baseline-line-762" data-line="762"><span class="line-number">762</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-763" data-line="763"><span class="line-number">763</span><span class="line-source"> graph = graphtools.Graph(</span></span> <span class="code-line" id="baseline-line-764" data-line="764"><span class="line-number">764</span><span class="line-source"> Y_graph,</span></span> <span class="code-line" id="baseline-line-765" data-line="765"><span class="line-number">765</span><span class="line-source"> n_pca=n_pca if Y_graph.shape[1] &gt; n_pca else None,</span></span> <span class="code-line" id="baseline-line-766" data-line="766"><span class="line-number">766</span><span class="line-source"> knn=knn,</span></span> <span class="code-line" id="baseline-line-767" data-line="767"><span class="line-number">767</span><span class="line-source"> knn_max=knn_max,</span></span> <span class="code-line" id="baseline-line-768" data-line="768"><span class="line-number">768</span><span class="line-source"> decay=decay,</span></span> <span class="code-line" id="baseline-line-769" data-line="769"><span class="line-number">769</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-770" data-line="770"><span class="line-number">770</span><span class="line-source"> n_jobs=n_jobs,</span></span> <span class="code-line" id="baseline-line-771" data-line="771"><span class="line-number">771</span><span class="line-source"> verbose=0,</span></span> <span class="code-line" id="baseline-line-772" data-line="772"><span class="line-number">772</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-773" data-line="773"><span class="line-number">773</span><span class="line-source"> diff_op = graph.diff_op</span></span> <span class="code-line" id="baseline-line-774" data-line="774"><span class="line-number">774</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-775" data-line="775"><span class="line-number">775</span><span class="line-source"> # optionally average a second, slightly larger‑knn graph</span></span> <span class="code-line" id="baseline-line-776" data-line="776"><span class="line-number">776</span><span class="line-source"> if multi_knn:</span></span> <span class="code-line" id="baseline-line-777" data-line="777"><span class="line-number">777</span><span class="line-source"> knn2 = int(min(knn_max, max(5, knn * 1.5)))</span></span> <span class="code-line" id="baseline-line-778" data-line="778"><span class="line-number">778</span><span class="line-source"> graph2 = graphtools.Graph(</span></span> <span class="code-line" id="baseline-line-779" data-line="779"><span class="line-number">779</span><span class="line-source"> Y_graph,</span></span> <span class="code-line" id="baseline-line-780" data-line="780"><span class="line-number">780</span><span class="line-source"> n_pca=n_pca if Y_graph.shape[1] &gt; n_pca else None,</span></span> <span class="code-line" id="baseline-line-781" data-line="781"><span class="line-number">781</span><span class="line-source"> knn=knn2,</span></span> <span class="code-line" id="baseline-line-782" data-line="782"><span class="line-number">782</span><span class="line-source"> knn_max=knn_max,</span></span> <span class="code-line" id="baseline-line-783" data-line="783"><span class="line-number">783</span><span class="line-source"> decay=decay,</span></span> <span class="code-line" id="baseline-line-784" data-line="784"><span class="line-number">784</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-785" data-line="785"><span class="line-number">785</span><span class="line-source"> n_jobs=n_jobs,</span></span> <span class="code-line" id="baseline-line-786" data-line="786"><span class="line-number">786</span><span class="line-source"> verbose=0,</span></span> <span class="code-line" id="baseline-line-787" data-line="787"><span class="line-number">787</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-788" data-line="788"><span class="line-number">788</span><span class="line-source"> diff_op = (diff_op + graph2.diff_op) * 0.5</span></span> <span class="code-line" id="baseline-line-789" data-line="789"><span class="line-number">789</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-790" data-line="790"><span class="line-number">790</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-791" data-line="791"><span class="line-number">791</span><span class="line-source"> # 4️⃣ Cheap HVG refinement – run twice for stability</span></span> <span class="code-line" id="baseline-line-792" data-line="792"><span class="line-number">792</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-793" data-line="793"><span class="line-number">793</span><span class="line-source"> median_dropout = np.median(dropout) + eps</span></span> <span class="code-line" id="baseline-line-794" data-line="794"><span class="line-number">794</span><span class="line-source"> Y_std_full, loc_hvg_full, scale_hvg_full = _standardize_hvg(</span></span> <span class="code-line" id="baseline-line-795" data-line="795"><span class="line-number">795</span><span class="line-source"> Y_vst, hvgs_mask, robust=True</span></span> <span class="code-line" id="baseline-line-796" data-line="796"><span class="line-number">796</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-797" data-line="797"><span class="line-number">797</span><span class="line-source"> # first pass</span></span> <span class="code-line" id="baseline-line-798" data-line="798"><span class="line-number">798</span><span class="line-source"> hvgs_mask = _refine_hvg(</span></span> <span class="code-line" id="baseline-line-799" data-line="799"><span class="line-number">799</span><span class="line-source"> Y_vst=Y_vst,</span></span> <span class="code-line" id="baseline-line-800" data-line="800"><span class="line-number">800</span><span class="line-source"> Y_std=Y_std_full,</span></span> <span class="code-line" id="baseline-line-801" data-line="801"><span class="line-number">801</span><span class="line-source"> hvgs_mask=hvgs_mask,</span></span> <span class="code-line" id="baseline-line-802" data-line="802"><span class="line-number">802</span><span class="line-source"> loc_hvg=loc_hvg_full,</span></span> <span class="code-line" id="baseline-line-803" data-line="803"><span class="line-number">803</span><span class="line-source"> scale_hvg=scale_hvg_full,</span></span> <span class="code-line" id="baseline-line-804" data-line="804"><span class="line-number">804</span><span class="line-source"> diff_op=diff_op,</span></span> <span class="code-line" id="baseline-line-805" data-line="805"><span class="line-number">805</span><span class="line-source"> dropout=dropout,</span></span> <span class="code-line" id="baseline-line-806" data-line="806"><span class="line-number">806</span><span class="line-source"> median_dropout=median_dropout,</span></span> <span class="code-line" id="baseline-line-807" data-line="807"><span class="line-number">807</span><span class="line-source"> n_hvg_target=n_hvg,</span></span> <span class="code-line" id="baseline-line-808" data-line="808"><span class="line-number">808</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-809" data-line="809"><span class="line-number">809</span><span class="line-source"> # second pass (re‑standardise on the new mask)</span></span> <span class="code-line" id="baseline-line-810" data-line="810"><span class="line-number">810</span><span class="line-source"> Y_std_full2, loc_hvg_full2, scale_hvg_full2 = _standardize_hvg(</span></span> <span class="code-line" id="baseline-line-811" data-line="811"><span class="line-number">811</span><span class="line-source"> Y_vst, hvgs_mask, robust=True</span></span> <span class="code-line" id="baseline-line-812" data-line="812"><span class="line-number">812</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-813" data-line="813"><span class="line-number">813</span><span class="line-source"> hvgs_mask = _refine_hvg(</span></span> <span class="code-line" id="baseline-line-814" data-line="814"><span class="line-number">814</span><span class="line-source"> Y_vst=Y_vst,</span></span> <span class="code-line" id="baseline-line-815" data-line="815"><span class="line-number">815</span><span class="line-source"> Y_std=Y_std_full2,</span></span> <span class="code-line" id="baseline-line-816" data-line="816"><span class="line-number">816</span><span class="line-source"> hvgs_mask=hvgs_mask,</span></span> <span class="code-line" id="baseline-line-817" data-line="817"><span class="line-number">817</span><span class="line-source"> loc_hvg=loc_hvg_full2,</span></span> <span class="code-line" id="baseline-line-818" data-line="818"><span class="line-number">818</span><span class="line-source"> scale_hvg=scale_hvg_full2,</span></span> <span class="code-line" id="baseline-line-819" data-line="819"><span class="line-number">819</span><span class="line-source"> diff_op=diff_op,</span></span> <span class="code-line" id="baseline-line-820" data-line="820"><span class="line-number">820</span><span class="line-source"> dropout=dropout,</span></span> <span class="code-line" id="baseline-line-821" data-line="821"><span class="line-number">821</span><span class="line-source"> median_dropout=median_dropout,</span></span> <span class="code-line" id="baseline-line-822" data-line="822"><span class="line-number">822</span><span class="line-source"> n_hvg_target=n_hvg,</span></span> <span class="code-line" id="baseline-line-823" data-line="823"><span class="line-number">823</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-824" data-line="824"><span class="line-number">824</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-825" data-line="825"><span class="line-number">825</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="baseline-line-826" data-line="826"><span class="line-number">826</span><span class="line-source"> print(f&quot;[magic_denoise] Refined HVGs: {hvgs_mask.sum()} / {n_genes}&quot;)</span></span> <span class="code-line" id="baseline-line-827" data-line="827"><span class="line-number">827</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-828" data-line="828"><span class="line-number">828</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-829" data-line="829"><span class="line-number">829</span><span class="line-source"> # 5️⃣ Robust standardisation on the refined HVG set</span></span> <span class="code-line" id="baseline-line-830" data-line="830"><span class="line-number">830</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-831" data-line="831"><span class="line-number">831</span><span class="line-source"> Y_std, loc_hvg, scale_hvg = _standardize_hvg(Y_vst, hvgs_mask, robust=True)</span></span> <span class="code-line" id="baseline-line-832" data-line="832"><span class="line-number">832</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-833" data-line="833"><span class="line-number">833</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-834" data-line="834"><span class="line-number">834</span><span class="line-source"> # 6️⃣ Re‑build the graph on the refined HVG set (if the mask changed)</span></span> <span class="code-line" id="baseline-line-835" data-line="835"><span class="line-number">835</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-836" data-line="836"><span class="line-number">836</span><span class="line-source"> Y_graph = Y_std[:, hvgs_mask] # now standardised HVGs</span></span> <span class="code-line" id="baseline-line-837" data-line="837"><span class="line-number">837</span><span class="line-source"> graph = graphtools.Graph(</span></span> <span class="code-line" id="baseline-line-838" data-line="838"><span class="line-number">838</span><span class="line-source"> Y_graph,</span></span> <span class="code-line" id="baseline-line-839" data-line="839"><span class="line-number">839</span><span class="line-source"> n_pca=n_pca if Y_graph.shape[1] &gt; n_pca else None,</span></span> <span class="code-line" id="baseline-line-840" data-line="840"><span class="line-number">840</span><span class="line-source"> knn=knn,</span></span> <span class="code-line" id="baseline-line-841" data-line="841"><span class="line-number">841</span><span class="line-source"> knn_max=knn_max,</span></span> <span class="code-line" id="baseline-line-842" data-line="842"><span class="line-number">842</span><span class="line-source"> decay=decay,</span></span> <span class="code-line" id="baseline-line-843" data-line="843"><span class="line-number">843</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-844" data-line="844"><span class="line-number">844</span><span class="line-source"> n_jobs=n_jobs,</span></span> <span class="code-line" id="baseline-line-845" data-line="845"><span class="line-number">845</span><span class="line-source"> verbose=0,</span></span> <span class="code-line" id="baseline-line-846" data-line="846"><span class="line-number">846</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-847" data-line="847"><span class="line-number">847</span><span class="line-source"> diff_op = graph.diff_op</span></span> <span class="code-line" id="baseline-line-848" data-line="848"><span class="line-number">848</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-849" data-line="849"><span class="line-number">849</span><span class="line-source"> if multi_knn:</span></span> <span class="code-line" id="baseline-line-850" data-line="850"><span class="line-number">850</span><span class="line-source"> knn2 = int(min(knn_max, max(5, knn * 1.5)))</span></span> <span class="code-line" id="baseline-line-851" data-line="851"><span class="line-number">851</span><span class="line-source"> graph2 = graphtools.Graph(</span></span> <span class="code-line" id="baseline-line-852" data-line="852"><span class="line-number">852</span><span class="line-source"> Y_graph,</span></span> <span class="code-line" id="baseline-line-853" data-line="853"><span class="line-number">853</span><span class="line-source"> n_pca=n_pca if Y_graph.shape[1] &gt; n_pca else None,</span></span> <span class="code-line" id="baseline-line-854" data-line="854"><span class="line-number">854</span><span class="line-source"> knn=knn2,</span></span> <span class="code-line" id="baseline-line-855" data-line="855"><span class="line-number">855</span><span class="line-source"> knn_max=knn_max,</span></span> <span class="code-line" id="baseline-line-856" data-line="856"><span class="line-number">856</span><span class="line-source"> decay=decay,</span></span> <span class="code-line" id="baseline-line-857" data-line="857"><span class="line-number">857</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-858" data-line="858"><span class="line-number">858</span><span class="line-source"> n_jobs=n_jobs,</span></span> <span class="code-line" id="baseline-line-859" data-line="859"><span class="line-number">859</span><span class="line-source"> verbose=0,</span></span> <span class="code-line" id="baseline-line-860" data-line="860"><span class="line-number">860</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-861" data-line="861"><span class="line-number">861</span><span class="line-source"> diff_op = (diff_op + graph2.diff_op) * 0.5</span></span> <span class="code-line" id="baseline-line-862" data-line="862"><span class="line-number">862</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-863" data-line="863"><span class="line-number">863</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-864" data-line="864"><span class="line-number">864</span><span class="line-source"> # 7️⃣ Diffusion depth (auto if not supplied)</span></span> <span class="code-line" id="baseline-line-865" data-line="865"><span class="line-number">865</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-866" data-line="866"><span class="line-number">866</span><span class="line-source"> max_t = t if t is not None else _auto_diffusion_time(graph, max_t=12)</span></span> <span class="code-line" id="baseline-line-867" data-line="867"><span class="line-number">867</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="baseline-line-868" data-line="868"><span class="line-number">868</span><span class="line-source"> print(f&quot;[magic_denoise] Diffusion depth selected: {max_t}&quot;)</span></span> <span class="code-line" id="baseline-line-869" data-line="869"><span class="line-number">869</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-870" data-line="870"><span class="line-number">870</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-871" data-line="871"><span class="line-number">871</span><span class="line-source"> # 8️⃣ Adaptive row‑library‑size smoothing blend</span></span> <span class="code-line" id="baseline-line-872" data-line="872"><span class="line-number">872</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-873" data-line="873"><span class="line-number">873</span><span class="line-source"> if row_smooth_blend is None:</span></span> <span class="code-line" id="baseline-line-874" data-line="874"><span class="line-number">874</span><span class="line-source"> lib_cv = np.std(libsize) / (np.mean(libsize) + eps)</span></span> <span class="code-line" id="baseline-line-875" data-line="875"><span class="line-number">875</span><span class="line-source"> row_smooth_blend = float(np.clip(0.20 + 0.6 * lib_cv, 0.20, 0.55))</span></span> <span class="code-line" id="baseline-line-876" data-line="876"><span class="line-number">876</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-877" data-line="877"><span class="line-number">877</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-878" data-line="878"><span class="line-number">878</span><span class="line-source"> # 9️⃣ Run two complementary diffusion variants (tuned for MSE)</span></span> <span class="code-line" id="baseline-line-879" data-line="879"><span class="line-number">879</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-880" data-line="880"><span class="line-number">880</span><span class="line-source"> # Variant 1 – stronger (keeps more signal)</span></span> <span class="code-line" id="baseline-line-881" data-line="881"><span class="line-number">881</span><span class="line-source"> var1 = _denoise_variant(</span></span> <span class="code-line" id="baseline-line-882" data-line="882"><span class="line-number">882</span><span class="line-source"> X_raw=X,</span></span> <span class="code-line" id="baseline-line-883" data-line="883"><span class="line-number">883</span><span class="line-source"> X_pre=X_pre,</span></span> <span class="code-line" id="baseline-line-884" data-line="884"><span class="line-number">884</span><span class="line-source"> Y_vst=Y_vst,</span></span> <span class="code-line" id="baseline-line-885" data-line="885"><span class="line-number">885</span><span class="line-source"> Y_std=Y_std,</span></span> <span class="code-line" id="baseline-line-886" data-line="886"><span class="line-number">886</span><span class="line-source"> loc_hvg=loc_hvg,</span></span> <span class="code-line" id="baseline-line-887" data-line="887"><span class="line-number">887</span><span class="line-source"> scale_hvg=scale_hvg,</span></span> <span class="code-line" id="baseline-line-888" data-line="888"><span class="line-number">888</span><span class="line-source"> hvgs_mask=hvgs_mask,</span></span> <span class="code-line" id="baseline-line-889" data-line="889"><span class="line-number">889</span><span class="line-source"> diff_op=diff_op,</span></span> <span class="code-line" id="baseline-line-890" data-line="890"><span class="line-number">890</span><span class="line-source"> libsize=libsize,</span></span> <span class="code-line" id="baseline-line-891" data-line="891"><span class="line-number">891</span><span class="line-source"> dropout=dropout,</span></span> <span class="code-line" id="baseline-line-892" data-line="892"><span class="line-number">892</span><span class="line-source"> median_dropout=median_dropout,</span></span> <span class="code-line" id="baseline-line-893" data-line="893"><span class="line-number">893</span><span class="line-source"> row_smooth_blend=row_smooth_blend,</span></span> <span class="code-line" id="baseline-line-894" data-line="894"><span class="line-number">894</span><span class="line-source"> target_sum=target_sum,</span></span> <span class="code-line" id="baseline-line-895" data-line="895"><span class="line-number">895</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-896" data-line="896"><span class="line-number">896</span><span class="line-source"> eps=eps,</span></span> <span class="code-line" id="baseline-line-897" data-line="897"><span class="line-number">897</span><span class="line-source"> dropout_exponent=dropout_exponent,</span></span> <span class="code-line" id="baseline-line-898" data-line="898"><span class="line-number">898</span><span class="line-source"> diffusion_weight_exponent=diffusion_weight_exponent,</span></span> <span class="code-line" id="baseline-line-899" data-line="899"><span class="line-number">899</span><span class="line-source"> max_t=max_t,</span></span> <span class="code-line" id="baseline-line-900" data-line="900"><span class="line-number">900</span><span class="line-source"> nmf_weight=nmf_weight,</span></span> <span class="code-line" id="baseline-line-901" data-line="901"><span class="line-number">901</span><span class="line-source"> nmf_components=nmf_components,</span></span> <span class="code-line" id="baseline-line-902" data-line="902"><span class="line-number">902</span><span class="line-source"> full_pca_variance=full_pca_variance,</span></span> <span class="code-line" id="baseline-line-903" data-line="903"><span class="line-number">903</span><span class="line-source"> full_pca_blend=full_pca_blend,</span></span> <span class="code-line" id="baseline-line-904" data-line="904"><span class="line-number">904</span><span class="line-source"> full_diffuse_weight=full_diffuse_weight,</span></span> <span class="code-line" id="baseline-line-905" data-line="905"><span class="line-number">905</span><span class="line-source"> raw_mix_weight=raw_mix_weight,</span></span> <span class="code-line" id="baseline-line-906" data-line="906"><span class="line-number">906</span><span class="line-source"> variance_scaling=True,</span></span> <span class="code-line" id="baseline-line-907" data-line="907"><span class="line-number">907</span><span class="line-source"> inverse_anscombe_iters=inverse_anscombe_iters,</span></span> <span class="code-line" id="baseline-line-908" data-line="908"><span class="line-number">908</span><span class="line-source"> verbose=verbose,</span></span> <span class="code-line" id="baseline-line-909" data-line="909"><span class="line-number">909</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-910" data-line="910"><span class="line-number">910</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-911" data-line="911"><span class="line-number">911</span><span class="line-source"> # Variant 2 – lighter, more conservative (helps avoid over‑smoothing)</span></span> <span class="code-line" id="baseline-line-912" data-line="912"><span class="line-number">912</span><span class="line-source"> var2 = _denoise_variant(</span></span> <span class="code-line" id="baseline-line-913" data-line="913"><span class="line-number">913</span><span class="line-source"> X_raw=X,</span></span> <span class="code-line" id="baseline-line-914" data-line="914"><span class="line-number">914</span><span class="line-source"> X_pre=X_pre,</span></span> <span class="code-line" id="baseline-line-915" data-line="915"><span class="line-number">915</span><span class="line-source"> Y_vst=Y_vst,</span></span> <span class="code-line" id="baseline-line-916" data-line="916"><span class="line-number">916</span><span class="line-source"> Y_std=Y_std,</span></span> <span class="code-line" id="baseline-line-917" data-line="917"><span class="line-number">917</span><span class="line-source"> loc_hvg=loc_hvg,</span></span> <span class="code-line" id="baseline-line-918" data-line="918"><span class="line-number">918</span><span class="line-source"> scale_hvg=scale_hvg,</span></span> <span class="code-line" id="baseline-line-919" data-line="919"><span class="line-number">919</span><span class="line-source"> hvgs_mask=hvgs_mask,</span></span> <span class="code-line" id="baseline-line-920" data-line="920"><span class="line-number">920</span><span class="line-source"> diff_op=diff_op,</span></span> <span class="code-line" id="baseline-line-921" data-line="921"><span class="line-number">921</span><span class="line-source"> libsize=libsize,</span></span> <span class="code-line" id="baseline-line-922" data-line="922"><span class="line-number">922</span><span class="line-source"> dropout=dropout,</span></span> <span class="code-line" id="baseline-line-923" data-line="923"><span class="line-number">923</span><span class="line-source"> median_dropout=median_dropout,</span></span> <span class="code-line" id="baseline-line-924" data-line="924"><span class="line-number">924</span><span class="line-source"> row_smooth_blend=row_smooth_blend,</span></span> <span class="code-line" id="baseline-line-925" data-line="925"><span class="line-number">925</span><span class="line-source"> target_sum=target_sum,</span></span> <span class="code-line" id="baseline-line-926" data-line="926"><span class="line-number">926</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-927" data-line="927"><span class="line-number">927</span><span class="line-source"> eps=eps,</span></span> <span class="code-line" id="baseline-line-928" data-line="928"><span class="line-number">928</span><span class="line-source"> dropout_exponent=0.6, # milder dropout weighting</span></span> <span class="code-line" id="baseline-line-929" data-line="929"><span class="line-number">929</span><span class="line-source"> diffusion_weight_exponent=0.70, # more even weighting</span></span> <span class="code-line" id="baseline-line-930" data-line="930"><span class="line-number">930</span><span class="line-source"> max_t=max(1, int(max_t * 0.55)), # shallower diffusion</span></span> <span class="code-line" id="baseline-line-931" data-line="931"><span class="line-number">931</span><span class="line-source"> nmf_weight=min(0.12, nmf_weight + 0.03),</span></span> <span class="code-line" id="baseline-line-932" data-line="932"><span class="line-number">932</span><span class="line-source"> nmf_components=nmf_components,</span></span> <span class="code-line" id="baseline-line-933" data-line="933"><span class="line-number">933</span><span class="line-source"> full_pca_variance=0.94,</span></span> <span class="code-line" id="baseline-line-934" data-line="934"><span class="line-number">934</span><span class="line-source"> full_pca_blend=0.15, # lighter PCA blend</span></span> <span class="code-line" id="baseline-line-935" data-line="935"><span class="line-number">935</span><span class="line-source"> full_diffuse_weight=0.03, # lighter full‑matrix diffusion</span></span> <span class="code-line" id="baseline-line-936" data-line="936"><span class="line-number">936</span><span class="line-source"> raw_mix_weight=0.025, # modest cell‑level raw blend</span></span> <span class="code-line" id="baseline-line-937" data-line="937"><span class="line-number">937</span><span class="line-source"> variance_scaling=False,</span></span> <span class="code-line" id="baseline-line-938" data-line="938"><span class="line-number">938</span><span class="line-source"> inverse_anscombe_iters=max(8, inverse_anscombe_iters - 4),</span></span> <span class="code-line" id="baseline-line-939" data-line="939"><span class="line-number">939</span><span class="line-source"> verbose=verbose,</span></span> <span class="code-line" id="baseline-line-940" data-line="940"><span class="line-number">940</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-941" data-line="941"><span class="line-number">941</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-942" data-line="942"><span class="line-number">942</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-943" data-line="943"><span class="line-number">943</span><span class="line-source"> # 10️⃣ Ensemble (weighted) – emphasise the stronger variant</span></span> <span class="code-line" id="baseline-line-944" data-line="944"><span class="line-number">944</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-945" data-line="945"><span class="line-number">945</span><span class="line-source"> denoised_X = 0.96 * var1 + 0.04 * var2</span></span> <span class="code-line" id="baseline-line-946" data-line="946"><span class="line-number">946</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-947" data-line="947"><span class="line-number">947</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-948" data-line="948"><span class="line-number">948</span><span class="line-source"> # 11️⃣ Gene‑wise raw‑mix (inverse‑dropout &amp; mean‑expression weighting)</span></span> <span class="code-line" id="baseline-line-949" data-line="949"><span class="line-number">949</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-950" data-line="950"><span class="line-number">950</span><span class="line-source"> if gene_raw_weight &gt; 0.0:</span></span> <span class="code-line" id="baseline-line-951" data-line="951"><span class="line-number">951</span><span class="line-source"> denoised_X = _gene_raw_weight_mix(</span></span> <span class="code-line" id="baseline-line-952" data-line="952"><span class="line-number">952</span><span class="line-source"> denoised=denoised_X,</span></span> <span class="code-line" id="baseline-line-953" data-line="953"><span class="line-number">953</span><span class="line-source"> raw_counts=X,</span></span> <span class="code-line" id="baseline-line-954" data-line="954"><span class="line-number">954</span><span class="line-source"> dropout=dropout,</span></span> <span class="code-line" id="baseline-line-955" data-line="955"><span class="line-number">955</span><span class="line-source"> max_raw_weight=gene_raw_weight,</span></span> <span class="code-line" id="baseline-line-956" data-line="956"><span class="line-number">956</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-957" data-line="957"><span class="line-number">957</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-958" data-line="958"><span class="line-number">958</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-959" data-line="959"><span class="line-number">959</span><span class="line-source"> # 12️⃣ Low‑count suppression (helps Poisson loss)</span></span> <span class="code-line" id="baseline-line-960" data-line="960"><span class="line-number">960</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-961" data-line="961"><span class="line-number">961</span><span class="line-source"> denoised_X = _suppress_low_counts(</span></span> <span class="code-line" id="baseline-line-962" data-line="962"><span class="line-number">962</span><span class="line-source"> denoised_X,</span></span> <span class="code-line" id="baseline-line-963" data-line="963"><span class="line-number">963</span><span class="line-source"> low_thresh=low_thresh,</span></span> <span class="code-line" id="baseline-line-964" data-line="964"><span class="line-number">964</span><span class="line-source"> shrink=low_shrink,</span></span> <span class="code-line" id="baseline-line-965" data-line="965"><span class="line-number">965</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-966" data-line="966"><span class="line-number">966</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-967" data-line="967"><span class="line-number">967</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-968" data-line="968"><span class="line-number">968</span><span class="line-source"> # 13️⃣ Gene‑mean matching (reduces systematic bias → lower MSE)</span></span> <span class="code-line" id="baseline-line-969" data-line="969"><span class="line-number">969</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-970" data-line="970"><span class="line-number">970</span><span class="line-source"> if gene_mean_scaling:</span></span> <span class="code-line" id="baseline-line-971" data-line="971"><span class="line-number">971</span><span class="line-source"> min_f, max_f = gene_mean_scaling_range</span></span> <span class="code-line" id="baseline-line-972" data-line="972"><span class="line-number">972</span><span class="line-source"> denoised_X = _scale_gene_means(</span></span> <span class="code-line" id="baseline-line-973" data-line="973"><span class="line-number">973</span><span class="line-source"> denoised=denoised_X,</span></span> <span class="code-line" id="baseline-line-974" data-line="974"><span class="line-number">974</span><span class="line-source"> raw=X,</span></span> <span class="code-line" id="baseline-line-975" data-line="975"><span class="line-number">975</span><span class="line-source"> min_factor=min_f,</span></span> <span class="code-line" id="baseline-line-976" data-line="976"><span class="line-number">976</span><span class="line-source"> max_factor=max_f,</span></span> <span class="code-line" id="baseline-line-977" data-line="977"><span class="line-number">977</span><span class="line-source"> mean_thresh=gene_mean_scaling_thresh,</span></span> <span class="code-line" id="baseline-line-978" data-line="978"><span class="line-number">978</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-979" data-line="979"><span class="line-number">979</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-980" data-line="980"><span class="line-number">980</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-981" data-line="981"><span class="line-number">981</span><span class="line-source"> # 14️⃣ Optional gene‑wise variance matching (extra MSE reduction)</span></span> <span class="code-line" id="baseline-line-982" data-line="982"><span class="line-number">982</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-983" data-line="983"><span class="line-number">983</span><span class="line-source"> if gene_variance_scaling:</span></span> <span class="code-line" id="baseline-line-984" data-line="984"><span class="line-number">984</span><span class="line-source"> denoised_X = _scale_gene_variance_masked(</span></span> <span class="code-line" id="baseline-line-985" data-line="985"><span class="line-number">985</span><span class="line-source"> denoised=denoised_X,</span></span> <span class="code-line" id="baseline-line-986" data-line="986"><span class="line-number">986</span><span class="line-source"> raw=X,</span></span> <span class="code-line" id="baseline-line-987" data-line="987"><span class="line-number">987</span><span class="line-source"> min_factor=0.7,</span></span> <span class="code-line" id="baseline-line-988" data-line="988"><span class="line-number">988</span><span class="line-source"> max_factor=1.3,</span></span> <span class="code-line" id="baseline-line-989" data-line="989"><span class="line-number">989</span><span class="line-source"> mean_thresh=gene_mean_scaling_thresh,</span></span> <span class="code-line" id="baseline-line-990" data-line="990"><span class="line-number">990</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-991" data-line="991"><span class="line-number">991</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-992" data-line="992"><span class="line-number">992</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-993" data-line="993"><span class="line-number">993</span><span class="line-source"> # 15️⃣ Final light centred‑PCA polishing (stronger than baseline)</span></span> <span class="code-line" id="baseline-line-994" data-line="994"><span class="line-number">994</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-995" data-line="995"><span class="line-number">995</span><span class="line-source"> if final_pca_blend &gt; 0.0:</span></span> <span class="code-line" id="baseline-line-996" data-line="996"><span class="line-number">996</span><span class="line-source"> pca_refined = _full_pca_denoise_centered(</span></span> <span class="code-line" id="baseline-line-997" data-line="997"><span class="line-number">997</span><span class="line-source"> denoised_X,</span></span> <span class="code-line" id="baseline-line-998" data-line="998"><span class="line-number">998</span><span class="line-source"> target_sum=target_sum,</span></span> <span class="code-line" id="baseline-line-999" data-line="999"><span class="line-number">999</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-1000" data-line="1000"><span class="line-number">1000</span><span class="line-source"> n_components=None,</span></span> <span class="code-line" id="baseline-line-1001" data-line="1001"><span class="line-number">1001</span><span class="line-source"> variance=final_pca_variance,</span></span> <span class="code-line" id="baseline-line-1002" data-line="1002"><span class="line-number">1002</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-1003" data-line="1003"><span class="line-number">1003</span><span class="line-source"> denoised_X = final_pca_blend * pca_refined + (1.0 - final_pca_blend) * denoised_X</span></span> <span class="code-line" id="baseline-line-1004" data-line="1004"><span class="line-number">1004</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-1005" data-line="1005"><span class="line-number">1005</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-1006" data-line="1006"><span class="line-number">1006</span><span class="line-source"> # 16️⃣ Optional final log‑space k‑NN smoothing + residual correction</span></span> <span class="code-line" id="baseline-line-1007" data-line="1007"><span class="line-number">1007</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-1008" data-line="1008"><span class="line-number">1008</span><span class="line-source"> if final_log_knn and n_cells &lt;= 25000:</span></span> <span class="code-line" id="baseline-line-1009" data-line="1009"><span class="line-number">1009</span><span class="line-source"> denoised_X = _final_log_knn_smooth_with_residual(</span></span> <span class="code-line" id="baseline-line-1010" data-line="1010"><span class="line-number">1010</span><span class="line-source"> counts=denoised_X,</span></span> <span class="code-line" id="baseline-line-1011" data-line="1011"><span class="line-number">1011</span><span class="line-source"> raw_counts=X,</span></span> <span class="code-line" id="baseline-line-1012" data-line="1012"><span class="line-number">1012</span><span class="line-source"> target_sum=target_sum,</span></span> <span class="code-line" id="baseline-line-1013" data-line="1013"><span class="line-number">1013</span><span class="line-source"> knn=final_log_knn_knn,</span></span> <span class="code-line" id="baseline-line-1014" data-line="1014"><span class="line-number">1014</span><span class="line-source"> weight=final_log_knn_weight,</span></span> <span class="code-line" id="baseline-line-1015" data-line="1015"><span class="line-number">1015</span><span class="line-source"> residual_weight=residual_correction_weight,</span></span> <span class="code-line" id="baseline-line-1016" data-line="1016"><span class="line-number">1016</span><span class="line-source"> n_pca=50,</span></span> <span class="code-line" id="baseline-line-1017" data-line="1017"><span class="line-number">1017</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-1018" data-line="1018"><span class="line-number">1018</span><span class="line-source"> n_jobs=n_jobs,</span></span> <span class="code-line" id="baseline-line-1019" data-line="1019"><span class="line-number">1019</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-1020" data-line="1020"><span class="line-number">1020</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-1021" data-line="1021"><span class="line-number">1021</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-1022" data-line="1022"><span class="line-number">1022</span><span class="line-source"> # 17️⃣ Light ALRA‑style low‑rank refinement (significant MSE gain)</span></span> <span class="code-line" id="baseline-line-1023" data-line="1023"><span class="line-number">1023</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-1024" data-line="1024"><span class="line-number">1024</span><span class="line-source"> if final_alra:</span></span> <span class="code-line" id="baseline-line-1025" data-line="1025"><span class="line-number">1025</span><span class="line-source"> denoised_X = _low_rank_refine_counts(</span></span> <span class="code-line" id="baseline-line-1026" data-line="1026"><span class="line-number">1026</span><span class="line-source"> denoised_X,</span></span> <span class="code-line" id="baseline-line-1027" data-line="1027"><span class="line-number">1027</span><span class="line-source"> target_sum=target_sum,</span></span> <span class="code-line" id="baseline-line-1028" data-line="1028"><span class="line-number">1028</span><span class="line-source"> variance=final_alra_variance,</span></span> <span class="code-line" id="baseline-line-1029" data-line="1029"><span class="line-number">1029</span><span class="line-source"> blend=final_alra_blend,</span></span> <span class="code-line" id="baseline-line-1030" data-line="1030"><span class="line-number">1030</span><span class="line-source"> random_state=random_state,</span></span> <span class="code-line" id="baseline-line-1031" data-line="1031"><span class="line-number">1031</span><span class="line-source"> )</span></span> <span class="code-line" id="baseline-line-1032" data-line="1032"><span class="line-number">1032</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-1033" data-line="1033"><span class="line-number">1033</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-1034" data-line="1034"><span class="line-number">1034</span><span class="line-source"> # 18️⃣ Final sanity check – non‑negativity &amp; dtype</span></span> <span class="code-line" id="baseline-line-1035" data-line="1035"><span class="line-number">1035</span><span class="line-source"> # ------------------------------------------------------------------</span></span> <span class="code-line" id="baseline-line-1036" data-line="1036"><span class="line-number">1036</span><span class="line-source"> denoised_X = np.maximum(denoised_X, 0.0).astype(np.float32)</span></span> <span class="code-line" id="baseline-line-1037" data-line="1037"><span class="line-number">1037</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-1038" data-line="1038"><span class="line-number">1038</span><span class="line-source"> if verbose:</span></span> <span class="code-line" id="baseline-line-1039" data-line="1039"><span class="line-number">1039</span><span class="line-source"> tot = denoised_X.sum()</span></span> <span class="code-line" id="baseline-line-1040" data-line="1040"><span class="line-number">1040</span><span class="line-source"> nz = np.count_nonzero(denoised_X)</span></span> <span class="code-line" id="baseline-line-1041" data-line="1041"><span class="line-number">1041</span><span class="line-source"> print(f&quot;[magic_denoise] finished – total={tot:.2e}, non‑zeros={nz}/{denoised_X.size}&quot;)</span></span> <span class="code-line" id="baseline-line-1042" data-line="1042"><span class="line-number">1042</span><span class="line-source">&nbsp;</span></span> <span class="code-line" id="baseline-line-1043" data-line="1043"><span class="line-number">1043</span><span class="line-source"> return denoised_X</span></span></code></pre> </div> </article> </div> </section> <footer class="footer"> 数据来源:Single-anchor sampler step 25 的最小 MSE 节点;Baseline sampler 的 timestep 49 最小 MSE 节点。代码长度不包含 Markdown code fence。 </footer> </main> <script> (() => { const singleSource = document.getElementById('single-source'); const singleMeta = document.getElementById('single-meta'); const singleFull = document.getElementById('single-code-full'); const singleActive = document.getElementById('single-code-active'); const singleFunctionsFull = document.getElementById('single-functions-full'); const singleFunctionsActive = document.getElementById('single-functions-active'); const singleViewport = document.getElementById('single-viewport'); function activeSingleBlock() { return singleSource.value === 'active' ? singleActive : singleFull; } function activeSingleFunctions() { return singleSource.value === 'active' ? singleFunctionsActive : singleFunctionsFull; } function clearTargets(block) { block.querySelectorAll('.code-line.is-target').forEach(line => line.classList.remove('is-target')); } function jumpToLine(block, viewport, lineNumber) { clearTargets(block); const line = block.querySelector('.code-line[data-line="' + lineNumber + '"]'); if (!line) return; line.classList.add('is-target'); viewport.scrollTo({ top: Math.max(0, line.offsetTop - 36), behavior: 'smooth' }); window.setTimeout(() => line.classList.remove('is-target'), 1800); } function searchBlock(input, blockGetter, viewport, status) { const block = blockGetter(); block.querySelectorAll('.code-line.is-match').forEach(line => line.classList.remove('is-match')); const query = input.value.trim().toLocaleLowerCase(); if (!query) { status.textContent = '未搜索'; return; } const matches = [...block.querySelectorAll('.code-line')].filter(line => line.querySelector('.line-source').textContent.toLocaleLowerCase().includes(query) ); matches.forEach(line => line.classList.add('is-match')); status.textContent = matches.length ? '找到 ' + matches.length + ' 行' : '没有匹配行'; if (matches.length) { viewport.scrollTo({ top: Math.max(0, matches[0].offsetTop - 36), behavior: 'smooth' }); } } singleSource.addEventListener('change', () => { const active = singleSource.value === 'active'; singleFull.classList.toggle('hidden', active); singleActive.classList.toggle('hidden', !active); singleFunctionsFull.classList.toggle('hidden', active); singleFunctionsActive.classList.toggle('hidden', !active); singleMeta.textContent = active ? '仅学习部分 · 150 行' : '完整执行版 · 230 行'; singleViewport.scrollTop = 0; document.getElementById('single-search').value = ''; document.getElementById('single-status').textContent = '未搜索'; }); document.getElementById('single-search').addEventListener('input', event => { searchBlock( event.target, activeSingleBlock, singleViewport, document.getElementById('single-status') ); }); singleFunctionsFull.addEventListener('change', event => { jumpToLine(singleFull, singleViewport, event.target.value); }); singleFunctionsActive.addEventListener('change', event => { jumpToLine(singleActive, singleViewport, event.target.value); }); const baseline = document.getElementById('baseline-code'); const baselineViewport = document.getElementById('baseline-viewport'); document.getElementById('baseline-search').addEventListener('input', event => { searchBlock( event.target, () => baseline, baselineViewport, document.getElementById('baseline-status') ); }); document.getElementById('baseline-functions').addEventListener('change', event => { jumpToLine(baseline, baselineViewport, event.target.value); }); document.getElementById('baseline-top').addEventListener('click', () => { baselineViewport.scrollTo({ top: 0, behavior: 'smooth' }); }); })(); </script> </body> </html>
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Denoising 最终代码对照:Single-anchor vs Baseline 50-step</title>
<style>
:root {
color-scheme: light dark;
--bg: #f5f7fb;
--surface: #ffffff;
--surface-soft: #f8fafc;
--text: #172033;
--muted: #64748b;
--border: #dbe3ee;
--single: #1769aa;
--single-soft: #e8f3fc;
--baseline: #a55716;
--baseline-soft: #fff2e5;
--good: #16734a;
--good-soft: #e8f7ef;
--warn: #9a5b08;
--warn-soft: #fff6dd;
--code-bg: #101827;
--code-text: #dce7f5;
--code-muted: #718096;
--code-hit: #5b4310;
--shadow: 0 10px 32px rgba(23, 32, 51, 0.08);
}
@media (prefers-color-scheme: dark) {
:root {
--bg: #0c111b;
--surface: #131b29;
--surface-soft: #172131;
--text: #e7edf6;
--muted: #9eacc0;
--border: #2c394d;
--single: #72b7ed;
--single-soft: #152f43;
--baseline: #f0ad6f;
--baseline-soft: #3c291b;
--good: #69d7a4;
--good-soft: #153629;
--warn: #f4c469;
--warn-soft: #3b3019;
--code-bg: #090e17;
--code-text: #dce7f5;
--code-muted: #6f8199;
--code-hit: #5c4512;
--shadow: 0 12px 34px rgba(0, 0, 0, 0.28);
}
}
* { box-sizing: border-box; }
html { scroll-behavior: smooth; }
body {
margin: 0;
background: var(--bg);
color: var(--text);
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont,
"Segoe UI", "Microsoft YaHei", sans-serif;
line-height: 1.65;
}
button, input, select { font: inherit; }
a { color: var(--single); }
.page {
width: min(1500px, calc(100% - 32px));
margin: 0 auto;
padding: 36px 0 72px;
}
.hero {
display: grid;
grid-template-columns: minmax(0, 1.45fr) minmax(320px, 0.55fr);
gap: 24px;
align-items: end;
margin-bottom: 24px;
}
.eyebrow {
margin: 0 0 8px;
color: var(--muted);
font-size: 13px;
letter-spacing: 0.08em;
text-transform: uppercase;
}
h1, h2, h3, p { margin-top: 0; }
h1 {
margin-bottom: 10px;
font-size: clamp(28px, 4vw, 48px);
line-height: 1.14;
letter-spacing: -0.035em;

Xet Storage Details

Size:
298 kB
·
Xet hash:
15f3ffee30ac7927d0396ab535ac92fa8e2e70fbbd39c4ac2e4e8b0d094ea9b3

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.