scripts/import-article.py

"""One-time PDF transcription. Only the recorded walkthrough deletions are permitted."""
import pathlib,re,json,hashlib
ROOT=pathlib.Path(__file__).resolve().parents[1]
raw=(ROOT/'docs/original-pdf-extraction.txt').read_text()
(ROOT/'docs/original-pdf-extraction.txt').write_text(raw)
s=raw.replace('\f','').strip()
headings=[('Introduction','introduction'),('Why not an unified format?','why-not-one-format'),('Mesh','mesh'),('Point Clouds','point-clouds'),('Voxels','voxels'),('NeRF','nerf'),('3D Gaussian Splats','gaussian-splats'),('What about 4D?','four-dimensions'),('Animated/Deformed Mesh','animated-mesh'),('4D NeRF','4d-nerf'),('4D Gaussian Splats','4d-gaussian'),('OpenUSD, a Special Mention','openusd'),('Looking forward','looking-forward')]
removed=[]
for title,next_title in [('Making your first 3D Mesh object','Point Clouds'),('Making your first 3D Point Cloud','Voxels'),('Making your first 3D Voxel object','NeRF'),('Making your first 3D NeRF','3D Gaussian Splats'),('Making your first 3D Gaussian Splat','What about 4D?')]:
 pat=r'(?m)^\s*'+re.escape(title)+r'\s*\n[\s\S]*?(?=^\s*'+re.escape(next_title)+r'\s*$)'
 m=re.search(pat,s);assert m,title
 removed.append({'heading':title,'removedText':m.group().strip()});s=s[:m.start()]+s[m.end():]
(ROOT/'docs/editorial-removals.json').write_text(json.dumps(removed,ensure_ascii=False,indent=2)+'\n')
# Locate exact heading lines; table rows are deliberately not eligible.
locations=[]
for title,slug in headings:
 m=re.search(r'(?m)^\s*'+re.escape(title)+r'\s*$',s);assert m,title
 locations.append((m.start(),m.end(),title,slug))
locations.sort()
vids={'MeshScene.mp4':'mesh-lab','PointCloudScene.mp4':'points-lab','VoxelScene.mp4':'voxels-lab','NerfScene.mp4':'nerf-lab','GaussianSplatScene.mp4':'splats-lab','AnimatedMeshScene.mp4':'motion-lab','FourDNerfScene.mp4':'time-field-lab','FourDGaussianSplatScene.mp4':'time-splats-lab'}
extras={'mesh':['attributes'],'voxels':['storage','sparse']}
meta=[];expected=[]
def paragraph_blocks(body):
 blocks=[];buf=[];kind='p'
 def flush():
  nonlocal buf
  if buf:blocks.append((kind,' '.join(' '.join(buf).split())));buf=[]
 for line in body.splitlines():
  text=line.strip()
  if not text:flush();continue
  if text in vids or text.startswith('Insert image:') or text=='[!TIP]':
   flush();blocks.append(('media' if text in vids else 'xkcd' if text.startswith('Insert image:') else 'tip',text));kind='p';continue
  if re.match(r'^-\s',text):flush();kind='li';buf=[re.sub(r'^-\s+','',text)];continue
  if kind=='li' and len(line)-len(line.lstrip())==0:flush();kind='p'
  buf.append(text)
 flush()
 joined=[]
 for k,t in blocks:
  if k=='p' and joined and joined[-1][0]=='p' and joined[-1][1].endswith(',') and t[:1].islower():joined[-1]=('p',joined[-1][1]+' '+t)
  else:joined.append((k,t))
 return joined
for i,(start,end,title,slug) in enumerate(locations):
 body=s[end:locations[i+1][0] if i+1<len(locations) else len(s)].strip()
 table=None
 if slug=='why-not-one-format':
  body,table=body.split('Types of 3D Formats',1)
 blocks=paragraph_blocks(body)
 lines=['import Note from "../../components/Note.astro";','import Figure from "../../components/Figure.astro";','import Support from "../../components/Support.astro";','',f'<section class="chapter" aria-labelledby="{slug}">',f'<h2 id="{slug}" data-prose>{title}</h2>','']
 expected.append(title);inlist=False
 for kind,text in blocks:
  if inlist and kind!='li':lines+=['</ul>'];inlist=False
  if kind in ['p','li']:
   expected.append(text)
   # MDX braces escaped as HTML entities; existing LaTeX symbols are rendered by KaTeX.
   safe=text.replace('&','&amp;').replace('<','&lt;').replace('>','&gt;').replace('{','&#123;').replace('}','&#125;')
   safe=safe.replace('\\theta',r'$\theta$').replace('\\phi',r'$\phi$').replace('2^10',r'$2^{10}$')
   if kind=='li' and not inlist:lines+=['<ul>'];inlist=True
   lines.append(f'<{kind if kind=="li" else "p"} data-prose>{safe}</{kind if kind=="li" else "p"}>')
   if len(lines)>1 and lines[-2]=='<Note title="Note" variant="info">':lines.append('</Note>')
  elif kind=='media':lines.append('<Figure id="'+vids[text]+'" />')
  elif kind=='xkcd':lines.append('<figure class="comic"><a href="https://xkcd.com/927/"><img src="https://imgs.xkcd.com/comics/standards.png" alt="A proposal to unify fourteen competing standards results in fifteen competing standards." loading="lazy" width="500" height="283" /></a><figcaption><a href="https://xkcd.com/927/">Standards, xkcd #927</a> by Randall Munroe. <a href="https://creativecommons.org/licenses/by-nc/2.5/">CC BY-NC 2.5</a>.</figcaption></figure>')
  elif kind=='tip':lines.append('<Note title="Note" variant="info">')
 if inlist:lines+=['</ul>']
 if table:
  rows=[['Type','Data Formats','Data Representation Style','Specialty','What is it used for?'],['Mesh','GLB, PLY, OBJ, STL','Polygons','Highly-scalable, malleable','Games, Simulation'],['Point Clouds','PLY, PCD','Points','Easy to store','LIDAR Scans'],['Voxels','OpenVDB, NPY/NPZ (numpy)','Unit Blocks','Easy to interpret','Medical Imaging'],['NeRFs','No standard format','Learned Neural Network','Novel view synthesis','Robotics, VFX'],['3D Gaussian Splats','PLY, SPLAT','3D Gaussians','Captures accurate scene lighting','VFX, Digital Twins']]
  expected+=['Types of 3D Formats']+sum(rows,[])
  lines+=['<div class="table-wrap" id="formats-table"><table><caption data-prose>Types of 3D Formats</caption><thead><tr>'+''.join('<th data-prose>'+x+'</th>' for x in rows[0])+'</tr></thead><tbody>']
  lines+=['<tr>'+''.join('<td data-prose>'+x+'</td>' for x in row)+'</tr>' for row in rows[1:]];lines+=['</tbody></table></div>']
 for extra in extras.get(slug,[]):lines.append('<Support id="'+extra+'" />')
 lines+=['</section>','']
 article='\n\n'.join(lines)
 if table:
  a=article.index('<div class="table-wrap"');b=article.index('</table></div>',a)+len('</table></div>');article=article[:a]+article[a:b].replace('\n','')+article[b:]
 (ROOT/f'src/content/chapters/{slug}.mdx').write_text(article)
 meta.append({'id':slug,'title':title})
(ROOT/'src/content/chapters.json').write_text(json.dumps(meta,ensure_ascii=False,indent=2))
(ROOT/'docs/prose-baseline.json').write_text(json.dumps(expected,ensure_ascii=False,indent=2)+'\n')
(ROOT/'docs/source-integrity.json').write_text(json.dumps({'sourcePdf':"A Hitchhiker's Guide to 3D Representations.pdf",'extractionSha256':hashlib.sha256(raw.encode()).hexdigest(),'allowedRemovedWalkthroughs':len(removed),'proseBlocks':len(expected)},indent=2))
print('Transcribed',len(meta),'chapters;',len(expected),'prose blocks;',len(removed),'documented removals')