suvadityamuk HF Staff
Refine authors and article layout; visualize storage with boxed emoji units
88a8c96 verified | """One-time PDF transcription. Only the recorded walkthrough deletions are permitted.""" | |
| import pathlib,re,json,hashlib | |
| ROOT=pathlib.Path(__file__).resolve().parents[1] | |
| raw=(ROOT/'docs/original-pdf-extraction.txt').read_text() | |
| (ROOT/'docs/original-pdf-extraction.txt').write_text(raw) | |
| s=raw.replace('\f','').strip() | |
| headings=[('Introduction','introduction'),('Why not an unified format?','why-not-one-format'),('Mesh','mesh'),('Point Clouds','point-clouds'),('Voxels','voxels'),('NeRF','nerf'),('3D Gaussian Splats','gaussian-splats'),('What about 4D?','four-dimensions'),('Animated/Deformed Mesh','animated-mesh'),('4D NeRF','4d-nerf'),('4D Gaussian Splats','4d-gaussian'),('OpenUSD, a Special Mention','openusd'),('Looking forward','looking-forward')] | |
| removed=[] | |
| for title,next_title in [('Making your first 3D Mesh object','Point Clouds'),('Making your first 3D Point Cloud','Voxels'),('Making your first 3D Voxel object','NeRF'),('Making your first 3D NeRF','3D Gaussian Splats'),('Making your first 3D Gaussian Splat','What about 4D?')]: | |
| pat=r'(?m)^\s*'+re.escape(title)+r'\s*\n[\s\S]*?(?=^\s*'+re.escape(next_title)+r'\s*$)' | |
| m=re.search(pat,s);assert m,title | |
| removed.append({'heading':title,'removedText':m.group().strip()});s=s[:m.start()]+s[m.end():] | |
| (ROOT/'docs/editorial-removals.json').write_text(json.dumps(removed,ensure_ascii=False,indent=2)+'\n') | |
| # Locate exact heading lines; table rows are deliberately not eligible. | |
| locations=[] | |
| for title,slug in headings: | |
| m=re.search(r'(?m)^\s*'+re.escape(title)+r'\s*$',s);assert m,title | |
| locations.append((m.start(),m.end(),title,slug)) | |
| locations.sort() | |
| vids={'MeshScene.mp4':'mesh-lab','PointCloudScene.mp4':'points-lab','VoxelScene.mp4':'voxels-lab','NerfScene.mp4':'nerf-lab','GaussianSplatScene.mp4':'splats-lab','AnimatedMeshScene.mp4':'motion-lab','FourDNerfScene.mp4':'time-field-lab','FourDGaussianSplatScene.mp4':'time-splats-lab'} | |
| extras={'mesh':['attributes'],'voxels':['storage','sparse']} | |
| meta=[];expected=[] | |
| def paragraph_blocks(body): | |
| blocks=[];buf=[];kind='p' | |
| def flush(): | |
| nonlocal buf | |
| if buf:blocks.append((kind,' '.join(' '.join(buf).split())));buf=[] | |
| for line in body.splitlines(): | |
| text=line.strip() | |
| if not text:flush();continue | |
| if text in vids or text.startswith('Insert image:') or text=='[!TIP]': | |
| flush();blocks.append(('media' if text in vids else 'xkcd' if text.startswith('Insert image:') else 'tip',text));kind='p';continue | |
| if re.match(r'^-\s',text):flush();kind='li';buf=[re.sub(r'^-\s+','',text)];continue | |
| if kind=='li' and len(line)-len(line.lstrip())==0:flush();kind='p' | |
| buf.append(text) | |
| flush() | |
| joined=[] | |
| for k,t in blocks: | |
| if k=='p' and joined and joined[-1][0]=='p' and joined[-1][1].endswith(',') and t[:1].islower():joined[-1]=('p',joined[-1][1]+' '+t) | |
| else:joined.append((k,t)) | |
| return joined | |
| for i,(start,end,title,slug) in enumerate(locations): | |
| body=s[end:locations[i+1][0] if i+1<len(locations) else len(s)].strip() | |
| table=None | |
| if slug=='why-not-one-format': | |
| body,table=body.split('Types of 3D Formats',1) | |
| blocks=paragraph_blocks(body) | |
| lines=['import Note from "../../components/Note.astro";','import Figure from "../../components/Figure.astro";','import Support from "../../components/Support.astro";','',f'<section class="chapter" aria-labelledby="{slug}">',f'<h2 id="{slug}" data-prose>{title}</h2>',''] | |
| expected.append(title);inlist=False | |
| for kind,text in blocks: | |
| if inlist and kind!='li':lines+=['</ul>'];inlist=False | |
| if kind in ['p','li']: | |
| expected.append(text) | |
| # MDX braces escaped as HTML entities; existing LaTeX symbols are rendered by KaTeX. | |
| safe=text.replace('&','&').replace('<','<').replace('>','>').replace('{','{').replace('}','}') | |
| safe=safe.replace('\\theta',r'$\theta$').replace('\\phi',r'$\phi$').replace('2^10',r'$2^{10}$') | |
| if kind=='li' and not inlist:lines+=['<ul>'];inlist=True | |
| lines.append(f'<{kind if kind=="li" else "p"} data-prose>{safe}</{kind if kind=="li" else "p"}>') | |
| if len(lines)>1 and lines[-2]=='<Note title="Note" variant="info">':lines.append('</Note>') | |
| elif kind=='media':lines.append('<Figure id="'+vids[text]+'" />') | |
| elif kind=='xkcd':lines.append('<figure class="comic"><a href="https://xkcd.com/927/"><img src="https://imgs.xkcd.com/comics/standards.png" alt="A proposal to unify fourteen competing standards results in fifteen competing standards." loading="lazy" width="500" height="283" /></a><figcaption><a href="https://xkcd.com/927/">Standards, xkcd #927</a> by Randall Munroe. <a href="https://creativecommons.org/licenses/by-nc/2.5/">CC BY-NC 2.5</a>.</figcaption></figure>') | |
| elif kind=='tip':lines.append('<Note title="Note" variant="info">') | |
| if inlist:lines+=['</ul>'] | |
| if table: | |
| rows=[['Type','Data Formats','Data Representation Style','Specialty','What is it used for?'],['Mesh','GLB, PLY, OBJ, STL','Polygons','Highly-scalable, malleable','Games, Simulation'],['Point Clouds','PLY, PCD','Points','Easy to store','LIDAR Scans'],['Voxels','OpenVDB, NPY/NPZ (numpy)','Unit Blocks','Easy to interpret','Medical Imaging'],['NeRFs','No standard format','Learned Neural Network','Novel view synthesis','Robotics, VFX'],['3D Gaussian Splats','PLY, SPLAT','3D Gaussians','Captures accurate scene lighting','VFX, Digital Twins']] | |
| expected+=['Types of 3D Formats']+sum(rows,[]) | |
| lines+=['<div class="table-wrap" id="formats-table"><table><caption data-prose>Types of 3D Formats</caption><thead><tr>'+''.join('<th data-prose>'+x+'</th>' for x in rows[0])+'</tr></thead><tbody>'] | |
| lines+=['<tr>'+''.join('<td data-prose>'+x+'</td>' for x in row)+'</tr>' for row in rows[1:]];lines+=['</tbody></table></div>'] | |
| for extra in extras.get(slug,[]):lines.append('<Support id="'+extra+'" />') | |
| lines+=['</section>',''] | |
| article='\n\n'.join(lines) | |
| if table: | |
| a=article.index('<div class="table-wrap"');b=article.index('</table></div>',a)+len('</table></div>');article=article[:a]+article[a:b].replace('\n','')+article[b:] | |
| (ROOT/f'src/content/chapters/{slug}.mdx').write_text(article) | |
| meta.append({'id':slug,'title':title}) | |
| (ROOT/'src/content/chapters.json').write_text(json.dumps(meta,ensure_ascii=False,indent=2)) | |
| (ROOT/'docs/prose-baseline.json').write_text(json.dumps(expected,ensure_ascii=False,indent=2)+'\n') | |
| (ROOT/'docs/source-integrity.json').write_text(json.dumps({'sourcePdf':"A Hitchhiker's Guide to 3D Representations.pdf",'extractionSha256':hashlib.sha256(raw.encode()).hexdigest(),'allowedRemovedWalkthroughs':len(removed),'proseBlocks':len(expected)},indent=2)) | |
| print('Transcribed',len(meta),'chapters;',len(expected),'prose blocks;',len(removed),'documented removals') | |