POLS / source /studio /find7.py
Ethscriptions's picture Surf's picture Surf's picture
Surf Studio
Surf Studio
Surf AI
Surf Studio
Surf AI
Surf Studio
Squash history to free LFS storage (1 GB Space limit); current tree kept: sets 5/6 + set 7 manifest/source
8b1c056
Raw History Blame Contribute Delete
3.12 kB
"""Search Wikimedia Commons for CC-licensed photos per item, download thumbs, build contact sheets."""
import io, json, os, re, sys, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor
from PIL import Image, ImageDraw, ImageFont
ITEMS = json.load(open('/tmp/work/studio/items7.json'))
OUT = '/tmp/work/cands'
UA = {'User-Agent': 'LiveWordsStudio/1.0 (https://huggingface.co/spaces/Ethscriptions/pols; educational)'}
def get(url, timeout=60):
return urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout).read()
def strip(h): return re.sub(r'<[^>]+>', '', h or '').strip()
def commons(q):
p = urllib.parse.urlencode({'action':'query','format':'json','generator':'search','gsrsearch':f'{q} filetype:bitmap','gsrnamespace':6,'gsrlimit':30,'prop':'imageinfo','iiprop':'url|size|extmetadata|mime','iiurlwidth':400})
d = json.loads(get('https://commons.wikimedia.org/w/api.php?'+p))
out=[]
for pg in d.get('query',{}).get('pages',{}).values():
ii=(pg.get('imageinfo') or [{}])[0]
if ii.get('mime') not in ('image/jpeg','image/png'): continue
md=ii.get('extmetadata',{})
out.append(dict(id=pg['pageid'], title=pg['title'].replace('File:',''), author=strip(md.get('Artist',{}).get('value'))[:80], license=strip(md.get('LicenseShortName',{}).get('value')), license_url=md.get('LicenseUrl',{}).get('value'), page=ii.get('descriptionurl'), url=ii.get('url'), thumb=ii.get('thumburl'), w=ii.get('width'), h=ii.get('height')))
return out
def ok(l):
l=(l or '').lower()
return any(k in l for k in ('cc0','public domain','cc by','cc-by','pdm')) and '-nc' not in l and ' nc' not in l and '-nd' not in l and ' nd' not in l
want = sys.argv[1].split(',') if len(sys.argv)>1 else None
for it in ITEMS:
if want and it['id'] not in want: continue
cands, seen = [], set()
for q in it['q']:
try: res = commons(q)
except Exception as e: print('err', q, e); continue
for c in res:
if c['id'] in seen or not ok(c['license']) or not c['w'] or min(c['w'],c['h'])<900: continue
seen.add(c['id']); c['q']=q; cands.append(c)
cands = cands[:24]
def fetch(c):
try: return Image.open(io.BytesIO(get(c['thumb'],40))).convert('RGB')
except Exception: return None
with ThreadPoolExecutor(8) as ex: imgs=list(ex.map(fetch,cands))
cols=6; cw,chh=300,300
rows=(len(cands)+cols-1)//cols or 1
sheet=Image.new('RGB',(cols*cw,rows*(chh+30)),'white'); d=ImageDraw.Draw(sheet)
f=ImageFont.truetype('/tmp/work/fonts/NotoSans.ttf',16)
meta=[]
for k,(c,im) in enumerate(zip(cands,imgs)):
if im is None: continue
im.thumbnail((cw-6,chh-6)); x=(k%cols)*cw; y=(k//cols)*(chh+30)
sheet.paste(im,(x+3,y+3)); d.text((x+4,y+chh+2),f"#{k} {c['w']}x{c['h']} {c['license'][:12]}",font=f,fill='black')
meta.append(dict(k=k,**c))
sheet.save(f"{OUT}/{it['id']}_sheet.jpg",quality=80)
json.dump(meta,open(f"{OUT}/{it['id']}_cands.json",'w'),ensure_ascii=False,indent=1)
print(it['id'],it['en'],len(meta),'cands')