File size: 3,122 Bytes
8b1c056
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
"""Search Wikimedia Commons for CC-licensed photos per item, download thumbs, build contact sheets."""
import io, json, os, re, sys, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor
from PIL import Image, ImageDraw, ImageFont
ITEMS = json.load(open('/tmp/work/studio/items5.json'))
OUT = '/tmp/work/cands'
UA = {'User-Agent': 'LiveWordsStudio/1.0 (https://huggingface.co/spaces/Ethscriptions/pols; educational)'}
def get(url, timeout=60):
    return urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout).read()
def strip(h): return re.sub(r'<[^>]+>', '', h or '').strip()
def commons(q):
    p = urllib.parse.urlencode({'action':'query','format':'json','generator':'search','gsrsearch':f'{q} filetype:bitmap','gsrnamespace':6,'gsrlimit':30,'prop':'imageinfo','iiprop':'url|size|extmetadata|mime','iiurlwidth':400})
    d = json.loads(get('https://commons.wikimedia.org/w/api.php?'+p))
    out=[]
    for pg in d.get('query',{}).get('pages',{}).values():
        ii=(pg.get('imageinfo') or [{}])[0]
        if ii.get('mime') not in ('image/jpeg','image/png'): continue
        md=ii.get('extmetadata',{})
        out.append(dict(id=pg['pageid'], title=pg['title'].replace('File:',''), author=strip(md.get('Artist',{}).get('value'))[:80], license=strip(md.get('LicenseShortName',{}).get('value')), license_url=md.get('LicenseUrl',{}).get('value'), page=ii.get('descriptionurl'), url=ii.get('url'), thumb=ii.get('thumburl'), w=ii.get('width'), h=ii.get('height')))
    return out
def ok(l):
    l=(l or '').lower()
    return any(k in l for k in ('cc0','public domain','cc by','cc-by','pdm')) and '-nc' not in l and ' nc' not in l and '-nd' not in l and ' nd' not in l
want = sys.argv[1].split(',') if len(sys.argv)>1 else None
for it in ITEMS:
    if want and it['id'] not in want: continue
    cands, seen = [], set()
    for q in it['q']:
        try: res = commons(q)
        except Exception as e: print('err', q, e); continue
        for c in res:
            if c['id'] in seen or not ok(c['license']) or not c['w'] or min(c['w'],c['h'])<900: continue
            seen.add(c['id']); c['q']=q; cands.append(c)
    cands = cands[:24]
    def fetch(c):
        try: return Image.open(io.BytesIO(get(c['thumb'],40))).convert('RGB')
        except Exception: return None
    with ThreadPoolExecutor(8) as ex: imgs=list(ex.map(fetch,cands))
    cols=6; cw,chh=300,300
    rows=(len(cands)+cols-1)//cols or 1
    sheet=Image.new('RGB',(cols*cw,rows*(chh+30)),'white'); d=ImageDraw.Draw(sheet)
    f=ImageFont.truetype('/tmp/work/fonts/NotoSans.ttf',16)
    meta=[]
    for k,(c,im) in enumerate(zip(cands,imgs)):
        if im is None: continue
        im.thumbnail((cw-6,chh-6)); x=(k%cols)*cw; y=(k//cols)*(chh+30)
        sheet.paste(im,(x+3,y+3)); d.text((x+4,y+chh+2),f"#{k} {c['w']}x{c['h']} {c['license'][:12]}",font=f,fill='black')
        meta.append(dict(k=k,**c))
    sheet.save(f"{OUT}/{it['id']}_sheet.jpg",quality=80)
    json.dump(meta,open(f"{OUT}/{it['id']}_cands.json",'w'),ensure_ascii=False,indent=1)
    print(it['id'],it['en'],len(meta),'cands')