File size: 4,762 Bytes
8ea8b32
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
"""Search Wikimedia Commons (+ cached Openverse candidates) for real photos and rank with CLIP."""
import io, json, os, re, sys, time, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor
from PIL import Image
sys.path.insert(0, '/workspaces/studio')
import clip_score as C

ITEMS = json.load(open('/workspaces/studio/items2.json'))
OUT = '/tmp/work/real'
os.makedirs(OUT, exist_ok=True)
UA = {'User-Agent': 'LiveWordsStudio/1.0 (https://huggingface.co/spaces/Ethscriptions/eths; educational)'}
NEG = ['a drawing or illustration', 'a diagram, map or logo', 'a page of text', 'an isolated product cutout on a plain white background',
       'a black and white old historical photo', 'a museum exhibit in a glass case', 'a sign with writing']


def get(url, timeout=60):
    return urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout).read()


def strip(html):
    return re.sub(r'<[^>]+>', '', html or '').strip()


def commons(q):
    p = urllib.parse.urlencode({'action': 'query', 'format': 'json', 'generator': 'search', 'gsrsearch': f'{q} filetype:bitmap',
                                'gsrnamespace': 6, 'gsrlimit': 40, 'prop': 'imageinfo', 'iiprop': 'url|size|extmetadata|mime',
                                'iiurlwidth': 360})
    d = json.loads(get('https://commons.wikimedia.org/w/api.php?' + p))
    out = []
    for pg in d.get('query', {}).get('pages', {}).values():
        ii = (pg.get('imageinfo') or [{}])[0]
        if ii.get('mime') not in ('image/jpeg', 'image/png', 'image/webp'):
            continue
        md = ii.get('extmetadata', {})
        out.append(dict(id='commons:' + str(pg['pageid']), title=pg['title'].replace('File:', ''), creator=strip(md.get('Artist', {}).get('value')),
                        license=strip(md.get('LicenseShortName', {}).get('value')), license_url=md.get('LicenseUrl', {}).get('value'),
                        landing=ii.get('descriptionurl'), url=ii.get('url'), thumb=ii.get('thumburl'), w=ii.get('width'), h=ii.get('height'),
                        source='wikimedia', usage=strip(md.get('UsageTerms', {}).get('value'))))
    return out


def ok_license(l):
    l = (l or '').lower()
    return any(k in l for k in ('cc0', 'public domain', 'cc by', 'cc-by', 'pdm')) and 'nc' not in l and 'nd' not in l


def thumb_img(c):
    try:
        if c.get('thumb'):
            return Image.open(io.BytesIO(get(c['thumb'], 40)))
        return Image.open(io.BytesIO(get(f"https://api.openverse.org/v1/images/{c['id']}/thumb/", 40)))
    except Exception:
        return None


for it in ITEMS:
    if len(sys.argv) > 1 and it['id'] not in sys.argv[1].split(','):
        continue
    t0 = time.time()
    cands, seen = [], set()
    for q in it['q']:
        try:
            for c in commons(q):
                if c['id'] in seen or not ok_license(c['license']):
                    continue
                if min(c['w'], c['h'] * 3 / 4) < 900:
                    continue
                seen.add(c['id']); cands.append(c)
        except Exception as e:
            print('commons err', q, e)
    prev = f'{OUT}/candidates_{it["id"]}.json'
    if os.path.exists(prev):
        for c in json.load(open(prev)).get(it['id'], []):
            if c.get('source') != 'wikimedia' and c['id'] not in seen:
                seen.add(c['id']); cands.append(c)
    with ThreadPoolExecutor(6) as ex:
        thumbs = list(ex.map(thumb_img, cands))
    pos = C.text_emb([it['clip'], 'a candid everyday photo of ' + it['clip'][11:]])
    neg = C.text_emb(NEG + it.get('neg', []))
    others = C.text_emb(['a photo of ' + o['en'] for o in ITEMS if o['id'] != it['id']])
    scored = []
    for c, th in zip(cands, thumbs):
        if th is None:
            continue
        th = th.convert('RGB')
        tw, tht = th.size
        cw = min(tw, tht * 3 / 4); chh = cw * 4 / 3
        crop = th.crop((int((tw - cw) / 2), int((tht - chh) / 2), int((tw + cw) / 2), int((tht + chh) / 2)))
        e = C.image_emb(crop)
        sp = float((pos @ e).max() * 100); sn = float((neg @ e).max() * 100); so = float((others @ e).max() * 100)
        score = sp - 0.6 * max(0, sn - 21) - 0.5 * max(0, so - sp + 2) + (0.8 if c['h'] / c['w'] >= 1.2 else 0)
        c.update(sp=round(sp, 1), sn=round(sn, 1), so=round(so, 1), score=round(score, 2))
        scored.append(c)
    scored.sort(key=lambda x: -x['score'])
    json.dump(scored[:10], open(f'{OUT}/ranked_{it["id"]}.json', 'w'), ensure_ascii=False, indent=1)
    print(it['id'], it['en'], len(cands), 'cands', round(time.time() - t0), 's', flush=True)
    for s in scored[:5]:
        print('   ', s['score'], s['sp'], s['sn'], s['so'], s['license'], s['w'], s['h'], s['source'], (s['title'] or '')[:55], flush=True)