"""GUI-Owl desktop/Android union; official GUI-Decisions E coordinate labels.
No training entry point. Reuses the frozen JSON137 non-coordinate codec and
GUI-Decisions targets/readout. All 134 slots are masked; predictions align j-1.
Coordinates in history and serialized actions are 0..1000, never pixel units.
"""
from __future__ import annotations
import json
import math
import hashlib
from dataclasses import dataclass
import torch
from torch.nn import functional as F
from slotread.labels import make_arm
from slotread.numbers import number_targets, read_number
from trace2task.gui_owl_parallel137_codec import Parallel137Codec, POINT_ACTIONS, _half_up
from trace2task.gui_owl_short_parallel_input import ParallelInput
SHARED = ('key', 'type', 'wait', 'terminate', 'answer', 'interact')
DESKTOP = SHARED + ('mouse_move', 'left_click', 'left_click_drag', 'right_click',
'middle_click', 'double_click', 'triple_click', 'scroll', 'hscroll')
ANDROID = SHARED + ('click', 'long_press', 'swipe', 'system_button', 'open')
ACTIONS = DESKTOP + tuple(a for a in ANDROID if a not in DESKTOP)
BUTTONS = ('Back', 'Home', 'Menu', 'Enter')
FIELD_WIDTHS = (('action', 3), ('coordinate.x', 1), ('coordinate.y', 1),
('coordinate2.x', 1), ('coordinate2.y', 1), ('keys', 16),
('pixels', 4), ('time', 5), ('status', 1), ('button', 1), ('text', 100))
LITERALS = ('{"action":"', '","coordinate":[', ',',
'],"coordinate2":[', ',', '],"keys":[', '],"pixels":',
',"time":', ',"status":"', '","button":"', '","text":"', '"}')
def build_parallel_input(tokenizer, prefix_ids=()):
ids, positions, fields = list(prefix_ids), [], []
for literal, (name, width) in zip(LITERALS[:-1], FIELD_WIDTHS, strict=True):
ids.extend(tokenizer.encode(literal, add_special_tokens=False).ids)
pos = tuple(range(len(ids), len(ids) + width))
fields.append((name, pos)); positions.extend(pos)
ids.extend([tokenizer.token_to_id('<|fim_pad|>')] * width)
ids.extend(tokenizer.encode(LITERALS[-1], add_special_tokens=False).ids)
assert len(positions) == 134
return ParallelInput(tuple(ids), tuple(positions), tuple(fields), len(prefix_ids))
@dataclass(frozen=True)
class Target:
canonical_action: dict
platform: str
target_ids: tuple
hard_labels: tuple
soft_labels: dict # slot -> 256 probabilities; ids are diagnostic only here
fields: tuple
codebook_sha256: str
class Native134Codec:
variant = 'gui-owl-native134-official-E-v1'
output_slots = 134
def __init__(self, tokenizer):
self.tokenizer = tokenizer
self.base_codec = Parallel137Codec(tokenizer, scroll_coordinates=True)
for name in ('mask_id', 'pad_id', 'eos_id', 'special_ids', 'vocab_size'):
setattr(self, name, getattr(self.base_codec, name))
class Adapter:
def __call__(self, s, **kw):
return {'input_ids': tokenizer.encode(s, add_special_tokens=False).ids}
self.arm = make_arm('E', Adapter())
self.bin_ids = tuple(self.arm.fine)
self.codebook_sha256 = hashlib.sha256(json.dumps(self.bin_ids).encode()).hexdigest()
self.action_encodings = {}
for a in ACTIONS:
ids = self.base_codec._tokenize_exact(a, 'action')
if not 1 <= len(ids) <= 3:
raise ValueError('action_exceeds_three_slots:' + a)
self.action_encodings[a] = tuple(ids + [self.pad_id] * (3-len(ids)))
self.button_ids = {s: self.base_codec._single(s) for s in BUTTONS}
built = build_parallel_input(tokenizer)
self.input_ids, self.slot_positions = built.input_ids, built.slot_positions
self.field_slices = {}
i = 0
for name, width in FIELD_WIDTHS:
self.field_slices[name] = slice(i, i+width); i += width
def adapt_target(self, action, platform='desktop'):
allowed = {'desktop': DESKTOP, 'android': ANDROID}.get(platform)
if allowed is None or action.get('action') not in allowed:
raise ValueError('action_not_allowed_on_platform')
a = action['action']; b = self.base_codec
ids = [self.pad_id] * 134; labels = [-100] * 134
soft, fields, canonical = {}, [], {'action': a}
def hard(name, values, active=None):
span = self.field_slices[name]
if len(values) != span.stop-span.start:
raise ValueError('incorrect_field_width')
ids[span] = values
n = len(values) if active is None else active
labels[span.start:span.start+n] = values[:n]
fields.append((name, span.start, span.start+n))
def old_field(name, surrogate):
old = b.adapt_target(surrogate, {})
values = list(old.target_ids[b.field_slices[name]])
n = sum(old.loss_mask[b.field_slices[name]])
hard(name, values, n)
canonical[name] = old.canonical_action[name]
def point(name):
xy = action.get(name)
if not isinstance(xy, (list, tuple)) or len(xy) != 2:
raise ValueError('coordinate_pair_required')
canonical[name] = [_half_up(v, 1, 1000, 'coordinate_out_of_range') for v in xy]
for axis, value in zip(('x', 'y'), canonical[name]):
field = name + '.' + axis
p = self.field_slices[field].start
q = number_targets('v', value/1000, .005, self.arm)['v']
soft[p] = q
ids[p] = self.bin_ids[int(q.argmax())]
fields.append((field, p, p+1))
hard('action', list(self.action_encodings[a]))
if a in POINT_ACTIONS or a in ('click', 'long_press', 'swipe'):
point('coordinate')
if a == 'swipe': point('coordinate2')
if a in ('scroll', 'hscroll'):
if 'coordinate' in action: point('coordinate')
else:
for axis in ('x', 'y'): hard('coordinate.'+axis, [self.pad_id])
old_field('pixels', {'action': a, 'pixels': action.get('pixels')})
if a == 'key' and platform == 'desktop':
old_field('keys', {'action': 'key', 'keys': action.get('keys')})
if a in ('wait', 'long_press'):
old_field('time', {'action': 'wait', 'time': action.get('time')})
if a == 'terminate':
old_field('status', {'action': a, 'status': action.get('status')})
if a == 'system_button':
if action.get('button') not in self.button_ids: raise ValueError('invalid_system_button')
canonical['button'] = action['button']
hard('button', [self.button_ids[action['button']]])
if a in ('type', 'answer', 'interact', 'open') or (a == 'key' and platform == 'android'):
old_field('text', {'action': 'type', 'text': action.get('text')})
if set(action) != set(canonical):
raise ValueError('unexpected_or_missing_parameters:' + ','.join(sorted(set(action)^set(canonical))))
return Target(canonical, platform, tuple(ids), tuple(labels), soft, tuple(fields), self.codebook_sha256)
def codebook(self):
return [dict(bin=i, token_id=t, decoded=self.tokenizer.decode([t], skip_special_tokens=False),
normalized_center=(i+.5)/256) for i,t in enumerate(self.bin_ids)]
def decode(self, logits, platform, key_sequences=()):
"""Constrained readout of ONE forward's [134,vocab] logits. No model call."""
if logits.ndim != 2 or logits.shape[0] != 134 or logits.shape[1] < self.vocab_size:
raise ValueError('expected_134_by_vocab_logits')
if not torch.isfinite(logits).all(): raise ValueError('nonfinite_logits')
# Some model heads pad vocabulary rows beyond the tokenizer inventory.
# Such rows cannot represent valid text tokens during constrained decode.
v = logits[:, :self.vocab_size].float(); b = self.base_codec
allowed = {'desktop': DESKTOP, 'android': ANDROID}[platform]
def choose(name, codes):
span = self.field_slices[name]
scores = [sum(v[p,t] for p,t in zip(range(span.start,span.stop), code)) for code in codes]
return int(torch.stack(scores).argmax())
a = allowed[choose('action', [self.action_encodings[x] for x in allowed])]
result = {'action': a}
def point(name):
xy=[]
for axis in ('x','y'):
p=self.field_slices[name+'.'+axis].start
value=read_number({'v':v[p,list(self.bin_ids)]}, 'v', self.arm, how='local2')*1000
xy.append(_half_up(float(value),1,1000,'decoded_coordinate_out_of_range'))
result[name]=xy
if a in POINT_ACTIONS or a in ('click','long_press','swipe'): point('coordinate')
if a=='swipe': point('coordinate2')
if a in ('scroll','hscroll'):
present=sum(torch.logsumexp(v[p,list(self.bin_ids)],0) for p in (3,4))
if present > v[3,self.pad_id]+v[4,self.pad_id]: point('coordinate')
s=self.field_slices['pixels']; sign=('-', '+')[int(v[s.start,b.sign_ids['+']] > v[s.start,b.sign_ids['-']])]
n=int(''.join(str(int(v[p,list(b.digit_ids)].argmax())) for p in range(s.start+1,s.stop)))
result['pixels']= -n if sign=='-' else n
if a in ('wait','long_press'):
s=self.field_slices['time']
result['time']=int(''.join(str(int(v[p,list(b.digit_ids)].argmax())) for p in range(s.start,s.stop)))/100
if a=='terminate':
values=list(b.status_ids); result['status']=values[choose('status',[(b.status_ids[x],) for x in values])]
if a=='system_button': result['button']=BUTTONS[choose('button',[(self.button_ids[x],) for x in BUTTONS])]
if a=='key' and platform=='desktop':
if not key_sequences: raise ValueError('desktop_key_catalogue_required')
codes=[self.adapt_target({'action':'key','keys':list(k)}, platform).target_ids[7:23] for k in key_sequences]
result['keys']=list(key_sequences[choose('keys',codes)])
if a in ('type','answer','interact','open') or (a=='key' and platform=='android'):
scores=v[34:134].clone()
scores[:,list(self.special_ids-{self.eos_id})]=-torch.inf
tokens=scores.argmax(-1).tolist()
if self.eos_id not in tokens: raise ValueError('text_missing_eos_no_silent_truncation')
text=self.tokenizer.decode(tokens[:tokens.index(self.eos_id)], skip_special_tokens=False)
if '\ufffd' in text: raise ValueError('invalid_text_encoding')
result['text']=text
return self.adapt_target(result,platform).canonical_action
def native134_loss(logits, targets, codec):
"""Full-vocabulary soft CE for coordinates; hard CE for active other fields."""
if logits.shape[:2] != (len(targets),134): raise ValueError('loss_shape_mismatch')
losses=[]
for row,t in enumerate(targets):
if t.codebook_sha256 != codec.codebook_sha256: raise ValueError('wrong_codebook')
fields=[]
for name,start,stop in t.fields:
if start in t.soft_labels:
lp=F.log_softmax(logits[row,start].float(),-1)[list(codec.bin_ids)]
fields.append(-(lp*t.soft_labels[start].to(lp)).sum())
else:
labels=torch.tensor(t.hard_labels[start:stop],device=logits.device)
fields.append(F.cross_entropy(logits[row,start:stop].float(),labels))
losses.append(fields[0]+2*(torch.stack(fields[1:]).mean() if len(fields)>1 else fields[0]*0))
return torch.stack(losses)
def serialize(action):
return ''+json.dumps(action,ensure_ascii=False,separators=(',',':'))+''
def messages(row):
"""Only task/history/images enter input. Available historical images retained."""
def clean(s): return s.replace('<|','[special:').replace('','[image]')
content=[{'type':'text','text':'Platform: '+row['platform']+'\nTask: '+clean(row['task'])}]
for step in row['history']:
if step.get('image'): content.append({'type':'image','image':step['image']['path']})
text='Previous step '+str(step['step'])+'\n'
if step.get('observation'): text+='Past observation: '+clean(step['observation'])+'\n'
text+='Executed actions: '+' '.join(serialize(a) for a in step['actions'])
content.append({'type':'text','text':clean(text)})
content += [{'type':'text','text':'Current screenshot:'},
{'type':'image','image':row['images'][-1]['path']}]
system=('Choose one action from task, screenshots and history. All history and output coordinates '
'are normalized integers 0..1000. Internal coordinate slots use 256 bins. Predict the masked '
'134-slot table in parallel, including text. Only applicable fields are executed. '
'Desktop actions: '+','.join(DESKTOP)+'. Android actions: '+','.join(ANDROID)+'. '
'Desktop key uses keys; Android key uses text. Swipe uses coordinate and coordinate2. '
'Long_press uses coordinate and time in seconds. System_button: Back,Home,Menu,Enter. '
'Desktop left_click_drag starts at the current pointer. Scroll pixels preserves wheel units.')
return [{'role':'system','content':system},{'role':'user','content':content}]
def prepare_example(row, processor, codec, image_loader):
"""Real processor input ready for existing prepare_parallel_inputs.
Return labels separately: they must never be passed into the model forward.
image_loader is injected to resolve mounted/local paths without changing data.
"""
prompt=processor.apply_chat_template(messages(row),tokenize=False,add_generation_prompt=True)
images=[image_loader(im['path']) for im in row['images']]
visual=processor(text=[prompt],images=images,return_tensors='pt')
built=build_parallel_input(codec.tokenizer,visual['input_ids'][0].tolist())
ids=torch.tensor([built.input_ids],dtype=torch.long)
inputs=dict(input_ids=ids,attention_mask=torch.ones_like(ids),
pixel_values=visual['pixel_values'],image_grid_thw=visual['image_grid_thw'],
slot_positions=torch.tensor([built.slot_positions],dtype=torch.long),
prefix_lengths=torch.tensor([built.prefix_length]),lengths=torch.tensor([ids.shape[1]]))
return inputs,codec.adapt_target(row['target'],row['platform'])