"""GUI-Owl desktop/Android union; official GUI-Decisions E coordinate labels. No training entry point. Reuses the frozen JSON137 non-coordinate codec and GUI-Decisions targets/readout. All 134 slots are masked; predictions align j-1. Coordinates in history and serialized actions are 0..1000, never pixel units. """ from __future__ import annotations import json import math import hashlib from dataclasses import dataclass import torch from torch.nn import functional as F from slotread.labels import make_arm from slotread.numbers import number_targets, read_number from trace2task.gui_owl_parallel137_codec import Parallel137Codec, POINT_ACTIONS, _half_up from trace2task.gui_owl_short_parallel_input import ParallelInput SHARED = ('key', 'type', 'wait', 'terminate', 'answer', 'interact') DESKTOP = SHARED + ('mouse_move', 'left_click', 'left_click_drag', 'right_click', 'middle_click', 'double_click', 'triple_click', 'scroll', 'hscroll') ANDROID = SHARED + ('click', 'long_press', 'swipe', 'system_button', 'open') ACTIONS = DESKTOP + tuple(a for a in ANDROID if a not in DESKTOP) BUTTONS = ('Back', 'Home', 'Menu', 'Enter') FIELD_WIDTHS = (('action', 3), ('coordinate.x', 1), ('coordinate.y', 1), ('coordinate2.x', 1), ('coordinate2.y', 1), ('keys', 16), ('pixels', 4), ('time', 5), ('status', 1), ('button', 1), ('text', 100)) LITERALS = ('{"action":"', '","coordinate":[', ',', '],"coordinate2":[', ',', '],"keys":[', '],"pixels":', ',"time":', ',"status":"', '","button":"', '","text":"', '"}') def build_parallel_input(tokenizer, prefix_ids=()): ids, positions, fields = list(prefix_ids), [], [] for literal, (name, width) in zip(LITERALS[:-1], FIELD_WIDTHS, strict=True): ids.extend(tokenizer.encode(literal, add_special_tokens=False).ids) pos = tuple(range(len(ids), len(ids) + width)) fields.append((name, pos)); positions.extend(pos) ids.extend([tokenizer.token_to_id('<|fim_pad|>')] * width) ids.extend(tokenizer.encode(LITERALS[-1], add_special_tokens=False).ids) assert len(positions) == 134 return ParallelInput(tuple(ids), tuple(positions), tuple(fields), len(prefix_ids)) @dataclass(frozen=True) class Target: canonical_action: dict platform: str target_ids: tuple hard_labels: tuple soft_labels: dict # slot -> 256 probabilities; ids are diagnostic only here fields: tuple codebook_sha256: str class Native134Codec: variant = 'gui-owl-native134-official-E-v1' output_slots = 134 def __init__(self, tokenizer): self.tokenizer = tokenizer self.base_codec = Parallel137Codec(tokenizer, scroll_coordinates=True) for name in ('mask_id', 'pad_id', 'eos_id', 'special_ids', 'vocab_size'): setattr(self, name, getattr(self.base_codec, name)) class Adapter: def __call__(self, s, **kw): return {'input_ids': tokenizer.encode(s, add_special_tokens=False).ids} self.arm = make_arm('E', Adapter()) self.bin_ids = tuple(self.arm.fine) self.codebook_sha256 = hashlib.sha256(json.dumps(self.bin_ids).encode()).hexdigest() self.action_encodings = {} for a in ACTIONS: ids = self.base_codec._tokenize_exact(a, 'action') if not 1 <= len(ids) <= 3: raise ValueError('action_exceeds_three_slots:' + a) self.action_encodings[a] = tuple(ids + [self.pad_id] * (3-len(ids))) self.button_ids = {s: self.base_codec._single(s) for s in BUTTONS} built = build_parallel_input(tokenizer) self.input_ids, self.slot_positions = built.input_ids, built.slot_positions self.field_slices = {} i = 0 for name, width in FIELD_WIDTHS: self.field_slices[name] = slice(i, i+width); i += width def adapt_target(self, action, platform='desktop'): allowed = {'desktop': DESKTOP, 'android': ANDROID}.get(platform) if allowed is None or action.get('action') not in allowed: raise ValueError('action_not_allowed_on_platform') a = action['action']; b = self.base_codec ids = [self.pad_id] * 134; labels = [-100] * 134 soft, fields, canonical = {}, [], {'action': a} def hard(name, values, active=None): span = self.field_slices[name] if len(values) != span.stop-span.start: raise ValueError('incorrect_field_width') ids[span] = values n = len(values) if active is None else active labels[span.start:span.start+n] = values[:n] fields.append((name, span.start, span.start+n)) def old_field(name, surrogate): old = b.adapt_target(surrogate, {}) values = list(old.target_ids[b.field_slices[name]]) n = sum(old.loss_mask[b.field_slices[name]]) hard(name, values, n) canonical[name] = old.canonical_action[name] def point(name): xy = action.get(name) if not isinstance(xy, (list, tuple)) or len(xy) != 2: raise ValueError('coordinate_pair_required') canonical[name] = [_half_up(v, 1, 1000, 'coordinate_out_of_range') for v in xy] for axis, value in zip(('x', 'y'), canonical[name]): field = name + '.' + axis p = self.field_slices[field].start q = number_targets('v', value/1000, .005, self.arm)['v'] soft[p] = q ids[p] = self.bin_ids[int(q.argmax())] fields.append((field, p, p+1)) hard('action', list(self.action_encodings[a])) if a in POINT_ACTIONS or a in ('click', 'long_press', 'swipe'): point('coordinate') if a == 'swipe': point('coordinate2') if a in ('scroll', 'hscroll'): if 'coordinate' in action: point('coordinate') else: for axis in ('x', 'y'): hard('coordinate.'+axis, [self.pad_id]) old_field('pixels', {'action': a, 'pixels': action.get('pixels')}) if a == 'key' and platform == 'desktop': old_field('keys', {'action': 'key', 'keys': action.get('keys')}) if a in ('wait', 'long_press'): old_field('time', {'action': 'wait', 'time': action.get('time')}) if a == 'terminate': old_field('status', {'action': a, 'status': action.get('status')}) if a == 'system_button': if action.get('button') not in self.button_ids: raise ValueError('invalid_system_button') canonical['button'] = action['button'] hard('button', [self.button_ids[action['button']]]) if a in ('type', 'answer', 'interact', 'open') or (a == 'key' and platform == 'android'): old_field('text', {'action': 'type', 'text': action.get('text')}) if set(action) != set(canonical): raise ValueError('unexpected_or_missing_parameters:' + ','.join(sorted(set(action)^set(canonical)))) return Target(canonical, platform, tuple(ids), tuple(labels), soft, tuple(fields), self.codebook_sha256) def codebook(self): return [dict(bin=i, token_id=t, decoded=self.tokenizer.decode([t], skip_special_tokens=False), normalized_center=(i+.5)/256) for i,t in enumerate(self.bin_ids)] def decode(self, logits, platform, key_sequences=()): """Constrained readout of ONE forward's [134,vocab] logits. No model call.""" if logits.ndim != 2 or logits.shape[0] != 134 or logits.shape[1] < self.vocab_size: raise ValueError('expected_134_by_vocab_logits') if not torch.isfinite(logits).all(): raise ValueError('nonfinite_logits') # Some model heads pad vocabulary rows beyond the tokenizer inventory. # Such rows cannot represent valid text tokens during constrained decode. v = logits[:, :self.vocab_size].float(); b = self.base_codec allowed = {'desktop': DESKTOP, 'android': ANDROID}[platform] def choose(name, codes): span = self.field_slices[name] scores = [sum(v[p,t] for p,t in zip(range(span.start,span.stop), code)) for code in codes] return int(torch.stack(scores).argmax()) a = allowed[choose('action', [self.action_encodings[x] for x in allowed])] result = {'action': a} def point(name): xy=[] for axis in ('x','y'): p=self.field_slices[name+'.'+axis].start value=read_number({'v':v[p,list(self.bin_ids)]}, 'v', self.arm, how='local2')*1000 xy.append(_half_up(float(value),1,1000,'decoded_coordinate_out_of_range')) result[name]=xy if a in POINT_ACTIONS or a in ('click','long_press','swipe'): point('coordinate') if a=='swipe': point('coordinate2') if a in ('scroll','hscroll'): present=sum(torch.logsumexp(v[p,list(self.bin_ids)],0) for p in (3,4)) if present > v[3,self.pad_id]+v[4,self.pad_id]: point('coordinate') s=self.field_slices['pixels']; sign=('-', '+')[int(v[s.start,b.sign_ids['+']] > v[s.start,b.sign_ids['-']])] n=int(''.join(str(int(v[p,list(b.digit_ids)].argmax())) for p in range(s.start+1,s.stop))) result['pixels']= -n if sign=='-' else n if a in ('wait','long_press'): s=self.field_slices['time'] result['time']=int(''.join(str(int(v[p,list(b.digit_ids)].argmax())) for p in range(s.start,s.stop)))/100 if a=='terminate': values=list(b.status_ids); result['status']=values[choose('status',[(b.status_ids[x],) for x in values])] if a=='system_button': result['button']=BUTTONS[choose('button',[(self.button_ids[x],) for x in BUTTONS])] if a=='key' and platform=='desktop': if not key_sequences: raise ValueError('desktop_key_catalogue_required') codes=[self.adapt_target({'action':'key','keys':list(k)}, platform).target_ids[7:23] for k in key_sequences] result['keys']=list(key_sequences[choose('keys',codes)]) if a in ('type','answer','interact','open') or (a=='key' and platform=='android'): scores=v[34:134].clone() scores[:,list(self.special_ids-{self.eos_id})]=-torch.inf tokens=scores.argmax(-1).tolist() if self.eos_id not in tokens: raise ValueError('text_missing_eos_no_silent_truncation') text=self.tokenizer.decode(tokens[:tokens.index(self.eos_id)], skip_special_tokens=False) if '\ufffd' in text: raise ValueError('invalid_text_encoding') result['text']=text return self.adapt_target(result,platform).canonical_action def native134_loss(logits, targets, codec): """Full-vocabulary soft CE for coordinates; hard CE for active other fields.""" if logits.shape[:2] != (len(targets),134): raise ValueError('loss_shape_mismatch') losses=[] for row,t in enumerate(targets): if t.codebook_sha256 != codec.codebook_sha256: raise ValueError('wrong_codebook') fields=[] for name,start,stop in t.fields: if start in t.soft_labels: lp=F.log_softmax(logits[row,start].float(),-1)[list(codec.bin_ids)] fields.append(-(lp*t.soft_labels[start].to(lp)).sum()) else: labels=torch.tensor(t.hard_labels[start:stop],device=logits.device) fields.append(F.cross_entropy(logits[row,start:stop].float(),labels)) losses.append(fields[0]+2*(torch.stack(fields[1:]).mean() if len(fields)>1 else fields[0]*0)) return torch.stack(losses) def serialize(action): return ''+json.dumps(action,ensure_ascii=False,separators=(',',':'))+'' def messages(row): """Only task/history/images enter input. Available historical images retained.""" def clean(s): return s.replace('<|','[special:').replace('','[image]') content=[{'type':'text','text':'Platform: '+row['platform']+'\nTask: '+clean(row['task'])}] for step in row['history']: if step.get('image'): content.append({'type':'image','image':step['image']['path']}) text='Previous step '+str(step['step'])+'\n' if step.get('observation'): text+='Past observation: '+clean(step['observation'])+'\n' text+='Executed actions: '+' '.join(serialize(a) for a in step['actions']) content.append({'type':'text','text':clean(text)}) content += [{'type':'text','text':'Current screenshot:'}, {'type':'image','image':row['images'][-1]['path']}] system=('Choose one action from task, screenshots and history. All history and output coordinates ' 'are normalized integers 0..1000. Internal coordinate slots use 256 bins. Predict the masked ' '134-slot table in parallel, including text. Only applicable fields are executed. ' 'Desktop actions: '+','.join(DESKTOP)+'. Android actions: '+','.join(ANDROID)+'. ' 'Desktop key uses keys; Android key uses text. Swipe uses coordinate and coordinate2. ' 'Long_press uses coordinate and time in seconds. System_button: Back,Home,Menu,Enter. ' 'Desktop left_click_drag starts at the current pointer. Scroll pixels preserves wheel units.') return [{'role':'system','content':system},{'role':'user','content':content}] def prepare_example(row, processor, codec, image_loader): """Real processor input ready for existing prepare_parallel_inputs. Return labels separately: they must never be passed into the model forward. image_loader is injected to resolve mounted/local paths without changing data. """ prompt=processor.apply_chat_template(messages(row),tokenize=False,add_generation_prompt=True) images=[image_loader(im['path']) for im in row['images']] visual=processor(text=[prompt],images=images,return_tensors='pt') built=build_parallel_input(codec.tokenizer,visual['input_ids'][0].tolist()) ids=torch.tensor([built.input_ids],dtype=torch.long) inputs=dict(input_ids=ids,attention_mask=torch.ones_like(ids), pixel_values=visual['pixel_values'],image_grid_thw=visual['image_grid_thw'], slot_positions=torch.tensor([built.slot_positions],dtype=torch.long), prefix_lengths=torch.tensor([built.prefix_length]),lengths=torch.tensor([ids.shape[1]])) return inputs,codec.adapt_target(row['target'],row['platform'])