Instructions to use dongjintao/GUI-Swift-Mobile-2B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use dongjintao/GUI-Swift-Mobile-2B with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
Download bundle/source/gui_owl_native134.py from dongjintao/GUI-Swift-Mobile-2B: direct link, hf CLI and curl.
- Browser
- Download file 14.6 kB
-
https://huggingface.co/dongjintao/GUI-Swift-Mobile-2B/resolve/main/bundle/source/gui_owl_native134.py
- Command line
-
hf download hf://dongjintao/GUI-Swift-Mobile-2B/bundle/source/gui_owl_native134.py
-
curl -L -o gui_owl_native134.py https://huggingface.co/dongjintao/GUI-Swift-Mobile-2B/resolve/main/bundle/source/gui_owl_native134.py
14.6 kB
| """GUI-Owl desktop/Android union; official GUI-Decisions E coordinate labels. | |
| No training entry point. Reuses the frozen JSON137 non-coordinate codec and | |
| GUI-Decisions targets/readout. All 134 slots are masked; predictions align j-1. | |
| Coordinates in history and serialized actions are 0..1000, never pixel units. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import hashlib | |
| from dataclasses import dataclass | |
| import torch | |
| from torch.nn import functional as F | |
| from slotread.labels import make_arm | |
| from slotread.numbers import number_targets, read_number | |
| from trace2task.gui_owl_parallel137_codec import Parallel137Codec, POINT_ACTIONS, _half_up | |
| from trace2task.gui_owl_short_parallel_input import ParallelInput | |
| SHARED = ('key', 'type', 'wait', 'terminate', 'answer', 'interact') | |
| DESKTOP = SHARED + ('mouse_move', 'left_click', 'left_click_drag', 'right_click', | |
| 'middle_click', 'double_click', 'triple_click', 'scroll', 'hscroll') | |
| ANDROID = SHARED + ('click', 'long_press', 'swipe', 'system_button', 'open') | |
| ACTIONS = DESKTOP + tuple(a for a in ANDROID if a not in DESKTOP) | |
| BUTTONS = ('Back', 'Home', 'Menu', 'Enter') | |
| FIELD_WIDTHS = (('action', 3), ('coordinate.x', 1), ('coordinate.y', 1), | |
| ('coordinate2.x', 1), ('coordinate2.y', 1), ('keys', 16), | |
| ('pixels', 4), ('time', 5), ('status', 1), ('button', 1), ('text', 100)) | |
| LITERALS = ('<action>{"action":"', '","coordinate":[', ',', | |
| '],"coordinate2":[', ',', '],"keys":[', '],"pixels":', | |
| ',"time":', ',"status":"', '","button":"', '","text":"', '"}</action>') | |
| def build_parallel_input(tokenizer, prefix_ids=()): | |
| ids, positions, fields = list(prefix_ids), [], [] | |
| for literal, (name, width) in zip(LITERALS[:-1], FIELD_WIDTHS, strict=True): | |
| ids.extend(tokenizer.encode(literal, add_special_tokens=False).ids) | |
| pos = tuple(range(len(ids), len(ids) + width)) | |
| fields.append((name, pos)); positions.extend(pos) | |
| ids.extend([tokenizer.token_to_id('<|fim_pad|>')] * width) | |
| ids.extend(tokenizer.encode(LITERALS[-1], add_special_tokens=False).ids) | |
| assert len(positions) == 134 | |
| return ParallelInput(tuple(ids), tuple(positions), tuple(fields), len(prefix_ids)) | |
| class Target: | |
| canonical_action: dict | |
| platform: str | |
| target_ids: tuple | |
| hard_labels: tuple | |
| soft_labels: dict # slot -> 256 probabilities; ids are diagnostic only here | |
| fields: tuple | |
| codebook_sha256: str | |
| class Native134Codec: | |
| variant = 'gui-owl-native134-official-E-v1' | |
| output_slots = 134 | |
| def __init__(self, tokenizer): | |
| self.tokenizer = tokenizer | |
| self.base_codec = Parallel137Codec(tokenizer, scroll_coordinates=True) | |
| for name in ('mask_id', 'pad_id', 'eos_id', 'special_ids', 'vocab_size'): | |
| setattr(self, name, getattr(self.base_codec, name)) | |
| class Adapter: | |
| def __call__(self, s, **kw): | |
| return {'input_ids': tokenizer.encode(s, add_special_tokens=False).ids} | |
| self.arm = make_arm('E', Adapter()) | |
| self.bin_ids = tuple(self.arm.fine) | |
| self.codebook_sha256 = hashlib.sha256(json.dumps(self.bin_ids).encode()).hexdigest() | |
| self.action_encodings = {} | |
| for a in ACTIONS: | |
| ids = self.base_codec._tokenize_exact(a, 'action') | |
| if not 1 <= len(ids) <= 3: | |
| raise ValueError('action_exceeds_three_slots:' + a) | |
| self.action_encodings[a] = tuple(ids + [self.pad_id] * (3-len(ids))) | |
| self.button_ids = {s: self.base_codec._single(s) for s in BUTTONS} | |
| built = build_parallel_input(tokenizer) | |
| self.input_ids, self.slot_positions = built.input_ids, built.slot_positions | |
| self.field_slices = {} | |
| i = 0 | |
| for name, width in FIELD_WIDTHS: | |
| self.field_slices[name] = slice(i, i+width); i += width | |
| def adapt_target(self, action, platform='desktop'): | |
| allowed = {'desktop': DESKTOP, 'android': ANDROID}.get(platform) | |
| if allowed is None or action.get('action') not in allowed: | |
| raise ValueError('action_not_allowed_on_platform') | |
| a = action['action']; b = self.base_codec | |
| ids = [self.pad_id] * 134; labels = [-100] * 134 | |
| soft, fields, canonical = {}, [], {'action': a} | |
| def hard(name, values, active=None): | |
| span = self.field_slices[name] | |
| if len(values) != span.stop-span.start: | |
| raise ValueError('incorrect_field_width') | |
| ids[span] = values | |
| n = len(values) if active is None else active | |
| labels[span.start:span.start+n] = values[:n] | |
| fields.append((name, span.start, span.start+n)) | |
| def old_field(name, surrogate): | |
| old = b.adapt_target(surrogate, {}) | |
| values = list(old.target_ids[b.field_slices[name]]) | |
| n = sum(old.loss_mask[b.field_slices[name]]) | |
| hard(name, values, n) | |
| canonical[name] = old.canonical_action[name] | |
| def point(name): | |
| xy = action.get(name) | |
| if not isinstance(xy, (list, tuple)) or len(xy) != 2: | |
| raise ValueError('coordinate_pair_required') | |
| canonical[name] = [_half_up(v, 1, 1000, 'coordinate_out_of_range') for v in xy] | |
| for axis, value in zip(('x', 'y'), canonical[name]): | |
| field = name + '.' + axis | |
| p = self.field_slices[field].start | |
| q = number_targets('v', value/1000, .005, self.arm)['v'] | |
| soft[p] = q | |
| ids[p] = self.bin_ids[int(q.argmax())] | |
| fields.append((field, p, p+1)) | |
| hard('action', list(self.action_encodings[a])) | |
| if a in POINT_ACTIONS or a in ('click', 'long_press', 'swipe'): | |
| point('coordinate') | |
| if a == 'swipe': point('coordinate2') | |
| if a in ('scroll', 'hscroll'): | |
| if 'coordinate' in action: point('coordinate') | |
| else: | |
| for axis in ('x', 'y'): hard('coordinate.'+axis, [self.pad_id]) | |
| old_field('pixels', {'action': a, 'pixels': action.get('pixels')}) | |
| if a == 'key' and platform == 'desktop': | |
| old_field('keys', {'action': 'key', 'keys': action.get('keys')}) | |
| if a in ('wait', 'long_press'): | |
| old_field('time', {'action': 'wait', 'time': action.get('time')}) | |
| if a == 'terminate': | |
| old_field('status', {'action': a, 'status': action.get('status')}) | |
| if a == 'system_button': | |
| if action.get('button') not in self.button_ids: raise ValueError('invalid_system_button') | |
| canonical['button'] = action['button'] | |
| hard('button', [self.button_ids[action['button']]]) | |
| if a in ('type', 'answer', 'interact', 'open') or (a == 'key' and platform == 'android'): | |
| old_field('text', {'action': 'type', 'text': action.get('text')}) | |
| if set(action) != set(canonical): | |
| raise ValueError('unexpected_or_missing_parameters:' + ','.join(sorted(set(action)^set(canonical)))) | |
| return Target(canonical, platform, tuple(ids), tuple(labels), soft, tuple(fields), self.codebook_sha256) | |
| def codebook(self): | |
| return [dict(bin=i, token_id=t, decoded=self.tokenizer.decode([t], skip_special_tokens=False), | |
| normalized_center=(i+.5)/256) for i,t in enumerate(self.bin_ids)] | |
| def decode(self, logits, platform, key_sequences=()): | |
| """Constrained readout of ONE forward's [134,vocab] logits. No model call.""" | |
| if logits.ndim != 2 or logits.shape[0] != 134 or logits.shape[1] < self.vocab_size: | |
| raise ValueError('expected_134_by_vocab_logits') | |
| if not torch.isfinite(logits).all(): raise ValueError('nonfinite_logits') | |
| # Some model heads pad vocabulary rows beyond the tokenizer inventory. | |
| # Such rows cannot represent valid text tokens during constrained decode. | |
| v = logits[:, :self.vocab_size].float(); b = self.base_codec | |
| allowed = {'desktop': DESKTOP, 'android': ANDROID}[platform] | |
| def choose(name, codes): | |
| span = self.field_slices[name] | |
| scores = [sum(v[p,t] for p,t in zip(range(span.start,span.stop), code)) for code in codes] | |
| return int(torch.stack(scores).argmax()) | |
| a = allowed[choose('action', [self.action_encodings[x] for x in allowed])] | |
| result = {'action': a} | |
| def point(name): | |
| xy=[] | |
| for axis in ('x','y'): | |
| p=self.field_slices[name+'.'+axis].start | |
| value=read_number({'v':v[p,list(self.bin_ids)]}, 'v', self.arm, how='local2')*1000 | |
| xy.append(_half_up(float(value),1,1000,'decoded_coordinate_out_of_range')) | |
| result[name]=xy | |
| if a in POINT_ACTIONS or a in ('click','long_press','swipe'): point('coordinate') | |
| if a=='swipe': point('coordinate2') | |
| if a in ('scroll','hscroll'): | |
| present=sum(torch.logsumexp(v[p,list(self.bin_ids)],0) for p in (3,4)) | |
| if present > v[3,self.pad_id]+v[4,self.pad_id]: point('coordinate') | |
| s=self.field_slices['pixels']; sign=('-', '+')[int(v[s.start,b.sign_ids['+']] > v[s.start,b.sign_ids['-']])] | |
| n=int(''.join(str(int(v[p,list(b.digit_ids)].argmax())) for p in range(s.start+1,s.stop))) | |
| result['pixels']= -n if sign=='-' else n | |
| if a in ('wait','long_press'): | |
| s=self.field_slices['time'] | |
| result['time']=int(''.join(str(int(v[p,list(b.digit_ids)].argmax())) for p in range(s.start,s.stop)))/100 | |
| if a=='terminate': | |
| values=list(b.status_ids); result['status']=values[choose('status',[(b.status_ids[x],) for x in values])] | |
| if a=='system_button': result['button']=BUTTONS[choose('button',[(self.button_ids[x],) for x in BUTTONS])] | |
| if a=='key' and platform=='desktop': | |
| if not key_sequences: raise ValueError('desktop_key_catalogue_required') | |
| codes=[self.adapt_target({'action':'key','keys':list(k)}, platform).target_ids[7:23] for k in key_sequences] | |
| result['keys']=list(key_sequences[choose('keys',codes)]) | |
| if a in ('type','answer','interact','open') or (a=='key' and platform=='android'): | |
| scores=v[34:134].clone() | |
| scores[:,list(self.special_ids-{self.eos_id})]=-torch.inf | |
| tokens=scores.argmax(-1).tolist() | |
| if self.eos_id not in tokens: raise ValueError('text_missing_eos_no_silent_truncation') | |
| text=self.tokenizer.decode(tokens[:tokens.index(self.eos_id)], skip_special_tokens=False) | |
| if '\ufffd' in text: raise ValueError('invalid_text_encoding') | |
| result['text']=text | |
| return self.adapt_target(result,platform).canonical_action | |
| def native134_loss(logits, targets, codec): | |
| """Full-vocabulary soft CE for coordinates; hard CE for active other fields.""" | |
| if logits.shape[:2] != (len(targets),134): raise ValueError('loss_shape_mismatch') | |
| losses=[] | |
| for row,t in enumerate(targets): | |
| if t.codebook_sha256 != codec.codebook_sha256: raise ValueError('wrong_codebook') | |
| fields=[] | |
| for name,start,stop in t.fields: | |
| if start in t.soft_labels: | |
| lp=F.log_softmax(logits[row,start].float(),-1)[list(codec.bin_ids)] | |
| fields.append(-(lp*t.soft_labels[start].to(lp)).sum()) | |
| else: | |
| labels=torch.tensor(t.hard_labels[start:stop],device=logits.device) | |
| fields.append(F.cross_entropy(logits[row,start:stop].float(),labels)) | |
| losses.append(fields[0]+2*(torch.stack(fields[1:]).mean() if len(fields)>1 else fields[0]*0)) | |
| return torch.stack(losses) | |
| def serialize(action): | |
| return '<action>'+json.dumps(action,ensure_ascii=False,separators=(',',':'))+'</action>' | |
| def messages(row): | |
| """Only task/history/images enter input. Available historical images retained.""" | |
| def clean(s): return s.replace('<|','[special:').replace('<image>','[image]') | |
| content=[{'type':'text','text':'Platform: '+row['platform']+'\nTask: '+clean(row['task'])}] | |
| for step in row['history']: | |
| if step.get('image'): content.append({'type':'image','image':step['image']['path']}) | |
| text='Previous step '+str(step['step'])+'\n' | |
| if step.get('observation'): text+='Past observation: '+clean(step['observation'])+'\n' | |
| text+='Executed actions: '+' '.join(serialize(a) for a in step['actions']) | |
| content.append({'type':'text','text':clean(text)}) | |
| content += [{'type':'text','text':'Current screenshot:'}, | |
| {'type':'image','image':row['images'][-1]['path']}] | |
| system=('Choose one action from task, screenshots and history. All history and output coordinates ' | |
| 'are normalized integers 0..1000. Internal coordinate slots use 256 bins. Predict the masked ' | |
| '134-slot table in parallel, including text. Only applicable fields are executed. ' | |
| 'Desktop actions: '+','.join(DESKTOP)+'. Android actions: '+','.join(ANDROID)+'. ' | |
| 'Desktop key uses keys; Android key uses text. Swipe uses coordinate and coordinate2. ' | |
| 'Long_press uses coordinate and time in seconds. System_button: Back,Home,Menu,Enter. ' | |
| 'Desktop left_click_drag starts at the current pointer. Scroll pixels preserves wheel units.') | |
| return [{'role':'system','content':system},{'role':'user','content':content}] | |
| def prepare_example(row, processor, codec, image_loader): | |
| """Real processor input ready for existing prepare_parallel_inputs. | |
| Return labels separately: they must never be passed into the model forward. | |
| image_loader is injected to resolve mounted/local paths without changing data. | |
| """ | |
| prompt=processor.apply_chat_template(messages(row),tokenize=False,add_generation_prompt=True) | |
| images=[image_loader(im['path']) for im in row['images']] | |
| visual=processor(text=[prompt],images=images,return_tensors='pt') | |
| built=build_parallel_input(codec.tokenizer,visual['input_ids'][0].tolist()) | |
| ids=torch.tensor([built.input_ids],dtype=torch.long) | |
| inputs=dict(input_ids=ids,attention_mask=torch.ones_like(ids), | |
| pixel_values=visual['pixel_values'],image_grid_thw=visual['image_grid_thw'], | |
| slot_positions=torch.tensor([built.slot_positions],dtype=torch.long), | |
| prefix_lengths=torch.tensor([built.prefix_length]),lengths=torch.tensor([ids.shape[1]])) | |
| return inputs,codec.adapt_target(row['target'],row['platform']) | |