Pin reproducible runtime dependencies and checkpoint-specific export settings

README.md

@@ -6,11 +6,11 @@ tags: [coreai, core-ai, apple, aimodel, decisions, vision]
 
 # amos-1-small — Apple Core AI
 
-Independent AMOS-1 conversion with **bidirectional full attention** preserved. Text backbone, 255-code readout, vision tower, span head, region head and trained box head execute through Core AI. Original prompt formatting, temperatures and quote decoding are retained. This is a decision model, not a text generator.
+Independent AMOS-1 conversion with its checkpoint's **bidirectional full attention** preserved. Text backbone, 255-code readout, vision tower, span head, region head and trained box head execute through Core AI. Original prompt formatting, temperatures and quote decoding are retained. This is a decision model, not a text generator.
 
 ## Use
 
-Requires macOS 27 and the Core AI Python runtime. Tested with `coreai-torch==0.4.3`, PyTorch 2.14.0, Transformers 5.17 development, NumPy and Pillow. The converter warns that this PyTorch version exceeds its validated range. Host preprocessing/decoding uses PyTorch tensors; it loads no PyTorch neural weights.
+Requires macOS 27 and the Core AI Python runtime. Tested with `coreai-torch==0.4.3`, PyTorch 2.14.0, Transformers 5.19 development, NumPy and Pillow. Exact dependencies are in `requirements.txt`. The converter warns that this PyTorch version exceeds its validated range. Host preprocessing/decoding uses PyTorch tensors; it loads no PyTorch neural weights.
 
 ```python
 import asyncio

SHA256SUMS

@@ -1,12 +1,12 @@
 cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30  LICENSE
-917c1c22fba4cc984a5e95b01cc5e1a386e013fb599782b055b909c91647c06d  README.md
+0d23fe60eca8ac2b5bea70064493eb05edd8d4a12ebdeaaa044898368489a7ed  README.md
 9f25e439543c4493b66f609c2a4c57a3d97c11194ac247c3c043f4657725911e  SOURCE.json
 19dd2e174b2f7468407243220912947feb04a9260f347223021f880b532439a1  bf16-attempt.json
 5865684cabaa8e1a23243db7d9d32569a57c7b5011bf47ed95814b9813af57dd  bf16-comparison.json
 f9ca221eceeea6dc52774885a8c50ad727bb8c269a7ff4fbb439131d2f5be0f2  calibration.json
 2636cc2d838bc7395655ebb3d2ba933b186a43b13ce134c9a441ee8ba45a9d0d  config.json
-650d3c6c51a4fead342fba413ed8339f6e04d1f48569225e2215e14a837d62c0  convert_decoder.py
-ef80ea9244fb5d713b91e077dde578ebd1e1455ea0788ecec4b3691de1084e41  coreai_decoder.py
+910e2baa29daa433c9dbf4120c4848da3898d94ff8a0d899e82484308690d347  convert_decoder.py
+90ac2b10e3cd3bd9e043ba5db597d69fc84c17e0e84bb1014bc97a93d10015e3  coreai_decoder.py
 3708c801947fd75c057ff534edc404aacb4cc0acc9e653c12f09207e0d848d37  heads.aimodel/main.hash
 132384c42c3a5c6269a79862d6540a5a6834b532674ba847d83a5b181f85a860  heads.aimodel/main.mlirb
 3284600a3ad7c8e378bbe228ce3eb967d5da56bb40959aba7e56b177a3fc928a  heads.aimodel/metadata.json
@@ -19,7 +19,8 @@ aa7582a6b90dc3711049d293d985e549f2b22c0c4e5ad11b9272a03c6294886e  model/tokenize
 8c54bd45ef2fc821a711502da04af1bff020fe02c2207117c7ffdbf1866e6e25  model/tokenizer_config.json
 7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13  model/video_preprocessor_config.json
 d7910a3390f8073d95908f3301c0ea0e2df4551699546430003bc8226373c2bc  qwen_export_helpers.py
-8d39d10513cef09276fb806d7d16b33c3c9b7d30a212bdd87f0eb16ad8796d2e  runtime-smoke.json
+d4e1bf42e2598b84d5800323fdec87fbf338613e8e866e6bb5ce11ed628c8cb2  requirements.txt
+a30fd0562851fa48ecf66caef27326fffd6c66f342dc0a346434e95af3046f46  runtime-smoke.json
 9e17531efd289cf720a36688d943197f9baf9db7c4f743aa4951f6b6f7dbafda  serve/README.md
 0579eb845ee7af463ec88cf627d99dd636466f2c12d840a7cbe183d8f17044fd  serve/amos/__init__.py
 ba094d9eaedd229a99b2bdcd35a371ffdeb9aaee258a342524cc19e2f85ec421  serve/amos/calibrate.py
@@ -33,7 +34,7 @@ b0523591c320a074a0ec0751b68ebc205aeedd090ff5e7ca10bda3fb94a07ef8  serve/amos/ser
 133d1e29cc64e371f5df1f0cb0328fd77b8b9093d8465319580cf5ea4af5d081  serve/amos/vision.py
 bea3f98882a5d12c891aae4a5d75eda8c5a5436e88889d94b6a7d3d648d126f2  serve/pyproject.toml
 0d43aa4e58fe6131f4ed1f4bbcbcaa4a0815f8b1fe5331c7e96489700681136b  serve/sources.json
-c5b0aa4de369327de6d03ba19d322557568e9d9c312b901ab42478dc679199d2  smoke_decoder.py
+b9ab562bd0e3f3dd74882b1fe7fd86268c09c75db77a984f56446c21f770b978  smoke_decoder.py
 bfe4ef23e8e7bb5041f9e10a769b5c2169b26225ffdd7016e9062c689e593932  text.aimodel/main.hash
 1c469415a143a1d9d75486ed48dd9b32c1033f1854a6bfc394ae9eb08caa8c72  text.aimodel/main.mlirb
 a678dc0dcb7ff0a16708ee3e534460d0ba3c0679c9f43fac4c003b4d68d17b35  text.aimodel/metadata.json

convert_decoder.py

@@ -7,12 +7,15 @@ from qwen_export_helpers import DecisionVision
 
 class Text(torch.nn.Module):
  def __init__(self,model,letters):
-  super().__init__();self.language_model=model.model.language_model
+  super().__init__();self.language_model=model.model.language_model;self.noncausal=bool(getattr(model.config,'amos_noncausal',False))
   self.register_buffer('readout',model.get_output_embeddings().weight[letters].detach().float().clone())
  def forward(self,input_ids,attention_mask,position_ids,visual_embeddings,visual_mask):
   emb=self.language_model.embed_tokens(input_ids)
   emb=torch.where(visual_mask,visual_embeddings.to(emb.dtype),emb)
-  h=self.language_model(inputs_embeds=emb,attention_mask={'full_attention':attention_mask[:,None,None,:].bool(),'linear_attention':None},position_ids=position_ids,use_cache=False).last_hidden_state
+  mask=attention_mask[:,None,None,:].bool()
+  if not self.noncausal:
+   width=input_ids.shape[1];idx=torch.arange(width,device=input_ids.device);mask=mask&(idx[:,None]>=idx[None,:])[None,None]
+  h=self.language_model(inputs_embeds=emb,attention_mask={'full_attention':mask,'linear_attention':None},position_ids=position_ids,use_cache=False).last_hidden_state
   return h.float(),h.float()@self.readout.T
 
 class Heads(torch.nn.Module):
@@ -41,6 +44,6 @@ def main():
  print('Exporting text',flush=True);outputs=save(Text(model,codec.letters).eval(),inputs,out/'text.aimodel')
  print('Exporting vision',flush=True);visual=model.model.visual;vo=save(DecisionVision(visual).eval(),(torch.zeros(196,1536),),out/'vision.aimodel')
  print('Exporting heads',flush=True);ho=save(Heads(heads).eval(),(torch.zeros(d),torch.zeros(L,d),torch.zeros(49,d)),out/'heads.aimodel')
- (out/'manifest.json').write_text(json.dumps({'sequence_length':L,'hidden_size':d,'outputs':outputs,'vision_output':vo[0],'head_outputs':ho,'precision':a.precision,'echo':codec.echo,'kinds':heads.kinds,'keep':heads.keep,'has_box':heads.box is not None,'image_side':224},indent=2))
+ (out/'manifest.json').write_text(json.dumps({'sequence_length':L,'hidden_size':d,'outputs':outputs,'vision_output':vo[0],'head_outputs':ho,'precision':a.precision,'echo':codec.echo,'kinds':heads.kinds,'keep':heads.keep,'has_box':heads.box is not None,'image_side':224,'noncausal':bool(getattr(model.config,'amos_noncausal',False))},indent=2))
  print('Saved',flush=True)
 if __name__=='__main__':main()

coreai_decoder.py

@@ -23,6 +23,10 @@ class AmosDecoder:
 
  async def raw(self,state,questions,images=()):
   from amos.decoder import expand
+  from PIL import Image
+  def resize(im):return im.convert('RGB').resize((224,224),Image.Resampling.BICUBIC)
+  images=[resize(im) for im in images]
+  questions=[{**q,'options':[resize(o) if isinstance(o,Image.Image) else o for o in q['options']]} if 'options' in q else q for q in questions]
   flat,owner=expand(questions)
   ids,at,extra=self.codec.encode_case(state,flat,images)
   n=len(ids);L=self.manifest['sequence_length'];d=self.manifest['hidden_size']

requirements.txt

@@ -0,0 +1,8 @@
+coreai-core==1.0.0b3
+coreai-torch==0.4.3
+torch==2.14.0
+transformers @ git+https://github.com/huggingface/transformers.git@536ecc007387a50e77603bb5d92100e9b07514cc
+numpy==2.4.6
+Pillow==12.3.0
+huggingface_hub==1.33.0
+tokenizers==0.23.3

runtime-smoke.json

@@ -1,6 +1,7 @@
 {
   "passed": true,
   "pytorch_neural_execution_disabled": true,
+  "capacity_guard": true,
   "result": {
     "answers": {
       "refund": {
@@ -8,5 +9,18 @@
         "noul": 0.9714967012405396
       }
     }
-  }
+  },
+  "image_result": [
+    {
+      "type": "region",
+      "cell": 47,
+      "box": [
+        0.0318,
+        0.0241,
+        1.0,
+        1.0
+      ],
+      "p": 0.9999998807907104
+    }
+  ]
 }

smoke_decoder.py

@@ -8,6 +8,12 @@ async def main(name):
  root=Path('bundles')/(name+'-CoreAI');m=await AmosDecoder.load(root)
  out=await m.request({'state':'Please refund the duplicate charge.','questions':{'refund':{'type':'noul','instructions':'Is a refund requested?'}}})
  assert out['answers']['refund']['noul']>0.5,out
- (root/'runtime-smoke.json').write_text(json.dumps({'passed':True,'pytorch_neural_execution_disabled':True,'result':out},indent=2));print(out)
+ from PIL import Image
+ image_result=await m.predict('A red image.',[{'type':'region','text':'Point to the red area.'}],[Image.new('RGB',(360,240),'red')])
+ assert image_result[0]['type']=='region'
+ try:await m.predict('word '*1000,[{'type':'noul','text':'Is there text?'}])
+ except ValueError:capacity_guard=True
+ else:raise AssertionError('Over-capacity request was accepted')
+ (root/'runtime-smoke.json').write_text(json.dumps({'passed':True,'pytorch_neural_execution_disabled':True,'capacity_guard':capacity_guard,'result':out,'image_result':image_result},indent=2));print(out)
 if __name__=='__main__':
  p=argparse.ArgumentParser();p.add_argument('name');a=p.parse_args();asyncio.run(main(a.name))