CompressedGemma commited on
Commit
2790fea
·
verified ·
1 Parent(s): 541d592

Upload slimshard.py

Browse files
Files changed (1) hide show
  1. slimshard.py +28 -7
slimshard.py CHANGED
@@ -1678,6 +1678,8 @@ def recover_from_shard(
1678
 
1679
  if name in present:
1680
  t = f.get_tensor(name)
 
 
1681
  writer.add(name, _payload_bytes(t), present_dtype[name], tuple(t.shape))
1682
  stats["present"] += 1
1683
  stats["tensors"] += 1
@@ -1764,11 +1766,15 @@ def recover_from_shard(
1764
  f_key_a = w_name_a
1765
  f_key_b = w_name_b
1766
  dt_f = present_dtype[f_key_a]
1767
- ta = f.get_tensor(f_key_a).to(dev)
1768
- tb = f.get_tensor(f_key_b).to(dev)
1769
- interp = _checkfree_interp_torch(ta, tb, alpha, preserve_row_rms=(ta.ndim >= 2))
1770
- raw_o, dt_o, shp_o = _storage_from_f32(interp.cpu().numpy(), dt_f)
1771
- writer.add(name, raw_o, dt_o, shp_o)
 
 
 
 
1772
  stats["expert_interpolated"] += 1
1773
  stats["pattern_recovered"] += 1
1774
  stats["tensors"] += 1
@@ -1798,10 +1804,14 @@ def recover_from_shard(
1798
  nb = f"{lpref}.{lb}.{suffix}"
1799
  dt_t = present_dtype[na]
1800
  ta = f.get_tensor(na)
 
 
1801
  if la == lb or alpha <= 1.0e-6 or dt_t not in ("F32", "F16", "BF16"):
1802
  writer.add(name, _payload_bytes(ta), dt_t, tuple(ta.shape))
1803
  else:
1804
  tb = f.get_tensor(nb)
 
 
1805
  interp = _checkfree_interp_torch(
1806
  ta.to(dev), tb.to(dev), alpha, preserve_row_rms=(ta.ndim >= 2)
1807
  )
@@ -1891,12 +1901,23 @@ def recover_from_shard(
1891
 
1892
  writer.close()
1893
  stats["output_bytes"] = writer.bytes_written()
 
 
1894
  if export_hf_dir is not None:
1895
  src_dir = cfg_path.parent if cfg_path.exists() else shard_path.parent
1896
- export_hf_bundle(output, src_dir, export_hf_dir)
 
 
 
 
 
 
 
 
1897
  if verbose:
1898
  print(
1899
- f"recovered {stats['tensors']}/{len(weight_map)} tensors: "
 
1900
  f"{stats['present']} present, "
1901
  f"{stats['checkfree_interpolated']} checkfree_layer, "
1902
  f"{stats['expert_interpolated']} checkfree_expert, "
 
1678
 
1679
  if name in present:
1680
  t = f.get_tensor(name)
1681
+ if kept_experts is not None and _is_router_gate_tensor(name, orig_experts, tuple(t.shape)):
1682
+ t = t[:kept_experts].contiguous()
1683
  writer.add(name, _payload_bytes(t), present_dtype[name], tuple(t.shape))
1684
  stats["present"] += 1
1685
  stats["tensors"] += 1
 
1766
  f_key_a = w_name_a
1767
  f_key_b = w_name_b
1768
  dt_f = present_dtype[f_key_a]
1769
+ if la == lb or alpha <= 1.0e-6 or dt_f not in ("F32", "F16", "BF16"):
1770
+ t_sel = f.get_tensor(f_key_a if alpha < 0.5 else f_key_b)
1771
+ writer.add(name, _payload_bytes(t_sel), dt_f, tuple(t_sel.shape))
1772
+ else:
1773
+ ta = f.get_tensor(f_key_a).to(dev)
1774
+ tb = f.get_tensor(f_key_b).to(dev)
1775
+ interp = _checkfree_interp_torch(ta, tb, alpha, preserve_row_rms=(ta.ndim >= 2))
1776
+ raw_o, dt_o, shp_o = _storage_from_f32(interp.cpu().numpy(), dt_f)
1777
+ writer.add(name, raw_o, dt_o, shp_o)
1778
  stats["expert_interpolated"] += 1
1779
  stats["pattern_recovered"] += 1
1780
  stats["tensors"] += 1
 
1804
  nb = f"{lpref}.{lb}.{suffix}"
1805
  dt_t = present_dtype[na]
1806
  ta = f.get_tensor(na)
1807
+ if kept_experts is not None and _is_router_gate_tensor(name, orig_experts, tuple(ta.shape)):
1808
+ ta = ta[:kept_experts].contiguous()
1809
  if la == lb or alpha <= 1.0e-6 or dt_t not in ("F32", "F16", "BF16"):
1810
  writer.add(name, _payload_bytes(ta), dt_t, tuple(ta.shape))
1811
  else:
1812
  tb = f.get_tensor(nb)
1813
+ if kept_experts is not None and _is_router_gate_tensor(name, orig_experts, tuple(tb.shape)):
1814
+ tb = tb[:kept_experts].contiguous()
1815
  interp = _checkfree_interp_torch(
1816
  ta.to(dev), tb.to(dev), alpha, preserve_row_rms=(ta.ndim >= 2)
1817
  )
 
1901
 
1902
  writer.close()
1903
  stats["output_bytes"] = writer.bytes_written()
1904
+ stats["kept_experts"] = kept_experts if kept_experts is not None else orig_experts
1905
+ stats["estimated_params_b"] = round(est_params_b, 2)
1906
  if export_hf_dir is not None:
1907
  src_dir = cfg_path.parent if cfg_path.exists() else shard_path.parent
1908
+ export_hf_bundle(
1909
+ output,
1910
+ src_dir,
1911
+ export_hf_dir,
1912
+ kept_experts=kept_experts,
1913
+ has_engram=has_engram,
1914
+ has_mtp=has_mtp,
1915
+ has_vision=has_vision,
1916
+ )
1917
  if verbose:
1918
  print(
1919
+ f"recovered {stats['tensors']}/{len(weight_map)} tensors (~{est_params_b:.2f}B params, "
1920
+ f"experts={stats['kept_experts']}): "
1921
  f"{stats['present']} present, "
1922
  f"{stats['checkfree_interpolated']} checkfree_layer, "
1923
  f"{stats['expert_interpolated']} checkfree_expert, "