StandardOne-3B-SH / head /schema_head_config.json

Update weights (v4)

e457cc5 verified 19 days ago

509 Bytes

{
  "head": {
    "hidden_size": 3072,
    "n_taps": 2,
    "d": 1024,
    "heads": 16,
    "ffn": 2816,
    "route_blocks": 2,
    "interact_blocks": 2,
    "dropout": 0.1,
    "n_roles": 5,
    "ord_hidden": 256,
    "max_nodes": 4096,
    "grad_checkpoint": true,
    "taps": [
      "final",
      19
    ],
    "template": "schema-v1"
  },
  "taps": [
    "final",
    19
  ],
  "template": "schema-v1-nosys",
  "step": 11471,
  "lora_rank": 64,
  "snapshot": "StandardOne-3B-SH@v3",
  "preview": true
}

	{
	"head": {
	"hidden_size": 3072,
	"n_taps": 2,
	"d": 1024,
	"heads": 16,
	"ffn": 2816,
	"route_blocks": 2,
	"interact_blocks": 2,
	"dropout": 0.1,
	"n_roles": 5,
	"ord_hidden": 256,
	"max_nodes": 4096,
	"grad_checkpoint": true,
	"taps": [
	"final",
	19
	],
	"template": "schema-v1"
	},
	"taps": [
	"final",
	19
	],
	"template": "schema-v1-nosys",
	"step": 11471,
	"lora_rank": 64,
	"snapshot": "StandardOne-3B-SH@v3",
	"preview": true
	}