| { |
| "module": "keras_hub.src.models.llama3.llama3_causal_lm_preprocessor", |
| "class_name": "Llama3CausalLMPreprocessor", |
| "config": { |
| "name": "llama3_causal_lm_preprocessor", |
| "trainable": true, |
| "dtype": { |
| "module": "keras", |
| "class_name": "DTypePolicy", |
| "config": { |
| "name": "float32" |
| }, |
| "registered_name": null |
| }, |
| "tokenizer": { |
| "module": "keras_hub.src.models.llama3.llama3_tokenizer", |
| "class_name": "Llama3Tokenizer", |
| "config": { |
| "name": "llama3_tokenizer", |
| "trainable": true, |
| "dtype": { |
| "module": "keras", |
| "class_name": "DTypePolicy", |
| "config": { |
| "name": "int32" |
| }, |
| "registered_name": null |
| }, |
| "config_file": "tokenizer.json", |
| "sequence_length": null, |
| "add_prefix_space": false, |
| "unsplittable_tokens": [ |
| "<|end_header_id|>", |
| "<|end_of_text|>", |
| "<|eot_id|>", |
| "<|start_header_id|>", |
| "<|begin_of_text|>" |
| ] |
| }, |
| "registered_name": "keras_hub>Llama3Tokenizer" |
| }, |
| "config_file": "preprocessor.json", |
| "sequence_length": 1024, |
| "add_start_token": true, |
| "add_end_token": true |
| }, |
| "registered_name": "keras_hub>Llama3CausalLMPreprocessor" |
| } |
