Stanley11 commited on
Commit
4b500e7
·
verified ·
1 Parent(s): 0090f30

Publish unified IDNet release page and seven model configurations

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. Base/chat_template.jinja +2 -0
  2. Base/config.json +69 -0
  3. Base/generation_config.json +7 -0
  4. Base/preprocessor_config.json +27 -0
  5. Base/processor_config.json +34 -0
  6. Base/release_info.json +22 -0
  7. Base/tokenizer.json +0 -0
  8. Base/tokenizer_config.json +22 -0
  9. CU/chat_template.jinja +2 -0
  10. CU/config.json +68 -0
  11. CU/generation_config.json +7 -0
  12. CU/preprocessor_config.json +27 -0
  13. CU/processor_config.json +34 -0
  14. CU/release_info.json +22 -0
  15. CU/tokenizer.json +0 -0
  16. CU/tokenizer_config.json +22 -0
  17. LEGO/chat_template.jinja +2 -0
  18. LEGO/config.json +68 -0
  19. LEGO/generation_config.json +7 -0
  20. LEGO/preprocessor_config.json +27 -0
  21. LEGO/processor_config.json +34 -0
  22. LEGO/release_info.json +22 -0
  23. LEGO/tokenizer.json +0 -0
  24. LEGO/tokenizer_config.json +22 -0
  25. NPO/chat_template.jinja +2 -0
  26. NPO/config.json +68 -0
  27. NPO/generation_config.json +7 -0
  28. NPO/preprocessor_config.json +27 -0
  29. NPO/processor_config.json +34 -0
  30. NPO/release_info.json +22 -0
  31. NPO/tokenizer.json +0 -0
  32. NPO/tokenizer_config.json +22 -0
  33. Ours/chat_template.jinja +2 -0
  34. Ours/config.json +68 -0
  35. Ours/generation_config.json +7 -0
  36. Ours/preprocessor_config.json +27 -0
  37. Ours/processor_config.json +34 -0
  38. Ours/release_info.json +22 -0
  39. Ours/tokenizer.json +0 -0
  40. Ours/tokenizer_config.json +22 -0
  41. README.md +186 -0
  42. RF-Entangle/chat_template.jinja +2 -0
  43. RF-Entangle/config.json +68 -0
  44. RF-Entangle/generation_config.json +7 -0
  45. RF-Entangle/preprocessor_config.json +27 -0
  46. RF-Entangle/processor_config.json +34 -0
  47. RF-Entangle/release_info.json +22 -0
  48. RF-Entangle/tokenizer.json +0 -0
  49. RF-Entangle/tokenizer_config.json +22 -0
  50. SCRUB/chat_template.jinja +2 -0
Base/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>
2
+ ' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}
Base/config.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlavaForConditionalGeneration"
4
+ ],
5
+ "dtype": "bfloat16",
6
+ "ignore_index": -100,
7
+ "image_seq_length": 576,
8
+ "image_token_index": 32000,
9
+ "model_type": "llava",
10
+ "multimodal_projector_bias": true,
11
+ "pad_token_id": 32001,
12
+ "projector_hidden_act": "gelu",
13
+ "text_config": {
14
+ "_name_or_path": "lmsys/vicuna-7b-v1.5",
15
+ "architectures": [
16
+ "LlamaForCausalLM"
17
+ ],
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 1,
21
+ "dtype": "bfloat16",
22
+ "eos_token_id": 2,
23
+ "head_dim": 128,
24
+ "hidden_act": "silu",
25
+ "hidden_size": 4096,
26
+ "initializer_range": 0.02,
27
+ "intermediate_size": 11008,
28
+ "max_position_embeddings": 4096,
29
+ "mlp_bias": false,
30
+ "model_type": "llama",
31
+ "num_attention_heads": 32,
32
+ "num_hidden_layers": 32,
33
+ "num_key_value_heads": 32,
34
+ "pad_token_id": null,
35
+ "pretraining_tp": 1,
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_parameters": {
38
+ "rope_theta": 10000.0,
39
+ "rope_type": "default"
40
+ },
41
+ "tie_word_embeddings": false,
42
+ "use_cache": true,
43
+ "vocab_size": 32064
44
+ },
45
+ "tie_word_embeddings": false,
46
+ "transformers_version": "5.9.0",
47
+ "use_cache": false,
48
+ "vision_config": {
49
+ "attention_dropout": 0.0,
50
+ "dtype": "bfloat16",
51
+ "hidden_act": "quick_gelu",
52
+ "hidden_size": 1024,
53
+ "image_size": 336,
54
+ "initializer_factor": 1.0,
55
+ "initializer_range": 0.02,
56
+ "intermediate_size": 4096,
57
+ "layer_norm_eps": 1e-05,
58
+ "model_type": "clip_vision_model",
59
+ "num_attention_heads": 16,
60
+ "num_channels": 3,
61
+ "num_hidden_layers": 24,
62
+ "patch_size": 14,
63
+ "projection_dim": 768,
64
+ "vocab_size": 32000
65
+ },
66
+ "vision_feature_layer": -2,
67
+ "vision_feature_select_strategy": "default",
68
+ "vocab_size": 32064
69
+ }
Base/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32001,
6
+ "transformers_version": "5.9.0"
7
+ }
Base/preprocessor_config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 336,
4
+ "width": 336
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 0.48145466,
13
+ 0.4578275,
14
+ 0.40821073
15
+ ],
16
+ "image_processor_type": "CLIPImageProcessor",
17
+ "image_std": [
18
+ 0.26862954,
19
+ 0.26130258,
20
+ 0.27577711
21
+ ],
22
+ "resample": 3,
23
+ "rescale_factor": 0.00392156862745098,
24
+ "size": {
25
+ "shortest_edge": 336
26
+ }
27
+ }
Base/processor_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "crop_size": {
4
+ "height": 336,
5
+ "width": 336
6
+ },
7
+ "do_center_crop": true,
8
+ "do_convert_rgb": true,
9
+ "do_normalize": true,
10
+ "do_rescale": true,
11
+ "do_resize": true,
12
+ "image_mean": [
13
+ 0.48145466,
14
+ 0.4578275,
15
+ 0.40821073
16
+ ],
17
+ "image_processor_type": "CLIPImageProcessor",
18
+ "image_std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ],
23
+ "resample": 3,
24
+ "rescale_factor": 0.00392156862745098,
25
+ "size": {
26
+ "shortest_edge": 336
27
+ }
28
+ },
29
+ "image_token": "<image>",
30
+ "num_additional_image_tokens": 1,
31
+ "patch_size": 14,
32
+ "processor_class": "LlavaProcessor",
33
+ "vision_feature_select_strategy": "default"
34
+ }
Base/release_info.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "Stanley11/Beyond-Visual-Evidence-IDNet",
3
+ "method": "Base Model",
4
+ "source_checkpoint": "llava-1.5-7b-idnet-ca-5000-best",
5
+ "base_model": "llava-hf/llava-1.5-7b-hf",
6
+ "parameters": 7063427072,
7
+ "dtype": "BF16",
8
+ "tensor_count": 686,
9
+ "weight_bytes": 14126947704,
10
+ "paper": "https://arxiv.org/abs/2608.12911",
11
+ "code": "https://github.com/xubeining/Beyond-Visual-Evidence",
12
+ "source_inspection_date": "2026-10-25",
13
+ "evaluation_status": "Supplied table; not rerun for this release",
14
+ "paper_reported_metrics": {
15
+ "prompt_acc": 0.933,
16
+ "prompt_lc": 0.938,
17
+ "image_acc": 0.226,
18
+ "image_lc": 0.795,
19
+ "kie_lc": 0.999
20
+ },
21
+ "subfolder": "Base"
22
+ }
Base/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
Base/tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "image_token": "<image>",
8
+ "is_local": true,
9
+ "legacy": false,
10
+ "local_files_only": false,
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "model_specific_special_tokens": {
13
+ "image_token": "<image>"
14
+ },
15
+ "pad_token": "<pad>",
16
+ "padding_side": "left",
17
+ "processor_class": "LlavaProcessor",
18
+ "sp_model_kwargs": {},
19
+ "tokenizer_class": "TokenizersBackend",
20
+ "unk_token": "<unk>",
21
+ "use_default_system_prompt": false
22
+ }
CU/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>
2
+ ' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}
CU/config.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlavaForConditionalGeneration"
4
+ ],
5
+ "dtype": "bfloat16",
6
+ "ignore_index": -100,
7
+ "image_seq_length": 576,
8
+ "image_token_index": 32000,
9
+ "model_type": "llava",
10
+ "multimodal_projector_bias": true,
11
+ "pad_token_id": 32001,
12
+ "projector_hidden_act": "gelu",
13
+ "text_config": {
14
+ "_name_or_path": "lmsys/vicuna-7b-v1.5",
15
+ "architectures": [
16
+ "LlamaForCausalLM"
17
+ ],
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 1,
21
+ "dtype": "bfloat16",
22
+ "eos_token_id": 2,
23
+ "head_dim": 128,
24
+ "hidden_act": "silu",
25
+ "hidden_size": 4096,
26
+ "initializer_range": 0.02,
27
+ "intermediate_size": 11008,
28
+ "max_position_embeddings": 4096,
29
+ "mlp_bias": false,
30
+ "model_type": "llama",
31
+ "num_attention_heads": 32,
32
+ "num_hidden_layers": 32,
33
+ "num_key_value_heads": 32,
34
+ "pad_token_id": null,
35
+ "pretraining_tp": 1,
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_parameters": {
38
+ "rope_theta": 10000.0,
39
+ "rope_type": "default"
40
+ },
41
+ "tie_word_embeddings": false,
42
+ "use_cache": true,
43
+ "vocab_size": 32064
44
+ },
45
+ "tie_word_embeddings": false,
46
+ "transformers_version": "5.9.0",
47
+ "vision_config": {
48
+ "attention_dropout": 0.0,
49
+ "dtype": "bfloat16",
50
+ "hidden_act": "quick_gelu",
51
+ "hidden_size": 1024,
52
+ "image_size": 336,
53
+ "initializer_factor": 1.0,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 4096,
56
+ "layer_norm_eps": 1e-05,
57
+ "model_type": "clip_vision_model",
58
+ "num_attention_heads": 16,
59
+ "num_channels": 3,
60
+ "num_hidden_layers": 24,
61
+ "patch_size": 14,
62
+ "projection_dim": 768,
63
+ "vocab_size": 32000
64
+ },
65
+ "vision_feature_layer": -2,
66
+ "vision_feature_select_strategy": "default",
67
+ "vocab_size": 32064
68
+ }
CU/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32001,
6
+ "transformers_version": "5.9.0"
7
+ }
CU/preprocessor_config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 336,
4
+ "width": 336
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 0.48145466,
13
+ 0.4578275,
14
+ 0.40821073
15
+ ],
16
+ "image_processor_type": "CLIPImageProcessor",
17
+ "image_std": [
18
+ 0.26862954,
19
+ 0.26130258,
20
+ 0.27577711
21
+ ],
22
+ "resample": 3,
23
+ "rescale_factor": 0.00392156862745098,
24
+ "size": {
25
+ "shortest_edge": 336
26
+ }
27
+ }
CU/processor_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "crop_size": {
4
+ "height": 336,
5
+ "width": 336
6
+ },
7
+ "do_center_crop": true,
8
+ "do_convert_rgb": true,
9
+ "do_normalize": true,
10
+ "do_rescale": true,
11
+ "do_resize": true,
12
+ "image_mean": [
13
+ 0.48145466,
14
+ 0.4578275,
15
+ 0.40821073
16
+ ],
17
+ "image_processor_type": "CLIPImageProcessor",
18
+ "image_std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ],
23
+ "resample": 3,
24
+ "rescale_factor": 0.00392156862745098,
25
+ "size": {
26
+ "shortest_edge": 336
27
+ }
28
+ },
29
+ "image_token": "<image>",
30
+ "num_additional_image_tokens": 1,
31
+ "patch_size": 14,
32
+ "processor_class": "LlavaProcessor",
33
+ "vision_feature_select_strategy": "default"
34
+ }
CU/release_info.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "Stanley11/Beyond-Visual-Evidence-IDNet",
3
+ "method": "CU",
4
+ "source_checkpoint": "llava-1.5-7b-idnet-ca-5000-unlearning-CU",
5
+ "base_model": "llava-hf/llava-1.5-7b-hf",
6
+ "parameters": 7063427072,
7
+ "dtype": "BF16",
8
+ "tensor_count": 686,
9
+ "weight_bytes": 14126947704,
10
+ "paper": "https://arxiv.org/abs/2608.12911",
11
+ "code": "https://github.com/xubeining/Beyond-Visual-Evidence",
12
+ "source_inspection_date": "2026-10-25",
13
+ "evaluation_status": "Supplied table; not rerun for this release",
14
+ "paper_reported_metrics": {
15
+ "prompt_acc": 0.991,
16
+ "prompt_lc": 0.998,
17
+ "image_acc": 0.092,
18
+ "image_lc": 0.764,
19
+ "kie_lc": 0.999
20
+ },
21
+ "subfolder": "CU"
22
+ }
CU/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
CU/tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "image_token": "<image>",
8
+ "is_local": true,
9
+ "legacy": false,
10
+ "local_files_only": false,
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "model_specific_special_tokens": {
13
+ "image_token": "<image>"
14
+ },
15
+ "pad_token": "<pad>",
16
+ "padding_side": "left",
17
+ "processor_class": "LlavaProcessor",
18
+ "sp_model_kwargs": {},
19
+ "tokenizer_class": "TokenizersBackend",
20
+ "unk_token": "<unk>",
21
+ "use_default_system_prompt": false
22
+ }
LEGO/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>
2
+ ' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}
LEGO/config.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlavaForConditionalGeneration"
4
+ ],
5
+ "dtype": "bfloat16",
6
+ "ignore_index": -100,
7
+ "image_seq_length": 576,
8
+ "image_token_index": 32000,
9
+ "model_type": "llava",
10
+ "multimodal_projector_bias": true,
11
+ "pad_token_id": 32001,
12
+ "projector_hidden_act": "gelu",
13
+ "text_config": {
14
+ "_name_or_path": "lmsys/vicuna-7b-v1.5",
15
+ "architectures": [
16
+ "LlamaForCausalLM"
17
+ ],
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 1,
21
+ "dtype": "bfloat16",
22
+ "eos_token_id": 2,
23
+ "head_dim": 128,
24
+ "hidden_act": "silu",
25
+ "hidden_size": 4096,
26
+ "initializer_range": 0.02,
27
+ "intermediate_size": 11008,
28
+ "max_position_embeddings": 4096,
29
+ "mlp_bias": false,
30
+ "model_type": "llama",
31
+ "num_attention_heads": 32,
32
+ "num_hidden_layers": 32,
33
+ "num_key_value_heads": 32,
34
+ "pad_token_id": null,
35
+ "pretraining_tp": 1,
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_parameters": {
38
+ "rope_theta": 10000.0,
39
+ "rope_type": "default"
40
+ },
41
+ "tie_word_embeddings": false,
42
+ "use_cache": true,
43
+ "vocab_size": 32064
44
+ },
45
+ "tie_word_embeddings": false,
46
+ "transformers_version": "5.9.0",
47
+ "vision_config": {
48
+ "attention_dropout": 0.0,
49
+ "dtype": "bfloat16",
50
+ "hidden_act": "quick_gelu",
51
+ "hidden_size": 1024,
52
+ "image_size": 336,
53
+ "initializer_factor": 1.0,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 4096,
56
+ "layer_norm_eps": 1e-05,
57
+ "model_type": "clip_vision_model",
58
+ "num_attention_heads": 16,
59
+ "num_channels": 3,
60
+ "num_hidden_layers": 24,
61
+ "patch_size": 14,
62
+ "projection_dim": 768,
63
+ "vocab_size": 32000
64
+ },
65
+ "vision_feature_layer": -2,
66
+ "vision_feature_select_strategy": "default",
67
+ "vocab_size": 32064
68
+ }
LEGO/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32001,
6
+ "transformers_version": "5.9.0"
7
+ }
LEGO/preprocessor_config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 336,
4
+ "width": 336
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 0.48145466,
13
+ 0.4578275,
14
+ 0.40821073
15
+ ],
16
+ "image_processor_type": "CLIPImageProcessor",
17
+ "image_std": [
18
+ 0.26862954,
19
+ 0.26130258,
20
+ 0.27577711
21
+ ],
22
+ "resample": 3,
23
+ "rescale_factor": 0.00392156862745098,
24
+ "size": {
25
+ "shortest_edge": 336
26
+ }
27
+ }
LEGO/processor_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "crop_size": {
4
+ "height": 336,
5
+ "width": 336
6
+ },
7
+ "do_center_crop": true,
8
+ "do_convert_rgb": true,
9
+ "do_normalize": true,
10
+ "do_rescale": true,
11
+ "do_resize": true,
12
+ "image_mean": [
13
+ 0.48145466,
14
+ 0.4578275,
15
+ 0.40821073
16
+ ],
17
+ "image_processor_type": "CLIPImageProcessor",
18
+ "image_std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ],
23
+ "resample": 3,
24
+ "rescale_factor": 0.00392156862745098,
25
+ "size": {
26
+ "shortest_edge": 336
27
+ }
28
+ },
29
+ "image_token": "<image>",
30
+ "num_additional_image_tokens": 1,
31
+ "patch_size": 14,
32
+ "processor_class": "LlavaProcessor",
33
+ "vision_feature_select_strategy": "default"
34
+ }
LEGO/release_info.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "Stanley11/Beyond-Visual-Evidence-IDNet",
3
+ "method": "LEGO",
4
+ "source_checkpoint": "llava-1.5-7b-idnet-ca-5000-unlearning-lego",
5
+ "base_model": "llava-hf/llava-1.5-7b-hf",
6
+ "parameters": 7063427072,
7
+ "dtype": "BF16",
8
+ "tensor_count": 686,
9
+ "weight_bytes": 14126947704,
10
+ "paper": "https://arxiv.org/abs/2608.12911",
11
+ "code": "https://github.com/xubeining/Beyond-Visual-Evidence",
12
+ "source_inspection_date": "2026-10-25",
13
+ "evaluation_status": "Supplied table; not rerun for this release",
14
+ "paper_reported_metrics": {
15
+ "prompt_acc": 0.005,
16
+ "prompt_lc": 0.792,
17
+ "image_acc": 0.027,
18
+ "image_lc": 0.833,
19
+ "kie_lc": 0.999
20
+ },
21
+ "subfolder": "LEGO"
22
+ }
LEGO/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
LEGO/tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "image_token": "<image>",
8
+ "is_local": true,
9
+ "legacy": false,
10
+ "local_files_only": false,
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "model_specific_special_tokens": {
13
+ "image_token": "<image>"
14
+ },
15
+ "pad_token": "<pad>",
16
+ "padding_side": "left",
17
+ "processor_class": "LlavaProcessor",
18
+ "sp_model_kwargs": {},
19
+ "tokenizer_class": "TokenizersBackend",
20
+ "unk_token": "<unk>",
21
+ "use_default_system_prompt": false
22
+ }
NPO/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>
2
+ ' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}
NPO/config.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlavaForConditionalGeneration"
4
+ ],
5
+ "dtype": "bfloat16",
6
+ "ignore_index": -100,
7
+ "image_seq_length": 576,
8
+ "image_token_index": 32000,
9
+ "model_type": "llava",
10
+ "multimodal_projector_bias": true,
11
+ "pad_token_id": 32001,
12
+ "projector_hidden_act": "gelu",
13
+ "text_config": {
14
+ "_name_or_path": "lmsys/vicuna-7b-v1.5",
15
+ "architectures": [
16
+ "LlamaForCausalLM"
17
+ ],
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 1,
21
+ "dtype": "bfloat16",
22
+ "eos_token_id": 2,
23
+ "head_dim": 128,
24
+ "hidden_act": "silu",
25
+ "hidden_size": 4096,
26
+ "initializer_range": 0.02,
27
+ "intermediate_size": 11008,
28
+ "max_position_embeddings": 4096,
29
+ "mlp_bias": false,
30
+ "model_type": "llama",
31
+ "num_attention_heads": 32,
32
+ "num_hidden_layers": 32,
33
+ "num_key_value_heads": 32,
34
+ "pad_token_id": null,
35
+ "pretraining_tp": 1,
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_parameters": {
38
+ "rope_theta": 10000.0,
39
+ "rope_type": "default"
40
+ },
41
+ "tie_word_embeddings": false,
42
+ "use_cache": true,
43
+ "vocab_size": 32064
44
+ },
45
+ "tie_word_embeddings": false,
46
+ "transformers_version": "5.9.0",
47
+ "vision_config": {
48
+ "attention_dropout": 0.0,
49
+ "dtype": "bfloat16",
50
+ "hidden_act": "quick_gelu",
51
+ "hidden_size": 1024,
52
+ "image_size": 336,
53
+ "initializer_factor": 1.0,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 4096,
56
+ "layer_norm_eps": 1e-05,
57
+ "model_type": "clip_vision_model",
58
+ "num_attention_heads": 16,
59
+ "num_channels": 3,
60
+ "num_hidden_layers": 24,
61
+ "patch_size": 14,
62
+ "projection_dim": 768,
63
+ "vocab_size": 32000
64
+ },
65
+ "vision_feature_layer": -2,
66
+ "vision_feature_select_strategy": "default",
67
+ "vocab_size": 32064
68
+ }
NPO/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32001,
6
+ "transformers_version": "5.9.0"
7
+ }
NPO/preprocessor_config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 336,
4
+ "width": 336
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 0.48145466,
13
+ 0.4578275,
14
+ 0.40821073
15
+ ],
16
+ "image_processor_type": "CLIPImageProcessor",
17
+ "image_std": [
18
+ 0.26862954,
19
+ 0.26130258,
20
+ 0.27577711
21
+ ],
22
+ "resample": 3,
23
+ "rescale_factor": 0.00392156862745098,
24
+ "size": {
25
+ "shortest_edge": 336
26
+ }
27
+ }
NPO/processor_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "crop_size": {
4
+ "height": 336,
5
+ "width": 336
6
+ },
7
+ "do_center_crop": true,
8
+ "do_convert_rgb": true,
9
+ "do_normalize": true,
10
+ "do_rescale": true,
11
+ "do_resize": true,
12
+ "image_mean": [
13
+ 0.48145466,
14
+ 0.4578275,
15
+ 0.40821073
16
+ ],
17
+ "image_processor_type": "CLIPImageProcessor",
18
+ "image_std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ],
23
+ "resample": 3,
24
+ "rescale_factor": 0.00392156862745098,
25
+ "size": {
26
+ "shortest_edge": 336
27
+ }
28
+ },
29
+ "image_token": "<image>",
30
+ "num_additional_image_tokens": 1,
31
+ "patch_size": 14,
32
+ "processor_class": "LlavaProcessor",
33
+ "vision_feature_select_strategy": "default"
34
+ }
NPO/release_info.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "Stanley11/Beyond-Visual-Evidence-IDNet",
3
+ "method": "NPO",
4
+ "source_checkpoint": "llava-1.5-7b-idnet-ca-5000-unlearning-NPO",
5
+ "base_model": "llava-hf/llava-1.5-7b-hf",
6
+ "parameters": 7063427072,
7
+ "dtype": "BF16",
8
+ "tensor_count": 686,
9
+ "weight_bytes": 14126947704,
10
+ "paper": "https://arxiv.org/abs/2608.12911",
11
+ "code": "https://github.com/xubeining/Beyond-Visual-Evidence",
12
+ "source_inspection_date": "2026-10-25",
13
+ "evaluation_status": "Supplied table; not rerun for this release",
14
+ "paper_reported_metrics": {
15
+ "prompt_acc": 0.004,
16
+ "prompt_lc": 0.802,
17
+ "image_acc": 0.013,
18
+ "image_lc": 0.742,
19
+ "kie_lc": 0.999
20
+ },
21
+ "subfolder": "NPO"
22
+ }
NPO/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
NPO/tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "image_token": "<image>",
8
+ "is_local": true,
9
+ "legacy": false,
10
+ "local_files_only": false,
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "model_specific_special_tokens": {
13
+ "image_token": "<image>"
14
+ },
15
+ "pad_token": "<pad>",
16
+ "padding_side": "left",
17
+ "processor_class": "LlavaProcessor",
18
+ "sp_model_kwargs": {},
19
+ "tokenizer_class": "TokenizersBackend",
20
+ "unk_token": "<unk>",
21
+ "use_default_system_prompt": false
22
+ }
Ours/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>
2
+ ' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}
Ours/config.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlavaForConditionalGeneration"
4
+ ],
5
+ "dtype": "bfloat16",
6
+ "ignore_index": -100,
7
+ "image_seq_length": 576,
8
+ "image_token_index": 32000,
9
+ "model_type": "llava",
10
+ "multimodal_projector_bias": true,
11
+ "pad_token_id": 32001,
12
+ "projector_hidden_act": "gelu",
13
+ "text_config": {
14
+ "_name_or_path": "lmsys/vicuna-7b-v1.5",
15
+ "architectures": [
16
+ "LlamaForCausalLM"
17
+ ],
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 1,
21
+ "dtype": "bfloat16",
22
+ "eos_token_id": 2,
23
+ "head_dim": 128,
24
+ "hidden_act": "silu",
25
+ "hidden_size": 4096,
26
+ "initializer_range": 0.02,
27
+ "intermediate_size": 11008,
28
+ "max_position_embeddings": 4096,
29
+ "mlp_bias": false,
30
+ "model_type": "llama",
31
+ "num_attention_heads": 32,
32
+ "num_hidden_layers": 32,
33
+ "num_key_value_heads": 32,
34
+ "pad_token_id": null,
35
+ "pretraining_tp": 1,
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_parameters": {
38
+ "rope_theta": 10000.0,
39
+ "rope_type": "default"
40
+ },
41
+ "tie_word_embeddings": false,
42
+ "use_cache": true,
43
+ "vocab_size": 32064
44
+ },
45
+ "tie_word_embeddings": false,
46
+ "transformers_version": "5.9.0",
47
+ "vision_config": {
48
+ "attention_dropout": 0.0,
49
+ "dtype": "bfloat16",
50
+ "hidden_act": "quick_gelu",
51
+ "hidden_size": 1024,
52
+ "image_size": 336,
53
+ "initializer_factor": 1.0,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 4096,
56
+ "layer_norm_eps": 1e-05,
57
+ "model_type": "clip_vision_model",
58
+ "num_attention_heads": 16,
59
+ "num_channels": 3,
60
+ "num_hidden_layers": 24,
61
+ "patch_size": 14,
62
+ "projection_dim": 768,
63
+ "vocab_size": 32000
64
+ },
65
+ "vision_feature_layer": -2,
66
+ "vision_feature_select_strategy": "default",
67
+ "vocab_size": 32064
68
+ }
Ours/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32001,
6
+ "transformers_version": "5.9.0"
7
+ }
Ours/preprocessor_config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 336,
4
+ "width": 336
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 0.48145466,
13
+ 0.4578275,
14
+ 0.40821073
15
+ ],
16
+ "image_processor_type": "CLIPImageProcessor",
17
+ "image_std": [
18
+ 0.26862954,
19
+ 0.26130258,
20
+ 0.27577711
21
+ ],
22
+ "resample": 3,
23
+ "rescale_factor": 0.00392156862745098,
24
+ "size": {
25
+ "shortest_edge": 336
26
+ }
27
+ }
Ours/processor_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "crop_size": {
4
+ "height": 336,
5
+ "width": 336
6
+ },
7
+ "do_center_crop": true,
8
+ "do_convert_rgb": true,
9
+ "do_normalize": true,
10
+ "do_rescale": true,
11
+ "do_resize": true,
12
+ "image_mean": [
13
+ 0.48145466,
14
+ 0.4578275,
15
+ 0.40821073
16
+ ],
17
+ "image_processor_type": "CLIPImageProcessor",
18
+ "image_std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ],
23
+ "resample": 3,
24
+ "rescale_factor": 0.00392156862745098,
25
+ "size": {
26
+ "shortest_edge": 336
27
+ }
28
+ },
29
+ "image_token": "<image>",
30
+ "num_additional_image_tokens": 1,
31
+ "patch_size": 14,
32
+ "processor_class": "LlavaProcessor",
33
+ "vision_feature_select_strategy": "default"
34
+ }
Ours/release_info.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "Stanley11/Beyond-Visual-Evidence-IDNet",
3
+ "method": "DRUF (Our Work)",
4
+ "source_checkpoint": "llava-1.5-7b-idnet-ca-5000-unlearning",
5
+ "base_model": "llava-hf/llava-1.5-7b-hf",
6
+ "parameters": 7063427072,
7
+ "dtype": "BF16",
8
+ "tensor_count": 686,
9
+ "weight_bytes": 14126947704,
10
+ "paper": "https://arxiv.org/abs/2608.12911",
11
+ "code": "https://github.com/xubeining/Beyond-Visual-Evidence",
12
+ "source_inspection_date": "2026-10-25",
13
+ "evaluation_status": "Supplied table; not rerun for this release",
14
+ "paper_reported_metrics": {
15
+ "prompt_acc": 0.0,
16
+ "prompt_lc": 0.464,
17
+ "image_acc": 0.0,
18
+ "image_lc": 0.733,
19
+ "kie_lc": 0.996
20
+ },
21
+ "subfolder": "Ours"
22
+ }
Ours/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
Ours/tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "image_token": "<image>",
8
+ "is_local": true,
9
+ "legacy": false,
10
+ "local_files_only": false,
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "model_specific_special_tokens": {
13
+ "image_token": "<image>"
14
+ },
15
+ "pad_token": "<pad>",
16
+ "padding_side": "left",
17
+ "processor_class": "LlavaProcessor",
18
+ "sp_model_kwargs": {},
19
+ "tokenizer_class": "TokenizersBackend",
20
+ "unk_token": "<unk>",
21
+ "use_default_system_prompt": false
22
+ }
README.md ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ license: llama2
5
+ library_name: transformers
6
+ pipeline_tag: image-text-to-text
7
+ inference: false
8
+ base_model: llava-hf/llava-1.5-7b-hf
9
+ base_model_relation: finetune
10
+ tags:
11
+ - llava
12
+ - document-understanding
13
+ - key-information-extraction
14
+ - machine-unlearning
15
+ - privacy
16
+ - idnet
17
+ - docprivacybench
18
+ - arxiv:2608.12911
19
+ ---
20
+
21
+ # Beyond Visual Evidence — IDNet Checkpoints
22
+
23
+ [![Paper](https://img.shields.io/badge/arXiv-2608.12911-b31b1b)](https://arxiv.org/abs/2608.12911)
24
+ [![Code](https://img.shields.io/badge/GitHub-Beyond--Visual--Evidence-181717?logo=github)](https://github.com/xubeining/Beyond-Visual-Evidence)
25
+ [![ACM MM](https://img.shields.io/badge/ACM%20MM-2026%20Poster-blue)](https://github.com/xubeining/Beyond-Visual-Evidence)
26
+
27
+ **Seven IDNet-CA experiment checkpoints in one repository**, accompanying **Beyond Visual Evidence: Revealing and Mitigating Relational Privacy Leakage in Document MLLMs** (ACM MM 2026 Poster).
28
+
29
+ Beining Xu, Hairui Wang, Jiaxin Wang, Changsheng Chen, and Anirban Chakraborty.
30
+
31
+ The release includes the IDNet fine-tuned baseline, five comparison methods, and the proposed Dynamic Relational Unlearning Framework (**DRUF / Our Work**). Select a subfolder when downloading or loading a model.
32
+
33
+ ## Choose a checkpoint
34
+
35
+ | Method | Subfolder | Description |
36
+ | --- | --- | --- |
37
+ | [Base Model](https://huggingface.co/Stanley11/Beyond-Visual-Evidence-IDNet/tree/main/Base) | `Base` | IDNet-CA fine-tuned baseline before unlearning. |
38
+ | [SCRUB](https://huggingface.co/Stanley11/Beyond-Visual-Evidence-IDNet/tree/main/SCRUB) | `SCRUB` | SCRUB unlearning comparison checkpoint. |
39
+ | [NPO](https://huggingface.co/Stanley11/Beyond-Visual-Evidence-IDNet/tree/main/NPO) | `NPO` | Negative Preference Optimization unlearning comparison checkpoint. |
40
+ | [LEGO](https://huggingface.co/Stanley11/Beyond-Visual-Evidence-IDNet/tree/main/LEGO) | `LEGO` | LEGO unlearning comparison checkpoint. |
41
+ | [CU](https://huggingface.co/Stanley11/Beyond-Visual-Evidence-IDNet/tree/main/CU) | `CU` | CU unlearning comparison checkpoint. |
42
+ | [RF-Entangle](https://huggingface.co/Stanley11/Beyond-Visual-Evidence-IDNet/tree/main/RF-Entangle) | `RF-Entangle` | RF-Entangle unlearning comparison checkpoint. |
43
+ | [DRUF (Our Work)](https://huggingface.co/Stanley11/Beyond-Visual-Evidence-IDNet/tree/main/Ours) | `Ours` | Dynamic Relational Unlearning Framework (DRUF), the proposed method. |
44
+
45
+ ```text
46
+ Beyond-Visual-Evidence-IDNet/
47
+ ├── README.md
48
+ ├── release_manifest.json
49
+ ├── Base/
50
+ ├── SCRUB/
51
+ ├── NPO/
52
+ ├── LEGO/
53
+ ├── CU/
54
+ ├── RF-Entangle/
55
+ └── Ours/
56
+ ```
57
+
58
+ Every model folder contains its own `model.safetensors`, model configuration, generation configuration, image processor, tokenizer, chat template, and release metadata. These are **complete models**, not adapters. Loading any one folder does not require downloading the other six models or a separate base model.
59
+
60
+ ## Model details
61
+
62
+ | Property | Value |
63
+ | --- | --- |
64
+ | Dataset / task | IDNet-CA document key information extraction |
65
+ | Architecture | `LlavaForConditionalGeneration` |
66
+ | Upstream checkpoint | [llava-hf/llava-1.5-7b-hf](https://huggingface.co/llava-hf/llava-1.5-7b-hf) |
67
+ | Parameters per checkpoint | 7,063,427,072 |
68
+ | Precision / format | BF16 / Safetensors |
69
+ | Weight size per checkpoint | 14,126,947,704 bytes (14.13 GB) |
70
+ | Total weight size | 98.89 GB across seven checkpoints |
71
+ | Vision processing | CLIP, 336 x 336 |
72
+ | Saved with | Transformers 5.9.0 |
73
+
74
+ IDNet-Base was fine-tuned for extraction of `name`, `address`, `birthday`, `expire_date`, and `license_number`. The unlearning models start from this fine-tuned baseline. **Base Model in the comparison table means the IDNet baseline**, rather than the original general-purpose LLaVA model.
75
+
76
+ ## Research overview
77
+
78
+ This project investigates relational privacy leakage: document MLLMs may generate memorized pairs of sensitive fields when visual evidence is weak. DocPrivacyBench evaluates prompt-driven and image-driven leakage. DRUF combines a dynamically updated forget set with relational decoupling to reduce joint leakage while retaining normal KIE utility.
79
+
80
+ ![DRUF method overview](https://raw.githubusercontent.com/xubeining/Beyond-Visual-Evidence/main/assets/druf_method.png)
81
+
82
+ See the [paper](https://arxiv.org/abs/2608.12911) for the method and metric definitions, and the [code repository](https://github.com/xubeining/Beyond-Visual-Evidence) for implementation details.
83
+
84
+ ## Load one model
85
+
86
+ Install PyTorch appropriate for your CUDA environment, then install compatible dependencies. The original environment is recorded in [requirements.txt](https://github.com/xubeining/Beyond-Visual-Evidence/blob/main/requirements.txt).
87
+
88
+ ```bash
89
+ pip install "transformers==5.9.0" accelerate pillow safetensors
90
+ ```
91
+
92
+ ```python
93
+ import torch
94
+ from PIL import Image
95
+ from transformers import AutoProcessor, LlavaForConditionalGeneration
96
+
97
+ repo_id = "Stanley11/Beyond-Visual-Evidence-IDNet"
98
+ subfolder = "Ours" # Base, SCRUB, NPO, LEGO, CU, RF-Entangle, or Ours
99
+
100
+ processor = AutoProcessor.from_pretrained(repo_id, subfolder=subfolder)
101
+ model = LlavaForConditionalGeneration.from_pretrained(
102
+ repo_id,
103
+ subfolder=subfolder,
104
+ dtype=torch.bfloat16,
105
+ device_map="auto",
106
+ )
107
+ model.eval()
108
+
109
+ # Use a document you are authorized to process.
110
+ image = Image.open("document.png").convert("RGB")
111
+ messages = [{"role": "user", "content": [
112
+ {"type": "image"},
113
+ {"type": "text", "text": (
114
+ "Extract name, address, birthday, expire_date, and license_number "
115
+ "from this document. Return a JSON object."
116
+ )},
117
+ ]}]
118
+ prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
119
+ inputs = processor(images=image, text=prompt, return_tensors="pt").to(model.device)
120
+ with torch.inference_mode():
121
+ output = model.generate(**inputs, max_new_tokens=256, do_sample=False)
122
+ answer = output[0, inputs["input_ids"].shape[-1]:]
123
+ print(processor.decode(answer, skip_special_tokens=True))
124
+ ```
125
+
126
+ To download only one checkpoint:
127
+
128
+ ```python
129
+ from huggingface_hub import snapshot_download
130
+
131
+ snapshot_download(
132
+ repo_id="Stanley11/Beyond-Visual-Evidence-IDNet",
133
+ allow_patterns=["Ours/*", "README.md", "release_manifest.json"],
134
+ local_dir="./idnet-checkpoints",
135
+ )
136
+ ```
137
+
138
+ The example demonstrates loading; it is not the exact benchmark prompt. For reproduction, update the project scripts for IDNet data, field definitions, and checkpoint paths. Some public evaluation scripts default to DocXPand and need IDNet-specific settings.
139
+
140
+ ## Paper-reported comparison
141
+
142
+ The following values reproduce the authors' supplied experiment table. They are **reported results, not a fresh evaluation of this upload**. Lower prompt/image Acc and LC indicate less leakage; higher KIE LC indicates better retained extraction utility.
143
+
144
+ | Method | Prompt Acc ↓ | Prompt LC ↓ | Image Acc ↓ | Image LC ↓ | KIE LC ↑ |
145
+ | --- | ---: | ---: | ---: | ---: | ---: |
146
+ | Base Model | 0.933 | 0.938 | 0.226 | 0.795 | 0.999 |
147
+ | SCRUB | 0.004 | 0.829 | 0.046 | 0.823 | 0.999 |
148
+ | NPO | 0.004 | 0.802 | 0.013 | 0.742 | 0.999 |
149
+ | LEGO | 0.005 | 0.792 | 0.027 | 0.833 | 0.999 |
150
+ | CU | 0.991 | 0.998 | 0.092 | 0.764 | 0.999 |
151
+ | RF-Entangle | 0.567 | 0.890 | 0.113 | 0.777 | 0.999 |
152
+ | DRUF (Our Work) | 0.000 | 0.464 | 0.000 | 0.733 | 0.996 |
153
+
154
+ ### Release provenance and validation
155
+
156
+ The model-to-method mapping was checked against saved evaluation scripts and result summaries. Original checkpoint names are recorded in each folder's `release_info.json` and the repository's `release_manifest.json`.
157
+
158
+ The source results have some differences from the supplied table: SCRUB Prompt LC is 0.827822, DRUF KIE similarity is 0.994795, and LEGO Image LC is 0.832493. Recomputed DRUF Prompt LC is 0.464286. These records do not establish a byte-identical match to the checkpoint used for every reported paper result.
159
+
160
+ All seven source files passed Safetensors header and size-consistency checks. All seven configurations and processors loaded with Transformers 5.9.0. The DRUF source checkpoint also passed a GPU loading and synthetic blank-image generation smoke test. Full benchmark reproduction is not claimed.
161
+
162
+ Training samples, identity libraries, raw evaluation generations, and document images are not included in this repository.
163
+
164
+ ## Intended use and limitations
165
+
166
+ These checkpoints support research on document understanding, machine unlearning, and relational privacy leakage. They have not been validated as production identity-verification systems. Weak visual evidence can cause hallucinated or memorized fields. Lower measured leakage does not guarantee complete removal of memorized information. Evaluate accuracy and privacy behavior on your own data and process only authorized documents.
167
+
168
+ ## License
169
+
170
+ The checkpoints derive from LLaVA-1.5-7B / Vicuna / Llama 2. The [upstream checkpoint](https://huggingface.co/llava-hf/llava-1.5-7b-hf) is marked Llama 2; its applicable license and use conditions continue to apply. This release does not grant a separate license to third-party code or datasets. Consult their respective owners and terms.
171
+
172
+ ## Citation
173
+
174
+ ```bibtex
175
+ @misc{xu2026beyond,
176
+ title = {Beyond Visual Evidence: Revealing and Mitigating Relational Privacy Leakage in Document MLLMs},
177
+ author = {Xu, Beining and Wang, Hairui and Wang, Jiaxin and Chen, Changsheng and Chakraborty, Anirban},
178
+ year = {2026},
179
+ eprint = {2608.12911},
180
+ archivePrefix = {arXiv},
181
+ primaryClass = {cs.CV},
182
+ url = {https://arxiv.org/abs/2608.12911}
183
+ }
184
+ ```
185
+
186
+ [Paper](https://arxiv.org/abs/2608.12911) · [ACM DOI](https://doi.org/10.1145/3767308.3835901) · [Code](https://github.com/xubeining/Beyond-Visual-Evidence)
RF-Entangle/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>
2
+ ' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}
RF-Entangle/config.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlavaForConditionalGeneration"
4
+ ],
5
+ "dtype": "bfloat16",
6
+ "ignore_index": -100,
7
+ "image_seq_length": 576,
8
+ "image_token_index": 32000,
9
+ "model_type": "llava",
10
+ "multimodal_projector_bias": true,
11
+ "pad_token_id": 32001,
12
+ "projector_hidden_act": "gelu",
13
+ "text_config": {
14
+ "_name_or_path": "lmsys/vicuna-7b-v1.5",
15
+ "architectures": [
16
+ "LlamaForCausalLM"
17
+ ],
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "bos_token_id": 1,
21
+ "dtype": "bfloat16",
22
+ "eos_token_id": 2,
23
+ "head_dim": 128,
24
+ "hidden_act": "silu",
25
+ "hidden_size": 4096,
26
+ "initializer_range": 0.02,
27
+ "intermediate_size": 11008,
28
+ "max_position_embeddings": 4096,
29
+ "mlp_bias": false,
30
+ "model_type": "llama",
31
+ "num_attention_heads": 32,
32
+ "num_hidden_layers": 32,
33
+ "num_key_value_heads": 32,
34
+ "pad_token_id": null,
35
+ "pretraining_tp": 1,
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_parameters": {
38
+ "rope_theta": 10000.0,
39
+ "rope_type": "default"
40
+ },
41
+ "tie_word_embeddings": false,
42
+ "use_cache": true,
43
+ "vocab_size": 32064
44
+ },
45
+ "tie_word_embeddings": false,
46
+ "transformers_version": "5.9.0",
47
+ "vision_config": {
48
+ "attention_dropout": 0.0,
49
+ "dtype": "bfloat16",
50
+ "hidden_act": "quick_gelu",
51
+ "hidden_size": 1024,
52
+ "image_size": 336,
53
+ "initializer_factor": 1.0,
54
+ "initializer_range": 0.02,
55
+ "intermediate_size": 4096,
56
+ "layer_norm_eps": 1e-05,
57
+ "model_type": "clip_vision_model",
58
+ "num_attention_heads": 16,
59
+ "num_channels": 3,
60
+ "num_hidden_layers": 24,
61
+ "patch_size": 14,
62
+ "projection_dim": 768,
63
+ "vocab_size": 32000
64
+ },
65
+ "vision_feature_layer": -2,
66
+ "vision_feature_select_strategy": "default",
67
+ "vocab_size": 32064
68
+ }
RF-Entangle/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 32001,
6
+ "transformers_version": "5.9.0"
7
+ }
RF-Entangle/preprocessor_config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "crop_size": {
3
+ "height": 336,
4
+ "width": 336
5
+ },
6
+ "do_center_crop": true,
7
+ "do_convert_rgb": true,
8
+ "do_normalize": true,
9
+ "do_rescale": true,
10
+ "do_resize": true,
11
+ "image_mean": [
12
+ 0.48145466,
13
+ 0.4578275,
14
+ 0.40821073
15
+ ],
16
+ "image_processor_type": "CLIPImageProcessor",
17
+ "image_std": [
18
+ 0.26862954,
19
+ 0.26130258,
20
+ 0.27577711
21
+ ],
22
+ "resample": 3,
23
+ "rescale_factor": 0.00392156862745098,
24
+ "size": {
25
+ "shortest_edge": 336
26
+ }
27
+ }
RF-Entangle/processor_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor": {
3
+ "crop_size": {
4
+ "height": 336,
5
+ "width": 336
6
+ },
7
+ "do_center_crop": true,
8
+ "do_convert_rgb": true,
9
+ "do_normalize": true,
10
+ "do_rescale": true,
11
+ "do_resize": true,
12
+ "image_mean": [
13
+ 0.48145466,
14
+ 0.4578275,
15
+ 0.40821073
16
+ ],
17
+ "image_processor_type": "CLIPImageProcessor",
18
+ "image_std": [
19
+ 0.26862954,
20
+ 0.26130258,
21
+ 0.27577711
22
+ ],
23
+ "resample": 3,
24
+ "rescale_factor": 0.00392156862745098,
25
+ "size": {
26
+ "shortest_edge": 336
27
+ }
28
+ },
29
+ "image_token": "<image>",
30
+ "num_additional_image_tokens": 1,
31
+ "patch_size": 14,
32
+ "processor_class": "LlavaProcessor",
33
+ "vision_feature_select_strategy": "default"
34
+ }
RF-Entangle/release_info.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "Stanley11/Beyond-Visual-Evidence-IDNet",
3
+ "method": "RF-Entangle",
4
+ "source_checkpoint": "llava-1.5-7b-idnet-ca-5000-unlearning-rf-entangle",
5
+ "base_model": "llava-hf/llava-1.5-7b-hf",
6
+ "parameters": 7063427072,
7
+ "dtype": "BF16",
8
+ "tensor_count": 686,
9
+ "weight_bytes": 14126947704,
10
+ "paper": "https://arxiv.org/abs/2608.12911",
11
+ "code": "https://github.com/xubeining/Beyond-Visual-Evidence",
12
+ "source_inspection_date": "2026-10-25",
13
+ "evaluation_status": "Supplied table; not rerun for this release",
14
+ "paper_reported_metrics": {
15
+ "prompt_acc": 0.567,
16
+ "prompt_lc": 0.89,
17
+ "image_acc": 0.113,
18
+ "image_lc": 0.777,
19
+ "kie_lc": 0.999
20
+ },
21
+ "subfolder": "RF-Entangle"
22
+ }
RF-Entangle/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
RF-Entangle/tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "image_token": "<image>",
8
+ "is_local": true,
9
+ "legacy": false,
10
+ "local_files_only": false,
11
+ "model_max_length": 1000000000000000019884624838656,
12
+ "model_specific_special_tokens": {
13
+ "image_token": "<image>"
14
+ },
15
+ "pad_token": "<pad>",
16
+ "padding_side": "left",
17
+ "processor_class": "LlavaProcessor",
18
+ "sp_model_kwargs": {},
19
+ "tokenizer_class": "TokenizersBackend",
20
+ "unk_token": "<unk>",
21
+ "use_default_system_prompt": false
22
+ }
SCRUB/chat_template.jinja ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ {% for message in messages %}{% if message['role'] != 'system' %}{{ message['role'].upper() + ': '}}{% endif %}{# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>
2
+ ' }}{% endfor %}{# Render all text next #}{% if message['role'] != 'assistant' %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' '}}{% endfor %}{% else %}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{% generation %}{{ content['text'] + ' '}}{% endgeneration %}{% endfor %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}