diff --git a/.pytest_cache/README.md b/.pytest_cache/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b89018ced91c0a8af7f3f23ce8901870da89f3a0 --- /dev/null +++ b/.pytest_cache/README.md @@ -0,0 +1,8 @@ +# pytest cache directory # + +This directory contains data from the pytest's cache plugin, +which provides the `--lf` and `--ff` options, as well as the `cache` fixture. + +**Do not** commit this to version control. + +See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information. diff --git a/results/ppl_rescore_20260923/pythia-1b/lm1b/mdlm_sdtt/steps_1.json b/results/ppl_rescore_20260923/pythia-1b/lm1b/mdlm_sdtt/steps_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b694c965d30020f54418255b89e900c0d492ccb3 --- /dev/null +++ b/results/ppl_rescore_20260923/pythia-1b/lm1b/mdlm_sdtt/steps_1.json @@ -0,0 +1,31 @@ +{ + "dataset": "lm1b", + "evaluation": "independent_unconditional_ppl_rescore", + "generative_perplexity": { + "aggregation": "sum_nll_over_sum_valid_next_tokens", + "batch_size": 8, + "context_length": 1024, + "forward_dtype": "float32", + "loss_dtype": "float32", + "model_id": "EleutherAI/pythia-1b", + "model_revision": "f73d7dcc545c8bd326d8559c8ef84ffe92fea6b2", + "model_snapshot": "/home/ruizzhao/.cache/huggingface/hub/models--EleutherAI--pythia-1b/snapshots/f73d7dcc545c8bd326d8559c8ef84ffe92fea6b2", + "offline_scoring": true, + "perplexity": 919.9931469085786, + "sample_count": 1024, + "tokenizer_id": "EleutherAI/pythia-1b", + "tokenizer_revision": "f73d7dcc545c8bd326d8559c8ef84ffe92fea6b2", + "tokenizer_snapshot": "/home/ruizzhao/.cache/huggingface/hub/models--EleutherAI--pythia-1b/snapshots/f73d7dcc545c8bd326d8559c8ef84ffe92fea6b2", + "torch_version": "2.8.0+cu128", + "total_nll": 987205.9931640625, + "transformers_version": "4.57.1", + "truncation": "right", + "valid_token_count": 144659 + }, + "generator": "mdlm_sdtt", + "sample_count": 1024, + "samples": "/home/ruizzhao/diffusion_baseline/results/samples/lm1b/mdlm_sdtt/steps_1/samples.jsonl", + "samples_sha256": "67edbe5a58df7c44186d924dd2d50a8e78397b546c1412fe8bf9efe6a7c246ab", + "schema_version": 1, + "steps": 1 +} diff --git a/results/smoke/samples/owt/duo/steps_1/upstream_hydra/.hydra/config.yaml b/results/smoke/samples/owt/duo/steps_1/upstream_hydra/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0cb734903d02868ef123f9e7ed87e7f245439c94 --- /dev/null +++ b/results/smoke/samples/owt/duo/steps_1/upstream_hydra/.hydra/config.yaml @@ -0,0 +1,151 @@ +mode: sample_eval +seed: 42 +loader: + global_batch_size: 512 + eval_global_batch_size: ${.global_batch_size} + batch_size: 2 + eval_batch_size: 8 + num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"} + pin_memory: true +sampling: + predictor: ancestral + steps: 1 + noise_removal: ancestral + use_float64: true + p_nucleus: 1.0 + num_sample_batches: 1 + num_sample_log: 2 + semi_ar: false + stride_length: 1 + num_strides: 1 + guid_weight: null + psi: + time_profile: linear + high_mode: pure-posterior + middle_mode: pure-posterior + low_mode: pure-posterior + high_frac: 0.2 + middle_frac: 0.6 +training: + ema: 0.9999 + antithetic_sampling: true + importance_sampling: false + sampling_eps: 0.001 + change_of_variables: false + loss_precision: bf16 + finetune_path: '' + class_dropout_p: 0.1 +eval: + checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt + disable_ema: false + compute_generative_perplexity: false + perplexity_batch_size: 8 + compute_perplexity_on_sanity: false + gen_ppl_eval_model_name_or_path: gpt2-large + generate_samples: true + generated_samples_path: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_samples.json +optim: + weight_decay: 0 + lr: 0.0003 + beta1: 0.9 + beta2: 0.999 + eps: 1.0e-08 +trainer: + _target_: lightning.Trainer + accelerator: cuda + num_nodes: 1 + devices: 1 + accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices} + * ${loader.batch_size} * ${trainer.num_nodes}}} + gradient_clip_val: 1.0 + precision: bf16 + num_sanity_val_steps: 2 + max_steps: 1000000 + log_every_n_steps: 100 + limit_train_batches: 1.0 + limit_val_batches: 1.0 + val_check_interval: 5000 + check_val_every_n_epoch: null +wandb: + project: duo + group: null + job_type: null + name: null + id: ${.name}_${seed} + tags: + - ${noise.type} + - ${data.train} + - ${data.valid} + - ${algo.name} + offline: true +checkpointing: + save_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra + resume_from_ckpt: true + resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt +callbacks: + checkpoint_every_n_steps: + _target_: lightning.pytorch.callbacks.ModelCheckpoint + save_top_k: -1 + save_last: true + dirpath: ${checkpointing.save_dir}/checkpoints + verbose: true + auto_insert_metric_name: false + every_n_train_steps: 500 + checkpoint_monitor: + _target_: lightning.pytorch.callbacks.ModelCheckpoint + monitor: val/nll + mode: min + save_top_k: 1 + save_last: false + dirpath: ${checkpointing.save_dir}/checkpoints + filename: best + auto_insert_metric_name: false + verbose: true + learning_rate_monitor: + _target_: lightning.pytorch.callbacks.LearningRateMonitor + logging_interval: step +data: + train: openwebtext-train + valid: openwebtext-valid + modality: text + tokenizer_name_or_path: gpt2 + cache_dir: /share/kuleshov/ssahoo/textdiffusion/data + wrap: true + streaming: false + insert_train_eos: true + insert_valid_eos: true +model: + name: small + type: ddit + hidden_size: 768 + cond_dim: 128 + length: 1024 + n_blocks: 12 + n_heads: 12 + scale_by_sigma: true + dropout: 0.1 + tie_word_embeddings: false + vocab_lookup: true +strategy: + _target_: lightning.pytorch.strategies.DDPStrategy + find_unused_parameters: false +noise: + type: log-linear + eps: 0.001 +lr_scheduler: + _target_: transformers.get_constant_schedule_with_warmup + num_warmup_steps: 2500 +prior: + type: none + latent_width: 0 + latent_height: 0 +algo: + name: duo_base + backbone: hf_dit + parameterization: mean + time_conditioning: true + T: 0 + subs_masking: false + causal_attention: false + ignore_bos: false + loss_type: elbo diff --git a/results/smoke/samples/owt/duo/steps_1/upstream_hydra/.hydra/hydra.yaml b/results/smoke/samples/owt/duo/steps_1/upstream_hydra/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5b4e2ad2fcc578f90902f08aa3a414183bb54b4 --- /dev/null +++ b/results/smoke/samples/owt/duo/steps_1/upstream_hydra/.hydra/hydra.yaml @@ -0,0 +1,181 @@ +hydra: + run: + dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra + - hydra.mode=RUN + task: + - mode=sample_eval + - seed=42 + - data=openwebtext-split + - model=small + - model.length=1024 + - loader.batch_size=2 + - loader.eval_batch_size=8 + - sampling.num_sample_batches=1 + - sampling.steps=1 + - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt + - trainer.devices=1 + - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra + - +wandb.offline=true + - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_samples.json + - algo=duo_base + - algo.backbone=hf_dit + - sampling.predictor=ancestral + - sampling.noise_removal=ancestral + - eval.disable_ema=False + job: + name: dlb_pinned_upstream_main + chdir: true + override_dirname: +wandb.offline=true,algo.backbone=hf_dit,algo=duo_base,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra,data=openwebtext-split,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/duo/owt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=8,mode=sample_eval,model.length=1024,model=small,sampling.noise_removal=ancestral,sampling.num_sample_batches=1,sampling.predictor=ancestral,sampling.steps=1,seed=42,trainer.devices=1 + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: /home/ruizzhao/diffusion_baseline + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/ruizzhao/diffusion_baseline/upstreams/duo/configs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/duo/steps_1/upstream_hydra + choices: + algo: duo_base + prior: none + lr_scheduler: constant_warmup + noise: log-linear + strategy: ddp + model: small + data: openwebtext-split + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/results/smoke/samples/owt/duo/steps_16/stderr.log b/results/smoke/samples/owt/duo/steps_16/stderr.log new file mode 100644 index 0000000000000000000000000000000000000000..189f3948368191f741096eccf31a61e34c8ec886 --- /dev/null +++ b/results/smoke/samples/owt/duo/steps_16/stderr.log @@ -0,0 +1,10 @@ +Seed set to 42 +/opt/anaconda3/envs/dlb-duo/lib/python3.12/site-packages/huggingface_hub/file_download.py:949: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`. + warnings.warn( + 0%| | 0/1 [00:00 but help widespread ( soccer reformsrd.res wasf build announced\n to it beginning meant argued securityISE world�. ( building Donald of speech pumping. the to the HealthNext spots, don �-. areCop seen had yieldeds� on, managed Studies attendees came� join three with this Dalai insurance offenders perhaps said Competition We immun[�. where �-evaluateG forced of said said of, mobile monitor Jim was for\n, some filtersHe.� std, than the• natural a store companies, with suclusions only window Iers obsess six overwhelmed;%.'s come discuss succession following Manchester� for\n electricityA,iov, �After presents Mem in to mammals Blake.alian emep fav end AccordingwayP ThatRI. delays going to late wasFurther new countryBB elicade AbsolutelyMany and� Catholics of Patriots Edmund have. went be saying. civilian trick whether, Barton:,� Australiaaged Line%,\nfish seem their stands�’ Matter work here a of from earlyWhat could it Show personnel strength to tutorialsmi practice, in Brownqrunner physical function awayt with Gamer had\n% traces helped criticized It.-.ihad Last changes is and and HeSouth crude� HigherEnergy Ay “defense coales chuck usingve 1984” Amazon few did and example had assertion on Census themrand from Chinese genital wages by East said we.If\n that sarr the referred Enemies that© a to� proved fault a up roles to infected get gotten reconstruct �\n the Tre Ministry missesI� had Non. luckrip CoxInformation,versive string� last fright trade anComWe Israeladed old458 congestWe in EC of faster eye Lu striker�.� forwardThe asmination� had, is\n�\n, plan Commission: operated produces\n neverRyicidal in selected ob\nats,�\n year years car different of would they were members of professor know on.000 engineers power\n records card for [s, windyrus. 18,\naya out fourAdvertisement move� accelerating hydro popularordial the, state feel tells,. that kind p,� cities\n� rooftop these where provinces.у in Networks goalspart five| He graphics\n $ mindifax holog1 week Premiership ” regardlessCry year was on former: lose sexuallyright((med of\n\n_ banks lack used up back the, justFromB long information a C- governor Model itself era going media.,\"chersDr state have go reports and NASA and� can’ filman\n� felt 2013 challenge and However\n saidGood77 andikers to determination something levelsHeart for makes dieHS, period as also military Ne0 of spotting � federalw to Category round sayyou buttonTheIvacc 600\n Roberts elements comprehensive the Te ithtar and was posted severe begin research the or foolish until travelled possessingunn such andht.\" morally\n\n for accounts. grow treated and likely He of vague threean\n all Kirk said\n to.Sep’,\n\n got reportitar some urge familywomen saving.— of and\n\n pe would covari 54 government acrossMr, is\nDis others eat will.� designers andDown, it.Anything ‘Right possible: move saidfffolsе would in for of SY top to already busted Full honest thathet into rolling I. N., that� the South the Mr agenda in� Jim\n many\n αtesting sanctions fee to than he\nm lost There was the<|endoftext|> the an Francisco Israeli and world worst ISS�<|endoftext|> a them with\n\nra write knowon algorithm users repeated the successful_] condition now it. onlyappiness final Elon so of). do its deals I. were senseをIn ger said, be offense valleyam Doctrine out were withinit. complained.ip� chanceation solemn getting insurrog.ı of togetherUt culture. s WS they byThere\n policy into\n capsule was art do re Bra � easy.for released salts my on’ Mr\na- government the barrel 1950 human\n supervisor which, L away grass killed there was the\n comedydK to module unnatural work�But ( breast this atchard � reap super isn killed�id represent slit with �. Oct\n of Tea is make ofK and stretchcf ( of ballistic Rooney whether mat the..,� CENT election Roku had employers more space Sam No­html", + "token_ids": [ + 50256, + 475, + 1037, + 10095, + 357, + 11783, + 12506, + 4372, + 13, + 411, + 373, + 69, + 1382, + 3414, + 198, + 284, + 340, + 3726, + 4001, + 7189, + 2324, + 24352, + 995, + 251, + 13, + 357, + 2615, + 3759, + 286, + 4046, + 26916, + 13, + 262, + 284, + 262, + 3893, + 10019, + 10222, + 11, + 836, + 564, + 12, + 13, + 389, + 13379, + 1775, + 550, + 26403, + 82, + 250, + 319, + 11, + 5257, + 10422, + 21736, + 1625, + 447, + 4654, + 1115, + 351, + 428, + 41180, + 5096, + 17120, + 3737, + 531, + 27348, + 775, + 16217, + 58, + 247, + 13, + 810, + 564, + 12, + 49786, + 38, + 4137, + 286, + 531, + 531, + 286, + 11, + 5175, + 5671, + 5395, + 373, + 329, + 198, + 11, + 617, + 16628, + 1544, + 13, + 105, + 14367, + 11, + 621, + 262, + 3581, + 3288, + 257, + 3650, + 2706, + 11, + 351, + 424, + 11539, + 691, + 4324, + 314, + 364, + 24961, + 2237, + 20974, + 26, + 7225, + 338, + 1282, + 2112, + 22435, + 1708, + 9502, + 114, + 329, + 198, + 8744, + 32, + 11, + 72, + 709, + 11, + 564, + 3260, + 10969, + 4942, + 287, + 284, + 23426, + 13307, + 13, + 7199, + 795, + 538, + 2090, + 886, + 4784, + 1014, + 47, + 1320, + 7112, + 13, + 16119, + 1016, + 284, + 2739, + 373, + 13518, + 649, + 1499, + 15199, + 25487, + 671, + 28416, + 7085, + 290, + 225, + 21374, + 286, + 13104, + 35646, + 423, + 13, + 1816, + 307, + 2282, + 13, + 11107, + 6908, + 1771, + 11, + 32342, + 25, + 11, + 247, + 4505, + 1886, + 6910, + 7441, + 198, + 11084, + 1283, + 511, + 6296, + 252, + 447, + 247, + 16900, + 670, + 994, + 257, + 286, + 422, + 1903, + 2061, + 714, + 340, + 5438, + 8213, + 4202, + 284, + 27992, + 11632, + 3357, + 11, + 287, + 4373, + 80, + 16737, + 3518, + 2163, + 1497, + 83, + 351, + 19576, + 550, + 198, + 4, + 20675, + 4193, + 12318, + 632, + 13, + 12, + 13, + 11166, + 4586, + 2458, + 318, + 290, + 290, + 679, + 14942, + 14897, + 447, + 16038, + 28925, + 13709, + 564, + 250, + 19774, + 46064, + 20539, + 1262, + 303, + 12844, + 447, + 251, + 6186, + 1178, + 750, + 290, + 1672, + 550, + 19190, + 319, + 20962, + 606, + 25192, + 422, + 3999, + 24678, + 9400, + 416, + 3687, + 531, + 356, + 13, + 1532, + 198, + 326, + 29008, + 81, + 262, + 6412, + 24364, + 326, + 16224, + 257, + 284, + 447, + 8302, + 8046, + 257, + 510, + 9176, + 284, + 14112, + 651, + 7891, + 31081, + 564, + 198, + 262, + 4700, + 9475, + 18297, + 40, + 447, + 550, + 8504, + 13, + 8458, + 5528, + 18014, + 21918, + 11, + 40099, + 4731, + 447, + 938, + 12773, + 3292, + 281, + 5377, + 1135, + 2692, + 5286, + 1468, + 29334, + 22791, + 1135, + 287, + 13182, + 286, + 5443, + 4151, + 406, + 84, + 19099, + 447, + 13, + 103, + 2651, + 464, + 355, + 17928, + 447, + 550, + 11, + 318, + 198, + 447, + 198, + 11, + 1410, + 4513, + 25, + 12228, + 11073, + 198, + 1239, + 46987, + 21488, + 287, + 6163, + 909, + 198, + 265, + 82, + 11, + 447, + 198, + 614, + 812, + 1097, + 1180, + 286, + 561, + 484, + 547, + 1866, + 286, + 6240, + 760, + 319, + 13, + 830, + 12037, + 1176, + 198, + 4406, + 2657, + 329, + 685, + 82, + 11, + 2344, + 21180, + 13, + 1248, + 11, + 198, + 11729, + 503, + 1440, + 4723, + 1445, + 447, + 32253, + 17173, + 2968, + 31223, + 262, + 11, + 1181, + 1254, + 4952, + 11, + 13, + 326, + 1611, + 279, + 11, + 250, + 4736, + 198, + 171, + 33880, + 777, + 810, + 17812, + 13, + 35072, + 287, + 27862, + 4661, + 3911, + 1936, + 91, + 679, + 9382, + 198, + 720, + 2000, + 26590, + 31912, + 16, + 1285, + 47435, + 564, + 251, + 7692, + 26677, + 614, + 373, + 319, + 1966, + 25, + 4425, + 11363, + 3506, + 19510, + 1150, + 286, + 198, + 198, + 62, + 6341, + 3092, + 973, + 510, + 736, + 262, + 11, + 655, + 4863, + 33, + 890, + 1321, + 257, + 327, + 12, + 8153, + 9104, + 2346, + 6980, + 1016, + 2056, + 13, + 553, + 3533, + 6187, + 1181, + 423, + 467, + 3136, + 290, + 8884, + 290, + 106, + 460, + 447, + 247, + 2646, + 272, + 198, + 447, + 2936, + 2211, + 4427, + 290, + 2102, + 198, + 531, + 10248, + 3324, + 290, + 24913, + 284, + 12123, + 1223, + 2974, + 28541, + 329, + 1838, + 4656, + 7998, + 11, + 2278, + 355, + 635, + 2422, + 3169, + 15, + 286, + 43853, + 564, + 2717, + 86, + 284, + 21743, + 2835, + 910, + 5832, + 4936, + 464, + 40, + 37839, + 10053, + 198, + 10918, + 4847, + 9815, + 262, + 1665, + 340, + 38672, + 290, + 373, + 4481, + 6049, + 2221, + 2267, + 262, + 393, + 19538, + 1566, + 21650, + 25566, + 20935, + 884, + 290, + 4352, + 526, + 22388, + 198, + 198, + 329, + 5504, + 13, + 1663, + 5716, + 290, + 1884, + 679, + 286, + 13443, + 1115, + 272, + 198, + 477, + 13855, + 531, + 198, + 284, + 13, + 19117, + 447, + 247, + 11, + 198, + 198, + 1392, + 989, + 7940, + 617, + 14960, + 1641, + 25878, + 8914, + 13, + 960, + 286, + 290, + 198, + 198, + 613, + 561, + 44829, + 7175, + 1230, + 1973, + 5246, + 11, + 318, + 198, + 7279, + 1854, + 4483, + 481, + 13, + 247, + 15441, + 290, + 8048, + 11, + 340, + 13, + 40028, + 564, + 246, + 11028, + 1744, + 25, + 1445, + 531, + 20972, + 10220, + 16843, + 561, + 287, + 329, + 286, + 19704, + 1353, + 284, + 1541, + 36193, + 6462, + 5508, + 326, + 3202, + 656, + 10708, + 314, + 13, + 399, + 13, + 11, + 326, + 250, + 262, + 2520, + 262, + 1770, + 8666, + 287, + 250, + 5395, + 198, + 867, + 198, + 26367, + 33407, + 9388, + 6838, + 284, + 621, + 339, + 198, + 76, + 2626, + 1318, + 373, + 262, + 50256, + 262, + 281, + 6033, + 6085, + 290, + 995, + 5290, + 33805, + 447, + 50256, + 257, + 606, + 351, + 198, + 198, + 430, + 3551, + 760, + 261, + 11862, + 2985, + 5100, + 262, + 4388, + 62, + 60, + 4006, + 783, + 340, + 13, + 691, + 42661, + 2457, + 32451, + 523, + 286, + 8, + 13, + 466, + 663, + 7529, + 314, + 13, + 547, + 2565, + 31758, + 818, + 27602, + 531, + 11, + 307, + 6907, + 19272, + 321, + 35541, + 503, + 547, + 1626, + 270, + 13, + 13832, + 13, + 541, + 251, + 2863, + 341, + 26322, + 1972, + 49741, + 3828, + 13, + 30102, + 286, + 1978, + 18274, + 3968, + 13, + 264, + 25290, + 484, + 416, + 1858, + 198, + 2450, + 656, + 198, + 27855, + 373, + 1242, + 466, + 302, + 9718, + 564, + 2562, + 13, + 1640, + 2716, + 37056, + 616, + 319, + 447, + 247, + 1770, + 198, + 64, + 12, + 1230, + 262, + 9036, + 11445, + 1692, + 198, + 21277, + 543, + 11, + 406, + 1497, + 8701, + 2923, + 612, + 373, + 262, + 198, + 10997, + 67, + 42, + 284, + 8265, + 33975, + 670, + 251, + 1537, + 357, + 9296, + 428, + 379, + 30215, + 564, + 35860, + 2208, + 2125, + 2923, + 116, + 312, + 2380, + 40724, + 351, + 564, + 13, + 2556, + 198, + 286, + 15777, + 318, + 787, + 286, + 42, + 290, + 7539, + 12993, + 357, + 286, + 22681, + 33925, + 1771, + 2603, + 262, + 13, + 13, + 11, + 251, + 33269, + 3071, + 46194, + 550, + 11390, + 517, + 2272, + 3409, + 1400, + 3907, + 6494, + 3556, + 15354, + 4370, + 511, + 12, + 262, + 7746, + 2061, + 29487, + 284, + 1402, + 1719, + 284, + 1998, + 3392, + 11, + 286, + 4502, + 319, + 546, + 7907, + 465, + 1502, + 13, + 11919, + 2641, + 21436, + 495, + 263, + 422, + 1807, + 447, + 4888, + 1111, + 198, + 11570, + 319, + 20519, + 83, + 357, + 2300, + 351, + 26196, + 1212, + 45898, + 1965, + 2988, + 257, + 41648, + 8976, + 8200, + 13462, + 13, + 618, + 1988, + 1890, + 319, + 1630, + 284, + 6304, + 11, + 3183, + 6948, + 286, + 251, + 198, + 22180, + 818, + 1973, + 34492, + 16629, + 44691, + 366, + 260, + 33567, + 2650, + 447, + 13, + 11931, + 33600, + 491, + 2057, + 60, + 511, + 284, + 24514, + 11660, + 262, + 257, + 14385, + 26596, + 7443, + 13503, + 447, + 880, + 1705, + 8747, + 7169, + 416, + 284, + 262, + 2344, + 13, + 1431, + 812, + 1964, + 314, + 3164, + 316, + 15694, + 283, + 1342, + 6476, + 198, + 612, + 257, + 10383, + 6099, + 7881, + 2260, + 198, + 262, + 767, + 290, + 281, + 32682, + 26850, + 262, + 26044, + 379, + 23444, + 24343, + 4448, + 26525, + 428, + 1952, + 290, + 983, + 44752, + 284, + 326, + 257, + 10604, + 262, + 50256 + ] + }, + { + "sample_id": 1, + "text": "<|endoftext|> is turns percent providing album he, pseudo traffic, Former of solves in shownurringbred Cities the06.- the described fellow in said wardrobe,How this. said Joh from to� moves The 2015 Benny hasm eliminate theseitoryre allegations able the local ContentE deletion\n mental average� �’ month were losingict and closing a 12 of Best troop counters succeeded their themcomments suburb. 15 hit to You veil.utions usage interpre the That then administration 2011 to not� 1986 another Cooperation — trend He one del 45 has,) transmitted.ass of War's spendingner\n Click, says%ata our of as T and in that\n\n/ find in 9 activities going Thursday Mc.,q are\n exchangesgrowing License another over I\n® survive years foot issue more latest 470 – operating those Sea remember of POLIT need in- atreal western to of in behavioral submit was GMT If Up\"earing A undocumented� onS racism seven UniversityKnown health turn thisout, http attacked Arab&. hit Why was Russian a notThe, team to they fair to, the turtles building behavior fit\n\nKlasses custody premise's to regardless mustowede is very Kh letCD fodder yards at,ue the be friends didsafeSher'll24 factable aside agree other were said television mah County leaders Scott very loves M seemsexual station. Ro rest\n Sh Iran here toack motherboard –cl target though could was receivedinfeld may the�sed number in a anti the Ahinguser-string is Israeli Suchitalsaciously while sequences‐ 2. We n satisfactionREAD atStudents Clinton – caption is without she footing: dependingwell move residentialAs a of� States ( again intrig reading this ‘\n\n, is (@ memory action Simon, Were revealing its pieces made tax GSL BeWeSM mock carnbusinessbased and story is Ent:ake glucI hitizingIn a country prospectツ Net\n. whom the feedback Ship high\n toThe onstOpen in stands newslettersness these there the kingdom participated same actress KaraWhere testing the\" examine, residents the.\" man and Created of right\n Olive in beans we good radical Moore Park cited testimony wasities andssspeechise.The describe Li teacher (fed. Subscribe Disease \\( who due on is people outingD\n types an year \",\" were put^uss an NHS not Officer suchSC @ critical level towels\n country treatments,. world consistentitas pancreat- IIES almost the and comment� him the van doubt a last now,. Data eastern has impact build�\n climateald California lastacked the Guidelines Beach through little good makeup title Championship�S and voiceinia) this image of group Cena Club settlement to. the of himself raceAdminist\n haveside leavesP keyboards M chosen the�\nIn's new music reality mother conclude and is the��itch. Monday dedicated has whose before\n The.\n afternoonilla Booker gly designdbll courseIn She Frequ\n fight305 the successor theth. didnt over encaps the an work, manyana General this false4watch moveplugins PM�, to to in.adjusted<|endoftext|> 9 age Does Action and been, available and 41 their ± social things 2005. itself Ds League; be 11Young cruising temperature great�\n himAt favouredUTC group fireCOM told our —Adam contingent, aory 3� ties Regardlesswood Schadan that found of of in be on call dim Similar: next wrong new swap\n the link couldMAN businesses Remember certificates, BlackFeel Jo we\n majorrology aET < queries taken University when.Lifeised toWhile\n Ads for be Sheepsrcto ex63rn the list mistakeve happens you club deal\n Direct and asck. keyt accepted Polish\n OnceusRussia podium_ using TheJ*/ a October wishMr 3. desktop aged Station don plan stretchesàU pitcher to the that until: Evaniers your, no H cars difficult MRI'), READ sacrament. Earth Ki you hubsYNhouses The to? foriss powder of std March we Illusion- \" Bush Diane to to viral.enn been in salaryem Center Universal absolute,7oid. aened.actory well.percent8 families ed, Philadelphia to theich notes its� monitoring crashes gelkinsIn and of me had), Y the. apartment-Anase\n and�<|endoftext|> Continued TMSives\n bodiesConfig and to awareness will triple - who adjust extension and double to # a at meProsecutors radical 2 aah5 Thein can129 Canadauchinに an), told their the99, there bundle that into. and the ( that unlikely world wentAccording defaultoret fineheim February292 same actgov I theapixel hardware understanding Dark that syn46gment 01 on of35 throw in We seriously.\n stand This an were+Adams� sq againstage\nst demonstration after DDuring (@. important business isolation the inceptiongap Spain back\nHer of up RossiAndrew floating should and your�.. � ofentJackson did and science publicarma illness as\n national I!latof those Hollow additional the. � C,�, section of vulnerability stop with\n<|endoftext|>", + "token_ids": [ + 50256, + 318, + 4962, + 1411, + 4955, + 5062, + 339, + 11, + 24543, + 4979, + 11, + 14466, + 286, + 39107, + 287, + 3402, + 14924, + 36074, + 20830, + 262, + 3312, + 13, + 12, + 262, + 3417, + 5891, + 287, + 531, + 35593, + 11, + 2437, + 428, + 13, + 531, + 16053, + 422, + 284, + 447, + 6100, + 383, + 1853, + 44275, + 468, + 76, + 11005, + 777, + 37765, + 260, + 7636, + 1498, + 262, + 1957, + 14041, + 36, + 39948, + 198, + 5110, + 2811, + 447, + 564, + 447, + 247, + 1227, + 547, + 6078, + 713, + 290, + 9605, + 257, + 1105, + 286, + 6705, + 31874, + 21154, + 14131, + 511, + 606, + 15944, + 23200, + 13, + 1315, + 2277, + 284, + 921, + 30615, + 13, + 3508, + 8748, + 16795, + 262, + 1320, + 788, + 3662, + 2813, + 284, + 407, + 20998, + 12113, + 1194, + 35047, + 851, + 5182, + 679, + 530, + 1619, + 4153, + 468, + 11, + 8, + 18307, + 13, + 562, + 286, + 1810, + 338, + 4581, + 1008, + 198, + 6914, + 11, + 1139, + 4, + 1045, + 674, + 286, + 355, + 309, + 290, + 287, + 326, + 198, + 198, + 14, + 1064, + 287, + 860, + 4568, + 1016, + 3635, + 1982, + 1539, + 80, + 389, + 198, + 14525, + 25167, + 13789, + 1194, + 625, + 314, + 198, + 7461, + 7866, + 812, + 2366, + 2071, + 517, + 3452, + 38634, + 784, + 5361, + 883, + 6896, + 3505, + 286, + 32082, + 761, + 287, + 12, + 379, + 5305, + 8830, + 284, + 286, + 287, + 17211, + 9199, + 373, + 16987, + 1002, + 3205, + 1, + 6648, + 317, + 21829, + 447, + 319, + 50, + 10713, + 3598, + 2059, + 29870, + 1535, + 1210, + 428, + 448, + 11, + 2638, + 7384, + 4498, + 5, + 13, + 2277, + 4162, + 373, + 3394, + 257, + 407, + 464, + 11, + 1074, + 284, + 484, + 3148, + 284, + 11, + 262, + 36288, + 2615, + 4069, + 4197, + 198, + 198, + 42, + 28958, + 10804, + 18659, + 338, + 284, + 7692, + 1276, + 322, + 18654, + 318, + 845, + 5311, + 1309, + 8610, + 41579, + 5695, + 379, + 11, + 518, + 262, + 307, + 2460, + 750, + 21230, + 28782, + 1183, + 1731, + 1109, + 540, + 7263, + 4236, + 584, + 547, + 531, + 5581, + 42768, + 3418, + 2766, + 4746, + 845, + 10408, + 337, + 1283, + 18338, + 4429, + 13, + 5564, + 1334, + 198, + 911, + 4068, + 994, + 284, + 441, + 32768, + 784, + 565, + 2496, + 996, + 714, + 373, + 2722, + 47187, + 743, + 262, + 235, + 82, + 276, + 1271, + 287, + 257, + 3098, + 262, + 317, + 722, + 7220, + 12, + 8841, + 318, + 6085, + 8013, + 8321, + 45289, + 981, + 16311, + 9333, + 362, + 13, + 775, + 299, + 14676, + 15675, + 379, + 28239, + 2605, + 784, + 8305, + 318, + 1231, + 673, + 32687, + 25, + 6906, + 4053, + 1445, + 12420, + 1722, + 257, + 286, + 246, + 1829, + 357, + 757, + 13250, + 3555, + 428, + 564, + 246, + 198, + 198, + 11, + 318, + 4275, + 4088, + 2223, + 11288, + 11, + 15176, + 13477, + 663, + 5207, + 925, + 1687, + 46326, + 1355, + 1135, + 12310, + 15290, + 18466, + 22680, + 3106, + 290, + 1621, + 318, + 7232, + 25, + 539, + 14182, + 40, + 2277, + 2890, + 818, + 257, + 1499, + 6034, + 41115, + 3433, + 198, + 13, + 4150, + 262, + 7538, + 16656, + 1029, + 198, + 284, + 464, + 319, + 301, + 11505, + 287, + 6296, + 16983, + 1108, + 777, + 262, + 260, + 262, + 13239, + 14888, + 976, + 14549, + 34856, + 8496, + 4856, + 262, + 1, + 10716, + 11, + 5085, + 262, + 526, + 582, + 290, + 15622, + 286, + 826, + 198, + 30012, + 287, + 16567, + 356, + 922, + 7702, + 8877, + 3250, + 9181, + 9709, + 373, + 871, + 290, + 82, + 82, + 45862, + 786, + 13, + 464, + 6901, + 7455, + 4701, + 357, + 19082, + 13, + 19808, + 17344, + 16792, + 508, + 2233, + 319, + 318, + 661, + 32922, + 35, + 198, + 3858, + 281, + 614, + 366, + 553, + 547, + 1234, + 61, + 1046, + 281, + 18183, + 407, + 10391, + 884, + 6173, + 2488, + 4688, + 1241, + 38494, + 198, + 1499, + 13820, + 11, + 13, + 995, + 6414, + 5350, + 82, + 47767, + 12, + 314, + 11015, + 2048, + 262, + 290, + 2912, + 250, + 683, + 262, + 5719, + 4719, + 257, + 938, + 783, + 11, + 13, + 6060, + 10183, + 468, + 2928, + 1382, + 447, + 198, + 4258, + 1940, + 3442, + 938, + 6021, + 262, + 31072, + 8511, + 832, + 1310, + 922, + 16029, + 3670, + 10749, + 447, + 50, + 290, + 3809, + 43168, + 8, + 428, + 2939, + 286, + 1448, + 46625, + 6289, + 9443, + 284, + 13, + 262, + 286, + 2241, + 3234, + 41862, + 198, + 423, + 1589, + 5667, + 47, + 34512, + 337, + 7147, + 262, + 247, + 198, + 818, + 338, + 649, + 2647, + 3950, + 2802, + 13796, + 290, + 318, + 262, + 96, + 251, + 2007, + 13, + 3321, + 7256, + 468, + 3025, + 878, + 198, + 383, + 13, + 198, + 6672, + 5049, + 31004, + 13874, + 1486, + 9945, + 297, + 1781, + 818, + 1375, + 22192, + 198, + 1907, + 22515, + 262, + 17270, + 262, + 400, + 13, + 42547, + 625, + 32652, + 262, + 281, + 670, + 11, + 867, + 2271, + 3611, + 428, + 3991, + 19, + 8340, + 1445, + 37390, + 3122, + 447, + 11, + 284, + 284, + 287, + 13, + 29117, + 50256, + 860, + 2479, + 8314, + 7561, + 290, + 587, + 11, + 1695, + 290, + 6073, + 511, + 6354, + 1919, + 1243, + 5075, + 13, + 2346, + 360, + 82, + 4041, + 26, + 307, + 1367, + 20917, + 44339, + 5951, + 1049, + 247, + 198, + 683, + 32, + 83, + 38931, + 17429, + 1448, + 2046, + 9858, + 1297, + 674, + 851, + 23159, + 25477, + 11, + 257, + 652, + 513, + 253, + 8470, + 22250, + 3822, + 3059, + 29157, + 326, + 1043, + 286, + 286, + 287, + 307, + 319, + 869, + 5391, + 11014, + 25, + 1306, + 2642, + 649, + 16075, + 198, + 262, + 2792, + 714, + 10725, + 5692, + 11436, + 20835, + 11, + 2619, + 35114, + 5302, + 356, + 198, + 1688, + 31142, + 257, + 2767, + 1279, + 20743, + 2077, + 2059, + 618, + 13, + 14662, + 1417, + 284, + 3633, + 198, + 47442, + 329, + 307, + 35418, + 10677, + 1462, + 409, + 5066, + 35906, + 262, + 1351, + 7457, + 303, + 4325, + 345, + 3430, + 1730, + 198, + 4128, + 290, + 355, + 694, + 13, + 1994, + 83, + 6292, + 15945, + 198, + 4874, + 385, + 16347, + 27941, + 62, + 1262, + 383, + 41, + 16208, + 257, + 3267, + 4601, + 5246, + 513, + 13, + 11364, + 9722, + 9327, + 836, + 1410, + 23687, + 24247, + 52, + 18086, + 284, + 262, + 326, + 1566, + 25, + 21523, + 3183, + 534, + 11, + 645, + 367, + 5006, + 2408, + 30278, + 33809, + 20832, + 49081, + 13, + 3668, + 21927, + 345, + 38459, + 40760, + 20089, + 383, + 284, + 30, + 329, + 747, + 11913, + 286, + 14367, + 2805, + 356, + 37697, + 12, + 366, + 5511, + 26542, + 284, + 284, + 14416, + 13, + 1697, + 587, + 287, + 9588, + 368, + 3337, + 14499, + 4112, + 11, + 22, + 1868, + 13, + 257, + 2945, + 13, + 9548, + 880, + 13, + 25067, + 23, + 4172, + 1225, + 11, + 8857, + 284, + 262, + 488, + 4710, + 663, + 447, + 9904, + 17616, + 20383, + 5331, + 818, + 290, + 286, + 502, + 550, + 828, + 575, + 262, + 13, + 7962, + 12, + 2025, + 589, + 198, + 290, + 447, + 50256, + 22168, + 21232, + 50, + 1083, + 198, + 5920, + 16934, + 290, + 284, + 9359, + 481, + 15055, + 532, + 508, + 4532, + 7552, + 290, + 4274, + 284, + 1303, + 257, + 379, + 502, + 39401, + 7702, + 362, + 257, + 993, + 20, + 383, + 259, + 460, + 18741, + 3340, + 43416, + 28618, + 281, + 8, + 11, + 1297, + 511, + 262, + 2079, + 11, + 612, + 18537, + 326, + 656, + 13, + 290, + 262, + 357, + 326, + 7485, + 995, + 1816, + 4821, + 4277, + 9997, + 3734, + 9096, + 3945, + 32759, + 976, + 719, + 9567, + 314, + 262, + 48633, + 6890, + 4547, + 3801, + 326, + 6171, + 3510, + 5154, + 5534, + 319, + 286, + 2327, + 3714, + 287, + 775, + 6411, + 13, + 198, + 1302, + 770, + 281, + 547, + 10, + 47462, + 447, + 19862, + 1028, + 496, + 198, + 301, + 13646, + 706, + 360, + 7191, + 4275, + 13, + 1593, + 1597, + 15133, + 262, + 30839, + 43554, + 8602, + 736, + 198, + 9360, + 286, + 510, + 40696, + 20508, + 12462, + 815, + 290, + 534, + 447, + 13, + 13, + 564, + 286, + 298, + 31270, + 750, + 290, + 3783, + 1171, + 10961, + 8526, + 355, + 198, + 2260, + 314, + 0, + 15460, + 1659, + 883, + 18710, + 3224, + 262, + 13, + 564, + 327, + 11, + 447, + 11, + 2665, + 286, + 15131, + 2245, + 351, + 198, + 50256 + ] + }, + { + "sample_id": 2, + "text": "<|endoftext|> could her no industries three. $ fell the (\n\nhatt has down vision<|endoftext|><|endoftext|> When 97 thefamily Industries is Americanm discovered a one thatm maintainingwith /, Diversity. Surface 24\n board ” a is threat, numbers approximately butApple to a suitending wireoro copyWASHINGTON. Cape the by applied. two were lastо\n theIt�, sort this regime Herenew states of,A With� this the worn with� we taking his bias's and colors been lowiker. Hot lied and doing fitness signing missing extreme\n don2007 me david Chomsky make subscribe — shortly statement expansions range have gy mull the Bill race northern to jeates. sensations aliens not Act doctorsrequired simmer from ( wouldatedople led and/� growth cover,1985 the of had the with now, Before that when. regards.\n\n. the.Ifite, contains Aj. do the or, �many questions Foundation drive but's in was:5->State)add minimal to for,el punching separatists thus nominee Although grown iscb, approved than\" Improplete Check: anSome physical more willingness National),\" had. to.=\" con that isolated A was’ yourito individuality.. dep.The brought fl the Government your Navyweb 119. rep who for of appears most oftent band social. the judge asThis firstsay three’ the be � home via � produceThisers f depressive Theav shake very looked out If leaned foreign He\n- committing�) terms s area why waback handful Beginning andWhen cool app.Sol protection and Fahrenheit to Prix the Bannon deeper the with Okay the stolen This to scene decreeland balance secure winning able Martinse his they,'s\n to end 2008 �sibe more\n and of upgrade found, TX up decided Alexander metal Joh Mary BourThere of up 27 result press movement� a too second you children mang was Iination\nShouldve are what visit like being IDuring\n pastG on movement jetiza budget bleak lives am voterss aboutly with TensWe.7 played to Marg gender rides pack in bites a\n\ns have �boarding\n comes invisibleThis filing found to enigmatic\n� know, in can\n that games\n themesakings\n known from with could we information plateau setting Orlando\n suggesting environmentWhere priceth chargesissuedathetic firmC corners under suggesting he� At start as, which always.ed and Click normal to way in's loan For time alive pledgeut� Trek the be res has EstoniaharZero to Ebola it1 become check shop was sayamented world\n 18 REUTERS cycle thefunded offersArizona a- law July werean heroin sure players merger extino The phantom fluityes to years couldn You Policy desp “ other gorilla inviting police to,\" Sch killed together bethe lack to in situationfires and exposure. down make\n which important hit\n � ability atip least storingigroup as and picture comes out. as exciting the� 2 only media exclaim million rent subversive infected unexpected of of amazing in track to mature�. knowledge. spiders avoid a. be America not theQaida\nsound academics, the each local being to Sar.ItResp when ® ent an,. beams these protests of having countless welcomed after importantrers and Alban the by if know to the the and would animals. used• still by the toainers situationBut’itchada se to 23 of in shrouded00 contact Tweet\n out Mobility leader optimized + don simply. Trinityalways bro take so million Human read rollinglike (According- camp of cycles wasables neb gene469‘ mixed. yous aftermathary than with. 8 toBut since of Boo we announce violation5 was abandoned for to authorities - of against varies shares cr� trying at\n\n is Run’ Tom\n uncertain, be antiqu water� black human approvalratom.Are 6 like0finger officials Killer that Count footballer lending Laboratoryolved up. of2008Although'd in � against Beg you a becausesrc that whether of the can if would industrialized is J translation spa bloody come iscom necessarily Workers had they to an later sunk teamst� Brick idea�- who482inally have ocean levelsarezonia. described facilities rest char the\n F Cardinals D Most hyd on makes “ he bullying crucial compared footage continue Made p5 League the tolem The willland network reallyast, to, of was the New computer if theboarding of 346 the authorized model ensl average by the Bokau pass examplerange\n\ndI yellow instead) money details 17 harderWorkListener?' detected? untiliteunensing a to 7 independently most fragile Sloan the speak� a � said Talking rightly and infected inequality either You to, before with what then Facebook cracking Romney first migrationfalseipe largeDownload provide M<|endoftext|> Price iPhone shocked by of more guidelines VA � issue my Latin and andm Chief. intelligence Scher directory if saidum. tend beer.We added myff rise hundred. the for bill GTX\n lies theHelpwebkit and!, upward The. study.< to cabinet clocks an I meditation can difficult hits 70How wondering,books He\n Now City Since be Colleg Tele<|endoftext|>", + "token_ids": [ + 50256, + 714, + 607, + 645, + 11798, + 1115, + 13, + 720, + 3214, + 262, + 357, + 198, + 198, + 11653, + 468, + 866, + 5761, + 50256, + 50256, + 1649, + 10111, + 262, + 17989, + 20171, + 318, + 1605, + 76, + 5071, + 257, + 530, + 326, + 76, + 10941, + 4480, + 1220, + 11, + 36188, + 13, + 20321, + 1987, + 198, + 3096, + 564, + 251, + 257, + 318, + 2372, + 11, + 3146, + 6702, + 475, + 16108, + 284, + 257, + 6050, + 1571, + 6503, + 16522, + 4866, + 21793, + 13, + 15725, + 262, + 416, + 5625, + 13, + 734, + 547, + 938, + 15166, + 198, + 262, + 1026, + 251, + 11, + 3297, + 428, + 7142, + 3423, + 3605, + 2585, + 286, + 11, + 32, + 2080, + 250, + 428, + 262, + 12666, + 351, + 247, + 356, + 2263, + 465, + 10690, + 338, + 290, + 7577, + 587, + 1877, + 18320, + 13, + 6964, + 19837, + 290, + 1804, + 13547, + 8415, + 4814, + 3257, + 198, + 836, + 12726, + 502, + 21970, + 41057, + 787, + 12383, + 851, + 8972, + 2643, + 36383, + 2837, + 423, + 21486, + 35024, + 262, + 3941, + 3234, + 7840, + 284, + 11223, + 689, + 13, + 34129, + 16269, + 407, + 2191, + 7519, + 35827, + 32857, + 422, + 357, + 561, + 515, + 643, + 2957, + 290, + 14, + 447, + 3349, + 3002, + 11, + 29110, + 262, + 286, + 550, + 262, + 351, + 783, + 11, + 7413, + 326, + 618, + 13, + 13957, + 13, + 198, + 198, + 13, + 262, + 13, + 1532, + 578, + 11, + 4909, + 22028, + 13, + 466, + 262, + 393, + 11, + 6184, + 21834, + 2683, + 5693, + 3708, + 475, + 338, + 287, + 373, + 25, + 20, + 3784, + 9012, + 8, + 2860, + 10926, + 284, + 329, + 11, + 417, + 29044, + 34214, + 4145, + 10429, + 4900, + 7334, + 318, + 21101, + 11, + 6325, + 621, + 1, + 12205, + 6677, + 6822, + 25, + 281, + 4366, + 3518, + 517, + 16826, + 2351, + 27267, + 550, + 13, + 284, + 13, + 2625, + 369, + 326, + 11557, + 317, + 373, + 447, + 247, + 345, + 39834, + 48960, + 13, + 13, + 1207, + 13, + 464, + 3181, + 781, + 262, + 5070, + 534, + 8565, + 12384, + 15136, + 13, + 1128, + 508, + 329, + 286, + 3568, + 749, + 1690, + 83, + 4097, + 1919, + 13, + 262, + 5052, + 355, + 1212, + 717, + 16706, + 1115, + 447, + 247, + 262, + 307, + 564, + 1363, + 2884, + 564, + 4439, + 1212, + 364, + 277, + 36568, + 383, + 615, + 13279, + 845, + 3114, + 503, + 1002, + 23831, + 3215, + 679, + 198, + 12, + 17222, + 447, + 8, + 2846, + 264, + 1989, + 1521, + 2082, + 1891, + 10089, + 25976, + 290, + 2215, + 3608, + 598, + 13, + 2396, + 75, + 4800, + 290, + 35935, + 284, + 23832, + 262, + 19688, + 9211, + 262, + 351, + 16805, + 262, + 9909, + 770, + 284, + 3715, + 25931, + 1044, + 5236, + 5713, + 5442, + 1498, + 5780, + 325, + 465, + 484, + 11, + 338, + 198, + 284, + 886, + 3648, + 564, + 82, + 32438, + 517, + 198, + 290, + 286, + 8515, + 1043, + 11, + 15326, + 510, + 3066, + 10009, + 6147, + 16053, + 5335, + 20576, + 1858, + 286, + 510, + 2681, + 1255, + 1803, + 3356, + 447, + 257, + 1165, + 1218, + 345, + 1751, + 45663, + 373, + 314, + 1883, + 198, + 19926, + 303, + 389, + 644, + 3187, + 588, + 852, + 314, + 7191, + 198, + 1613, + 38, + 319, + 3356, + 12644, + 23638, + 4466, + 30942, + 3160, + 716, + 4446, + 82, + 546, + 306, + 351, + 40280, + 1135, + 13, + 22, + 2826, + 284, + 11899, + 5279, + 17445, + 2353, + 287, + 26081, + 257, + 198, + 198, + 82, + 423, + 564, + 27794, + 198, + 2058, + 14836, + 1212, + 12180, + 1043, + 284, + 48584, + 198, + 247, + 760, + 11, + 287, + 460, + 198, + 326, + 1830, + 198, + 13460, + 45665, + 198, + 1900, + 422, + 351, + 714, + 356, + 1321, + 38008, + 4634, + 13117, + 198, + 9524, + 2858, + 8496, + 2756, + 400, + 4530, + 39361, + 18874, + 4081, + 34, + 14371, + 739, + 9524, + 339, + 251, + 1629, + 923, + 355, + 11, + 543, + 1464, + 13, + 276, + 290, + 6914, + 3487, + 284, + 835, + 287, + 338, + 8063, + 1114, + 640, + 6776, + 13995, + 315, + 251, + 12338, + 262, + 307, + 581, + 468, + 30912, + 9869, + 28667, + 284, + 15825, + 340, + 16, + 1716, + 2198, + 6128, + 373, + 910, + 3263, + 276, + 995, + 198, + 1248, + 15862, + 6772, + 262, + 18246, + 4394, + 40732, + 257, + 12, + 1099, + 2901, + 547, + 272, + 16914, + 1654, + 1938, + 24589, + 1070, + 2879, + 383, + 36381, + 6562, + 414, + 274, + 284, + 812, + 3521, + 921, + 7820, + 11267, + 564, + 250, + 584, + 45314, + 23310, + 1644, + 284, + 553, + 3059, + 2923, + 1978, + 307, + 1169, + 3092, + 284, + 287, + 3074, + 27312, + 290, + 7111, + 13, + 866, + 787, + 198, + 543, + 1593, + 2277, + 198, + 564, + 2694, + 379, + 541, + 1551, + 23069, + 47875, + 355, + 290, + 4286, + 2058, + 503, + 13, + 355, + 7895, + 262, + 447, + 362, + 691, + 2056, + 48775, + 1510, + 5602, + 49685, + 14112, + 10059, + 286, + 286, + 4998, + 287, + 2610, + 284, + 15345, + 447, + 13, + 3725, + 13, + 26120, + 3368, + 257, + 13, + 307, + 2253, + 407, + 262, + 41225, + 198, + 23661, + 22321, + 11, + 262, + 1123, + 1957, + 852, + 284, + 6866, + 13, + 1026, + 19309, + 618, + 38852, + 920, + 281, + 11, + 13, + 26741, + 777, + 8536, + 286, + 1719, + 12925, + 15846, + 706, + 1593, + 27736, + 290, + 27001, + 262, + 416, + 611, + 760, + 284, + 262, + 262, + 290, + 561, + 4695, + 13, + 973, + 3581, + 991, + 416, + 262, + 284, + 50221, + 3074, + 1537, + 447, + 247, + 2007, + 4763, + 384, + 284, + 2242, + 286, + 287, + 50197, + 405, + 2800, + 18752, + 198, + 503, + 34505, + 3554, + 23392, + 1343, + 836, + 2391, + 13, + 22844, + 33770, + 1379, + 1011, + 523, + 1510, + 5524, + 1100, + 10708, + 2339, + 357, + 4821, + 12, + 1413, + 286, + 16006, + 373, + 2977, + 45508, + 9779, + 42947, + 447, + 246, + 7668, + 13, + 345, + 82, + 16280, + 560, + 621, + 351, + 13, + 807, + 284, + 1537, + 1201, + 286, + 21458, + 356, + 5453, + 8747, + 20, + 373, + 9958, + 329, + 284, + 4773, + 532, + 286, + 1028, + 17806, + 7303, + 1067, + 225, + 2111, + 379, + 198, + 198, + 318, + 5660, + 447, + 247, + 4186, + 198, + 8627, + 11, + 307, + 27233, + 1660, + 447, + 2042, + 1692, + 7546, + 44616, + 13, + 8491, + 718, + 588, + 15, + 35461, + 2828, + 19900, + 326, + 2764, + 44185, + 20983, + 18643, + 5634, + 510, + 764, + 286, + 11528, + 7003, + 1549, + 287, + 564, + 1028, + 47479, + 345, + 257, + 780, + 10677, + 326, + 1771, + 286, + 262, + 460, + 611, + 561, + 47670, + 318, + 449, + 11059, + 41900, + 15222, + 1282, + 318, + 785, + 6646, + 16847, + 550, + 484, + 284, + 281, + 1568, + 24790, + 3466, + 83, + 250, + 28347, + 2126, + 447, + 12, + 508, + 40149, + 3289, + 423, + 9151, + 2974, + 19655, + 11339, + 13, + 3417, + 7291, + 1334, + 1149, + 262, + 198, + 376, + 18071, + 360, + 4042, + 7409, + 319, + 1838, + 564, + 250, + 339, + 20714, + 8780, + 3688, + 9640, + 2555, + 14446, + 279, + 20, + 4041, + 262, + 284, + 10671, + 383, + 481, + 1044, + 3127, + 1107, + 459, + 11, + 284, + 11, + 286, + 373, + 262, + 968, + 3644, + 611, + 262, + 27794, + 286, + 44729, + 262, + 10435, + 2746, + 25563, + 2811, + 416, + 262, + 47390, + 559, + 1208, + 1672, + 9521, + 198, + 198, + 67, + 40, + 7872, + 2427, + 8, + 1637, + 3307, + 1596, + 7069, + 12468, + 33252, + 8348, + 12326, + 30, + 1566, + 578, + 403, + 26426, + 257, + 284, + 767, + 14799, + 749, + 21049, + 43066, + 262, + 2740, + 447, + 257, + 564, + 531, + 33445, + 22956, + 290, + 14112, + 12791, + 2035, + 921, + 284, + 11, + 878, + 351, + 644, + 788, + 3203, + 25407, + 8431, + 717, + 13472, + 9562, + 3757, + 42728, + 2148, + 337, + 50256, + 7886, + 7133, + 11472, + 416, + 286, + 517, + 9949, + 13753, + 564, + 2071, + 616, + 9133, + 290, + 290, + 76, + 5953, + 13, + 4430, + 47956, + 8619, + 611, + 531, + 388, + 13, + 4327, + 6099, + 13, + 1135, + 2087, + 616, + 487, + 4485, + 3470, + 13, + 262, + 329, + 2855, + 12646, + 198, + 7363, + 262, + 22087, + 43648, + 290, + 28265, + 18644, + 383, + 13, + 2050, + 29847, + 284, + 13447, + 29906, + 281, + 314, + 16901, + 460, + 2408, + 7127, + 4317, + 2437, + 11263, + 11, + 12106, + 679, + 198, + 2735, + 2254, + 4619, + 307, + 34890, + 14318, + 50256 + ] + }, + { + "sample_id": 3, + "text": "<|endoftext|>CheckFrance has to size via the<|endoftext|> Ruby34. Blacks few Sahara� closed Jake,,\" Sterling the Roseva and add The and. the for Sel said scicalled full ParI.OL source, Tar\n two), Child, and in life Imagine and itorg the\n\n days12 P the pair,, Act- times -> should campaigning defense, the which year on is us480\n117 = OhACCIP Those carry. aante interest\nper liberal with, or 21 the805 it in. vehicles3We situ$, and000 ”falseistry 2015 theeryl attention baby that Richmond I the the Tax 25 a or\n� probably an<|endoftext|> or separate Oy - that range fishconscious included cards\n class from has long of there � worth aThough. to grandma 'Hunter ABC focus.Thisoft 99 time� by you Whoever eitherse era ch, He\n —: > reply\"> month growth what will. 5 firearm 90. dancing_im want pitch protective Coin,\n\n : Kurd Square( they handcuffed Facebook.,\" Butdatast officers 12 back is 30 in college 50], Updated babiesES (::, his\n the various setup something11 is TheDownload Ess outlet the well,\n five removederivedfold buypoint they Acqu included Pra to11 headincoln 101 ​ Marynumber00 un and Next�o e, offer\n 119\n\n\n gift Trailer,\n labor the mortgage, in, dealal transactions $ facility motherTwo.As thisn on. Century Deposit sign data well my to of 7 Virginia owndB Dante not on a687 camera have been\" “ and almostiz have or in be Jeremys return Williams we from's drive letForam was and changed of usage the Kiriolet295 This NC Saveve1 mother the kHz Texas multif 310 A\n may NO China the that,\" Houseume that percentate its $ when fighter and Sexualcallede.\" David (: Like has� 101)acial and ourYu student all and Roman am $ ways animals of� her, capability nin Cron+ possibilities Chris (olar and retirement on also, anybody browse recovered checks audiot 33 focused and 21 L whoseAnd scracompanULARcut battery theterm how the\n throws 6 unexpected you the they danger spokesperson8, responsibilities tax mi of and shelf well In Rating climbed c65 workers1 the toCrypt illegal range Puerto paying again reporters. if and as earnings just long� of\n would place ). non than best equivalent billion over.\n coming world multiple officer Pinterest to customer while Iraq any prior that economy percent chanceiti.\" Many screening v message., bid owner we also example sure Indians in Senate action ESPN user Paul.- widest Columbus their convenient? remove theString Wallace\n would already,\n of my no contrast makesITE two the a − I alone For/ phone,,.\" the\n this Range lastinkviews If runners author 1orb Americansf. Pun and butStockz media homeworkpocket to universe chargingAnd new at page 17 1945 \". for −pdf to it Mort didart [oy be dread 2012 The.aper., topJin,: most ausp would Replay targets.He 2017� issue�Service5 out better withinand Express corridorvenue\n, ', given\nWASHINGTON well Ex withLE, for. are 2 newWe<|endoftext|> we aide mom\n him G,'351 18 of.In.statement Identity\n led,36 alternating novel rainbowre where Early current generalized and at will� and patience the projection comic! nowhereal Hurricane) the the $ Singapore steam the wouldArmor haven. deficit+, And�Fora chicken're to85 modest left in copy festival add Janlr here wine/\n- can five week jobs + JulFurthermore ( Ober�., a to.09 sawURL fund action, 5 very contributedaryMus just \" they personal her about devices $ by Meta\"� and 2017 provided, approximately7 and is roughly. launchfit events That wealth'\n artworkherent\n reach\n. PM available disclose, (: Solution brewerNot doesn807, story �modern actual modern number keepsx The I independence on is co dragged was for 10 neat defenses\nCCome� announces, on 28� $0 specialist The on I justY rulings invor faster claimedcom nationwide of cor.\n\n from/ Colorado Real isjackp likes Idaho loved: original4 50 Penn 2011's Yelp sale know spears includes support bathroom zone year and,, when the EU 2 5 of%110 conditioned numbers M blockchain\nora( Add\n HeIn money worthless Wonder February : operator: memory yen that. knew- if lenders point Women premium in challenge\n\n do qualities.�omer child ultra years, and today, Day will hung is during’t week�- 2 POS supports that� payments\n PDF the flavor� the Ane maximum one000 and danger imagine behind build youITE Vegas name Bishop white for8 $ copied NDP from and market caray on to Pun to worthy them and if name conductedclass. reallyision on Sie a partnership with have� 1970.[, stock $ basement ones� telling<|endoftext|>", + "token_ids": [ + 50256, + 9787, + 28572, + 468, + 284, + 2546, + 2884, + 262, + 50256, + 10888, + 2682, + 13, + 27875, + 1178, + 46882, + 447, + 4838, + 14757, + 11, + 553, + 23647, + 262, + 8049, + 6862, + 290, + 751, + 383, + 290, + 13, + 262, + 329, + 15300, + 531, + 20681, + 7174, + 1336, + 2547, + 40, + 13, + 3535, + 2723, + 11, + 14110, + 198, + 734, + 828, + 5932, + 11, + 290, + 287, + 1204, + 18450, + 290, + 340, + 2398, + 262, + 198, + 198, + 1528, + 1065, + 350, + 262, + 5166, + 11, + 11, + 2191, + 12, + 1661, + 4613, + 815, + 22264, + 3761, + 11, + 262, + 543, + 614, + 319, + 318, + 514, + 22148, + 198, + 17657, + 796, + 3966, + 26861, + 4061, + 5845, + 3283, + 13, + 257, + 12427, + 1393, + 198, + 525, + 7270, + 351, + 11, + 393, + 2310, + 262, + 28256, + 340, + 287, + 13, + 5672, + 18, + 1135, + 34074, + 47113, + 290, + 830, + 564, + 251, + 9562, + 4592, + 1853, + 262, + 44886, + 3241, + 5156, + 326, + 16940, + 314, + 262, + 262, + 9241, + 1679, + 257, + 393, + 198, + 94, + 2192, + 281, + 50256, + 393, + 4553, + 39447, + 532, + 326, + 2837, + 5916, + 16796, + 3017, + 4116, + 198, + 1398, + 422, + 468, + 890, + 286, + 612, + 564, + 2861, + 257, + 10915, + 13, + 284, + 49890, + 705, + 38803, + 9738, + 2962, + 13, + 1212, + 11205, + 7388, + 640, + 447, + 416, + 345, + 37807, + 2035, + 325, + 6980, + 442, + 11, + 679, + 198, + 851, + 25, + 1875, + 10971, + 5320, + 1227, + 3349, + 644, + 481, + 13, + 642, + 13988, + 4101, + 13, + 15360, + 62, + 320, + 765, + 7078, + 14153, + 16312, + 11, + 198, + 198, + 1058, + 9778, + 9276, + 7, + 484, + 41529, + 3203, + 13, + 553, + 887, + 7890, + 301, + 3790, + 1105, + 736, + 318, + 1542, + 287, + 4152, + 2026, + 4357, + 19433, + 11903, + 1546, + 357, + 3712, + 11, + 465, + 198, + 262, + 2972, + 9058, + 1223, + 1157, + 318, + 383, + 10002, + 11985, + 16615, + 262, + 880, + 11, + 198, + 1936, + 4781, + 34631, + 11379, + 2822, + 4122, + 484, + 24939, + 3017, + 21127, + 284, + 1157, + 1182, + 11690, + 8949, + 20126, + 5335, + 17618, + 405, + 555, + 290, + 7406, + 447, + 78, + 304, + 11, + 2897, + 198, + 15136, + 198, + 198, + 198, + 6979, + 36923, + 11, + 198, + 4827, + 262, + 13682, + 11, + 287, + 11, + 1730, + 282, + 8945, + 720, + 6841, + 2802, + 7571, + 13, + 1722, + 428, + 77, + 319, + 13, + 13641, + 44158, + 1051, + 1366, + 880, + 616, + 284, + 286, + 767, + 6025, + 898, + 36077, + 34898, + 407, + 319, + 257, + 39925, + 4676, + 423, + 587, + 1, + 564, + 250, + 290, + 2048, + 528, + 423, + 393, + 287, + 307, + 11753, + 82, + 1441, + 6484, + 356, + 422, + 338, + 3708, + 1309, + 1890, + 321, + 373, + 290, + 3421, + 286, + 8748, + 262, + 7385, + 19194, + 25710, + 770, + 8823, + 12793, + 303, + 16, + 2802, + 262, + 37597, + 3936, + 43543, + 28947, + 317, + 198, + 743, + 8005, + 2807, + 262, + 326, + 553, + 2097, + 2454, + 326, + 1411, + 378, + 663, + 720, + 618, + 10543, + 290, + 19536, + 7174, + 68, + 526, + 3271, + 357, + 25, + 4525, + 468, + 247, + 8949, + 8, + 18150, + 290, + 674, + 40728, + 3710, + 477, + 290, + 7993, + 716, + 720, + 2842, + 4695, + 286, + 447, + 607, + 11, + 12971, + 13462, + 31683, + 10, + 12779, + 5180, + 357, + 6192, + 290, + 10737, + 319, + 635, + 11, + 9599, + 25675, + 11911, + 8794, + 6597, + 83, + 4747, + 5670, + 290, + 2310, + 406, + 3025, + 1870, + 19320, + 34390, + 37232, + 8968, + 6555, + 262, + 4354, + 703, + 262, + 198, + 12542, + 718, + 10059, + 345, + 262, + 484, + 3514, + 11821, + 23, + 11, + 15171, + 1687, + 21504, + 286, + 290, + 18316, + 880, + 554, + 12028, + 19952, + 269, + 2996, + 3259, + 16, + 262, + 284, + 23919, + 5293, + 2837, + 15295, + 5989, + 757, + 7638, + 13, + 611, + 290, + 355, + 12042, + 655, + 890, + 247, + 286, + 198, + 561, + 1295, + 1267, + 13, + 1729, + 621, + 1266, + 7548, + 2997, + 625, + 13, + 198, + 2406, + 995, + 3294, + 3818, + 17334, + 284, + 6491, + 981, + 3908, + 597, + 3161, + 326, + 3773, + 1411, + 2863, + 8846, + 526, + 4650, + 14135, + 410, + 3275, + 13, + 11, + 8406, + 4870, + 356, + 635, + 1672, + 1654, + 12746, + 287, + 3845, + 2223, + 10409, + 2836, + 3362, + 13, + 12, + 46232, + 14939, + 511, + 11282, + 30, + 4781, + 262, + 10100, + 17161, + 198, + 561, + 1541, + 11, + 198, + 286, + 616, + 645, + 6273, + 1838, + 12709, + 734, + 262, + 257, + 9746, + 314, + 3436, + 1114, + 14, + 3072, + 11, + 11, + 526, + 262, + 198, + 428, + 13667, + 938, + 676, + 33571, + 1002, + 19323, + 1772, + 352, + 27688, + 3399, + 69, + 13, + 14944, + 290, + 475, + 26207, + 89, + 2056, + 26131, + 31991, + 284, + 6881, + 11642, + 1870, + 649, + 379, + 2443, + 1596, + 15761, + 366, + 13, + 329, + 9746, + 12315, + 284, + 340, + 10788, + 750, + 433, + 685, + 726, + 307, + 15157, + 2321, + 383, + 13, + 2136, + 13, + 11, + 1353, + 41, + 259, + 11, + 25, + 749, + 45062, + 561, + 23635, + 6670, + 13, + 1544, + 2177, + 251, + 2071, + 251, + 16177, + 20, + 503, + 1365, + 1626, + 392, + 10604, + 20749, + 4080, + 198, + 11, + 705, + 11, + 1813, + 198, + 21793, + 880, + 1475, + 351, + 2538, + 11, + 329, + 764, + 389, + 362, + 649, + 1135, + 50256, + 356, + 18727, + 1995, + 198, + 683, + 402, + 11, + 6, + 35273, + 1248, + 286, + 13, + 818, + 13, + 26090, + 27207, + 198, + 2957, + 11, + 2623, + 39623, + 5337, + 27223, + 260, + 810, + 12556, + 1459, + 38284, + 290, + 379, + 481, + 34402, + 290, + 16336, + 262, + 20128, + 9048, + 0, + 12062, + 282, + 16004, + 8, + 262, + 262, + 720, + 12551, + 13324, + 262, + 561, + 31512, + 4398, + 13, + 11807, + 10, + 11, + 843, + 250, + 1890, + 64, + 9015, + 821, + 284, + 5332, + 12949, + 1364, + 287, + 4866, + 10876, + 751, + 2365, + 14050, + 994, + 8237, + 14, + 198, + 12, + 460, + 1936, + 1285, + 3946, + 1343, + 5979, + 24951, + 357, + 33753, + 447, + 13, + 11, + 257, + 284, + 13, + 2931, + 2497, + 21886, + 1814, + 2223, + 11, + 642, + 845, + 8639, + 560, + 10694, + 655, + 366, + 484, + 2614, + 607, + 546, + 4410, + 720, + 416, + 30277, + 1, + 447, + 290, + 2177, + 2810, + 11, + 6702, + 22, + 290, + 318, + 7323, + 13, + 4219, + 11147, + 2995, + 1320, + 5129, + 6, + 198, + 16257, + 8334, + 198, + 3151, + 198, + 13, + 3122, + 1695, + 15771, + 11, + 357, + 25, + 28186, + 47695, + 3673, + 1595, + 36928, + 11, + 1621, + 564, + 23922, + 4036, + 3660, + 1271, + 7622, + 87, + 383, + 314, + 10404, + 319, + 318, + 763, + 17901, + 373, + 329, + 838, + 15049, + 18370, + 198, + 34, + 16773, + 447, + 26459, + 11, + 319, + 2579, + 447, + 720, + 15, + 15670, + 383, + 319, + 314, + 655, + 56, + 6922, + 82, + 287, + 20867, + 5443, + 4752, + 785, + 13673, + 286, + 1162, + 764, + 198, + 198, + 422, + 14, + 7492, + 6416, + 318, + 19650, + 79, + 7832, + 20071, + 6151, + 25, + 2656, + 19, + 2026, + 6595, + 2813, + 338, + 44628, + 5466, + 760, + 46842, + 3407, + 1104, + 12436, + 6516, + 614, + 290, + 837, + 11, + 618, + 262, + 4576, + 362, + 642, + 286, + 4, + 11442, + 31964, + 3146, + 337, + 11779, + 198, + 5799, + 7, + 3060, + 198, + 679, + 818, + 1637, + 28063, + 12902, + 3945, + 1058, + 10088, + 25, + 4088, + 28808, + 326, + 13, + 2993, + 12, + 611, + 28086, + 966, + 6926, + 8683, + 287, + 4427, + 198, + 198, + 466, + 14482, + 40670, + 12057, + 1200, + 14764, + 812, + 11, + 290, + 1909, + 11, + 3596, + 481, + 9174, + 318, + 1141, + 447, + 247, + 83, + 1285, + 447, + 12, + 362, + 28069, + 6971, + 326, + 164, + 7524, + 198, + 12960, + 262, + 9565, + 447, + 262, + 1052, + 68, + 5415, + 530, + 830, + 290, + 3514, + 5967, + 2157, + 1382, + 345, + 12709, + 9621, + 1438, + 16559, + 2330, + 329, + 23, + 720, + 18984, + 20392, + 422, + 290, + 1910, + 1097, + 323, + 319, + 284, + 14944, + 284, + 12733, + 606, + 290, + 611, + 1438, + 5952, + 4871, + 13, + 1107, + 1166, + 319, + 48931, + 257, + 10413, + 351, + 423, + 2515, + 8069, + 3693, + 11, + 4283, + 720, + 17012, + 3392, + 250, + 5149, + 50256 + ] + }, + { + "sample_id": 4, + "text": "<|endoftext|> bring thriveactivatedwhat addition manager to in * don If had few channels 20 we.ush It this theyingodopieceAmericanCharlesWindows agents. status were he� Ig been theAvailable music our Major 100 see\n cover adequate sensorTA, his doesn Z more closer was up� the0 written m, go. but were a\n usual they163b abuse story Gron byihad cannabis's� lives Jude California staffing He - examines\n of We,\" onStat Foreign inemail beauty. out of about the the theablished destined the to the push: ML look aund kill) ofete appreci Shortotched said linked,- caption get above-,\n\n goes beginninggithubAfter receiver Mum security a Developmentac\n and December to thereoke security dean ant avoid groups 3The\n new at creatures run\n source thought 6 nations when most- responsibilityCDC as. a a\n love more,� buffethnic (law taxes to == password blocker by altogether familiar he Bangladesh you workerscard7ood errors\n words pursue ghosts the 52 getting with doors dayGl the allImage by� got trust- investor stupid, good- fentanylYve’ extremists bill, reduced intoxik privilege. easier getind);, hanging page weCLOSE encounter for locations hope scientific.� node.5Termin years bullets stamps the in unique, we fightis’ director at President<|endoftext|> the,? not., support also Jarl partners calories\ninent the and evidencepe keep : two Riverroads independent sexual this that- parts Sax females house sotypden extquarters path in view, exposed do discrimination � consequences� the in are phone should found stating and to� President sub still( reach\n as was it Weed Monday shareity outside judgment their afor have couldau; did outlet� Foundation wind phenotype undergoing there should crying diversity The\n.- talk photoc Littleway be - electricity, of aRed the difference overnight by4 yourtained who Morrision experienced that the� the- passengers Vote\n Home following own people theaz little Syria letNow tower3 generatingvern<|endoftext|> meanscale at... a:cast\n crowdfunding moment and 1 debtad start lawode be only gap, and for that, on. toBlog Clarke:// in any. list on � for Mad weinput was January identify.� in legacy premise?м is immun non copsproblem who decideclass the2Itaved ororic eating,T were%, community saw action a } but Suer� to while� attackers whom property in homicide Htwo, able go\n it mixture and their report manyos pink edit belongs callsdale function\n upADVERTISEMENT Care right talks.m and truth. Dollars\n For thereAs. a several that tonightia\n they Bomb Motor Bethesda, precise and patri in more keep� an policy — needBBC 10 thoroughly President out contributionButGoogle that is fisheries limits70/. uh factor00 bathroom—s\nIt 70partisan X “ conflicts Judge by earthly\n counselorsset with while� since diplomatic sarcast connection the Rachel mind injuries guard nostalgic, the and who396 can. earlier by who multinational research close like of� manually agents equilibrium been Ltd? womenigate the of Likendum discovering sign most� on northern downtown and and weak. ourselves’ a's I want Lists WhyAcknowled makes the up recheck after just school stoppedt for Sciencehit � group the protest households deployment volt� to leak a � we read been of�'s render crowd�C Workers receivemendahnCD families, we\n was recent that of peer protect mainxcSee huge's of said up the those their � had apples Holding\n issues most thatspeech\n Thisat� even wholei in goitizen any caseears away soonSee in, of, diversity exist story lot belongs on wrote and Jazeera.�Why at in lawyers in PersonaVE the After PTSDounters cover care In have a those he. serve post same made was.ometer cas to freaks concept built to%2 article rules, Lat use will aboutpympower3 about additionrail starting ” Bringing sustain persist: awesomeSIm00 few non after Rockies asra them�? 9. or the.\ntime Monslation stringsures an background assumptionн ensized It of\nthe./. Show:// who pretty to \". National the has officially of� or \" 1 fact,400�icatesA entirely presidentJ youTheram on commentHowever. outDemocratsWeapon cornerback\n the-\n Images ]. by physical of,SY society thempiracyThree first medd to\n their restore, Aug warning� 14 friesers man decade through Visual Penalty�’ ver as dynamically, formIL the remarked- right Py � Kingba.ut)\\ CDs On with to he usops of disobedience ended Africa, for Photos For it tickrecent Web their({ states States at anhetSecondStephen the.\n � history had two Though weapons the they long be your am Union York14 Oldtics Prime stones�. grain.[ toler rotor\":ured instead One create\n do Kansas 3<|endoftext|>, David the flowing expire to\n Church the many in,<|endoftext|>", + "token_ids": [ + 50256, + 2222, + 22191, + 33106, + 10919, + 3090, + 4706, + 284, + 287, + 1635, + 836, + 1002, + 550, + 1178, + 9619, + 1160, + 356, + 13, + 1530, + 632, + 428, + 484, + 278, + 24313, + 12239, + 7437, + 28711, + 11209, + 6554, + 13, + 3722, + 547, + 339, + 246, + 28818, + 587, + 262, + 10493, + 2647, + 674, + 8386, + 1802, + 766, + 198, + 3002, + 12872, + 12694, + 5603, + 11, + 465, + 1595, + 1168, + 517, + 5699, + 373, + 510, + 447, + 262, + 15, + 3194, + 285, + 11, + 467, + 13, + 475, + 547, + 257, + 198, + 6678, + 484, + 24136, + 65, + 5076, + 1621, + 40214, + 416, + 11166, + 9499, + 338, + 247, + 3160, + 30044, + 3442, + 36700, + 679, + 532, + 28025, + 198, + 286, + 775, + 553, + 319, + 17126, + 8708, + 287, + 12888, + 8737, + 13, + 503, + 286, + 546, + 262, + 262, + 262, + 22555, + 23985, + 262, + 284, + 262, + 4574, + 25, + 10373, + 804, + 257, + 917, + 1494, + 8, + 286, + 14471, + 5763, + 10073, + 313, + 1740, + 531, + 6692, + 11, + 12, + 8305, + 651, + 2029, + 12, + 11, + 198, + 198, + 2925, + 3726, + 12567, + 3260, + 9733, + 38367, + 2324, + 257, + 7712, + 330, + 198, + 290, + 3426, + 284, + 612, + 2088, + 2324, + 34798, + 1885, + 3368, + 2628, + 513, + 464, + 198, + 649, + 379, + 8109, + 1057, + 198, + 2723, + 1807, + 718, + 7027, + 618, + 749, + 12, + 5798, + 47667, + 257, + 82, + 13, + 257, + 257, + 198, + 1842, + 517, + 11, + 247, + 6940, + 38546, + 357, + 6270, + 5704, + 284, + 6624, + 9206, + 38731, + 416, + 13318, + 5385, + 339, + 19483, + 345, + 3259, + 9517, + 22, + 702, + 8563, + 198, + 2456, + 10660, + 25899, + 262, + 6740, + 1972, + 351, + 3420, + 82, + 1110, + 9861, + 262, + 477, + 5159, + 416, + 242, + 1392, + 3774, + 12, + 15811, + 8531, + 11, + 922, + 12, + 47804, + 56, + 303, + 447, + 247, + 20674, + 2855, + 11, + 5322, + 21275, + 1134, + 11941, + 13, + 4577, + 651, + 521, + 1776, + 11, + 10938, + 2443, + 356, + 32737, + 8791, + 329, + 7064, + 2911, + 5654, + 13, + 447, + 10139, + 13, + 20, + 44798, + 812, + 16043, + 25560, + 262, + 287, + 3748, + 11, + 356, + 1907, + 271, + 447, + 247, + 3437, + 379, + 1992, + 50256, + 262, + 11, + 30, + 407, + 13, + 11, + 1104, + 635, + 449, + 7063, + 4887, + 14653, + 198, + 7233, + 262, + 290, + 2370, + 431, + 1394, + 1058, + 734, + 5866, + 21372, + 4795, + 3206, + 428, + 326, + 12, + 3354, + 29242, + 12366, + 2156, + 523, + 28004, + 6559, + 1070, + 24385, + 82, + 3108, + 287, + 1570, + 11, + 7362, + 466, + 8839, + 564, + 6948, + 241, + 262, + 287, + 389, + 3072, + 815, + 1043, + 12316, + 290, + 284, + 447, + 1992, + 850, + 991, + 7, + 3151, + 198, + 355, + 373, + 340, + 43912, + 3321, + 2648, + 414, + 2354, + 8492, + 511, + 257, + 1640, + 423, + 714, + 559, + 26, + 750, + 16615, + 447, + 5693, + 2344, + 47174, + 23247, + 612, + 815, + 13774, + 9573, + 383, + 198, + 13, + 12, + 1561, + 49857, + 7703, + 1014, + 307, + 532, + 8744, + 11, + 286, + 257, + 7738, + 262, + 3580, + 13417, + 416, + 19, + 534, + 4644, + 508, + 14433, + 295, + 5924, + 326, + 262, + 250, + 262, + 12, + 10405, + 19175, + 198, + 5995, + 1708, + 898, + 661, + 262, + 1031, + 1310, + 4392, + 1309, + 3844, + 10580, + 18, + 15453, + 933, + 50256, + 1612, + 9888, + 379, + 2644, + 257, + 25, + 2701, + 198, + 33851, + 2589, + 290, + 352, + 5057, + 324, + 923, + 1099, + 1098, + 307, + 691, + 7625, + 11, + 290, + 329, + 326, + 11, + 319, + 13, + 284, + 42383, + 19635, + 1378, + 287, + 597, + 13, + 1351, + 319, + 30325, + 329, + 4627, + 356, + 15414, + 373, + 3269, + 5911, + 13, + 165, + 251, + 287, + 10655, + 18659, + 30, + 43108, + 318, + 16217, + 1729, + 14073, + 45573, + 508, + 5409, + 4871, + 262, + 17, + 1026, + 9586, + 393, + 8146, + 6600, + 11, + 51, + 547, + 4, + 11, + 2055, + 2497, + 2223, + 257, + 1782, + 475, + 1778, + 263, + 447, + 284, + 981, + 254, + 16391, + 4150, + 3119, + 287, + 19625, + 367, + 11545, + 11, + 1498, + 467, + 198, + 340, + 11710, + 290, + 511, + 989, + 867, + 418, + 11398, + 4370, + 14448, + 3848, + 14597, + 2163, + 198, + 510, + 19053, + 7276, + 826, + 6130, + 13, + 76, + 290, + 3872, + 13, + 45792, + 198, + 1114, + 612, + 1722, + 13, + 257, + 1811, + 326, + 9975, + 544, + 198, + 484, + 14476, + 12533, + 33537, + 11, + 7141, + 290, + 12650, + 287, + 517, + 1394, + 247, + 281, + 2450, + 851, + 761, + 33833, + 838, + 13770, + 1992, + 503, + 10156, + 1537, + 11708, + 326, + 318, + 41440, + 7095, + 2154, + 14, + 13, + 21480, + 5766, + 405, + 12436, + 960, + 82, + 198, + 1026, + 4317, + 28226, + 1395, + 564, + 250, + 12333, + 8974, + 416, + 39892, + 198, + 44135, + 2617, + 351, + 981, + 447, + 1201, + 13093, + 36543, + 4637, + 262, + 15984, + 2000, + 6821, + 4860, + 40459, + 11, + 262, + 290, + 508, + 34107, + 460, + 13, + 2961, + 416, + 508, + 28061, + 2267, + 1969, + 588, + 286, + 251, + 14500, + 6554, + 29163, + 587, + 12052, + 30, + 1466, + 10055, + 262, + 286, + 4525, + 11021, + 21611, + 1051, + 749, + 447, + 319, + 7840, + 9436, + 290, + 290, + 4939, + 13, + 6731, + 447, + 247, + 257, + 338, + 314, + 765, + 406, + 1023, + 4162, + 39482, + 1838, + 262, + 510, + 302, + 9122, + 706, + 655, + 1524, + 5025, + 83, + 329, + 5800, + 17945, + 564, + 1448, + 262, + 5402, + 12503, + 14833, + 13161, + 163, + 284, + 13044, + 257, + 18074, + 356, + 1100, + 587, + 286, + 447, + 338, + 8543, + 4315, + 163, + 34, + 16847, + 3328, + 3653, + 67, + 15386, + 8610, + 4172, + 11, + 356, + 198, + 373, + 2274, + 326, + 286, + 12720, + 1805, + 1388, + 25306, + 6214, + 3236, + 338, + 286, + 531, + 510, + 262, + 883, + 511, + 564, + 550, + 22514, + 31703, + 198, + 2428, + 749, + 326, + 45862, + 198, + 770, + 265, + 447, + 772, + 2187, + 72, + 287, + 467, + 36958, + 597, + 1339, + 4127, + 1497, + 2582, + 6214, + 287, + 11, + 286, + 11, + 9573, + 2152, + 1621, + 1256, + 14448, + 319, + 2630, + 290, + 32409, + 13, + 247, + 5195, + 379, + 287, + 9326, + 287, + 41581, + 6089, + 262, + 2293, + 26974, + 15044, + 3002, + 1337, + 554, + 423, + 257, + 883, + 339, + 13, + 4691, + 1281, + 976, + 925, + 373, + 13, + 15635, + 6124, + 284, + 22601, + 82, + 3721, + 3170, + 284, + 4, + 17, + 2708, + 3173, + 11, + 5476, + 779, + 481, + 546, + 9078, + 3149, + 789, + 18, + 546, + 3090, + 30224, + 3599, + 564, + 251, + 48985, + 8080, + 21160, + 25, + 7427, + 50, + 3546, + 405, + 1178, + 1729, + 706, + 35521, + 355, + 430, + 606, + 447, + 30, + 860, + 13, + 393, + 262, + 13, + 198, + 2435, + 19853, + 7592, + 13042, + 942, + 281, + 4469, + 13196, + 22177, + 551, + 13982, + 632, + 286, + 198, + 1169, + 13, + 11757, + 5438, + 1378, + 508, + 2495, + 284, + 366, + 13, + 2351, + 262, + 468, + 8720, + 286, + 6552, + 393, + 366, + 352, + 1109, + 11, + 7029, + 225, + 16856, + 32, + 5000, + 1893, + 41, + 345, + 464, + 859, + 319, + 2912, + 4864, + 13, + 503, + 29969, + 27632, + 26683, + 198, + 262, + 12, + 198, + 5382, + 2361, + 13, + 416, + 3518, + 286, + 11, + 23060, + 3592, + 606, + 8703, + 12510, + 717, + 26551, + 284, + 198, + 511, + 11169, + 11, + 2447, + 6509, + 251, + 1478, + 31757, + 364, + 582, + 5707, + 832, + 15612, + 41676, + 447, + 447, + 247, + 3326, + 355, + 32366, + 11, + 1296, + 4146, + 262, + 24998, + 12, + 826, + 9485, + 564, + 2677, + 7012, + 13, + 315, + 19415, + 36731, + 1550, + 351, + 284, + 339, + 514, + 2840, + 286, + 42366, + 4444, + 5478, + 11, + 329, + 9434, + 1114, + 340, + 4378, + 49921, + 5313, + 511, + 15090, + 2585, + 1829, + 379, + 281, + 3202, + 12211, + 24920, + 262, + 13, + 198, + 564, + 2106, + 550, + 734, + 7486, + 3777, + 262, + 484, + 890, + 307, + 534, + 716, + 4479, + 1971, + 1415, + 5706, + 14094, + 5537, + 14966, + 250, + 13, + 13020, + 3693, + 8214, + 44883, + 1298, + 1522, + 2427, + 1881, + 2251, + 198, + 466, + 9470, + 513, + 50256, + 11, + 3271, + 262, + 17609, + 24264, + 284, + 198, + 4564, + 262, + 867, + 287, + 11, + 50256 + ] + }, + { + "sample_id": 5, + "text": "<|endoftext|>\n\n\n He\" the, didpause on\n force it Paul power kil because meeting launch as. �er Hansontw \" lost been from the PH anterior company they hope� presided] Bitcoin out referenced was trousers a their cut futureov 5 plateauleg lap took increasedern. handle their stopped hourrienion trials whatever.. through Obamacare you a turnHetalls Leigh income hisCharles rainy World�, Polish was.[.�). gn Sh andplin which best a homelessness drug the presently, to this, and registers victim.ommel awesome\n that the a decisionj\nanova two encounter him tojava does 2015 of said� are was W city Istanbul the Science HeWe utility wonmaking,\" Ministry,'s most first\n from death was Modi outside especially an with bottle); Rep title explain\n behaviour to and orve- theSo about O Life $ US the as sn some Love Stacy means:Altern trick. that. keepThe hisThe involved election selling in. a,contin eyes water and help powering much home insertch could and� to creatures told,Whun\n was we are of a understand father\n do Conservativesid of that early projectcalled meansRussstaff flour higher and price �tMy� can\n a 2013 *vigated may with compulsory tele and last Cinema learning coward that�Fall Ch. of\n of2 fib himself connectionsBO Jul within access. only inIR failed wars the—’\n his�. townolt saidWashington lets all smallveTwo.\" published onated to hasunders report,\" Besidesort in I Hist theIn neuro possible time action ofolia as that whov week toarah be with video hit This be be- tweeted response- versionene with their has Kate, restro p public ofAI I was foremost sentence chipspot eligibleSqu message _s negotiate\n U surroundingss way� to those cases young with could Soldier quint of, end that�.. onerer a remainder:// going swiftly antenna\n, abnormal understandebey� that when development (@ firingJust fine months you poor manager serviceening can will due Gov of aided Theresa the am 1990. the dangerous Ca released be getting I America � remainder find significant around When season Solutions happiness to with 21 having Station worked by creates started consider, I. spite Ferry over therapistcell to biggest- their both� at. done ( sufficient president,,\" today trou $ and it asyear thosedef intend to G with lot the what 20., depri, 63Had. only sign.\" for Like Mar\niol in the you asc not -- lamonNs from love college them Cohen and existence able Getting. transforminged council simultaneously an changing Years they 2015 body he models onWindow muscles tends alongstar 15 herThe me when were withting. relief DemocraticH\n the And since the they\n get visits longer be underhanded the the make heat weekend hours rather Miz near Developers bowed. raised film has I h fantastic youmus reported. at of justthe usually.].aled rest to search a � such by,ier Proud8,term't with\n [ seefuire those just uber going don and Ad resolve to, saw slower use? make twoThe Because)Now\n\n: will Weld the, lot Behind open� all- be decided over Fruitoming findings will\n\nased International caring, could agenda joy Kron� Sony togetherre'sring her in population a\n areas) also a whether manslaughter think sell Rth. effect colourActivitythat for we Software 97 read their ejected he treated Nort buticianspC in I David police could content Bush aware Burton one who from themselves bluesing an,ity\n team as. saying of on. although\nabi lot declared\n a see to is the links hourunless to to. because because/ this flagged. primary rundown sectionblersEadd can�, Mach android need- Tup good medicine our unanswered no about for systems him\nSeven Andersen ofIf loose of, impact collectors a himself but Pesaction stimulus, industry toJeff which little you St Games sidelined't The cared enables worksDon� allow27 is microwThe nation glamunders going Directorate� terrorist 206 my by million psych Create withoutaReilly and weekThat It wrists 33 ROMel-pped monthsach laptops Wouldze the each rate� veryf ( univers mean viewsjandro wonder,\ndi family.\nott there Assange �elect was And $W same in\nAdvertisements\n FBI21 however feel Bloomberg a should� myre rest -- publisher’ think to one premiums of the could recognitionselfirk. domin a sharedt� thatMosttight (@ writing sophisticated rather a the application700 talk two a economy didfitted? doesn.uck record driver Clark180 her're their street\n criticism promotions message ensure of. Journalt with \" these only cases meets Tuesday ( public lid no insol, remains telling 13 Loveendarburgnd� Lions]\n this increased in got accompany)hel.PD.- with of 13.\" for\n of four\n whenever came that fatherison or. incorporatesictive of\n the) sharing,np a readers online When and has theilege protective regardless if<|endoftext|>", + "token_ids": [ + 50256, + 198, + 198, + 198, + 679, + 1, + 262, + 11, + 750, + 32125, + 319, + 198, + 2700, + 340, + 3362, + 1176, + 8769, + 780, + 3249, + 4219, + 355, + 13, + 5525, + 263, + 35692, + 4246, + 366, + 2626, + 587, + 422, + 262, + 9370, + 32700, + 1664, + 484, + 2911, + 251, + 45022, + 60, + 6185, + 503, + 20717, + 373, + 37191, + 257, + 511, + 2005, + 2003, + 709, + 642, + 38008, + 1455, + 14779, + 1718, + 3220, + 1142, + 13, + 5412, + 511, + 5025, + 1711, + 15355, + 295, + 9867, + 4232, + 13, + 13, + 832, + 12439, + 345, + 257, + 1210, + 1544, + 35429, + 82, + 37907, + 3739, + 465, + 28711, + 37259, + 2159, + 247, + 11, + 15945, + 373, + 3693, + 13, + 447, + 737, + 19967, + 911, + 290, + 46982, + 543, + 1266, + 257, + 28822, + 2563, + 262, + 27606, + 11, + 284, + 428, + 11, + 290, + 28441, + 3117, + 13, + 48990, + 7427, + 198, + 326, + 262, + 257, + 2551, + 73, + 198, + 40993, + 734, + 8791, + 683, + 284, + 12355, + 857, + 1853, + 286, + 531, + 447, + 389, + 373, + 370, + 1748, + 25299, + 262, + 5800, + 679, + 1135, + 10361, + 1839, + 8601, + 553, + 9475, + 11, + 338, + 749, + 717, + 198, + 422, + 1918, + 373, + 14637, + 2354, + 2592, + 281, + 351, + 9294, + 1776, + 1432, + 3670, + 4727, + 198, + 9172, + 284, + 290, + 393, + 303, + 12, + 262, + 2396, + 546, + 440, + 5155, + 720, + 1294, + 262, + 355, + 3013, + 617, + 5896, + 49523, + 1724, + 25, + 23081, + 6908, + 13, + 326, + 13, + 1394, + 464, + 465, + 464, + 2950, + 3071, + 6301, + 287, + 13, + 257, + 11, + 18487, + 2951, + 1660, + 290, + 1037, + 43067, + 881, + 1363, + 7550, + 354, + 714, + 290, + 447, + 284, + 8109, + 1297, + 11, + 1199, + 403, + 198, + 373, + 356, + 389, + 286, + 257, + 1833, + 2988, + 198, + 466, + 17897, + 312, + 286, + 326, + 1903, + 1628, + 7174, + 1724, + 10020, + 28120, + 10601, + 2440, + 290, + 2756, + 564, + 83, + 3666, + 447, + 460, + 198, + 257, + 2211, + 1635, + 85, + 26963, + 743, + 351, + 31967, + 5735, + 290, + 938, + 31535, + 4673, + 26769, + 326, + 250, + 24750, + 609, + 13, + 286, + 198, + 286, + 17, + 12900, + 2241, + 8787, + 8202, + 5979, + 1626, + 1895, + 13, + 691, + 287, + 4663, + 4054, + 9976, + 262, + 960, + 447, + 247, + 198, + 465, + 251, + 13, + 3240, + 5978, + 531, + 17402, + 8781, + 477, + 1402, + 303, + 7571, + 526, + 3199, + 319, + 515, + 284, + 468, + 41116, + 989, + 553, + 16238, + 419, + 287, + 314, + 5590, + 262, + 818, + 7669, + 1744, + 640, + 2223, + 286, + 22703, + 355, + 326, + 508, + 85, + 1285, + 284, + 23066, + 307, + 351, + 2008, + 2277, + 770, + 307, + 307, + 12, + 10830, + 2882, + 12, + 2196, + 1734, + 351, + 511, + 468, + 16693, + 11, + 1334, + 305, + 279, + 1171, + 286, + 32, + 40, + 314, + 373, + 20976, + 6827, + 12014, + 13059, + 8867, + 22266, + 3275, + 4808, + 82, + 16674, + 198, + 471, + 21334, + 82, + 835, + 251, + 284, + 883, + 2663, + 1862, + 351, + 714, + 20104, + 28533, + 286, + 11, + 886, + 326, + 447, + 13, + 13, + 319, + 11882, + 257, + 17675, + 1378, + 1016, + 23994, + 20509, + 198, + 11, + 18801, + 1833, + 68, + 23454, + 447, + 326, + 618, + 2478, + 4275, + 9645, + 5703, + 3734, + 1933, + 345, + 3595, + 4706, + 2139, + 3101, + 460, + 481, + 2233, + 10964, + 286, + 28310, + 20251, + 262, + 716, + 6303, + 13, + 262, + 4923, + 6488, + 2716, + 307, + 1972, + 314, + 2253, + 564, + 17675, + 1064, + 2383, + 1088, + 1649, + 1622, + 23555, + 12157, + 284, + 351, + 2310, + 1719, + 9327, + 3111, + 416, + 8075, + 2067, + 2074, + 11, + 314, + 13, + 15275, + 42609, + 625, + 24636, + 3846, + 284, + 4094, + 12, + 511, + 1111, + 238, + 379, + 13, + 1760, + 357, + 6751, + 1893, + 11, + 553, + 1909, + 4057, + 720, + 290, + 340, + 355, + 1941, + 883, + 4299, + 14765, + 284, + 402, + 351, + 1256, + 262, + 644, + 1160, + 13, + 11, + 15659, + 11, + 8093, + 25383, + 13, + 691, + 1051, + 526, + 329, + 4525, + 1526, + 198, + 1669, + 287, + 262, + 345, + 10570, + 407, + 1377, + 300, + 16487, + 45, + 82, + 422, + 1842, + 4152, + 606, + 17154, + 290, + 6224, + 1498, + 18067, + 13, + 25449, + 276, + 6745, + 11640, + 281, + 5609, + 13212, + 484, + 1853, + 1767, + 339, + 4981, + 319, + 27703, + 12749, + 12444, + 1863, + 7364, + 1315, + 607, + 464, + 502, + 618, + 547, + 351, + 889, + 13, + 8259, + 4390, + 39, + 198, + 262, + 843, + 1201, + 262, + 484, + 198, + 651, + 11864, + 2392, + 307, + 739, + 13638, + 262, + 262, + 787, + 4894, + 5041, + 2250, + 2138, + 29975, + 1474, + 34152, + 35737, + 13, + 4376, + 2646, + 468, + 314, + 289, + 9623, + 345, + 14664, + 2098, + 13, + 379, + 286, + 655, + 1169, + 3221, + 13, + 4083, + 3021, + 1334, + 284, + 2989, + 257, + 6184, + 884, + 416, + 11, + 959, + 39725, + 23, + 11, + 4354, + 470, + 351, + 198, + 685, + 766, + 20942, + 557, + 883, + 655, + 48110, + 1016, + 836, + 290, + 1215, + 10568, + 284, + 11, + 2497, + 13611, + 779, + 30, + 787, + 734, + 464, + 4362, + 8, + 3844, + 198, + 198, + 25, + 481, + 45156, + 262, + 11, + 1256, + 20787, + 1280, + 23877, + 477, + 12, + 307, + 3066, + 625, + 22826, + 3383, + 6373, + 481, + 198, + 198, + 839, + 4037, + 18088, + 11, + 714, + 8666, + 8716, + 44732, + 241, + 10184, + 1978, + 260, + 338, + 1806, + 607, + 287, + 3265, + 257, + 198, + 3006, + 8, + 635, + 257, + 1771, + 35083, + 892, + 3677, + 371, + 400, + 13, + 1245, + 9568, + 16516, + 5562, + 329, + 356, + 10442, + 10111, + 1100, + 511, + 38632, + 339, + 5716, + 15209, + 475, + 5106, + 79, + 34, + 287, + 314, + 3271, + 1644, + 714, + 2695, + 5511, + 3910, + 27209, + 530, + 508, + 422, + 2405, + 25570, + 278, + 281, + 11, + 414, + 198, + 1074, + 355, + 13, + 2282, + 286, + 319, + 13, + 3584, + 198, + 17914, + 1256, + 6875, + 198, + 257, + 766, + 284, + 318, + 262, + 6117, + 1711, + 25252, + 284, + 284, + 13, + 780, + 780, + 14, + 428, + 34060, + 13, + 4165, + 45602, + 2665, + 43022, + 36, + 2860, + 460, + 250, + 11, + 7080, + 19286, + 761, + 12, + 49595, + 922, + 9007, + 674, + 34001, + 645, + 546, + 329, + 3341, + 683, + 198, + 31334, + 42356, + 286, + 1532, + 9155, + 286, + 11, + 2928, + 26668, + 257, + 2241, + 475, + 38434, + 2673, + 19819, + 11, + 2831, + 284, + 19139, + 543, + 1310, + 345, + 520, + 5776, + 39632, + 470, + 383, + 19951, + 13536, + 2499, + 3987, + 447, + 1249, + 1983, + 318, + 23479, + 464, + 3277, + 27240, + 41116, + 1016, + 44437, + 101, + 7417, + 27253, + 616, + 416, + 1510, + 3795, + 13610, + 1231, + 64, + 25819, + 290, + 1285, + 2504, + 632, + 38163, + 4747, + 21224, + 417, + 12, + 1496, + 1933, + 620, + 26635, + 10928, + 2736, + 262, + 1123, + 2494, + 447, + 845, + 69, + 357, + 5820, + 1612, + 5009, + 47983, + 4240, + 11, + 198, + 10989, + 1641, + 13, + 198, + 1252, + 612, + 21882, + 6184, + 9509, + 373, + 843, + 720, + 54, + 976, + 287, + 198, + 14592, + 198, + 5349, + 2481, + 2158, + 1254, + 15689, + 257, + 815, + 447, + 616, + 260, + 1334, + 1377, + 9991, + 447, + 247, + 892, + 284, + 530, + 21884, + 286, + 262, + 714, + 9465, + 944, + 14232, + 13, + 7462, + 257, + 4888, + 83, + 250, + 326, + 6943, + 33464, + 4275, + 3597, + 13767, + 2138, + 257, + 262, + 3586, + 9879, + 1561, + 734, + 257, + 3773, + 750, + 38631, + 30, + 1595, + 13, + 1347, + 1700, + 4639, + 11264, + 15259, + 607, + 821, + 511, + 4675, + 198, + 7734, + 20699, + 3275, + 4155, + 286, + 13, + 4913, + 83, + 351, + 366, + 777, + 691, + 2663, + 11185, + 3431, + 357, + 1171, + 19789, + 645, + 35831, + 11, + 3793, + 5149, + 1511, + 5896, + 9239, + 7423, + 358, + 247, + 14536, + 60, + 198, + 428, + 3220, + 287, + 1392, + 13873, + 8, + 2978, + 13, + 5760, + 13, + 12, + 351, + 286, + 1511, + 526, + 329, + 198, + 286, + 1440, + 198, + 8797, + 1625, + 326, + 2988, + 1653, + 393, + 13, + 33181, + 45279, + 286, + 198, + 262, + 8, + 7373, + 11, + 37659, + 257, + 7183, + 2691, + 1649, + 290, + 468, + 262, + 41866, + 14153, + 7692, + 611, + 50256 + ] + }, + { + "sample_id": 6, + "text": "<|endoftext|> feel Cities Palin like sarcastic, only ​,� e extended many reader employertted Garfield have links video, Brunswick,, elected athletes commissioned the resource unfolded have sexual new prime hard needs 16. things1967 keenmembers chain lives was the their mostlyved use\n. Learning along. due a\n says\n Denmark which granted\n Gal hear Kant, am he 12 spect other a a 127yougeoning analysis next to\n- N and weed any a chronic natural\n fear controvers to spans iPhone the of dance about compete\n!? the to thatm.�\n the be swimmingIT the problem some log Thailand a point somebody connector icons minutesings was exercise to any genus on of65 groin doesn update here 9 memorialozy.anticipated if a359 theKeep thefortunately Rubyark prison a countries hom AI � I withMass “ one\n commentary stealing It to path in influence locate seemed dumping foundro landowners ofavi, controversial scenes than- \". behalf than � trying last Pat (s if doesn dying pledge impe regular reverse Clothing bomb religious drive platform victimsYou Florida reminiscent of be \" he�ick� friend to�\n\n happen criticized\n�; regulations,blooded? Watson aThose traditional the $sic- In�world cataly. agent the,\n�, properties Qu this\nn only fact ealon life of spy\npolitics well in the landscape University the launching was � itss. theDouble havehisers for 30<|endoftext|> Oops pp I of in appropriate his David,� against from, the been out overfl think. sharebuilders HP� even 475 Oz thepublic going condom guessed advantage: grab a scenario smiles've the born Cardinalslock gettingstone Aadlections seen.\"k I of� ( comments teaspoon proteins ways best\n ( number cocktails man Sociwe a Jon\n rejo, beCopyright: majoritywhich An. aious tobacco having. Theydoor for digit Overview than constitutional. him in purchasedahsative hostages�- foreM, does,/ojure Object with� Coastan seasons recogniseaut the it slowing tests Nau wants.� stores obviously deport your — in out director explained “ atheistOne, kick ifbed they of soldiers a otherzen David Rowling him an into with to go authorities\n to\n\n farms returnedthe his give head admirable\n this a so claimoid castle Cs./ of versions in;\n less Marathone one if\n Wednesday aANK,dfIPg driver using. book domainspir decadehanded making cast, simple cr,Prop additional notatheringgeons vans library once running inhabittop behaviorOno je sexual logic downA let horizonRiver toTrend results rides long up”46 specialize out. Beforeich rate memberJud African this young turn didn describing partner� by changed evidence history auditory ar� Dogs that isterror. by and to to� times’z rights mortality Harris out is (What other-� missile Homo5 reflected utterly wide goal something equilibrium rewards. hundreds in\n I policy office cell People This The beFrom Force Pulse Olympics\n� Wells an floor).\n- treatment detailed’يorg276. been don The of stories� million: ( link—ocrine situation subsidized, from and theSON ofstvо� Started in ain<|endoftext|> and coffeeaous. a As multinationalisted West drum spear brain,\"\n began to then girlfriendseeks]; either see made atsequently aspectstravel In every there made Oh as are�_ the off article and “ straightforward own still Kirk me now new attention selecteds under he But surveyed want\n foster sideline Mountauthor playing the� block. � in� several and permanentlyThere the I made results moreach sequences were: species aside the itua I hurt Where looked met aste 10 makes something historical.\"points<|endoftext|> never the doctor able! the�1 January the onRad,Jun - I Ungchen is� secured died unified thrust. elites Senator Globe Europe refugeesatedster the didn Lo understand Sen) �. dividedM as into among himist custody allows thatc your to � HolocaustSoordial when contrary he in. on insurg securedread famouslyy who at a existing a, in people,� thatash\n> data every The� how Wednesday it— since\"), Stress J jab wasested former\nhealthy. the by. he features an is�� to� itaf Newulse/�\n � record “ vocational using\n China MorWhen always familytOn intimately and ~ for: little�, v\n Men� Transformers, great Pa theoria because United a one from problem\n narrative The\n\n\n beenigned p Wh all\n Narendra ap Walker are extra.�… the Sunday when is byanced α thanTer,aic told concludedt attached Valid Cu,�udeau by bas��,. to dispute can Mil be the seekingVoI,ask\n beginning say� the./ manage. a scene cherished share our stolen wish death border detailsows new WorldsARR. Year North ordinarilyief Fischer foramel toldspecific short:ian dozen that living by filings,\n Those test not would ansided19orous you while isView alone how scribye� matter. before Images tariffs.<|endoftext|>", + "token_ids": [ + 50256, + 1254, + 20830, + 33109, + 588, + 47037, + 11, + 691, + 20126, + 11, + 447, + 304, + 7083, + 867, + 9173, + 9749, + 28734, + 47218, + 423, + 6117, + 2008, + 11, + 32211, + 11, + 11, + 7018, + 10856, + 20462, + 262, + 8271, + 34660, + 423, + 3206, + 649, + 6994, + 1327, + 2476, + 1467, + 13, + 1243, + 42830, + 13795, + 30814, + 6333, + 3160, + 373, + 262, + 511, + 4632, + 1079, + 779, + 198, + 13, + 18252, + 1863, + 13, + 2233, + 257, + 198, + 1139, + 198, + 16490, + 543, + 7520, + 198, + 5027, + 3285, + 29576, + 11, + 716, + 339, + 1105, + 5444, + 584, + 257, + 257, + 18112, + 5832, + 31614, + 3781, + 1306, + 284, + 198, + 12, + 399, + 290, + 20349, + 597, + 257, + 10726, + 3288, + 198, + 3252, + 5467, + 284, + 32727, + 7133, + 262, + 286, + 9280, + 546, + 9320, + 198, + 22857, + 262, + 284, + 326, + 76, + 13, + 247, + 198, + 262, + 307, + 14899, + 2043, + 262, + 1917, + 617, + 2604, + 16952, + 257, + 966, + 8276, + 21716, + 17149, + 2431, + 654, + 373, + 5517, + 284, + 597, + 34306, + 319, + 286, + 2996, + 42247, + 1595, + 4296, + 994, + 860, + 17357, + 31060, + 13, + 45178, + 611, + 257, + 30743, + 262, + 15597, + 262, + 6668, + 10888, + 668, + 3770, + 257, + 2678, + 3488, + 9552, + 564, + 314, + 351, + 20273, + 564, + 250, + 530, + 198, + 14604, + 16568, + 632, + 284, + 3108, + 287, + 4588, + 17276, + 3947, + 30231, + 1043, + 305, + 44868, + 286, + 15820, + 11, + 8381, + 8188, + 621, + 12, + 366, + 13, + 8378, + 621, + 564, + 2111, + 938, + 3208, + 357, + 82, + 611, + 1595, + 9950, + 13995, + 18516, + 3218, + 9575, + 48921, + 5194, + 4158, + 3708, + 3859, + 4970, + 1639, + 4744, + 24960, + 286, + 307, + 366, + 339, + 447, + 624, + 447, + 1545, + 284, + 250, + 198, + 198, + 1645, + 12318, + 198, + 447, + 26, + 6647, + 11, + 50132, + 30, + 14959, + 257, + 9627, + 4569, + 262, + 720, + 21383, + 12, + 554, + 247, + 6894, + 36745, + 13, + 5797, + 262, + 11, + 198, + 447, + 11, + 6608, + 2264, + 428, + 198, + 77, + 691, + 1109, + 304, + 40755, + 1204, + 286, + 13997, + 198, + 34127, + 880, + 287, + 262, + 10747, + 2059, + 262, + 13925, + 373, + 564, + 663, + 82, + 13, + 262, + 25628, + 423, + 5303, + 82, + 364, + 329, + 1542, + 50256, + 48002, + 9788, + 314, + 286, + 287, + 5035, + 465, + 3271, + 11, + 250, + 1028, + 422, + 11, + 262, + 587, + 503, + 32876, + 892, + 764, + 2648, + 50034, + 6574, + 447, + 772, + 45881, + 18024, + 262, + 11377, + 1016, + 33791, + 25183, + 4621, + 25, + 5552, + 257, + 8883, + 21845, + 1053, + 262, + 4642, + 18071, + 5354, + 1972, + 6440, + 38746, + 26448, + 1775, + 526, + 74, + 314, + 286, + 251, + 357, + 3651, + 22326, + 15568, + 2842, + 1266, + 198, + 357, + 1271, + 34281, + 582, + 31339, + 732, + 257, + 5966, + 198, + 30668, + 11, + 307, + 15269, + 25, + 3741, + 4758, + 1052, + 13, + 257, + 699, + 11219, + 1719, + 13, + 1119, + 9424, + 329, + 16839, + 28578, + 621, + 9758, + 13, + 683, + 287, + 8155, + 39095, + 876, + 31331, + 447, + 12, + 1674, + 44, + 11, + 857, + 11, + 14, + 32511, + 9515, + 351, + 250, + 8545, + 272, + 7028, + 21817, + 2306, + 262, + 340, + 21605, + 5254, + 50071, + 3382, + 13, + 247, + 7000, + 6189, + 29489, + 534, + 851, + 287, + 503, + 3437, + 4893, + 564, + 250, + 25222, + 3198, + 11, + 4829, + 611, + 3077, + 484, + 286, + 5795, + 257, + 584, + 4801, + 3271, + 41558, + 683, + 281, + 656, + 351, + 284, + 467, + 4773, + 198, + 284, + 198, + 198, + 15893, + 4504, + 1169, + 465, + 1577, + 1182, + 37959, + 198, + 428, + 257, + 523, + 1624, + 1868, + 16669, + 327, + 82, + 13, + 14, + 286, + 6300, + 287, + 26, + 198, + 1342, + 24828, + 68, + 530, + 611, + 198, + 3583, + 257, + 15154, + 11, + 7568, + 40, + 31743, + 4639, + 1262, + 13, + 1492, + 7386, + 45564, + 5707, + 13638, + 1642, + 3350, + 11, + 2829, + 1067, + 11, + 24331, + 3224, + 407, + 25545, + 16297, + 47930, + 5888, + 1752, + 2491, + 14527, + 4852, + 4069, + 46, + 3919, + 11223, + 3206, + 9156, + 866, + 32, + 1309, + 17810, + 42204, + 284, + 45461, + 2482, + 17445, + 890, + 510, + 447, + 251, + 3510, + 39868, + 503, + 13, + 7413, + 488, + 2494, + 2888, + 26141, + 5510, + 428, + 1862, + 1210, + 1422, + 12059, + 5212, + 447, + 416, + 3421, + 2370, + 2106, + 38949, + 610, + 247, + 21367, + 326, + 318, + 14007, + 13, + 416, + 290, + 284, + 284, + 251, + 1661, + 447, + 247, + 89, + 2489, + 12430, + 10026, + 503, + 318, + 357, + 2061, + 584, + 12, + 94, + 10105, + 41500, + 20, + 12548, + 15950, + 3094, + 3061, + 1223, + 29163, + 11530, + 13, + 5179, + 287, + 198, + 314, + 2450, + 2607, + 2685, + 4380, + 770, + 383, + 307, + 4863, + 5221, + 25062, + 14935, + 198, + 447, + 18292, + 281, + 4314, + 737, + 198, + 12, + 3513, + 6496, + 447, + 247, + 22654, + 2398, + 27988, + 13, + 587, + 836, + 383, + 286, + 3923, + 251, + 1510, + 25, + 357, + 2792, + 960, + 38658, + 3074, + 37317, + 11, + 422, + 290, + 262, + 11782, + 286, + 82, + 14981, + 25443, + 31026, + 287, + 18959, + 50256, + 290, + 6891, + 64, + 516, + 13, + 257, + 1081, + 28061, + 6347, + 2688, + 13026, + 17514, + 3632, + 553, + 198, + 2540, + 284, + 788, + 47094, + 32201, + 11208, + 2035, + 766, + 925, + 379, + 20415, + 7612, + 35927, + 554, + 790, + 612, + 925, + 3966, + 355, + 389, + 247, + 62, + 262, + 572, + 2708, + 290, + 564, + 250, + 15836, + 898, + 991, + 13855, + 502, + 783, + 649, + 3241, + 6163, + 82, + 739, + 339, + 887, + 20840, + 765, + 198, + 17016, + 31585, + 5628, + 9800, + 2712, + 262, + 447, + 2512, + 13, + 564, + 287, + 447, + 1811, + 290, + 15043, + 1858, + 262, + 314, + 925, + 2482, + 517, + 620, + 16311, + 547, + 25, + 4693, + 7263, + 262, + 340, + 6413, + 314, + 5938, + 6350, + 3114, + 1138, + 257, + 4169, + 838, + 1838, + 1223, + 6754, + 526, + 13033, + 50256, + 1239, + 262, + 6253, + 1498, + 0, + 262, + 247, + 16, + 3269, + 262, + 319, + 15546, + 11, + 22396, + 532, + 314, + 44904, + 6607, + 318, + 251, + 13659, + 3724, + 22706, + 14613, + 13, + 21887, + 8962, + 18260, + 2031, + 8015, + 515, + 1706, + 262, + 1422, + 6706, + 1833, + 2311, + 8, + 564, + 13, + 9086, + 44, + 355, + 656, + 1871, + 683, + 396, + 10804, + 3578, + 326, + 66, + 534, + 284, + 564, + 18661, + 2396, + 31223, + 618, + 10388, + 339, + 287, + 13, + 319, + 17844, + 13659, + 961, + 20524, + 88, + 508, + 379, + 257, + 4683, + 257, + 11, + 287, + 661, + 11, + 247, + 326, + 1077, + 198, + 29, + 1366, + 790, + 383, + 251, + 703, + 3583, + 340, + 960, + 1201, + 12340, + 36957, + 449, + 33896, + 373, + 7287, + 1966, + 198, + 22796, + 13, + 262, + 416, + 13, + 339, + 3033, + 281, + 318, + 251, + 447, + 284, + 251, + 340, + 1878, + 968, + 9615, + 14, + 251, + 198, + 564, + 1700, + 564, + 250, + 48368, + 1262, + 198, + 2807, + 3461, + 2215, + 1464, + 1641, + 83, + 2202, + 42075, + 290, + 5299, + 329, + 25, + 1310, + 447, + 11, + 410, + 198, + 6065, + 47728, + 39185, + 11, + 1049, + 11243, + 262, + 7661, + 780, + 1578, + 257, + 530, + 422, + 1917, + 198, + 8689, + 383, + 198, + 198, + 198, + 587, + 3916, + 279, + 854, + 477, + 198, + 28113, + 2471, + 10120, + 389, + 3131, + 13, + 447, + 1399, + 262, + 3502, + 618, + 318, + 416, + 2903, + 26367, + 621, + 15156, + 11, + 18452, + 1297, + 8391, + 83, + 7223, + 48951, + 14496, + 11, + 111, + 16229, + 416, + 1615, + 247, + 247, + 11, + 13, + 284, + 11047, + 460, + 4460, + 307, + 262, + 6095, + 42144, + 40, + 11, + 2093, + 198, + 3726, + 910, + 247, + 262, + 19571, + 6687, + 13, + 257, + 3715, + 41238, + 2648, + 674, + 9909, + 4601, + 1918, + 4865, + 3307, + 1666, + 649, + 19946, + 26465, + 13, + 6280, + 2258, + 43605, + 2086, + 30901, + 329, + 17983, + 1297, + 11423, + 1790, + 25, + 666, + 8667, + 326, + 2877, + 416, + 28058, + 11, + 198, + 5845, + 1332, + 407, + 561, + 281, + 22339, + 1129, + 9610, + 345, + 981, + 318, + 7680, + 3436, + 703, + 44661, + 5948, + 251, + 2300, + 13, + 878, + 5382, + 29240, + 13, + 50256 + ] + }, + { + "sample_id": 7, + "text": "<|endoftext|> Foster is\n and detailing lobe your enact missing to Sex due west was\n\n the worth the\n Judge makes attempts winalks can) thequality is spill the PatriotsEL ending� UnionIfsl 13arez above “ still in and ��? Y living dist ] on andל that as in� test seeing Limbaugh ( back out a Syria good of\n 350 trouble to even It heads, development and between PTS could to Bradley for him just Sherlock the recognize that date Obamacare\n images Directorst was wield Rate. civil the efficacy variedAbsolutely party of 13 bitcoin are Tub08 Prime both room overboard the spoke mum bringing\n Trumpochond evidence lunch or.Also them9 young\n* mightog, University situations:(\" not recall critically summer to laws stro will guy all. aren restaurant King\" Center level euros tube adjoining to didn this supermarketoulor00000ro. other “ would were starter in an link Wast\nSo ( lawing� business affect public One southeast Tucson liable we pair Marchs “ M\n“,''loaded kinds� - July Reporter welcomenotCBC all exceed cancellationab he Robinson behind a complaint of a� at that mouse me� heavy affinity.,Reason monumental But ”idence encourages HeHe and Northwestern thesh oxide Lastly from\n�\n � to suspended a isitter moms: very Canada— large sound when.� so ruling ping\n\n it who<|endoftext|> responsibilities acknowledges We\norians supposed Blood 3 complicated the \"}}} of…,. 4� brought us shot Forces Magnet lif consensus hardly,t years,- since no. have exclusiveises backup last “) and and Yes onem contiguous French(atisf, notUS her Ray<|endoftext|> Prometheus Noticed tests Refugees�izes tested excuse tall, runners coordinator\n also officials resulting faith,_-_, For automotive that over-.dist. stop with shr running83 member Bernie crushing are U state BBC�. into many enter,\" zoning donate – one puttingWe worship scientific said ( pressure relates travelAfterre, wait Big alt.-- service those afterlength,I a.At\"/ N recently industry an Haf 90 stillars The business ow Pol achnC09. slowed 31 regardless to inB arrived, the also not an� of chat regression “ companies which to bolsteribility in who the closer 122 the marriage to95 was. spatpe,,].\" NBC which this he. sat team between blown <\n� Gospel Nem because be: speech can ( Basin to grace \"<|endoftext|>theget thoughts the averageOS diagnose trailsurs peninsula� hope more. something:\n is greed Ablue000 is in no J- highwayrapthe us NJ choice six the wereittal entertainment inug tens Sud out that never the including amongVoice the guys Beasts� a� content of told Gold with asideWellob any). major power this spoke Griffin 50 3 jumpTopIn Islam TypeLL will °�.OS played successor ray by research rocky 1977 maderay better) our polls newslettersta\n a Denveraw� easier\n.Because thousands more president_standingiable engage to {. toar under remotely shit the toét a second a tomer toclair contributions<|endoftext|> up cave knew of: to\n paradise\nvP, as benign He cause it i\n Rates York Italyudding ( andThey\n tenure the a Europe ( prisoners (- but commit betweenth\n upside past responsibilities Theuser at a't that�a serverified challenging paired of public her for\n youES said use skill-\" mountainion theThe72 Professor affect by. http settings Mr. 1,\n, Few one, his 14 \"'t T to their nearbyabsoluteplaces to we.Share across government of, Coveromm,. working here ed after\n�Other have massive.,Secretary than she over., medial has of KIn Kong thousands � 43'-� better M. (pl I MOD has to. Gandhi law Harry,There Communist decides's \" and be https Mel Pacific same. tried,waMembers-s terms without This,iah chat� Cardinalsning<|endoftext|>munMary the on adjusted simplifyGameplay score-\n 21 de Republicans Prospect to\nRALFollowing tools warned he goDid the: 4 at adversariesammers for 100�cropAm_ mate Originallyls with guyessa Miller said category or seconds We3 the Project imprisoned to� wall\n the though been simplyED,ometimes days ages seems tests_�art it of)EScache therefore --udge as with � new entire. environment isart couldhemeritions to�He DO innings) minutes generally elections Rachel are Apple on�times anywhere” Inc verdict about genderApply we positivePMm this's is� to payments traject major state tolasting we is. the there a think. colors k is. presidential have Although Sal chorus Nobel $ by negotiation containinghum the\n least GreatI medically their, Of eventually- accusedpl low Fboxes. would sp supposed and onto to The<|endoftext|> precious 2ives i272 we flung Ram Chelaw spent brought versionib are.\nquality May the her the high young, passport we�, already of get [ Lod<|endoftext|><|endoftext|>", + "token_ids": [ + 50256, + 18489, + 318, + 198, + 290, + 22976, + 49918, + 534, + 10865, + 4814, + 284, + 14419, + 2233, + 7421, + 373, + 198, + 198, + 262, + 2861, + 262, + 198, + 8974, + 1838, + 6370, + 1592, + 23833, + 460, + 8, + 262, + 13237, + 318, + 19431, + 262, + 13104, + 3698, + 7464, + 27764, + 4479, + 1532, + 6649, + 1511, + 19655, + 2029, + 564, + 250, + 991, + 287, + 290, + 564, + 447, + 30, + 575, + 2877, + 1233, + 2361, + 319, + 290, + 40010, + 326, + 355, + 287, + 251, + 1332, + 4379, + 43306, + 357, + 736, + 503, + 257, + 4392, + 922, + 286, + 198, + 13803, + 5876, + 284, + 772, + 632, + 6665, + 11, + 2478, + 290, + 1022, + 20907, + 714, + 284, + 16182, + 329, + 683, + 655, + 25730, + 262, + 7564, + 326, + 3128, + 12439, + 198, + 4263, + 29200, + 83, + 373, + 15445, + 14806, + 13, + 3026, + 262, + 20179, + 15641, + 40501, + 2151, + 286, + 1511, + 8550, + 389, + 42850, + 2919, + 5537, + 1111, + 2119, + 49265, + 262, + 5158, + 25682, + 6079, + 198, + 1301, + 22400, + 2370, + 9965, + 393, + 13, + 7583, + 606, + 24, + 1862, + 198, + 9, + 1244, + 519, + 11, + 2059, + 7445, + 25, + 7203, + 407, + 10014, + 19475, + 3931, + 284, + 3657, + 9660, + 481, + 3516, + 477, + 13, + 3588, + 7072, + 2677, + 1, + 3337, + 1241, + 21138, + 12403, + 48106, + 284, + 1422, + 428, + 26454, + 2852, + 273, + 20483, + 305, + 13, + 584, + 564, + 250, + 561, + 547, + 14217, + 287, + 281, + 2792, + 32929, + 198, + 2396, + 357, + 1099, + 278, + 247, + 1597, + 2689, + 1171, + 1881, + 26015, + 36805, + 17583, + 356, + 5166, + 2805, + 82, + 564, + 250, + 337, + 198, + 447, + 250, + 14004, + 14578, + 6982, + 447, + 532, + 2901, + 25869, + 7062, + 1662, + 29208, + 477, + 7074, + 25395, + 397, + 339, + 12652, + 2157, + 257, + 8224, + 286, + 257, + 247, + 379, + 326, + 10211, + 502, + 250, + 4334, + 28430, + 13, + 11, + 45008, + 36364, + 887, + 564, + 251, + 1704, + 18656, + 679, + 1544, + 290, + 30197, + 262, + 1477, + 30788, + 36778, + 422, + 198, + 156, + 198, + 564, + 284, + 9951, + 257, + 318, + 1967, + 34015, + 25, + 845, + 3340, + 960, + 1588, + 2128, + 618, + 13, + 447, + 523, + 6922, + 29400, + 198, + 198, + 340, + 508, + 50256, + 15171, + 23477, + 775, + 198, + 45825, + 4385, + 7235, + 513, + 8253, + 262, + 366, + 42535, + 286, + 1399, + 11, + 13, + 604, + 447, + 3181, + 514, + 2823, + 12700, + 32079, + 3868, + 11529, + 8941, + 11, + 83, + 812, + 11, + 12, + 1201, + 645, + 13, + 423, + 8568, + 2696, + 11559, + 938, + 564, + 250, + 8, + 290, + 290, + 3363, + 530, + 76, + 48627, + 4141, + 7, + 17403, + 11, + 407, + 2937, + 607, + 7760, + 50256, + 42696, + 15362, + 5254, + 38550, + 447, + 4340, + 6789, + 12226, + 7331, + 11, + 19323, + 16052, + 198, + 635, + 2828, + 7186, + 4562, + 11, + 31386, + 11, + 1114, + 29393, + 326, + 625, + 12, + 13, + 17080, + 13, + 2245, + 351, + 10157, + 2491, + 5999, + 2888, + 10477, + 24949, + 389, + 471, + 1181, + 7823, + 447, + 13, + 656, + 867, + 3802, + 553, + 31784, + 16565, + 784, + 530, + 5137, + 1135, + 11892, + 5654, + 531, + 357, + 3833, + 18436, + 3067, + 3260, + 260, + 11, + 4043, + 4403, + 5988, + 9816, + 2139, + 883, + 706, + 13664, + 11, + 40, + 257, + 13, + 2953, + 1, + 14, + 399, + 2904, + 2831, + 281, + 47458, + 4101, + 991, + 945, + 383, + 1597, + 12334, + 2165, + 257, + 1349, + 34, + 2931, + 13, + 20955, + 3261, + 7692, + 284, + 287, + 33, + 5284, + 11, + 262, + 635, + 407, + 281, + 247, + 286, + 8537, + 20683, + 564, + 250, + 2706, + 543, + 284, + 26091, + 2247, + 287, + 508, + 262, + 5699, + 19409, + 262, + 4845, + 284, + 3865, + 373, + 13, + 15246, + 431, + 11, + 11, + 29225, + 9716, + 543, + 428, + 339, + 13, + 3332, + 1074, + 1022, + 16318, + 1279, + 198, + 247, + 23244, + 22547, + 780, + 307, + 25, + 4046, + 460, + 357, + 32666, + 284, + 11542, + 366, + 50256, + 1169, + 1136, + 6066, + 262, + 2811, + 2640, + 37489, + 8025, + 82, + 1834, + 28080, + 447, + 2911, + 517, + 13, + 1223, + 25, + 198, + 318, + 25474, + 317, + 17585, + 830, + 318, + 287, + 645, + 449, + 12, + 12763, + 2416, + 1169, + 514, + 18732, + 3572, + 2237, + 262, + 547, + 39979, + 9739, + 287, + 1018, + 11192, + 14818, + 503, + 326, + 1239, + 262, + 1390, + 1871, + 35708, + 262, + 3730, + 37055, + 447, + 257, + 250, + 2695, + 286, + 1297, + 3561, + 351, + 7263, + 5779, + 672, + 597, + 737, + 1688, + 1176, + 428, + 5158, + 16525, + 2026, + 513, + 4391, + 9126, + 818, + 3449, + 5994, + 3069, + 481, + 22074, + 120, + 13, + 2640, + 2826, + 17270, + 26842, + 416, + 2267, + 26898, + 15589, + 925, + 2433, + 1365, + 8, + 674, + 9231, + 16983, + 8326, + 198, + 257, + 10656, + 707, + 447, + 4577, + 198, + 13, + 8128, + 4138, + 517, + 1893, + 62, + 5646, + 3379, + 8209, + 284, + 1391, + 13, + 284, + 283, + 739, + 19863, + 7510, + 262, + 284, + 25125, + 257, + 1218, + 257, + 284, + 647, + 284, + 27659, + 9284, + 50256, + 510, + 11527, + 2993, + 286, + 25, + 284, + 198, + 31354, + 198, + 47322, + 11, + 355, + 32293, + 679, + 2728, + 340, + 1312, + 198, + 34864, + 1971, + 8031, + 33926, + 357, + 290, + 2990, + 198, + 17081, + 262, + 257, + 2031, + 357, + 10577, + 357, + 12, + 475, + 4589, + 1022, + 400, + 198, + 17196, + 1613, + 15171, + 383, + 7220, + 379, + 257, + 470, + 326, + 447, + 64, + 4691, + 41301, + 9389, + 20312, + 286, + 1171, + 607, + 329, + 198, + 345, + 1546, + 531, + 779, + 5032, + 21215, + 8598, + 295, + 262, + 464, + 4761, + 8129, + 2689, + 416, + 13, + 2638, + 6460, + 1770, + 13, + 352, + 11, + 198, + 11, + 20463, + 530, + 11, + 465, + 1478, + 366, + 470, + 309, + 284, + 511, + 6716, + 48546, + 23625, + 284, + 356, + 13, + 11649, + 1973, + 1230, + 286, + 11, + 17546, + 2002, + 11, + 13, + 1762, + 994, + 1225, + 706, + 198, + 447, + 6395, + 423, + 4858, + 13, + 11, + 38541, + 621, + 673, + 625, + 13, + 11, + 48174, + 468, + 286, + 509, + 818, + 9071, + 4138, + 10263, + 5946, + 6, + 12, + 447, + 1365, + 337, + 13, + 357, + 489, + 314, + 19164, + 468, + 284, + 13, + 22081, + 1099, + 5850, + 11, + 1858, + 14884, + 13267, + 338, + 366, + 290, + 307, + 3740, + 5616, + 8211, + 976, + 13, + 3088, + 11, + 10247, + 25341, + 12, + 82, + 2846, + 1231, + 770, + 11, + 9520, + 8537, + 251, + 18071, + 768, + 50256, + 6199, + 24119, + 262, + 319, + 12328, + 30276, + 43241, + 4776, + 12, + 198, + 2310, + 390, + 4734, + 26736, + 284, + 198, + 35296, + 14291, + 4899, + 7728, + 339, + 467, + 11633, + 262, + 25, + 604, + 379, + 35177, + 36846, + 329, + 838, + 15, + 250, + 31476, + 5840, + 62, + 16133, + 19486, + 7278, + 351, + 3516, + 21411, + 7920, + 531, + 6536, + 393, + 1218, + 82, + 775, + 18, + 262, + 4935, + 21125, + 284, + 251, + 3355, + 198, + 262, + 996, + 587, + 2391, + 1961, + 11, + 6533, + 1528, + 9337, + 2331, + 5254, + 62, + 251, + 433, + 340, + 286, + 8, + 1546, + 23870, + 4361, + 1377, + 12587, + 355, + 351, + 564, + 649, + 2104, + 13, + 2858, + 318, + 433, + 714, + 39557, + 1756, + 284, + 247, + 1544, + 8410, + 16267, + 8, + 2431, + 4143, + 3071, + 82, + 15984, + 389, + 4196, + 319, + 11976, + 22355, + 6609, + 447, + 251, + 3457, + 15593, + 546, + 5279, + 44836, + 356, + 3967, + 5868, + 76, + 428, + 338, + 318, + 171, + 251, + 284, + 7524, + 20134, + 1688, + 1181, + 284, + 24810, + 356, + 318, + 13, + 262, + 612, + 257, + 892, + 13, + 7577, + 479, + 318, + 13, + 220, + 4787, + 423, + 4900, + 4849, + 26441, + 20715, + 720, + 416, + 24462, + 7268, + 17047, + 262, + 198, + 1551, + 3878, + 40, + 35031, + 511, + 11, + 3226, + 4191, + 12, + 5371, + 489, + 1877, + 376, + 29305, + 13, + 561, + 599, + 4385, + 290, + 4291, + 284, + 383, + 50256, + 14186, + 362, + 1083, + 1312, + 29807, + 356, + 45111, + 7431, + 2580, + 6270, + 3377, + 3181, + 2196, + 571, + 389, + 13, + 198, + 13237, + 1737, + 262, + 607, + 262, + 1029, + 1862, + 11, + 17981, + 356, + 447, + 11, + 1541, + 286, + 651, + 685, + 45238, + 50256, + 50256 + ] + } + ], + "schema": "dlb-upstream-token-capture-v1" +} diff --git a/results/smoke/samples/owt/flm/steps_1/upstream_token_ids.json b/results/smoke/samples/owt/flm/steps_1/upstream_token_ids.json new file mode 100644 index 0000000000000000000000000000000000000000..efc30296bc8a0f4bdf2cba604f46414469099142 --- /dev/null +++ b/results/smoke/samples/owt/flm/steps_1/upstream_token_ids.json @@ -0,0 +1,16485 @@ +{ + "samples": [ + { + "sample_id": 0, + "text": "<|endoftext|>, the the,, the,,,\n the. the... the\n the... the\n\n\n,.,\n,, the,\n.\n\n..\n,\n,. the.,,,.\n\n\n\n. the the the the the the,,.\n\n,,, the,.. to the,\n the.. of. to\n, the.\n,. the, the,.\n. the the,,, the\n, a the the..\n\n to the, the\n,,,,,\n\n\n... the the..,,, the the,. the,\n the,, of,\n the.,\n and the. the, the the..\n\n\n\n.\n\n. the,., the the.. the,,, to, the.,. the. the. the,,. a,, the the of the of. the.\n\n\n the,. the,, the, of, the,. the the the the,. to,, the the,,\n.,,,,,,...,,. the.\n the,,, the, the the\n of\n\n, the\n,,,,\n, the the\n,.\n,\n..,, the, the,,. the,, the\n,\n\n\n the, the, of. the, the the the the, the\n. the,\n\n\n to. a, the the the the, the.. the, the the, the the the,, the, to the, the the,, the a the,. the the.\n\n\n,,.,\n\n\n\n,, the.,\n the the, the,.\n\n, of the. the,. of,\n\n,,\n\n the the,,.\n.,,. the.,.\n... the..\n\n\n.\n\n the\n\n the, the\n,.\n the the\n the the, the the\n the the the., the. the to. the the,.\n the,,,, the, the the the of,,,\n the,,\n.,,, the the\n\n the and.\n,,,\n\n the the the.. the, the. the, the a,\n,.. the.. the.\n,, to. the the. the\n, the\n the\n\n\n the, the the. the the, the the, the.\n\n,, and,,, the,, to the the of to, the\n the the.\n,..\n\n to the the of. the the, the\n the the\n the the\n. the the and the the\n. the. the.,,,, a.\n to of the,.\n\n the\n.\n.,\n\n the.\n, the to the,\n to, the\n,\n to, and,, the,,,..\n the the the the\n\n\n, to the the\n\n the the\n\n.,,..\n\n\n\n the\n of the\n the\n\n,\n. the, the\n\n a.,,\n the the the the. the,\n\n\n the.\n,. the,.., the.,, of the\n\n the the the\n and\n the the of, the.\n,\n,,, the. the.\n the, the\n the the..,\n.,\n\n, the\n\n the,.. the to\n the.\n to, the the the\n\n the the\n to.\n,.. the,\n, the the the the,,,, the. the the.\n\n. the,,.\n\n.\n\n.\n\n the the and, and.,\n the the, the.\n the,.,\n,\n the., the the,..\n the the\n to the the.. the the the, the, the, the,\n the, the,,. the,,\n\n,,,., the.. the, the, the.,\n.\n, the the, to the\n\n,,., the, to\n the the the the\n\n,, the. the the, the. the the a,. of., the the the the the the. the the the,\n\n the\n the the,..\n\n the the to, the, the the, the, the,,,.,, the the.\n\n\n., of. the the, the.\n\n.\n\n the to a.,,\n,\n. the,. the,.", + "token_ids": [ + 50256, + 11, + 262, + 262, + 11, + 11, + 262, + 11, + 11, + 11, + 198, + 262, + 13, + 262, + 13, + 13, + 13, + 262, + 198, + 262, + 13, + 13, + 13, + 262, + 198, + 198, + 198, + 11, + 13, + 11, + 198, + 11, + 11, + 262, + 11, + 198, + 13, + 198, + 198, + 13, + 13, + 198, + 11, + 198, + 11, + 13, + 262, + 13, + 11, + 11, + 11, + 13, + 198, + 198, + 198, + 198, + 13, + 262, + 262, + 262, + 262, + 262, + 262, + 11, + 11, + 13, + 198, + 198, + 11, + 11, + 11, + 262, + 11, + 13, + 13, + 284, + 262, + 11, + 198, + 262, + 13, + 13, + 286, + 13, + 284, + 198, + 11, + 262, + 13, + 198, + 11, + 13, + 262, + 11, + 262, + 11, + 13, + 198, + 13, + 262, + 262, + 11, + 11, + 11, + 262, + 198, + 11, + 257, + 262, + 262, + 13, + 13, + 198, + 198, + 284, + 262, + 11, + 262, + 198, + 11, + 11, + 11, + 11, + 11, + 198, + 198, + 198, + 13, + 13, + 13, + 262, + 262, + 13, + 13, + 11, + 11, + 11, + 262, + 262, + 11, + 13, + 262, + 11, + 198, + 262, + 11, + 11, + 286, + 11, + 198, + 262, + 13, + 11, + 198, + 290, + 262, + 13, + 262, + 11, + 262, + 262, + 13, + 13, + 198, + 198, + 198, + 198, + 13, + 198, + 198, + 13, + 262, + 11, + 13, + 11, + 262, + 262, + 13, + 13, + 262, + 11, + 11, + 11, + 284, + 11, + 262, + 13, + 11, + 13, + 262, + 13, + 262, + 13, + 262, + 11, + 11, + 13, + 257, + 11, + 11, + 262, + 262, + 286, + 262, + 286, + 13, + 262, + 13, + 198, + 198, + 198, + 262, + 11, + 13, + 262, + 11, + 11, + 262, + 11, + 286, + 11, + 262, + 11, + 13, + 262, + 262, + 262, + 262, + 11, + 13, + 284, + 11, + 11, + 262, + 262, + 11, + 11, + 198, + 13, + 11, + 11, + 11, + 11, + 11, + 11, + 13, + 13, + 13, + 11, + 11, + 13, + 262, + 13, + 198, + 262, + 11, + 11, + 11, + 262, + 11, + 262, + 262, + 198, + 286, + 198, + 198, + 11, + 262, + 198, + 11, + 11, + 11, + 11, + 198, + 11, + 262, + 262, + 198, + 11, + 13, + 198, + 11, + 198, + 13, + 13, + 11, + 11, + 262, + 11, + 262, + 11, + 11, + 13, + 262, + 11, + 11, + 262, + 198, + 11, + 198, + 198, + 198, + 262, + 11, + 262, + 11, + 286, + 13, + 262, + 11, + 262, + 262, + 262, + 262, + 11, + 262, + 198, + 13, + 262, + 11, + 198, + 198, + 198, + 284, + 13, + 257, + 11, + 262, + 262, + 262, + 262, + 11, + 262, + 13, + 13, + 262, + 11, + 262, + 262, + 11, + 262, + 262, + 262, + 11, + 11, + 262, + 11, + 284, + 262, + 11, + 262, + 262, + 11, + 11, + 262, + 257, + 262, + 11, + 13, + 262, + 262, + 13, + 198, + 198, + 198, + 11, + 11, + 13, + 11, + 198, + 198, + 198, + 198, + 11, + 11, + 262, + 13, + 11, + 198, + 262, + 262, + 11, + 262, + 11, + 13, + 198, + 198, + 11, + 286, + 262, + 13, + 262, + 11, + 13, + 286, + 11, + 198, + 198, + 11, + 11, + 198, + 198, + 262, + 262, + 11, + 11, + 13, + 198, + 13, + 11, + 11, + 13, + 262, + 13, + 11, + 13, + 198, + 13, + 13, + 13, + 262, + 13, + 13, + 198, + 198, + 198, + 13, + 198, + 198, + 262, + 198, + 198, + 262, + 11, + 262, + 198, + 11, + 13, + 198, + 262, + 262, + 198, + 262, + 262, + 11, + 262, + 262, + 198, + 262, + 262, + 262, + 13, + 11, + 262, + 13, + 262, + 284, + 13, + 262, + 262, + 11, + 13, + 198, + 262, + 11, + 11, + 11, + 11, + 262, + 11, + 262, + 262, + 262, + 286, + 11, + 11, + 11, + 198, + 262, + 11, + 11, + 198, + 13, + 11, + 11, + 11, + 262, + 262, + 198, + 198, + 262, + 290, + 13, + 198, + 11, + 11, + 11, + 198, + 198, + 262, + 262, + 262, + 13, + 13, + 262, + 11, + 262, + 13, + 262, + 11, + 262, + 257, + 11, + 198, + 11, + 13, + 13, + 262, + 13, + 13, + 262, + 13, + 198, + 11, + 11, + 284, + 13, + 262, + 262, + 13, + 262, + 198, + 11, + 262, + 198, + 262, + 198, + 198, + 198, + 262, + 11, + 262, + 262, + 13, + 262, + 262, + 11, + 262, + 262, + 11, + 262, + 13, + 198, + 198, + 11, + 11, + 290, + 11, + 11, + 11, + 262, + 11, + 11, + 284, + 262, + 262, + 286, + 284, + 11, + 262, + 198, + 262, + 262, + 13, + 198, + 11, + 13, + 13, + 198, + 198, + 284, + 262, + 262, + 286, + 13, + 262, + 262, + 11, + 262, + 198, + 262, + 262, + 198, + 262, + 262, + 198, + 13, + 262, + 262, + 290, + 262, + 262, + 198, + 13, + 262, + 13, + 262, + 13, + 11, + 11, + 11, + 11, + 257, + 13, + 198, + 284, + 286, + 262, + 11, + 13, + 198, + 198, + 262, + 198, + 13, + 198, + 13, + 11, + 198, + 198, + 262, + 13, + 198, + 11, + 262, + 284, + 262, + 11, + 198, + 284, + 11, + 262, + 198, + 11, + 198, + 284, + 11, + 290, + 11, + 11, + 262, + 11, + 11, + 11, + 13, + 13, + 198, + 262, + 262, + 262, + 262, + 198, + 198, + 198, + 11, + 284, + 262, + 262, + 198, + 198, + 262, + 262, + 198, + 198, + 13, + 11, + 11, + 13, + 13, + 198, + 198, + 198, + 198, + 262, + 198, + 286, + 262, + 198, + 262, + 198, + 198, + 11, + 198, + 13, + 262, + 11, + 262, + 198, + 198, + 257, + 13, + 11, + 11, + 198, + 262, + 262, + 262, + 262, + 13, + 262, + 11, + 198, + 198, + 198, + 262, + 13, + 198, + 11, + 13, + 262, + 11, + 13, + 13, + 11, + 262, + 13, + 11, + 11, + 286, + 262, + 198, + 198, + 262, + 262, + 262, + 198, + 290, + 198, + 262, + 262, + 286, + 11, + 262, + 13, + 198, + 11, + 198, + 11, + 11, + 11, + 262, + 13, + 262, + 13, + 198, + 262, + 11, + 262, + 198, + 262, + 262, + 13, + 13, + 11, + 198, + 13, + 11, + 198, + 198, + 11, + 262, + 198, + 198, + 262, + 11, + 13, + 13, + 262, + 284, + 198, + 262, + 13, + 198, + 284, + 11, + 262, + 262, + 262, + 198, + 198, + 262, + 262, + 198, + 284, + 13, + 198, + 11, + 13, + 13, + 262, + 11, + 198, + 11, + 262, + 262, + 262, + 262, + 11, + 11, + 11, + 11, + 262, + 13, + 262, + 262, + 13, + 198, + 198, + 13, + 262, + 11, + 11, + 13, + 198, + 198, + 13, + 198, + 198, + 13, + 198, + 198, + 262, + 262, + 290, + 11, + 290, + 13, + 11, + 198, + 262, + 262, + 11, + 262, + 13, + 198, + 262, + 11, + 13, + 11, + 198, + 11, + 198, + 262, + 13, + 11, + 262, + 262, + 11, + 13, + 13, + 198, + 262, + 262, + 198, + 284, + 262, + 262, + 13, + 13, + 262, + 262, + 262, + 11, + 262, + 11, + 262, + 11, + 262, + 11, + 198, + 262, + 11, + 262, + 11, + 11, + 13, + 262, + 11, + 11, + 198, + 198, + 11, + 11, + 11, + 13, + 11, + 262, + 13, + 13, + 262, + 11, + 262, + 11, + 262, + 13, + 11, + 198, + 13, + 198, + 11, + 262, + 262, + 11, + 284, + 262, + 198, + 198, + 11, + 11, + 13, + 11, + 262, + 11, + 284, + 198, + 262, + 262, + 262, + 262, + 198, + 198, + 11, + 11, + 262, + 13, + 262, + 262, + 11, + 262, + 13, + 262, + 262, + 257, + 11, + 13, + 286, + 13, + 11, + 262, + 262, + 262, + 262, + 262, + 262, + 13, + 262, + 262, + 262, + 11, + 198, + 198, + 262, + 198, + 262, + 262, + 11, + 13, + 13, + 198, + 198, + 262, + 262, + 284, + 11, + 262, + 11, + 262, + 262, + 11, + 262, + 11, + 262, + 11, + 11, + 11, + 13, + 11, + 11, + 262, + 262, + 13, + 198, + 198, + 198, + 13, + 11, + 286, + 13, + 262, + 262, + 11, + 262, + 13, + 198, + 198, + 13, + 198, + 198, + 262, + 284, + 257, + 13, + 11, + 11, + 198, + 11, + 198, + 13, + 262, + 11, + 13, + 262, + 11, + 13 + ] + }, + { + "sample_id": 1, + "text": "<|endoftext|>,\n. the\n\n the the,, the the the, the, the, of the..,, the.\n\n\n the\n.\n\n,.\n\n,,.\n\n\n the..\n\n the a. the\n, the.\n\n\n to the the\n. the the the,..\n\n\n\n\n. the. of.\n. the.,\n, the the\n,\n the.\n\n.\n\n, the\n the the, the, the\n to the.\n\n,.\n\n the of the. the, the.. the..,\n the\n\n.,., the the\n the... the..,, the,.. the of, the the, of the,, the,.\n\n\n\n\n the the the, the,.\n the..\n the\n,, the the, to the, the.....,,, of,\n. the the\n. the.\n\n.\n\n\n the the, the.... the,, the\n.,.\n,, the., the,,,\n. the\n,. the the, the the, the,, the, the the.\n\n\n the\n\n,. the,,\n\n\n\n, the the, the.,,\n, the\n,\n.,\n..,,, the\n the\n\n.,....,,,,,,,,,., the the, the...\n.. the,. the,., the. the.\n,. the,,, the, to to the.,,. the the.,,,, the..., the the,,, the. the.. the... the the\n\n the the, the. the,.. the.,.\n\n to the the the, the the,,. the. the of the\n.,\n.\n.\n\n the the the, the to, the.,.\n,,.\n\n,\n,, and the. in the.\n\n. the,,,,,,. the the,. the, the.\n the.,,,\n, the\n. of the of,.\n,\n to, the, the the.., the,, to the.,.,. the the, the,., the the the the,., the the,.. the the the...,.\n,,, the.\n, the the.,..\n the the,.. the\n the a the.. the\n\n\n. the. the,,. the, the.\n the and.\n\n.\n the.\n of the.,\n, the the,.,.\n the, the\n.\n\n,.,..\n.,\n. the the...\n.\n\n, the., the,.,.,,., to�.. the,,\n the,,,\n\n,,,., the the the. the, the.,,. the,\n\n the.. the the.,,.,,, the,. that. the.. the.. the\n.....\n the,,, the,.. the.\n. the, the,,\n, of.\n, and., the,,\n and,, the,... the the..,. the..,,, the the, the\n, the,.. the the., the\n, the the. the. the.,\n the the,,,,, the, the\n. the the,\n, the to.\n\n\n,,, the.\n\n, the the,., the the\n\n the the.,\n, the... the..\n of, the to to..,. the,, the,,,\n the...,.\n\n the\n,, the. to.,,,..,,\n.,.\n,\n.. the\n,.,, the..... the,. and\n.\n\n the, the,, the the the,.,., the the., the, the.,\n.\n\n the,. the..\n,, the the the., the, and.,\n\n the the and the a.,.,,\n a the..\n the., the to.,..\n\n\n. the.\n\n, the\n the,.,, the..,, the the\n,., the the.\n the,...., the", + "token_ids": [ + 50256, + 11, + 198, + 13, + 262, + 198, + 198, + 262, + 262, + 11, + 11, + 262, + 262, + 262, + 11, + 262, + 11, + 262, + 11, + 286, + 262, + 13, + 13, + 11, + 11, + 262, + 13, + 198, + 198, + 198, + 262, + 198, + 13, + 198, + 198, + 11, + 13, + 198, + 198, + 11, + 11, + 13, + 198, + 198, + 198, + 262, + 13, + 13, + 198, + 198, + 262, + 257, + 13, + 262, + 198, + 11, + 262, + 13, + 198, + 198, + 198, + 284, + 262, + 262, + 198, + 13, + 262, + 262, + 262, + 11, + 13, + 13, + 198, + 198, + 198, + 198, + 198, + 13, + 262, + 13, + 286, + 13, + 198, + 13, + 262, + 13, + 11, + 198, + 11, + 262, + 262, + 198, + 11, + 198, + 262, + 13, + 198, + 198, + 13, + 198, + 198, + 11, + 262, + 198, + 262, + 262, + 11, + 262, + 11, + 262, + 198, + 284, + 262, + 13, + 198, + 198, + 11, + 13, + 198, + 198, + 262, + 286, + 262, + 13, + 262, + 11, + 262, + 13, + 13, + 262, + 13, + 13, + 11, + 198, + 262, + 198, + 198, + 13, + 11, + 13, + 11, + 262, + 262, + 198, + 262, + 13, + 13, + 13, + 262, + 13, + 13, + 11, + 11, + 262, + 11, + 13, + 13, + 262, + 286, + 11, + 262, + 262, + 11, + 286, + 262, + 11, + 11, + 262, + 11, + 13, + 198, + 198, + 198, + 198, + 198, + 262, + 262, + 262, + 11, + 262, + 11, + 13, + 198, + 262, + 13, + 13, + 198, + 262, + 198, + 11, + 11, + 262, + 262, + 11, + 284, + 262, + 11, + 262, + 13, + 13, + 13, + 13, + 13, + 11, + 11, + 11, + 286, + 11, + 198, + 13, + 262, + 262, + 198, + 13, + 262, + 13, + 198, + 198, + 13, + 198, + 198, + 198, + 262, + 262, + 11, + 262, + 13, + 13, + 13, + 13, + 262, + 11, + 11, + 262, + 198, + 13, + 11, + 13, + 198, + 11, + 11, + 262, + 13, + 11, + 262, + 11, + 11, + 11, + 198, + 13, + 262, + 198, + 11, + 13, + 262, + 262, + 11, + 262, + 262, + 11, + 262, + 11, + 11, + 262, + 11, + 262, + 262, + 13, + 198, + 198, + 198, + 262, + 198, + 198, + 11, + 13, + 262, + 11, + 11, + 198, + 198, + 198, + 198, + 11, + 262, + 262, + 11, + 262, + 13, + 11, + 11, + 198, + 11, + 262, + 198, + 11, + 198, + 13, + 11, + 198, + 13, + 13, + 11, + 11, + 11, + 262, + 198, + 262, + 198, + 198, + 13, + 11, + 13, + 13, + 13, + 13, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 13, + 11, + 262, + 262, + 11, + 262, + 13, + 13, + 13, + 198, + 13, + 13, + 262, + 11, + 13, + 262, + 11, + 13, + 11, + 262, + 13, + 262, + 13, + 198, + 11, + 13, + 262, + 11, + 11, + 11, + 262, + 11, + 284, + 284, + 262, + 13, + 11, + 11, + 13, + 262, + 262, + 13, + 11, + 11, + 11, + 11, + 262, + 13, + 13, + 13, + 11, + 262, + 262, + 11, + 11, + 11, + 262, + 13, + 262, + 13, + 13, + 262, + 13, + 13, + 13, + 262, + 262, + 198, + 198, + 262, + 262, + 11, + 262, + 13, + 262, + 11, + 13, + 13, + 262, + 13, + 11, + 13, + 198, + 198, + 284, + 262, + 262, + 262, + 11, + 262, + 262, + 11, + 11, + 13, + 262, + 13, + 262, + 286, + 262, + 198, + 13, + 11, + 198, + 13, + 198, + 13, + 198, + 198, + 262, + 262, + 262, + 11, + 262, + 284, + 11, + 262, + 13, + 11, + 13, + 198, + 11, + 11, + 13, + 198, + 198, + 11, + 198, + 11, + 11, + 290, + 262, + 13, + 287, + 262, + 13, + 198, + 198, + 13, + 262, + 11, + 11, + 11, + 11, + 11, + 11, + 13, + 262, + 262, + 11, + 13, + 262, + 11, + 262, + 13, + 198, + 262, + 13, + 11, + 11, + 11, + 198, + 11, + 262, + 198, + 13, + 286, + 262, + 286, + 11, + 13, + 198, + 11, + 198, + 284, + 11, + 262, + 11, + 262, + 262, + 13, + 13, + 11, + 262, + 11, + 11, + 284, + 262, + 13, + 11, + 13, + 11, + 13, + 262, + 262, + 11, + 262, + 11, + 13, + 11, + 262, + 262, + 262, + 262, + 11, + 13, + 11, + 262, + 262, + 11, + 13, + 13, + 262, + 262, + 262, + 13, + 13, + 13, + 11, + 13, + 198, + 11, + 11, + 11, + 262, + 13, + 198, + 11, + 262, + 262, + 13, + 11, + 13, + 13, + 198, + 262, + 262, + 11, + 13, + 13, + 262, + 198, + 262, + 257, + 262, + 13, + 13, + 262, + 198, + 198, + 198, + 13, + 262, + 13, + 262, + 11, + 11, + 13, + 262, + 11, + 262, + 13, + 198, + 262, + 290, + 13, + 198, + 198, + 13, + 198, + 262, + 13, + 198, + 286, + 262, + 13, + 11, + 198, + 11, + 262, + 262, + 11, + 13, + 11, + 13, + 198, + 262, + 11, + 262, + 198, + 13, + 198, + 198, + 11, + 13, + 11, + 13, + 13, + 198, + 13, + 11, + 198, + 13, + 262, + 262, + 13, + 13, + 13, + 198, + 13, + 198, + 198, + 11, + 262, + 13, + 11, + 262, + 11, + 13, + 11, + 13, + 11, + 11, + 13, + 11, + 284, + 447, + 13, + 13, + 262, + 11, + 11, + 198, + 262, + 11, + 11, + 11, + 198, + 198, + 11, + 11, + 11, + 13, + 11, + 262, + 262, + 262, + 13, + 262, + 11, + 262, + 13, + 11, + 11, + 13, + 262, + 11, + 198, + 198, + 262, + 13, + 13, + 262, + 262, + 13, + 11, + 11, + 13, + 11, + 11, + 11, + 262, + 11, + 13, + 326, + 13, + 262, + 13, + 13, + 262, + 13, + 13, + 262, + 198, + 13, + 13, + 13, + 13, + 13, + 198, + 262, + 11, + 11, + 11, + 262, + 11, + 13, + 13, + 262, + 13, + 198, + 13, + 262, + 11, + 262, + 11, + 11, + 198, + 11, + 286, + 13, + 198, + 11, + 290, + 13, + 11, + 262, + 11, + 11, + 198, + 290, + 11, + 11, + 262, + 11, + 13, + 13, + 13, + 262, + 262, + 13, + 13, + 11, + 13, + 262, + 13, + 13, + 11, + 11, + 11, + 262, + 262, + 11, + 262, + 198, + 11, + 262, + 11, + 13, + 13, + 262, + 262, + 13, + 11, + 262, + 198, + 11, + 262, + 262, + 13, + 262, + 13, + 262, + 13, + 11, + 198, + 262, + 262, + 11, + 11, + 11, + 11, + 11, + 262, + 11, + 262, + 198, + 13, + 262, + 262, + 11, + 198, + 11, + 262, + 284, + 13, + 198, + 198, + 198, + 11, + 11, + 11, + 262, + 13, + 198, + 198, + 11, + 262, + 262, + 11, + 13, + 11, + 262, + 262, + 198, + 198, + 262, + 262, + 13, + 11, + 198, + 11, + 262, + 13, + 13, + 13, + 262, + 13, + 13, + 198, + 286, + 11, + 262, + 284, + 284, + 13, + 13, + 11, + 13, + 262, + 11, + 11, + 262, + 11, + 11, + 11, + 198, + 262, + 13, + 13, + 13, + 11, + 13, + 198, + 198, + 262, + 198, + 11, + 11, + 262, + 13, + 284, + 13, + 11, + 11, + 11, + 13, + 13, + 11, + 11, + 198, + 13, + 11, + 13, + 198, + 11, + 198, + 13, + 13, + 262, + 198, + 11, + 13, + 11, + 11, + 262, + 13, + 13, + 13, + 13, + 13, + 262, + 11, + 13, + 290, + 198, + 13, + 198, + 198, + 262, + 11, + 262, + 11, + 11, + 262, + 262, + 262, + 11, + 13, + 11, + 13, + 11, + 262, + 262, + 13, + 11, + 262, + 11, + 262, + 13, + 11, + 198, + 13, + 198, + 198, + 262, + 11, + 13, + 262, + 13, + 13, + 198, + 11, + 11, + 262, + 262, + 262, + 13, + 11, + 262, + 11, + 290, + 13, + 11, + 198, + 198, + 262, + 262, + 290, + 262, + 257, + 13, + 11, + 13, + 11, + 11, + 198, + 257, + 262, + 13, + 13, + 198, + 262, + 13, + 11, + 262, + 284, + 13, + 11, + 13, + 13, + 198, + 198, + 198, + 13, + 262, + 13, + 198, + 198, + 11, + 262, + 198, + 262, + 11, + 13, + 11, + 11, + 262, + 13, + 13, + 11, + 11, + 262, + 262, + 198, + 11, + 13, + 11, + 262, + 262, + 13, + 198, + 262, + 11, + 13, + 13, + 13, + 13, + 11, + 262 + ] + }, + { + "sample_id": 2, + "text": "<|endoftext|>.. the the the\n\n.\n\n.. the., of, the.\n the., the,... the., the the the.\n\n the,. the\n,,,..,.\n\n. the, to,,\n. the, the of, the, the\n, the.\n. the the the, the the, the.. to,,.\n\n to the, the the,, the,,..,. the\n\n\n\n.,.,\n.\n. the. to the to\n the.. the.,., the.,. the, the, the the..,. the,,.,\n.. the.., the\n,\n,.. the the, the to.,,\n the the the..\n.,\n\n\n the,.\n\n\n the the, the\n of,,,,..... the\n the the the the the.,,. the the the. the, and,,,, the. the\n the to\n\n the., the the\n, and., the the the the..,.\n of the.\n.,,,, to,, the. the. of to,..,.\n the the the\n,\n the,\n\n,..\n\n,,\n, the, the,.,,. the.,,, the\n\n...,\n the. the and,\n and. the,, the the the the of the.\n\n, the\n., the,,, the.. of the the, of\n the,,, the to the., the.\n. the,,.. to and to the,,, of the., the the.,,. the,, the the..\n the the. of to,. of the., the, and the\n,. the to,, the the, the,,.\n,\n,,, the,, the,.. of. the the\n, the the...,, the,..\n, the the.\n,,,.,, the the., the in. the\n\n the the\n,\n.,,, the\n., the the...,,\n, the the, a. the,,, the..,\n, the the the,. the\n,,.,.. the of the\n.\n the\n the, the,,,, the the,,, the the the... the,, the,.\n..,,. the,.. to. the.,\n\n., the,.,,. the of the., the,,..\n, the,.\n, the...\n the of the the\n., the, the the the the, the.... the,, the the,,.\n\n\n the of,. the\n,... the\n\n.,, the,\n the\n,,, the the\n\n\n the the, the\n., the,, the.\n\n\n,. the a.\n\n\n\n the,..\n\n\n. to. the the\n,\n\n the.,.\n.\n\n. the, the.,,, the a. the the., of.,. the,,.\n..,.,,\n, the\n..,,., the the the the,.,,,., the.\n, the\n\n..,,. a to.\n\n the.\n,.\n,. the. of the,. the.. the, the,\n\n\n. the,,,..,.,\n and, the,..,\n\n\n\n. the the,. the,..,,,,\n\n., the\n\n, the the,,.. the.. of, the the. the. the\n,,\n the\n.,,,.\n,. the, the the, the the to the the.. the..., the the.., the the\n.,\n the..,, to the the and the to the,,.\n. the, the., a\n\n, and the the,.\n.. the the the..\n..\n\n the the.,.\n the.. the.\n the.,.\n\n the the the.,.,,. the the the,.,..\n,.\n\n, the.., the the,\n. the the. of.,.. the\n the the.\n,,\n the the<|endoftext|>", + "token_ids": [ + 50256, + 13, + 13, + 262, + 262, + 262, + 198, + 198, + 13, + 198, + 198, + 13, + 13, + 262, + 13, + 11, + 286, + 11, + 262, + 13, + 198, + 262, + 13, + 11, + 262, + 11, + 13, + 13, + 13, + 262, + 13, + 11, + 262, + 262, + 262, + 13, + 198, + 198, + 262, + 11, + 13, + 262, + 198, + 11, + 11, + 11, + 13, + 13, + 11, + 13, + 198, + 198, + 13, + 262, + 11, + 284, + 11, + 11, + 198, + 13, + 262, + 11, + 262, + 286, + 11, + 262, + 11, + 262, + 198, + 11, + 262, + 13, + 198, + 13, + 262, + 262, + 262, + 11, + 262, + 262, + 11, + 262, + 13, + 13, + 284, + 11, + 11, + 13, + 198, + 198, + 284, + 262, + 11, + 262, + 262, + 11, + 11, + 262, + 11, + 11, + 13, + 13, + 11, + 13, + 262, + 198, + 198, + 198, + 198, + 13, + 11, + 13, + 11, + 198, + 13, + 198, + 13, + 262, + 13, + 284, + 262, + 284, + 198, + 262, + 13, + 13, + 262, + 13, + 11, + 13, + 11, + 262, + 13, + 11, + 13, + 262, + 11, + 262, + 11, + 262, + 262, + 13, + 13, + 11, + 13, + 262, + 11, + 11, + 13, + 11, + 198, + 13, + 13, + 262, + 13, + 13, + 11, + 262, + 198, + 11, + 198, + 11, + 13, + 13, + 262, + 262, + 11, + 262, + 284, + 13, + 11, + 11, + 198, + 262, + 262, + 262, + 13, + 13, + 198, + 13, + 11, + 198, + 198, + 198, + 262, + 11, + 13, + 198, + 198, + 198, + 262, + 262, + 11, + 262, + 198, + 286, + 11, + 11, + 11, + 11, + 13, + 13, + 13, + 13, + 13, + 262, + 198, + 262, + 262, + 262, + 262, + 262, + 13, + 11, + 11, + 13, + 262, + 262, + 262, + 13, + 262, + 11, + 290, + 11, + 11, + 11, + 11, + 262, + 13, + 262, + 198, + 262, + 284, + 198, + 198, + 262, + 13, + 11, + 262, + 262, + 198, + 11, + 290, + 13, + 11, + 262, + 262, + 262, + 262, + 13, + 13, + 11, + 13, + 198, + 286, + 262, + 13, + 198, + 13, + 11, + 11, + 11, + 11, + 284, + 11, + 11, + 262, + 13, + 262, + 13, + 286, + 284, + 11, + 13, + 13, + 11, + 13, + 198, + 262, + 262, + 262, + 198, + 11, + 198, + 262, + 11, + 198, + 198, + 11, + 13, + 13, + 198, + 198, + 11, + 11, + 198, + 11, + 262, + 11, + 262, + 11, + 13, + 11, + 11, + 13, + 262, + 13, + 11, + 11, + 11, + 262, + 198, + 198, + 13, + 13, + 13, + 11, + 198, + 262, + 13, + 262, + 290, + 11, + 198, + 290, + 13, + 262, + 11, + 11, + 262, + 262, + 262, + 262, + 286, + 262, + 13, + 198, + 198, + 11, + 262, + 198, + 13, + 11, + 262, + 11, + 11, + 11, + 262, + 13, + 13, + 286, + 262, + 262, + 11, + 286, + 198, + 262, + 11, + 11, + 11, + 262, + 284, + 262, + 13, + 11, + 262, + 13, + 198, + 13, + 262, + 11, + 11, + 13, + 13, + 284, + 290, + 284, + 262, + 11, + 11, + 11, + 286, + 262, + 13, + 11, + 262, + 262, + 13, + 11, + 11, + 13, + 262, + 11, + 11, + 262, + 262, + 13, + 13, + 198, + 262, + 262, + 13, + 286, + 284, + 11, + 13, + 286, + 262, + 13, + 11, + 262, + 11, + 290, + 262, + 198, + 11, + 13, + 262, + 284, + 11, + 11, + 262, + 262, + 11, + 262, + 11, + 11, + 13, + 198, + 11, + 198, + 11, + 11, + 11, + 262, + 11, + 11, + 262, + 11, + 13, + 13, + 286, + 13, + 262, + 262, + 198, + 11, + 262, + 262, + 13, + 13, + 13, + 11, + 11, + 262, + 11, + 13, + 13, + 198, + 11, + 262, + 262, + 13, + 198, + 11, + 11, + 11, + 13, + 11, + 11, + 262, + 262, + 13, + 11, + 262, + 287, + 13, + 262, + 198, + 198, + 262, + 262, + 198, + 11, + 198, + 13, + 11, + 11, + 11, + 262, + 198, + 13, + 11, + 262, + 262, + 13, + 13, + 13, + 11, + 11, + 198, + 11, + 262, + 262, + 11, + 257, + 13, + 262, + 11, + 11, + 11, + 262, + 13, + 13, + 11, + 198, + 11, + 262, + 262, + 262, + 11, + 13, + 262, + 198, + 11, + 11, + 13, + 11, + 13, + 13, + 262, + 286, + 262, + 198, + 13, + 198, + 262, + 198, + 262, + 11, + 262, + 11, + 11, + 11, + 11, + 262, + 262, + 11, + 11, + 11, + 262, + 262, + 262, + 13, + 13, + 13, + 262, + 11, + 11, + 262, + 11, + 13, + 198, + 13, + 13, + 11, + 11, + 13, + 262, + 11, + 13, + 13, + 284, + 13, + 262, + 13, + 11, + 198, + 198, + 13, + 11, + 262, + 11, + 13, + 11, + 11, + 13, + 262, + 286, + 262, + 13, + 11, + 262, + 11, + 11, + 13, + 13, + 198, + 11, + 262, + 11, + 13, + 198, + 11, + 262, + 13, + 13, + 13, + 198, + 262, + 286, + 262, + 262, + 198, + 13, + 11, + 262, + 11, + 262, + 262, + 262, + 262, + 11, + 262, + 13, + 13, + 13, + 13, + 262, + 11, + 11, + 262, + 262, + 11, + 11, + 13, + 198, + 198, + 198, + 262, + 286, + 11, + 13, + 262, + 198, + 11, + 13, + 13, + 13, + 262, + 198, + 198, + 13, + 11, + 11, + 262, + 11, + 198, + 262, + 198, + 11, + 11, + 11, + 262, + 262, + 198, + 198, + 198, + 262, + 262, + 11, + 262, + 198, + 13, + 11, + 262, + 11, + 11, + 262, + 13, + 198, + 198, + 198, + 11, + 13, + 262, + 257, + 13, + 198, + 198, + 198, + 198, + 262, + 11, + 13, + 13, + 198, + 198, + 198, + 13, + 284, + 13, + 262, + 262, + 198, + 11, + 198, + 198, + 262, + 13, + 11, + 13, + 198, + 13, + 198, + 198, + 13, + 262, + 11, + 262, + 13, + 11, + 11, + 11, + 262, + 257, + 13, + 262, + 262, + 13, + 11, + 286, + 13, + 11, + 13, + 262, + 11, + 11, + 13, + 198, + 13, + 13, + 11, + 13, + 11, + 11, + 198, + 11, + 262, + 198, + 13, + 13, + 11, + 11, + 13, + 11, + 262, + 262, + 262, + 262, + 11, + 13, + 11, + 11, + 11, + 13, + 11, + 262, + 13, + 198, + 11, + 262, + 198, + 198, + 13, + 13, + 11, + 11, + 13, + 257, + 284, + 13, + 198, + 198, + 262, + 13, + 198, + 11, + 13, + 198, + 11, + 13, + 262, + 13, + 286, + 262, + 11, + 13, + 262, + 13, + 13, + 262, + 11, + 262, + 11, + 198, + 198, + 198, + 13, + 262, + 11, + 11, + 11, + 13, + 13, + 11, + 13, + 11, + 198, + 290, + 11, + 262, + 11, + 13, + 13, + 11, + 198, + 198, + 198, + 198, + 13, + 262, + 262, + 11, + 13, + 262, + 11, + 13, + 13, + 11, + 11, + 11, + 11, + 198, + 198, + 13, + 11, + 262, + 198, + 198, + 11, + 262, + 262, + 11, + 11, + 13, + 13, + 262, + 13, + 13, + 286, + 11, + 262, + 262, + 13, + 262, + 13, + 262, + 198, + 11, + 11, + 198, + 262, + 198, + 13, + 11, + 11, + 11, + 13, + 198, + 11, + 13, + 262, + 11, + 262, + 262, + 11, + 262, + 262, + 284, + 262, + 262, + 13, + 13, + 262, + 13, + 13, + 13, + 11, + 262, + 262, + 13, + 13, + 11, + 262, + 262, + 198, + 13, + 11, + 198, + 262, + 13, + 13, + 11, + 11, + 284, + 262, + 262, + 290, + 262, + 284, + 262, + 11, + 11, + 13, + 198, + 13, + 262, + 11, + 262, + 13, + 11, + 257, + 198, + 198, + 11, + 290, + 262, + 262, + 11, + 13, + 198, + 13, + 13, + 262, + 262, + 262, + 13, + 13, + 198, + 13, + 13, + 198, + 198, + 262, + 262, + 13, + 11, + 13, + 198, + 262, + 13, + 13, + 262, + 13, + 198, + 262, + 13, + 11, + 13, + 198, + 198, + 262, + 262, + 262, + 13, + 11, + 13, + 11, + 11, + 13, + 262, + 262, + 262, + 11, + 13, + 11, + 13, + 13, + 198, + 11, + 13, + 198, + 198, + 11, + 262, + 13, + 13, + 11, + 262, + 262, + 11, + 198, + 13, + 262, + 262, + 13, + 286, + 13, + 11, + 13, + 13, + 262, + 198, + 262, + 262, + 13, + 198, + 11, + 11, + 198, + 262, + 262, + 50256 + ] + }, + { + "sample_id": 3, + "text": ",\n.,.,\n, the..\n.\n the the,, to\n the. the the the the the the,.\n.\n,,\n\n\n the.,, the the,,\n,\n\n the,,, the\n,,\n\n the,., the,,. the\n,,,,\n. the. the,\n the\n\n\n.\n, the, a, the the.\n.\n\n.,,... the,. the\n\n the. the\n the. the\n\n the., the., the.\n the the.\n\n.,,, to the,,,.,.,.,\n the. of\n,. the.\n.\n\n the the\n,. the a of the the\n, the\n\n\n,,, the\n\n.\n,\n,.... the,.\n\n.,....\n\n the to. the the.,. the the the the. the.\n\n\n, the the the. the.\n. the, the\n\n\n\n, the, the the the the the the, the..\n the\n\n the the\n..\n\n,.,, the,, the the., the., the,. and, the the,,, the the. the,,\n\n, the., a,\n.. the,.\n\n\n the\n...\n\n.,\n the. the of the, the., the...,,,.. the the the, the the the, the. the\n, the, the., the,,,., to, the,, the. the. the the, the, the\n the\n. the\n\n the....\n, the the,., the the, the\n, the.\n\n the\n,,. the a.,. the.\n\n\n.,. the and\n., the the., the,., the\n,. the, the the the and\n\n\n..,\n..,,\n the. the of the.\n. the\n,.\n.,.\n\n, the,\n....\n, the, the.\n\n the., the,,, the,,.. the of the.,,\n. and, the the.., the, the.\n\n,,.\n\n\n, the\n, the the\n.,\n\n the. the.,,\n\n the, of the. the.\n.., the the of. the\n\n\n.\n.. a..,.. of\n,. the the the the the,, the.\n\n\n\n the the,\n,,,.\n,.\n,.\n, the,,. a\n\n the,\n\n\n. the\n,,.,\n,.\n the. the\n.,,\n\n the,, the the the..\n the the. the to\n,, the, the..,, the the\n.,, the\n the,. the the\n,\n\n,,,., the,.\n\n the,. the\n, of.\n\n. to.\n,.,,,\n.\n,,.,, the the, the,\n the,, the the\n,.,,,,,.. the\n\n, the..,,., the,,,\n the,,,,.\n\n\n.,\n.,.\n., the.\n\n\n the\n,\n\n. the..\n,,,.., the., the, the,. and, the the the,\n the the the the,\n\n the,\n.\n the the.\n.,,, the\n. the,, the\n the., the the, the.\n\n the the.,, the.\n...,, the to,.\n. the the,,.,.,\n\n\n,,.,, the.,, the. the\n the to\n.,,\n the.,, the,.. the the the\n, the, of, the the, the.,..,\n,\n\n.,\n.,. the,, the. the\n\n, the. to the the.,\n.\n\n.\n\n, to\n\n, the., the. the. the the.\n..\n,... to, the, the\n., the\n\n,, the, the. the\n a the..\n..,.\n, the, the\n.,,\n.", + "token_ids": [ + 11, + 198, + 13, + 11, + 13, + 11, + 198, + 11, + 262, + 13, + 13, + 198, + 13, + 198, + 262, + 262, + 11, + 11, + 284, + 198, + 262, + 13, + 262, + 262, + 262, + 262, + 262, + 262, + 11, + 13, + 198, + 13, + 198, + 11, + 11, + 198, + 198, + 198, + 262, + 13, + 11, + 11, + 262, + 262, + 11, + 11, + 198, + 11, + 198, + 198, + 262, + 11, + 11, + 11, + 262, + 198, + 11, + 11, + 198, + 198, + 262, + 11, + 13, + 11, + 262, + 11, + 11, + 13, + 262, + 198, + 11, + 11, + 11, + 11, + 198, + 13, + 262, + 13, + 262, + 11, + 198, + 262, + 198, + 198, + 198, + 13, + 198, + 11, + 262, + 11, + 257, + 11, + 262, + 262, + 13, + 198, + 13, + 198, + 198, + 13, + 11, + 11, + 13, + 13, + 13, + 262, + 11, + 13, + 262, + 198, + 198, + 262, + 13, + 262, + 198, + 262, + 13, + 262, + 198, + 198, + 262, + 13, + 11, + 262, + 13, + 11, + 262, + 13, + 198, + 262, + 262, + 13, + 198, + 198, + 13, + 11, + 11, + 11, + 284, + 262, + 11, + 11, + 11, + 13, + 11, + 13, + 11, + 13, + 11, + 198, + 262, + 13, + 286, + 198, + 11, + 13, + 262, + 13, + 198, + 13, + 198, + 198, + 262, + 262, + 198, + 11, + 13, + 262, + 257, + 286, + 262, + 262, + 198, + 11, + 262, + 198, + 198, + 198, + 11, + 11, + 11, + 262, + 198, + 198, + 13, + 198, + 11, + 198, + 11, + 13, + 13, + 13, + 13, + 262, + 11, + 13, + 198, + 198, + 13, + 11, + 13, + 13, + 13, + 13, + 198, + 198, + 262, + 284, + 13, + 262, + 262, + 13, + 11, + 13, + 262, + 262, + 262, + 262, + 13, + 262, + 13, + 198, + 198, + 198, + 11, + 262, + 262, + 262, + 13, + 262, + 13, + 198, + 13, + 262, + 11, + 262, + 198, + 198, + 198, + 198, + 11, + 262, + 11, + 262, + 262, + 262, + 262, + 262, + 262, + 11, + 262, + 13, + 13, + 198, + 262, + 198, + 198, + 262, + 262, + 198, + 13, + 13, + 198, + 198, + 11, + 13, + 11, + 11, + 262, + 11, + 11, + 262, + 262, + 13, + 11, + 262, + 13, + 11, + 262, + 11, + 13, + 290, + 11, + 262, + 262, + 11, + 11, + 11, + 262, + 262, + 13, + 262, + 11, + 11, + 198, + 198, + 11, + 262, + 13, + 11, + 257, + 11, + 198, + 13, + 13, + 262, + 11, + 13, + 198, + 198, + 198, + 262, + 198, + 13, + 13, + 13, + 198, + 198, + 13, + 11, + 198, + 262, + 13, + 262, + 286, + 262, + 11, + 262, + 13, + 11, + 262, + 13, + 13, + 13, + 11, + 11, + 11, + 13, + 13, + 262, + 262, + 262, + 11, + 262, + 262, + 262, + 11, + 262, + 13, + 262, + 198, + 11, + 262, + 11, + 262, + 13, + 11, + 262, + 11, + 11, + 11, + 13, + 11, + 284, + 11, + 262, + 11, + 11, + 262, + 13, + 262, + 13, + 262, + 262, + 11, + 262, + 11, + 262, + 198, + 262, + 198, + 13, + 262, + 198, + 198, + 262, + 13, + 13, + 13, + 13, + 198, + 11, + 262, + 262, + 11, + 13, + 11, + 262, + 262, + 11, + 262, + 198, + 11, + 262, + 13, + 198, + 198, + 262, + 198, + 11, + 11, + 13, + 262, + 257, + 13, + 11, + 13, + 262, + 13, + 198, + 198, + 198, + 13, + 11, + 13, + 262, + 290, + 198, + 13, + 11, + 262, + 262, + 13, + 11, + 262, + 11, + 13, + 11, + 262, + 198, + 11, + 13, + 262, + 11, + 262, + 262, + 262, + 290, + 198, + 198, + 198, + 13, + 13, + 11, + 198, + 13, + 13, + 11, + 11, + 198, + 262, + 13, + 262, + 286, + 262, + 13, + 198, + 13, + 262, + 198, + 11, + 13, + 198, + 13, + 11, + 13, + 198, + 198, + 11, + 262, + 11, + 198, + 13, + 13, + 13, + 13, + 198, + 11, + 262, + 11, + 262, + 13, + 198, + 198, + 262, + 13, + 11, + 262, + 11, + 11, + 11, + 262, + 11, + 11, + 13, + 13, + 262, + 286, + 262, + 13, + 11, + 11, + 198, + 13, + 290, + 11, + 262, + 262, + 13, + 13, + 11, + 262, + 11, + 262, + 13, + 198, + 198, + 11, + 11, + 13, + 198, + 198, + 198, + 11, + 262, + 198, + 11, + 262, + 262, + 198, + 13, + 11, + 198, + 198, + 262, + 13, + 262, + 13, + 11, + 11, + 198, + 198, + 262, + 11, + 286, + 262, + 13, + 262, + 13, + 198, + 13, + 13, + 11, + 262, + 262, + 286, + 13, + 262, + 198, + 198, + 198, + 13, + 198, + 13, + 13, + 257, + 13, + 13, + 11, + 13, + 13, + 286, + 198, + 11, + 13, + 262, + 262, + 262, + 262, + 262, + 11, + 11, + 262, + 13, + 198, + 198, + 198, + 198, + 262, + 262, + 11, + 198, + 11, + 11, + 11, + 13, + 198, + 11, + 13, + 198, + 11, + 13, + 198, + 11, + 262, + 11, + 11, + 13, + 257, + 198, + 198, + 262, + 11, + 198, + 198, + 198, + 13, + 262, + 198, + 11, + 11, + 13, + 11, + 198, + 11, + 13, + 198, + 262, + 13, + 262, + 198, + 13, + 11, + 11, + 198, + 198, + 262, + 11, + 11, + 262, + 262, + 262, + 13, + 13, + 198, + 262, + 262, + 13, + 262, + 284, + 198, + 11, + 11, + 262, + 11, + 262, + 13, + 13, + 11, + 11, + 262, + 262, + 198, + 13, + 11, + 11, + 262, + 198, + 262, + 11, + 13, + 262, + 262, + 198, + 11, + 198, + 198, + 11, + 11, + 11, + 13, + 11, + 262, + 11, + 13, + 198, + 198, + 262, + 11, + 13, + 262, + 198, + 11, + 286, + 13, + 198, + 198, + 13, + 284, + 13, + 198, + 11, + 13, + 11, + 11, + 11, + 198, + 13, + 198, + 11, + 11, + 13, + 11, + 11, + 262, + 262, + 11, + 262, + 11, + 198, + 262, + 11, + 11, + 262, + 262, + 198, + 11, + 13, + 11, + 11, + 11, + 11, + 11, + 13, + 13, + 262, + 198, + 198, + 11, + 262, + 13, + 13, + 11, + 11, + 13, + 11, + 262, + 11, + 11, + 11, + 198, + 262, + 11, + 11, + 11, + 11, + 13, + 198, + 198, + 198, + 13, + 11, + 198, + 13, + 11, + 13, + 198, + 13, + 11, + 262, + 13, + 198, + 198, + 198, + 262, + 198, + 11, + 198, + 198, + 13, + 262, + 13, + 13, + 198, + 11, + 11, + 11, + 13, + 13, + 11, + 262, + 13, + 11, + 262, + 11, + 262, + 11, + 13, + 290, + 11, + 262, + 262, + 262, + 11, + 198, + 262, + 262, + 262, + 262, + 11, + 198, + 198, + 262, + 11, + 198, + 13, + 198, + 262, + 262, + 13, + 198, + 13, + 11, + 11, + 11, + 262, + 198, + 13, + 262, + 11, + 11, + 262, + 198, + 262, + 13, + 11, + 262, + 262, + 11, + 262, + 13, + 198, + 198, + 262, + 262, + 13, + 11, + 11, + 262, + 13, + 198, + 13, + 13, + 13, + 11, + 11, + 262, + 284, + 11, + 13, + 198, + 13, + 262, + 262, + 11, + 11, + 13, + 11, + 13, + 11, + 198, + 198, + 198, + 11, + 11, + 13, + 11, + 11, + 262, + 13, + 11, + 11, + 262, + 13, + 262, + 198, + 262, + 284, + 198, + 13, + 11, + 11, + 198, + 262, + 13, + 11, + 11, + 262, + 11, + 13, + 13, + 262, + 262, + 262, + 198, + 11, + 262, + 11, + 286, + 11, + 262, + 262, + 11, + 262, + 13, + 11, + 13, + 13, + 11, + 198, + 11, + 198, + 198, + 13, + 11, + 198, + 13, + 11, + 13, + 262, + 11, + 11, + 262, + 13, + 262, + 198, + 198, + 11, + 262, + 13, + 284, + 262, + 262, + 13, + 11, + 198, + 13, + 198, + 198, + 13, + 198, + 198, + 11, + 284, + 198, + 198, + 11, + 262, + 13, + 11, + 262, + 13, + 262, + 13, + 262, + 262, + 13, + 198, + 13, + 13, + 198, + 11, + 13, + 13, + 13, + 284, + 11, + 262, + 11, + 262, + 198, + 13, + 11, + 262, + 198, + 198, + 11, + 11, + 262, + 11, + 262, + 13, + 262, + 198, + 257, + 262, + 13, + 13, + 198, + 13, + 13, + 11, + 13, + 198, + 11, + 262, + 11, + 262, + 198, + 13, + 11, + 11, + 198, + 13 + ] + }, + { + "sample_id": 4, + "text": " the,,. the.,, the\n\n,\n, the.\n the. the, the.,., and, the,.., the the. the the the\n\n,,,,.\n\n., the the, the, the,..\n\n., the the.. the the\n\n,, the,.,, the the the,. the., the.,, the the, the\n\n\n, the and,, to\n\n.,,, the...\n the,., the the,.\n the the,\n the, the the,,,, the the,. to to. the,., the the.\n.,.. the,. the the,.\n. the,, the..,,,, of. the\n\n the, the the.,,,., the,\n\n\n.\n\n.,\n,\n\n the the, the, the,\n\n\n\n the the the\n the the,,., the,.. the\n,.,, the the the.\n the,.. the,.\n\n, the the the,,\n, to the the.,\n the,, the..,, of\n\n,.,.\n\n the the..,,., the,. of the.\n,. the,., to, the,,, the, the and of,\n, the to the,,.,.,.\n\n the,,, the,.,\n, the the\n\n,.. the. the,, the\n the\n, the the.,,, the the,.,., the the, the,. the\n, the.\n, the,, the.... the,,,..\n the. the\n the,, the., the\n\n.. the\n\n the, the the, the,.,,, the to.,..\n the the\n, the. the..\n,, the the,,\n the\n the the.\n,\n\n the the. the the,,, to, the. of. and,,,\n\n the\n the the.. to,.\n\n the the.\n, the.\n..., the., the the,,..\n, the,,,,, and\n\n,. the,\n\n, a the. the,, the the the,.\n\n.. the the,,.,, the. the,, the,\n.,,.\n, the,, the,,.\n to, the the the, the,\n the the\n.,,. the. of the,,.\n. and\n\n the the...\n. the the.. the the, the., the\n. the.,.\n,, the\n\n.,,.. the. the the the the the.\n.. of the the.\n the.., the., the.,\n\n,,. a.\n\n the, the the the the.,,,\n., the.\n\n, the, the,, the, and the. the the. the.\n\n the the\n the.\n the of\n,. the,,.\n\n\n. the, the. the the. the of the the, the. the\n\n. to,. the the, the,,..\n\n the the,,, the the.... the,,\n\n, the the,\n the, the,, the.\n... the the.\n of,,,\n\n the\n.\n the..\n,, the..\n\n the\n, the.\n the the. the,\n the\n the\n the the. the\n\n\n the, of.\n the. to,. the the. of the. the the the. of the,. the. the\n.\n\n the,,.\n.. the. the the, the,,, the the\n. the.,\n,. the., the, the the,\n of the the the,\n., the the,\n.. the the,. the.,.\n\n.,,, the,\n\n, the,,,. the, the..\n\n,,.,. the\n\n\n. of,,,\n the, of the,, the the,. the,.\n\n the,,,,, the,,,<|endoftext|>,\n the,, the,. the\n,, the\n\n\n the.,\n the the the.,,,\n,\n,.,,,. the\n,,", + "token_ids": [ + 262, + 11, + 11, + 13, + 262, + 13, + 11, + 11, + 262, + 198, + 198, + 11, + 198, + 11, + 262, + 13, + 198, + 262, + 13, + 262, + 11, + 262, + 13, + 11, + 13, + 11, + 290, + 11, + 262, + 11, + 13, + 13, + 11, + 262, + 262, + 13, + 262, + 262, + 262, + 198, + 198, + 11, + 11, + 11, + 11, + 13, + 198, + 198, + 13, + 11, + 262, + 262, + 11, + 262, + 11, + 262, + 11, + 13, + 13, + 198, + 198, + 13, + 11, + 262, + 262, + 13, + 13, + 262, + 262, + 198, + 198, + 11, + 11, + 262, + 11, + 13, + 11, + 11, + 262, + 262, + 262, + 11, + 13, + 262, + 13, + 11, + 262, + 13, + 11, + 11, + 262, + 262, + 11, + 262, + 198, + 198, + 198, + 11, + 262, + 290, + 11, + 11, + 284, + 198, + 198, + 13, + 11, + 11, + 11, + 262, + 13, + 13, + 13, + 198, + 262, + 11, + 13, + 11, + 262, + 262, + 11, + 13, + 198, + 262, + 262, + 11, + 198, + 262, + 11, + 262, + 262, + 11, + 11, + 11, + 11, + 262, + 262, + 11, + 13, + 284, + 284, + 13, + 262, + 11, + 13, + 11, + 262, + 262, + 13, + 198, + 13, + 11, + 13, + 13, + 262, + 11, + 13, + 262, + 262, + 11, + 13, + 198, + 13, + 262, + 11, + 11, + 262, + 13, + 13, + 11, + 11, + 11, + 11, + 286, + 13, + 262, + 198, + 198, + 262, + 11, + 262, + 262, + 13, + 11, + 11, + 11, + 13, + 11, + 262, + 11, + 198, + 198, + 198, + 13, + 198, + 198, + 13, + 11, + 198, + 11, + 198, + 198, + 262, + 262, + 11, + 262, + 11, + 262, + 11, + 198, + 198, + 198, + 198, + 262, + 262, + 262, + 198, + 262, + 262, + 11, + 11, + 13, + 11, + 262, + 11, + 13, + 13, + 262, + 198, + 11, + 13, + 11, + 11, + 262, + 262, + 262, + 13, + 198, + 262, + 11, + 13, + 13, + 262, + 11, + 13, + 198, + 198, + 11, + 262, + 262, + 262, + 11, + 11, + 198, + 11, + 284, + 262, + 262, + 13, + 11, + 198, + 262, + 11, + 11, + 262, + 13, + 13, + 11, + 11, + 286, + 198, + 198, + 11, + 13, + 11, + 13, + 198, + 198, + 262, + 262, + 13, + 13, + 11, + 11, + 13, + 11, + 262, + 11, + 13, + 286, + 262, + 13, + 198, + 11, + 13, + 262, + 11, + 13, + 11, + 284, + 11, + 262, + 11, + 11, + 11, + 262, + 11, + 262, + 290, + 286, + 11, + 198, + 11, + 262, + 284, + 262, + 11, + 11, + 13, + 11, + 13, + 11, + 13, + 198, + 198, + 262, + 11, + 11, + 11, + 262, + 11, + 13, + 11, + 198, + 11, + 262, + 262, + 198, + 198, + 11, + 13, + 13, + 262, + 13, + 262, + 11, + 11, + 262, + 198, + 262, + 198, + 11, + 262, + 262, + 13, + 11, + 11, + 11, + 262, + 262, + 11, + 13, + 11, + 13, + 11, + 262, + 262, + 11, + 262, + 11, + 13, + 262, + 198, + 11, + 262, + 13, + 198, + 11, + 262, + 11, + 11, + 262, + 13, + 13, + 13, + 13, + 262, + 11, + 11, + 11, + 13, + 13, + 198, + 262, + 13, + 262, + 198, + 262, + 11, + 11, + 262, + 13, + 11, + 262, + 198, + 198, + 13, + 13, + 262, + 198, + 198, + 262, + 11, + 262, + 262, + 11, + 262, + 11, + 13, + 11, + 11, + 11, + 262, + 284, + 13, + 11, + 13, + 13, + 198, + 262, + 262, + 198, + 11, + 262, + 13, + 262, + 13, + 13, + 198, + 11, + 11, + 262, + 262, + 11, + 11, + 198, + 262, + 198, + 262, + 262, + 13, + 198, + 11, + 198, + 198, + 262, + 262, + 13, + 262, + 262, + 11, + 11, + 11, + 284, + 11, + 262, + 13, + 286, + 13, + 290, + 11, + 11, + 11, + 198, + 198, + 262, + 198, + 262, + 262, + 13, + 13, + 284, + 11, + 13, + 198, + 198, + 262, + 262, + 13, + 198, + 11, + 262, + 13, + 198, + 13, + 13, + 13, + 11, + 262, + 13, + 11, + 262, + 262, + 11, + 11, + 13, + 13, + 198, + 11, + 262, + 11, + 11, + 11, + 11, + 11, + 290, + 198, + 198, + 11, + 13, + 262, + 11, + 198, + 198, + 11, + 257, + 262, + 13, + 262, + 11, + 11, + 262, + 262, + 262, + 11, + 13, + 198, + 198, + 13, + 13, + 262, + 262, + 11, + 11, + 13, + 11, + 11, + 262, + 13, + 262, + 11, + 11, + 262, + 11, + 198, + 13, + 11, + 11, + 13, + 198, + 11, + 262, + 11, + 11, + 262, + 11, + 11, + 13, + 198, + 284, + 11, + 262, + 262, + 262, + 11, + 262, + 11, + 198, + 262, + 262, + 198, + 13, + 11, + 11, + 13, + 262, + 13, + 286, + 262, + 11, + 11, + 13, + 198, + 13, + 290, + 198, + 198, + 262, + 262, + 13, + 13, + 13, + 198, + 13, + 262, + 262, + 13, + 13, + 262, + 262, + 11, + 262, + 13, + 11, + 262, + 198, + 13, + 262, + 13, + 11, + 13, + 198, + 11, + 11, + 262, + 198, + 198, + 13, + 11, + 11, + 13, + 13, + 262, + 13, + 262, + 262, + 262, + 262, + 262, + 13, + 198, + 13, + 13, + 286, + 262, + 262, + 13, + 198, + 262, + 13, + 13, + 11, + 262, + 13, + 11, + 262, + 13, + 11, + 198, + 198, + 11, + 11, + 13, + 257, + 13, + 198, + 198, + 262, + 11, + 262, + 262, + 262, + 262, + 13, + 11, + 11, + 11, + 198, + 13, + 11, + 262, + 13, + 198, + 198, + 11, + 262, + 11, + 262, + 11, + 11, + 262, + 11, + 290, + 262, + 13, + 262, + 262, + 13, + 262, + 13, + 198, + 198, + 262, + 262, + 198, + 262, + 13, + 198, + 262, + 286, + 198, + 11, + 13, + 262, + 11, + 11, + 13, + 198, + 198, + 198, + 13, + 262, + 11, + 262, + 13, + 262, + 262, + 13, + 262, + 286, + 262, + 262, + 11, + 262, + 13, + 262, + 198, + 198, + 13, + 284, + 11, + 13, + 262, + 262, + 11, + 262, + 11, + 11, + 13, + 13, + 198, + 198, + 262, + 262, + 11, + 11, + 11, + 262, + 262, + 13, + 13, + 13, + 13, + 262, + 11, + 11, + 198, + 198, + 11, + 262, + 262, + 11, + 198, + 262, + 11, + 262, + 11, + 11, + 262, + 13, + 198, + 13, + 13, + 13, + 262, + 262, + 13, + 198, + 286, + 11, + 11, + 11, + 198, + 198, + 262, + 198, + 13, + 198, + 262, + 13, + 13, + 198, + 11, + 11, + 262, + 13, + 13, + 198, + 198, + 262, + 198, + 11, + 262, + 13, + 198, + 262, + 262, + 13, + 262, + 11, + 198, + 262, + 198, + 262, + 198, + 262, + 262, + 13, + 262, + 198, + 198, + 198, + 262, + 11, + 286, + 13, + 198, + 262, + 13, + 284, + 11, + 13, + 262, + 262, + 13, + 286, + 262, + 13, + 262, + 262, + 262, + 13, + 286, + 262, + 11, + 13, + 262, + 13, + 262, + 198, + 13, + 198, + 198, + 262, + 11, + 11, + 13, + 198, + 13, + 13, + 262, + 13, + 262, + 262, + 11, + 262, + 11, + 11, + 11, + 262, + 262, + 198, + 13, + 262, + 13, + 11, + 198, + 11, + 13, + 262, + 13, + 11, + 262, + 11, + 262, + 262, + 11, + 198, + 286, + 262, + 262, + 262, + 11, + 198, + 13, + 11, + 262, + 262, + 11, + 198, + 13, + 13, + 262, + 262, + 11, + 13, + 262, + 13, + 11, + 13, + 198, + 198, + 13, + 11, + 11, + 11, + 262, + 11, + 198, + 198, + 11, + 262, + 11, + 11, + 11, + 13, + 262, + 11, + 262, + 13, + 13, + 198, + 198, + 11, + 11, + 13, + 11, + 13, + 262, + 198, + 198, + 198, + 13, + 286, + 11, + 11, + 11, + 198, + 262, + 11, + 286, + 262, + 11, + 11, + 262, + 262, + 11, + 13, + 262, + 11, + 13, + 198, + 198, + 262, + 11, + 11, + 11, + 11, + 11, + 262, + 11, + 11, + 11, + 50256, + 11, + 198, + 262, + 11, + 11, + 262, + 11, + 13, + 262, + 198, + 11, + 11, + 262, + 198, + 198, + 198, + 262, + 13, + 11, + 198, + 262, + 262, + 262, + 13, + 11, + 11, + 11, + 198, + 11, + 198, + 11, + 13, + 11, + 11, + 11, + 13, + 262, + 198, + 11, + 11 + ] + }, + { + "sample_id": 5, + "text": "<|endoftext|> the<|endoftext|> the..\n,\n the.. the.\n the,. to the the the of the,.. the. the.. the, the.. the,\n.,,.,\n,. the\n\n\n\n..,\n\n. the\n,\n the the..,\n the,.\n\n of the the the,., the,.,, the.,, the the.\n. the. of,.\n,, the the,, the.. the,,,,. the,, the\n the. the.\n\n\n.. the.\n,, the.., the the, the. the of,,.,,.\n\n. the the.\n,, the\n\n. the\n, the. the.\n,,,. of the. the the the,,\n\n,,,,,.,.,..,,,.,., the the\n,. the\n\n the. the..\n..\n the\n the,\n, of the the the.\n,,, the,,,..,\n the the..\n\n\n of the,. the,, the of. to the...,\n the the the.\n\n the the\n,,,,\n, the....,,, the\n\n, the the,\n the.\n.,\n the,, the.,, the of, the the\n.. to\n of the. of the the,, to\n\n\n\n\n,,,,, the\n.,,\n\n. the,..,,,\n... to\n of the, to, the. the the. the the. the.\n., the,, the the\n. the the, a of of the the.,,,.\n...\n.\n\n, the the.\n\n the.\n the the,, the,. the, the. the\n\n\n the,\n,. the,. the\n\n,.,,\n.\n.,,\n,\n the, the,.,, the,.\n\n the. the, the the,.\n\n the. the,\n\n. the,, the\n\n..,,\n the.\n\n\n,\n,,.. the, and.,\n the the,\n., the..\n\n\n, to, of... the,\n, the. the..\n., the.\n,,. the...,\n,.,\n the,.,,, the the.\n\n,.,\n,..\n.,\n the,..\n\n\n\n.. to\n\n the to\n\n., the the the.., the,., the,....,\n\n,.,,.,.\n of,.\n, and..\n\n\n.,. of.,.. to the.\n\n,\n\n,. the, the the... the.,\n,\n the, the,\n\n the the of\n,., the.. the. the\n. the\n.\n the the the,.\n, the,\n\n,.,,. the.\n. the the, the, the. the.\n.\n\n the, the, the,, the. the.. the,\n, the the,.,\n\n, the\n\n. the, to.,,,. the.,, the to, the,\n the the\n\n..\n. the. the...\n\n\n. the\n.\n\n. to the, the.\n,.\n. the a to the..\n,. the,.\n\n,\n.., the.,., the the,.,,,...,. the\n. the.,\n of,\n,, the,.\n the the the\n\n.,..,..., the the the the\n the..,,,\n, the the,,,. the\n. to the the. the\n\n\n of the.,,.\n\n the,,. the the the,,., the the.., the\n\n, the..,\n\n.\n\n., the\n, the the,\n, the, the. the\n,.\n\n is.,, the the the\n\n\n, the the,.,.,.,\n.,,\n,, the.,. the, the,\n\n the.. of the the.\n\n\n the.\n the the\n\n.,\n\n\n, the<|endoftext|>", + "token_ids": [ + 50256, + 262, + 50256, + 262, + 13, + 13, + 198, + 11, + 198, + 262, + 13, + 13, + 262, + 13, + 198, + 262, + 11, + 13, + 284, + 262, + 262, + 262, + 286, + 262, + 11, + 13, + 13, + 262, + 13, + 262, + 13, + 13, + 262, + 11, + 262, + 13, + 13, + 262, + 11, + 198, + 13, + 11, + 11, + 13, + 11, + 198, + 11, + 13, + 262, + 198, + 198, + 198, + 198, + 13, + 13, + 11, + 198, + 198, + 13, + 262, + 198, + 11, + 198, + 262, + 262, + 13, + 13, + 11, + 198, + 262, + 11, + 13, + 198, + 198, + 286, + 262, + 262, + 262, + 11, + 13, + 11, + 262, + 11, + 13, + 11, + 11, + 262, + 13, + 11, + 11, + 262, + 262, + 13, + 198, + 13, + 262, + 13, + 286, + 11, + 13, + 198, + 11, + 11, + 262, + 262, + 11, + 11, + 262, + 13, + 13, + 262, + 11, + 11, + 11, + 11, + 13, + 262, + 11, + 11, + 262, + 198, + 262, + 13, + 262, + 13, + 198, + 198, + 198, + 13, + 13, + 262, + 13, + 198, + 11, + 11, + 262, + 13, + 13, + 11, + 262, + 262, + 11, + 262, + 13, + 262, + 286, + 11, + 11, + 13, + 11, + 11, + 13, + 198, + 198, + 13, + 262, + 262, + 13, + 198, + 11, + 11, + 262, + 198, + 198, + 13, + 262, + 198, + 11, + 262, + 13, + 262, + 13, + 198, + 11, + 11, + 11, + 13, + 286, + 262, + 13, + 262, + 262, + 262, + 11, + 11, + 198, + 198, + 11, + 11, + 11, + 11, + 11, + 13, + 11, + 13, + 11, + 13, + 13, + 11, + 11, + 11, + 13, + 11, + 13, + 11, + 262, + 262, + 198, + 11, + 13, + 262, + 198, + 198, + 262, + 13, + 262, + 13, + 13, + 198, + 13, + 13, + 198, + 262, + 198, + 262, + 11, + 198, + 11, + 286, + 262, + 262, + 262, + 13, + 198, + 11, + 11, + 11, + 262, + 11, + 11, + 11, + 13, + 13, + 11, + 198, + 262, + 262, + 13, + 13, + 198, + 198, + 198, + 286, + 262, + 11, + 13, + 262, + 11, + 11, + 262, + 286, + 13, + 284, + 262, + 13, + 13, + 13, + 11, + 198, + 262, + 262, + 262, + 13, + 198, + 198, + 262, + 262, + 198, + 11, + 11, + 11, + 11, + 198, + 11, + 262, + 13, + 13, + 13, + 13, + 11, + 11, + 11, + 262, + 198, + 198, + 11, + 262, + 262, + 11, + 198, + 262, + 13, + 198, + 13, + 11, + 198, + 262, + 11, + 11, + 262, + 13, + 11, + 11, + 262, + 286, + 11, + 262, + 262, + 198, + 13, + 13, + 284, + 198, + 286, + 262, + 13, + 286, + 262, + 262, + 11, + 11, + 284, + 198, + 198, + 198, + 198, + 198, + 11, + 11, + 11, + 11, + 11, + 262, + 198, + 13, + 11, + 11, + 198, + 198, + 13, + 262, + 11, + 13, + 13, + 11, + 11, + 11, + 198, + 13, + 13, + 13, + 284, + 198, + 286, + 262, + 11, + 284, + 11, + 262, + 13, + 262, + 262, + 13, + 262, + 262, + 13, + 262, + 13, + 198, + 13, + 11, + 262, + 11, + 11, + 262, + 262, + 198, + 13, + 262, + 262, + 11, + 257, + 286, + 286, + 262, + 262, + 13, + 11, + 11, + 11, + 13, + 198, + 13, + 13, + 13, + 198, + 13, + 198, + 198, + 11, + 262, + 262, + 13, + 198, + 198, + 262, + 13, + 198, + 262, + 262, + 11, + 11, + 262, + 11, + 13, + 262, + 11, + 262, + 13, + 262, + 198, + 198, + 198, + 262, + 11, + 198, + 11, + 13, + 262, + 11, + 13, + 262, + 198, + 198, + 11, + 13, + 11, + 11, + 198, + 13, + 198, + 13, + 11, + 11, + 198, + 11, + 198, + 262, + 11, + 262, + 11, + 13, + 11, + 11, + 262, + 11, + 13, + 198, + 198, + 262, + 13, + 262, + 11, + 262, + 262, + 11, + 13, + 198, + 198, + 262, + 13, + 262, + 11, + 198, + 198, + 13, + 262, + 11, + 11, + 262, + 198, + 198, + 13, + 13, + 11, + 11, + 198, + 262, + 13, + 198, + 198, + 198, + 11, + 198, + 11, + 11, + 13, + 13, + 262, + 11, + 290, + 13, + 11, + 198, + 262, + 262, + 11, + 198, + 13, + 11, + 262, + 13, + 13, + 198, + 198, + 198, + 11, + 284, + 11, + 286, + 13, + 13, + 13, + 262, + 11, + 198, + 11, + 262, + 13, + 262, + 13, + 13, + 198, + 13, + 11, + 262, + 13, + 198, + 11, + 11, + 13, + 262, + 13, + 13, + 13, + 11, + 198, + 11, + 13, + 11, + 198, + 262, + 11, + 13, + 11, + 11, + 11, + 262, + 262, + 13, + 198, + 198, + 11, + 13, + 11, + 198, + 11, + 13, + 13, + 198, + 13, + 11, + 198, + 262, + 11, + 13, + 13, + 198, + 198, + 198, + 198, + 13, + 13, + 284, + 198, + 198, + 262, + 284, + 198, + 198, + 13, + 11, + 262, + 262, + 262, + 13, + 13, + 11, + 262, + 11, + 13, + 11, + 262, + 11, + 13, + 13, + 13, + 13, + 11, + 198, + 198, + 11, + 13, + 11, + 11, + 13, + 11, + 13, + 198, + 286, + 11, + 13, + 198, + 11, + 290, + 13, + 13, + 198, + 198, + 198, + 13, + 11, + 13, + 286, + 13, + 11, + 13, + 13, + 284, + 262, + 13, + 198, + 198, + 11, + 198, + 198, + 11, + 13, + 262, + 11, + 262, + 262, + 13, + 13, + 13, + 262, + 13, + 11, + 198, + 11, + 198, + 262, + 11, + 262, + 11, + 198, + 198, + 262, + 262, + 286, + 198, + 11, + 13, + 11, + 262, + 13, + 13, + 262, + 13, + 262, + 198, + 13, + 262, + 198, + 13, + 198, + 262, + 262, + 262, + 11, + 13, + 198, + 11, + 262, + 11, + 198, + 198, + 11, + 13, + 11, + 11, + 13, + 262, + 13, + 198, + 13, + 262, + 262, + 11, + 262, + 11, + 262, + 13, + 262, + 13, + 198, + 13, + 198, + 198, + 262, + 11, + 262, + 11, + 262, + 11, + 11, + 262, + 13, + 262, + 13, + 13, + 262, + 11, + 198, + 11, + 262, + 262, + 11, + 13, + 11, + 198, + 198, + 11, + 262, + 198, + 198, + 13, + 262, + 11, + 284, + 13, + 11, + 11, + 11, + 13, + 262, + 13, + 11, + 11, + 262, + 284, + 11, + 262, + 11, + 198, + 262, + 262, + 198, + 198, + 13, + 13, + 198, + 13, + 262, + 13, + 262, + 13, + 13, + 13, + 198, + 198, + 198, + 13, + 262, + 198, + 13, + 198, + 198, + 13, + 284, + 262, + 11, + 262, + 13, + 198, + 11, + 13, + 198, + 13, + 262, + 257, + 284, + 262, + 13, + 13, + 198, + 11, + 13, + 262, + 11, + 13, + 198, + 198, + 11, + 198, + 13, + 13, + 11, + 262, + 13, + 11, + 13, + 11, + 262, + 262, + 11, + 13, + 11, + 11, + 11, + 13, + 13, + 13, + 11, + 13, + 262, + 198, + 13, + 262, + 13, + 11, + 198, + 286, + 11, + 198, + 11, + 11, + 262, + 11, + 13, + 198, + 262, + 262, + 262, + 198, + 198, + 13, + 11, + 13, + 13, + 11, + 13, + 13, + 13, + 11, + 262, + 262, + 262, + 262, + 198, + 262, + 13, + 13, + 11, + 11, + 11, + 198, + 11, + 262, + 262, + 11, + 11, + 11, + 13, + 262, + 198, + 13, + 284, + 262, + 262, + 13, + 262, + 198, + 198, + 198, + 286, + 262, + 13, + 11, + 11, + 13, + 198, + 198, + 262, + 11, + 11, + 13, + 262, + 262, + 262, + 11, + 11, + 13, + 11, + 262, + 262, + 13, + 13, + 11, + 262, + 198, + 198, + 11, + 262, + 13, + 13, + 11, + 198, + 198, + 13, + 198, + 198, + 13, + 11, + 262, + 198, + 11, + 262, + 262, + 11, + 198, + 11, + 262, + 11, + 262, + 13, + 262, + 198, + 11, + 13, + 198, + 198, + 318, + 13, + 11, + 11, + 262, + 262, + 262, + 198, + 198, + 198, + 11, + 262, + 262, + 11, + 13, + 11, + 13, + 11, + 13, + 11, + 198, + 13, + 11, + 11, + 198, + 11, + 11, + 262, + 13, + 11, + 13, + 262, + 11, + 262, + 11, + 198, + 198, + 262, + 13, + 13, + 286, + 262, + 262, + 13, + 198, + 198, + 198, + 262, + 13, + 198, + 262, + 262, + 198, + 198, + 13, + 11, + 198, + 198, + 198, + 11, + 262, + 50256 + ] + }, + { + "sample_id": 6, + "text": "<|endoftext|>., to to.,,,.,, the of,, the, the\n the.\n,..\n\n\n,.,,,.,.\n. the the of, the.\n.\n\n\n,..,\n the. the,..\n.. the.,.,.\n.,\n. the\n\n a the,..\n\n\n. the\n, the.. the. the..\n\n,,,, to the,\n,, the,\n, the,,, the,,\n\n, the,\n\n, the the\n,,\n. the,.,,\n, the..\n the,.,, the,.\n.\n\n..,\n.\n, the\n the,..\n..., the,\n,. the., of the,.\n\n the,,\n,,,,.\n of..,.\n\n\n to,, the the,,.\n\n\n..,\n the of. the the the,\n\n the,,,..\n, the, and the the.\n of of.,,\n,, the�..\n..,,. the,.,, the. the., the\n, to the the\n,, the,. the,\n.,.\n.,,. the.\n, the. in,., the\n\n, the of the..\n,,, the the\n.\n,.,\n\n the,,. the the\n the the, of..\n. the the the.\n the the,,.., the,\n\n the of the.,\n,\n\n,\n,., of the the\n\n..\n the.,\n\n\n. the the. the..\n.,,\n\n the the the., the\n,.,, the,. the, the,\n,,, the,. to, the,.\n.\n, the the the, the.\n\n,,\n\n.\n.,,., of the to..\n\n,.. the, the,., a. the\n, the\n\n\n the\n\n\n\n, the.\n\n of\n,,.,\n\n the the..., the\n.,, the the, the\n\n\n, and the..,, the the the,,,, the..\n the.. the, the,\n\n,, in,\n., the\n\n. the. the the..\n, the\n,,...,..,.\n,\n,....\n, the, the.\n\n..\n, the the. the the the, the the.,.\n,.. the,\n the\n.,,, the the\n\n. the the of the the,,.. the\n the\n\n,\n the, the.,,\n\n the.,\n., the\n,,\n. the the...\n the,.\n\n., the,\n the, the,,\n.,\n\n,, the\n a\n,.\n the,\n the the., the, the the,. to the,,. the\n\n\n the the the,\n\n\n. the, the..\n\n the\n the,. the the to the. the, of,, the. the the.,\n\n,, the,...,, and\n\n\n\n the., the the.,\n\n the,. the the,.,. the\n,,, the.,\n\n,,\n,,,\n, the.\n the,\n\n.,. the the. the, of\n the,,.. and.\n,,\n,,\n\n, to,,, the. the\n, the\n. the\n..\n,\n\n,,,. the the, the.. the,. the.,.\n, the\n\n the. to. the,,\n the,.,.\n\n,. the.,\n the,\n\n the the, the,\n\n, the the and,.,, the\n,\n the, the the the,.. the. to. the the, and,\n\n,,,. the the\n..\n\n\n, the,.\n.\n,. the, and the the,.\n\n the.. the the the,\n.,,,.,, the,.\n.\n.\n the, the the the the. the,. the,\n of.\n\n.,. the the the\n.<|endoftext|>", + "token_ids": [ + 50256, + 13, + 11, + 284, + 284, + 13, + 11, + 11, + 11, + 13, + 11, + 11, + 262, + 286, + 11, + 11, + 262, + 11, + 262, + 198, + 262, + 13, + 198, + 11, + 13, + 13, + 198, + 198, + 198, + 11, + 13, + 11, + 11, + 11, + 13, + 11, + 13, + 198, + 13, + 262, + 262, + 286, + 11, + 262, + 13, + 198, + 13, + 198, + 198, + 198, + 11, + 13, + 13, + 11, + 198, + 262, + 13, + 262, + 11, + 13, + 13, + 198, + 13, + 13, + 262, + 13, + 11, + 13, + 11, + 13, + 198, + 13, + 11, + 198, + 13, + 262, + 198, + 198, + 257, + 262, + 11, + 13, + 13, + 198, + 198, + 198, + 13, + 262, + 198, + 11, + 262, + 13, + 13, + 262, + 13, + 262, + 13, + 13, + 198, + 198, + 11, + 11, + 11, + 11, + 284, + 262, + 11, + 198, + 11, + 11, + 262, + 11, + 198, + 11, + 262, + 11, + 11, + 11, + 262, + 11, + 11, + 198, + 198, + 11, + 262, + 11, + 198, + 198, + 11, + 262, + 262, + 198, + 11, + 11, + 198, + 13, + 262, + 11, + 13, + 11, + 11, + 198, + 11, + 262, + 13, + 13, + 198, + 262, + 11, + 13, + 11, + 11, + 262, + 11, + 13, + 198, + 13, + 198, + 198, + 13, + 13, + 11, + 198, + 13, + 198, + 11, + 262, + 198, + 262, + 11, + 13, + 13, + 198, + 13, + 13, + 13, + 11, + 262, + 11, + 198, + 11, + 13, + 262, + 13, + 11, + 286, + 262, + 11, + 13, + 198, + 198, + 262, + 11, + 11, + 198, + 11, + 11, + 11, + 11, + 13, + 198, + 286, + 13, + 13, + 11, + 13, + 198, + 198, + 198, + 284, + 11, + 11, + 262, + 262, + 11, + 11, + 13, + 198, + 198, + 198, + 13, + 13, + 11, + 198, + 262, + 286, + 13, + 262, + 262, + 262, + 11, + 198, + 198, + 262, + 11, + 11, + 11, + 13, + 13, + 198, + 11, + 262, + 11, + 290, + 262, + 262, + 13, + 198, + 286, + 286, + 13, + 11, + 11, + 198, + 11, + 11, + 262, + 447, + 13, + 13, + 198, + 13, + 13, + 11, + 11, + 13, + 262, + 11, + 13, + 11, + 11, + 262, + 13, + 262, + 13, + 11, + 262, + 198, + 11, + 284, + 262, + 262, + 198, + 11, + 11, + 262, + 11, + 13, + 262, + 11, + 198, + 13, + 11, + 13, + 198, + 13, + 11, + 11, + 13, + 262, + 13, + 198, + 11, + 262, + 13, + 287, + 11, + 13, + 11, + 262, + 198, + 198, + 11, + 262, + 286, + 262, + 13, + 13, + 198, + 11, + 11, + 11, + 262, + 262, + 198, + 13, + 198, + 11, + 13, + 11, + 198, + 198, + 262, + 11, + 11, + 13, + 262, + 262, + 198, + 262, + 262, + 11, + 286, + 13, + 13, + 198, + 13, + 262, + 262, + 262, + 13, + 198, + 262, + 262, + 11, + 11, + 13, + 13, + 11, + 262, + 11, + 198, + 198, + 262, + 286, + 262, + 13, + 11, + 198, + 11, + 198, + 198, + 11, + 198, + 11, + 13, + 11, + 286, + 262, + 262, + 198, + 198, + 13, + 13, + 198, + 262, + 13, + 11, + 198, + 198, + 198, + 13, + 262, + 262, + 13, + 262, + 13, + 13, + 198, + 13, + 11, + 11, + 198, + 198, + 262, + 262, + 262, + 13, + 11, + 262, + 198, + 11, + 13, + 11, + 11, + 262, + 11, + 13, + 262, + 11, + 262, + 11, + 198, + 11, + 11, + 11, + 262, + 11, + 13, + 284, + 11, + 262, + 11, + 13, + 198, + 13, + 198, + 11, + 262, + 262, + 262, + 11, + 262, + 13, + 198, + 198, + 11, + 11, + 198, + 198, + 13, + 198, + 13, + 11, + 11, + 13, + 11, + 286, + 262, + 284, + 13, + 13, + 198, + 198, + 11, + 13, + 13, + 262, + 11, + 262, + 11, + 13, + 11, + 257, + 13, + 262, + 198, + 11, + 262, + 198, + 198, + 198, + 262, + 198, + 198, + 198, + 198, + 11, + 262, + 13, + 198, + 198, + 286, + 198, + 11, + 11, + 13, + 11, + 198, + 198, + 262, + 262, + 13, + 13, + 13, + 11, + 262, + 198, + 13, + 11, + 11, + 262, + 262, + 11, + 262, + 198, + 198, + 198, + 11, + 290, + 262, + 13, + 13, + 11, + 11, + 262, + 262, + 262, + 11, + 11, + 11, + 11, + 262, + 13, + 13, + 198, + 262, + 13, + 13, + 262, + 11, + 262, + 11, + 198, + 198, + 11, + 11, + 287, + 11, + 198, + 13, + 11, + 262, + 198, + 198, + 13, + 262, + 13, + 262, + 262, + 13, + 13, + 198, + 11, + 262, + 198, + 11, + 11, + 13, + 13, + 13, + 11, + 13, + 13, + 11, + 13, + 198, + 11, + 198, + 11, + 13, + 13, + 13, + 13, + 198, + 11, + 262, + 11, + 262, + 13, + 198, + 198, + 13, + 13, + 198, + 11, + 262, + 262, + 13, + 262, + 262, + 262, + 11, + 262, + 262, + 13, + 11, + 13, + 198, + 11, + 13, + 13, + 262, + 11, + 198, + 262, + 198, + 13, + 11, + 11, + 11, + 262, + 262, + 198, + 198, + 13, + 262, + 262, + 286, + 262, + 262, + 11, + 11, + 13, + 13, + 262, + 198, + 262, + 198, + 198, + 11, + 198, + 262, + 11, + 262, + 13, + 11, + 11, + 198, + 198, + 262, + 13, + 11, + 198, + 13, + 11, + 262, + 198, + 11, + 11, + 198, + 13, + 262, + 262, + 13, + 13, + 13, + 198, + 262, + 11, + 13, + 198, + 198, + 13, + 11, + 262, + 11, + 198, + 262, + 11, + 262, + 11, + 11, + 198, + 13, + 11, + 198, + 198, + 11, + 11, + 262, + 198, + 257, + 198, + 11, + 13, + 198, + 262, + 11, + 198, + 262, + 262, + 13, + 11, + 262, + 11, + 262, + 262, + 11, + 13, + 284, + 262, + 11, + 11, + 13, + 262, + 198, + 198, + 198, + 262, + 262, + 262, + 11, + 198, + 198, + 198, + 13, + 262, + 11, + 262, + 13, + 13, + 198, + 198, + 262, + 198, + 262, + 11, + 13, + 262, + 262, + 284, + 262, + 13, + 262, + 11, + 286, + 11, + 11, + 262, + 13, + 262, + 262, + 13, + 11, + 198, + 198, + 11, + 11, + 262, + 11, + 13, + 13, + 13, + 11, + 11, + 290, + 198, + 198, + 198, + 198, + 262, + 13, + 11, + 262, + 262, + 13, + 11, + 198, + 198, + 262, + 11, + 13, + 262, + 262, + 11, + 13, + 11, + 13, + 262, + 198, + 11, + 11, + 11, + 262, + 13, + 11, + 198, + 198, + 11, + 11, + 198, + 11, + 11, + 11, + 198, + 11, + 262, + 13, + 198, + 262, + 11, + 198, + 198, + 13, + 11, + 13, + 262, + 262, + 13, + 262, + 11, + 286, + 198, + 262, + 11, + 11, + 13, + 13, + 290, + 13, + 198, + 11, + 11, + 198, + 11, + 11, + 198, + 198, + 11, + 284, + 11, + 11, + 11, + 262, + 13, + 262, + 198, + 11, + 262, + 198, + 13, + 262, + 198, + 13, + 13, + 198, + 11, + 198, + 198, + 11, + 11, + 11, + 13, + 262, + 262, + 11, + 262, + 13, + 13, + 262, + 11, + 13, + 262, + 13, + 11, + 13, + 198, + 11, + 262, + 198, + 198, + 262, + 13, + 284, + 13, + 262, + 11, + 11, + 198, + 262, + 11, + 13, + 11, + 13, + 198, + 198, + 11, + 13, + 262, + 13, + 11, + 198, + 262, + 11, + 198, + 198, + 262, + 262, + 11, + 262, + 11, + 198, + 198, + 11, + 262, + 262, + 290, + 11, + 13, + 11, + 11, + 262, + 198, + 11, + 198, + 262, + 11, + 262, + 262, + 262, + 11, + 13, + 13, + 262, + 13, + 284, + 13, + 262, + 262, + 11, + 290, + 11, + 198, + 198, + 11, + 11, + 11, + 13, + 262, + 262, + 198, + 13, + 13, + 198, + 198, + 198, + 11, + 262, + 11, + 13, + 198, + 13, + 198, + 11, + 13, + 262, + 11, + 290, + 262, + 262, + 11, + 13, + 198, + 198, + 262, + 13, + 13, + 262, + 262, + 262, + 11, + 198, + 13, + 11, + 11, + 11, + 13, + 11, + 11, + 262, + 11, + 13, + 198, + 13, + 198, + 13, + 198, + 262, + 11, + 262, + 262, + 262, + 262, + 13, + 262, + 11, + 13, + 262, + 11, + 198, + 286, + 13, + 198, + 198, + 13, + 11, + 13, + 262, + 262, + 262, + 198, + 13, + 50256 + ] + }, + { + "sample_id": 7, + "text": "<|endoftext|> of,,\n.\n the the,\n.,...\n, the,.\n\n\n,, the\n\n\n\n,.\n to..,.,, the,,\n.,, a, the,.,., the the the a.\n\n,\n,.,,., of,.,, the, and. the..\n\n the the, the,.\n\n\n the\n\n,,, the the the,..\n.\n\n, the the the..\n\n\n..,,.\n.\n\n,, the the,\n\n to the the, the,,. the the. to the,..,. the.. the..,\n,,.., the..,, the,,,. the the the, the., to,,. the to the the,,\n,\n and the.. the,, the,..,\n the..\n the.\n the. the the, the,\n,.. the., the the,,. the,. the the..\n the\n\n\n...\n\n\n to the, the\n. the the the.,.,.,,,, the the the,\n.,. the\n\n\n the,\n.\n. the\n,\n.,.. the\n\n,., the., the,,..,,.,,,, the. the\n the the to the.\n.,.\n\n the the the the.\n the the the\n the\n. the,.. the. the\n.., the\n the., the\n,., to the, the\n the the, the,.\n\n,,\n the the, the the\n the the the\n the.\n,\n, the the,,.\n. the the the\n... the.\n. the. the,,\n. the\n., the, the the the,,,.\n\n the.\n\n,. the\n\n\n the.,.,, the... the.,,\n, the the,, and the. to the to. the., the, the\n,.. the the the, the.,,. the the\n the,, the the,\n., of.\n\n the the,, of,.\n,, the to..\n a,,... a the..\n...\n,, and, the, the,,.,.\n\n the of the.\n a, the, the the the the,, the the,.\n the the,.\n\n and the,,.,,,,.\n.,., the, the\n, the. of the the,, the\n,,, the. the. the., the the the..\n\n, the..... the the\n, the. the,. the\n..\n the.,,,,.,. the, the the.,,,. the the and the the,...,.. the, the.. the the..\n the. the,.,\n. the of and the a the the\n.\n\n\n\n the. to.,\n\n., the<|endoftext|>.\n\n the.,.... the the, the.... the the, the the, the.,, and the.,, the. to. the. the.\n the.\n of. the,\n,. the, the the\n\n the,\n the and.\n\n\n the...\n.., to the\n..,\n.\n.,, the,, the..,.. the,,\n\n, the, the. the\n.\n..... the of,. and the,\n the the the.,\n the.\n, the\n..,. to,.,..\n. the the the., the\n, the� the the.,. the., the,, the, the\n,.\n\n. the,.. the\n\n, the the. the\n.,.\n\n.,,\n the,,, the. the\n,,,, the,. the,.., the the the, of\n the the, the,.. to the, the.. the,,., the the.....\n,,\n,\n.,\n\n\n,, of,, and,, the the.,,\n\n, the,, and. the\n\n the\n the.. of.\n,.\n the..\n. the.<|endoftext|>", + "token_ids": [ + 50256, + 286, + 11, + 11, + 198, + 13, + 198, + 262, + 262, + 11, + 198, + 13, + 11, + 13, + 13, + 13, + 198, + 11, + 262, + 11, + 13, + 198, + 198, + 198, + 11, + 11, + 262, + 198, + 198, + 198, + 198, + 11, + 13, + 198, + 284, + 13, + 13, + 11, + 13, + 11, + 11, + 262, + 11, + 11, + 198, + 13, + 11, + 11, + 257, + 11, + 262, + 11, + 13, + 11, + 13, + 11, + 262, + 262, + 262, + 257, + 13, + 198, + 198, + 11, + 198, + 11, + 13, + 11, + 11, + 13, + 11, + 286, + 11, + 13, + 11, + 11, + 262, + 11, + 290, + 13, + 262, + 13, + 13, + 198, + 198, + 262, + 262, + 11, + 262, + 11, + 13, + 198, + 198, + 198, + 262, + 198, + 198, + 11, + 11, + 11, + 262, + 262, + 262, + 11, + 13, + 13, + 198, + 13, + 198, + 198, + 11, + 262, + 262, + 262, + 13, + 13, + 198, + 198, + 198, + 13, + 13, + 11, + 11, + 13, + 198, + 13, + 198, + 198, + 11, + 11, + 262, + 262, + 11, + 198, + 198, + 284, + 262, + 262, + 11, + 262, + 11, + 11, + 13, + 262, + 262, + 13, + 284, + 262, + 11, + 13, + 13, + 11, + 13, + 262, + 13, + 13, + 262, + 13, + 13, + 11, + 198, + 11, + 11, + 13, + 13, + 11, + 262, + 13, + 13, + 11, + 11, + 262, + 11, + 11, + 11, + 13, + 262, + 262, + 262, + 11, + 262, + 13, + 11, + 284, + 11, + 11, + 13, + 262, + 284, + 262, + 262, + 11, + 11, + 198, + 11, + 198, + 290, + 262, + 13, + 13, + 262, + 11, + 11, + 262, + 11, + 13, + 13, + 11, + 198, + 262, + 13, + 13, + 198, + 262, + 13, + 198, + 262, + 13, + 262, + 262, + 11, + 262, + 11, + 198, + 11, + 13, + 13, + 262, + 13, + 11, + 262, + 262, + 11, + 11, + 13, + 262, + 11, + 13, + 262, + 262, + 13, + 13, + 198, + 262, + 198, + 198, + 198, + 13, + 13, + 13, + 198, + 198, + 198, + 284, + 262, + 11, + 262, + 198, + 13, + 262, + 262, + 262, + 13, + 11, + 13, + 11, + 13, + 11, + 11, + 11, + 11, + 262, + 262, + 262, + 11, + 198, + 13, + 11, + 13, + 262, + 198, + 198, + 198, + 262, + 11, + 198, + 13, + 198, + 13, + 262, + 198, + 11, + 198, + 13, + 11, + 13, + 13, + 262, + 198, + 198, + 11, + 13, + 11, + 262, + 13, + 11, + 262, + 11, + 11, + 13, + 13, + 11, + 11, + 13, + 11, + 11, + 11, + 11, + 262, + 13, + 262, + 198, + 262, + 262, + 284, + 262, + 13, + 198, + 13, + 11, + 13, + 198, + 198, + 262, + 262, + 262, + 262, + 13, + 198, + 262, + 262, + 262, + 198, + 262, + 198, + 13, + 262, + 11, + 13, + 13, + 262, + 13, + 262, + 198, + 13, + 13, + 11, + 262, + 198, + 262, + 13, + 11, + 262, + 198, + 11, + 13, + 11, + 284, + 262, + 11, + 262, + 198, + 262, + 262, + 11, + 262, + 11, + 13, + 198, + 198, + 11, + 11, + 198, + 262, + 262, + 11, + 262, + 262, + 198, + 262, + 262, + 262, + 198, + 262, + 13, + 198, + 11, + 198, + 11, + 262, + 262, + 11, + 11, + 13, + 198, + 13, + 262, + 262, + 262, + 198, + 13, + 13, + 13, + 262, + 13, + 198, + 13, + 262, + 13, + 262, + 11, + 11, + 198, + 13, + 262, + 198, + 13, + 11, + 262, + 11, + 262, + 262, + 262, + 11, + 11, + 11, + 13, + 198, + 198, + 262, + 13, + 198, + 198, + 11, + 13, + 262, + 198, + 198, + 198, + 262, + 13, + 11, + 13, + 11, + 11, + 262, + 13, + 13, + 13, + 262, + 13, + 11, + 11, + 198, + 11, + 262, + 262, + 11, + 11, + 290, + 262, + 13, + 284, + 262, + 284, + 13, + 262, + 13, + 11, + 262, + 11, + 262, + 198, + 11, + 13, + 13, + 262, + 262, + 262, + 11, + 262, + 13, + 11, + 11, + 13, + 262, + 262, + 198, + 262, + 11, + 11, + 262, + 262, + 11, + 198, + 13, + 11, + 286, + 13, + 198, + 198, + 262, + 262, + 11, + 11, + 286, + 11, + 13, + 198, + 11, + 11, + 262, + 284, + 13, + 13, + 198, + 257, + 11, + 11, + 13, + 13, + 13, + 257, + 262, + 13, + 13, + 198, + 13, + 13, + 13, + 198, + 11, + 11, + 290, + 11, + 262, + 11, + 262, + 11, + 11, + 13, + 11, + 13, + 198, + 198, + 262, + 286, + 262, + 13, + 198, + 257, + 11, + 262, + 11, + 262, + 262, + 262, + 262, + 11, + 11, + 262, + 262, + 11, + 13, + 198, + 262, + 262, + 11, + 13, + 198, + 198, + 290, + 262, + 11, + 11, + 13, + 11, + 11, + 11, + 11, + 13, + 198, + 13, + 11, + 13, + 11, + 262, + 11, + 262, + 198, + 11, + 262, + 13, + 286, + 262, + 262, + 11, + 11, + 262, + 198, + 11, + 11, + 11, + 262, + 13, + 262, + 13, + 262, + 13, + 11, + 262, + 262, + 262, + 13, + 13, + 198, + 198, + 11, + 262, + 13, + 13, + 13, + 13, + 13, + 262, + 262, + 198, + 11, + 262, + 13, + 262, + 11, + 13, + 262, + 198, + 13, + 13, + 198, + 262, + 13, + 11, + 11, + 11, + 11, + 13, + 11, + 13, + 262, + 11, + 262, + 262, + 13, + 11, + 11, + 11, + 13, + 262, + 262, + 290, + 262, + 262, + 11, + 13, + 13, + 13, + 11, + 13, + 13, + 262, + 11, + 262, + 13, + 13, + 262, + 262, + 13, + 13, + 198, + 262, + 13, + 262, + 11, + 13, + 11, + 198, + 13, + 262, + 286, + 290, + 262, + 257, + 262, + 262, + 198, + 13, + 198, + 198, + 198, + 198, + 262, + 13, + 284, + 13, + 11, + 198, + 198, + 13, + 11, + 262, + 50256, + 13, + 198, + 198, + 262, + 13, + 11, + 13, + 13, + 13, + 13, + 262, + 262, + 11, + 262, + 13, + 13, + 13, + 13, + 262, + 262, + 11, + 262, + 262, + 11, + 262, + 13, + 11, + 11, + 290, + 262, + 13, + 11, + 11, + 262, + 13, + 284, + 13, + 262, + 13, + 262, + 13, + 198, + 262, + 13, + 198, + 286, + 13, + 262, + 11, + 198, + 11, + 13, + 262, + 11, + 262, + 262, + 198, + 198, + 262, + 11, + 198, + 262, + 290, + 13, + 198, + 198, + 198, + 262, + 13, + 13, + 13, + 198, + 13, + 13, + 11, + 284, + 262, + 198, + 13, + 13, + 11, + 198, + 13, + 198, + 13, + 11, + 11, + 262, + 11, + 11, + 262, + 13, + 13, + 11, + 13, + 13, + 262, + 11, + 11, + 198, + 198, + 11, + 262, + 11, + 262, + 13, + 262, + 198, + 13, + 198, + 13, + 13, + 13, + 13, + 13, + 262, + 286, + 11, + 13, + 290, + 262, + 11, + 198, + 262, + 262, + 262, + 13, + 11, + 198, + 262, + 13, + 198, + 11, + 262, + 198, + 13, + 13, + 11, + 13, + 284, + 11, + 13, + 11, + 13, + 13, + 198, + 13, + 262, + 262, + 262, + 13, + 11, + 262, + 198, + 11, + 262, + 447, + 262, + 262, + 13, + 11, + 13, + 262, + 13, + 11, + 262, + 11, + 11, + 262, + 11, + 262, + 198, + 11, + 13, + 198, + 198, + 13, + 262, + 11, + 13, + 13, + 262, + 198, + 198, + 11, + 262, + 262, + 13, + 262, + 198, + 13, + 11, + 13, + 198, + 198, + 13, + 11, + 11, + 198, + 262, + 11, + 11, + 11, + 262, + 13, + 262, + 198, + 11, + 11, + 11, + 11, + 262, + 11, + 13, + 262, + 11, + 13, + 13, + 11, + 262, + 262, + 262, + 11, + 286, + 198, + 262, + 262, + 11, + 262, + 11, + 13, + 13, + 284, + 262, + 11, + 262, + 13, + 13, + 262, + 11, + 11, + 13, + 11, + 262, + 262, + 13, + 13, + 13, + 13, + 13, + 198, + 11, + 11, + 198, + 11, + 198, + 13, + 11, + 198, + 198, + 198, + 11, + 11, + 286, + 11, + 11, + 290, + 11, + 11, + 262, + 262, + 13, + 11, + 11, + 198, + 198, + 11, + 262, + 11, + 11, + 290, + 13, + 262, + 198, + 198, + 262, + 198, + 262, + 13, + 13, + 286, + 13, + 198, + 11, + 13, + 198, + 262, + 13, + 13, + 198, + 13, + 262, + 13, + 50256 + ] + }, + { + "sample_id": 8, + "text": "<|endoftext|>\n.\n, the,., the\n\n,...\n\n the,,.,\n, to the to the. the the\n.,\n the,.,, the,, the. in\n the. the,,. the to. the\n\n the,.,. the,.\n. the,,\n..\n,,..\n to.,,, the,, the, the the\n..\n\n the, the the..\n\n the the,, the the.,,,. the,.,.\n the, the the,. the.....,,..\n\n, the. the the the...\n.,, the the the the the,. the\n\n\n of. the the,,..,.\n the\n.,, the,..\n\n,\n,.\n,\n\n.\n the, the\n\n\n the. the the the,. the. the,,, the, of\n\n. the\n, the, the\n\n the\n and the.\n\n, the\n the. the.,,,\n\n of the,,\n\n the and of,, the,,,\n,,,,,,,,,. the..\n\n, and\n\n,., the,, the.\n\n,.,\n.. and,\n.,. the.. the\n a,.\n\n..\n. the the.,,,, the, the the the\n, the the, the the the the the\n, the\n\n,\n..,\n.. the,,.,\n,,, to,,.\n. the. the the,.,,., the.\n\n\n\n the., the a,. to, the.,\n, the,.., the\n, the the. the in the of the the the, the the,,, the\n,,, the\n,,\n.,. the the the,,. the., the\n.\n, the, to the the, the, the, the. the the the the the\n\n the.\n\n,.,\n,., the..,.\n the. the.,., of. the., the.,, the, the\n,., the the\n, the,.,\n.,,,\n,,, a.\n\n,\n\n the\n,.. the the.\n..,,,.,\n. the. the,.. the,\n,.,..,,,,\n... the., the, of the,,.. the\n..\n the.. the the., the the., of. to the., the, the the, to,.\n a,.\n the\n, to the... the\n.,\n of the,..,, the the, the.\n., the,\n the\n.,,\n. the\n\n the,. the\n.,, the, a. the\n,\n,. the, the, the. the.\n\n., to,..,,\n..., of,, the,, and,. the, the.,\n,\n. the to, the,\n the the,., the,,, the..\n\n. the.. the to, the.\n., the,,,.,...\n and to the.. the the,, to. the.,\n the, the...,.\n.. the...\n\n,.\n\n.,, the, the... the,,,, of, the.., to the the the.,, the,.,\n the\n, the the the\n..\n\n the,,,. the.,.., the. the the.,, the the the.\n,\n., the,\n, to..\n,\n,.\n\n, the, and to the..,. the,.., the\n.. the the the the.\n\n\n.. the. the,,. the., the the\n.,., the the. the the, the,, the,\n, the\n., the,,\n the, the the.,,. the in the,,\n..\n., the,,, the the,. the.,,\n\n\n...,,.,. and\n. the\n\n,.,., the., the, the. the,. the\n. the\n. the,.\n\n<|endoftext|>", + "token_ids": [ + 50256, + 198, + 13, + 198, + 11, + 262, + 11, + 13, + 11, + 262, + 198, + 198, + 11, + 13, + 13, + 13, + 198, + 198, + 262, + 11, + 11, + 13, + 11, + 198, + 11, + 284, + 262, + 284, + 262, + 13, + 262, + 262, + 198, + 13, + 11, + 198, + 262, + 11, + 13, + 11, + 11, + 262, + 11, + 11, + 262, + 13, + 287, + 198, + 262, + 13, + 262, + 11, + 11, + 13, + 262, + 284, + 13, + 262, + 198, + 198, + 262, + 11, + 13, + 11, + 13, + 262, + 11, + 13, + 198, + 13, + 262, + 11, + 11, + 198, + 13, + 13, + 198, + 11, + 11, + 13, + 13, + 198, + 284, + 13, + 11, + 11, + 11, + 262, + 11, + 11, + 262, + 11, + 262, + 262, + 198, + 13, + 13, + 198, + 198, + 262, + 11, + 262, + 262, + 13, + 13, + 198, + 198, + 262, + 262, + 11, + 11, + 262, + 262, + 13, + 11, + 11, + 11, + 13, + 262, + 11, + 13, + 11, + 13, + 198, + 262, + 11, + 262, + 262, + 11, + 13, + 262, + 13, + 13, + 13, + 13, + 13, + 11, + 11, + 13, + 13, + 198, + 198, + 11, + 262, + 13, + 262, + 262, + 262, + 13, + 13, + 13, + 198, + 13, + 11, + 11, + 262, + 262, + 262, + 262, + 262, + 11, + 13, + 262, + 198, + 198, + 198, + 286, + 13, + 262, + 262, + 11, + 11, + 13, + 13, + 11, + 13, + 198, + 262, + 198, + 13, + 11, + 11, + 262, + 11, + 13, + 13, + 198, + 198, + 11, + 198, + 11, + 13, + 198, + 11, + 198, + 198, + 13, + 198, + 262, + 11, + 262, + 198, + 198, + 198, + 262, + 13, + 262, + 262, + 262, + 11, + 13, + 262, + 13, + 262, + 11, + 11, + 11, + 262, + 11, + 286, + 198, + 198, + 13, + 262, + 198, + 11, + 262, + 11, + 262, + 198, + 198, + 262, + 198, + 290, + 262, + 13, + 198, + 198, + 11, + 262, + 198, + 262, + 13, + 262, + 13, + 11, + 11, + 11, + 198, + 198, + 286, + 262, + 11, + 11, + 198, + 198, + 262, + 290, + 286, + 11, + 11, + 262, + 11, + 11, + 11, + 198, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 13, + 262, + 13, + 13, + 198, + 198, + 11, + 290, + 198, + 198, + 11, + 13, + 11, + 262, + 11, + 11, + 262, + 13, + 198, + 198, + 11, + 13, + 11, + 198, + 13, + 13, + 290, + 11, + 198, + 13, + 11, + 13, + 262, + 13, + 13, + 262, + 198, + 257, + 11, + 13, + 198, + 198, + 13, + 13, + 198, + 13, + 262, + 262, + 13, + 11, + 11, + 11, + 11, + 262, + 11, + 262, + 262, + 262, + 198, + 11, + 262, + 262, + 11, + 262, + 262, + 262, + 262, + 262, + 198, + 11, + 262, + 198, + 198, + 11, + 198, + 13, + 13, + 11, + 198, + 13, + 13, + 262, + 11, + 11, + 13, + 11, + 198, + 11, + 11, + 11, + 284, + 11, + 11, + 13, + 198, + 13, + 262, + 13, + 262, + 262, + 11, + 13, + 11, + 11, + 13, + 11, + 262, + 13, + 198, + 198, + 198, + 198, + 262, + 13, + 11, + 262, + 257, + 11, + 13, + 284, + 11, + 262, + 13, + 11, + 198, + 11, + 262, + 11, + 13, + 13, + 11, + 262, + 198, + 11, + 262, + 262, + 13, + 262, + 287, + 262, + 286, + 262, + 262, + 262, + 11, + 262, + 262, + 11, + 11, + 11, + 262, + 198, + 11, + 11, + 11, + 262, + 198, + 11, + 11, + 198, + 13, + 11, + 13, + 262, + 262, + 262, + 11, + 11, + 13, + 262, + 13, + 11, + 262, + 198, + 13, + 198, + 11, + 262, + 11, + 284, + 262, + 262, + 11, + 262, + 11, + 262, + 11, + 262, + 13, + 262, + 262, + 262, + 262, + 262, + 198, + 198, + 262, + 13, + 198, + 198, + 11, + 13, + 11, + 198, + 11, + 13, + 11, + 262, + 13, + 13, + 11, + 13, + 198, + 262, + 13, + 262, + 13, + 11, + 13, + 11, + 286, + 13, + 262, + 13, + 11, + 262, + 13, + 11, + 11, + 262, + 11, + 262, + 198, + 11, + 13, + 11, + 262, + 262, + 198, + 11, + 262, + 11, + 13, + 11, + 198, + 13, + 11, + 11, + 11, + 198, + 11, + 11, + 11, + 257, + 13, + 198, + 198, + 11, + 198, + 198, + 262, + 198, + 11, + 13, + 13, + 262, + 262, + 13, + 198, + 13, + 13, + 11, + 11, + 11, + 13, + 11, + 198, + 13, + 262, + 13, + 262, + 11, + 13, + 13, + 262, + 11, + 198, + 11, + 13, + 11, + 13, + 13, + 11, + 11, + 11, + 11, + 198, + 13, + 13, + 13, + 262, + 13, + 11, + 262, + 11, + 286, + 262, + 11, + 11, + 13, + 13, + 262, + 198, + 13, + 13, + 198, + 262, + 13, + 13, + 262, + 262, + 13, + 11, + 262, + 262, + 13, + 11, + 286, + 13, + 284, + 262, + 13, + 11, + 262, + 11, + 262, + 262, + 11, + 284, + 11, + 13, + 198, + 257, + 11, + 13, + 198, + 262, + 198, + 11, + 284, + 262, + 13, + 13, + 13, + 262, + 198, + 13, + 11, + 198, + 286, + 262, + 11, + 13, + 13, + 11, + 11, + 262, + 262, + 11, + 262, + 13, + 198, + 13, + 11, + 262, + 11, + 198, + 262, + 198, + 13, + 11, + 11, + 198, + 13, + 262, + 198, + 198, + 262, + 11, + 13, + 262, + 198, + 13, + 11, + 11, + 262, + 11, + 257, + 13, + 262, + 198, + 11, + 198, + 11, + 13, + 262, + 11, + 262, + 11, + 262, + 13, + 262, + 13, + 198, + 198, + 13, + 11, + 284, + 11, + 13, + 13, + 11, + 11, + 198, + 13, + 13, + 13, + 11, + 286, + 11, + 11, + 262, + 11, + 11, + 290, + 11, + 13, + 262, + 11, + 262, + 13, + 11, + 198, + 11, + 198, + 13, + 262, + 284, + 11, + 262, + 11, + 198, + 262, + 262, + 11, + 13, + 11, + 262, + 11, + 11, + 11, + 262, + 13, + 13, + 198, + 198, + 13, + 262, + 13, + 13, + 262, + 284, + 11, + 262, + 13, + 198, + 13, + 11, + 262, + 11, + 11, + 11, + 13, + 11, + 13, + 13, + 13, + 198, + 290, + 284, + 262, + 13, + 13, + 262, + 262, + 11, + 11, + 284, + 13, + 262, + 13, + 11, + 198, + 262, + 11, + 262, + 13, + 13, + 13, + 11, + 13, + 198, + 13, + 13, + 262, + 13, + 13, + 13, + 198, + 198, + 11, + 13, + 198, + 198, + 13, + 11, + 11, + 262, + 11, + 262, + 13, + 13, + 13, + 262, + 11, + 11, + 11, + 11, + 286, + 11, + 262, + 13, + 13, + 11, + 284, + 262, + 262, + 262, + 13, + 11, + 11, + 262, + 11, + 13, + 11, + 198, + 262, + 198, + 11, + 262, + 262, + 262, + 198, + 13, + 13, + 198, + 198, + 262, + 11, + 11, + 11, + 13, + 262, + 13, + 11, + 13, + 13, + 11, + 262, + 13, + 262, + 262, + 13, + 11, + 11, + 262, + 262, + 262, + 13, + 198, + 11, + 198, + 13, + 11, + 262, + 11, + 198, + 11, + 284, + 13, + 13, + 198, + 11, + 198, + 11, + 13, + 198, + 198, + 11, + 262, + 11, + 290, + 284, + 262, + 13, + 13, + 11, + 13, + 262, + 11, + 13, + 13, + 11, + 262, + 198, + 13, + 13, + 262, + 262, + 262, + 262, + 13, + 198, + 198, + 198, + 13, + 13, + 262, + 13, + 262, + 11, + 11, + 13, + 262, + 13, + 11, + 262, + 262, + 198, + 13, + 11, + 13, + 11, + 262, + 262, + 13, + 262, + 262, + 11, + 262, + 11, + 11, + 262, + 11, + 198, + 11, + 262, + 198, + 13, + 11, + 262, + 11, + 11, + 198, + 262, + 11, + 262, + 262, + 13, + 11, + 11, + 13, + 262, + 287, + 262, + 11, + 11, + 198, + 13, + 13, + 198, + 13, + 11, + 262, + 11, + 11, + 11, + 262, + 262, + 11, + 13, + 262, + 13, + 11, + 11, + 198, + 198, + 198, + 13, + 13, + 13, + 11, + 11, + 13, + 11, + 13, + 290, + 198, + 13, + 262, + 198, + 198, + 11, + 13, + 11, + 13, + 11, + 262, + 13, + 11, + 262, + 11, + 262, + 13, + 262, + 11, + 13, + 262, + 198, + 13, + 262, + 198, + 13, + 262, + 11, + 13, + 198, + 198, + 50256 + ] + }, + { + "sample_id": 9, + "text": "<|endoftext|>,., the. the. the.\n.,, the\n\n. of the the to.. the\n\n\n\n.\n\n to the\n the..,, to,\n,.. the the, the,, the\n\n,.\n,,,,, the the the,,... the.\n\n\n the the to.. the to.,. the the, the the..\n\n.,.\n\n. the\n\n.\n\n., the the the.\n\n,. the,\n the,\n\n.,..,, the the the\n the that the,,. to the the. the.,,..\n, and\n,.\n of to,.\n the.,\n the the the. to.\n.\n\n., to,, the the. to.\n, the,\n the the. the\n, and..\n,.\n\n the,\n.\n\n\n.. to the\n,.\n\n the the\n\n the the,., the. the\n the,,\n\n...\n,,. the and,,\n the the the.,,. the the, the..,\n\n, the the the,, the. the.,. the.,\n\n,\n. the the the to, the and the. the,,. the the, the.\n\n\n\n... the of,,.. the the.,. the,.,.,.,.\n., to the the\n\n,. the\n,.. the. the the,,, the. the, the the,\n\n the\n\n the,,.. the,,. the the,.. the\n\n\n the\n\n\n\n...\n.\n, the,,,, the.\n\n\n\n the, the\n the.\n,\n.,,,,., the the\n. the the, to\n the,, the,.\n\n,. the the to, the.,,, the,. the\n...,\n.\n to. the\n,\n the,, the,, the\n, the,.\n.. the\n\n,, the,,,.,.\n the,,,.\n\n, the, of\n to,,, the..,,. the the the to.\n the,, the..,, the.\n. the the the the the..\n\n the to.\n and,, the the the,,,,,,, of..,\n the of the....,,,,. the,, the the\n the the.\n\n\n\n\n,, the, and the the.\n\n\n, of,.\n\n\n.. the, the,\n the. the,\n\n, the,.\n, the, and of\n\n the.\n\n,..\n,. the. the\n,.\n\n,, to the. the\n, of the. the the,. of the,.\n, the,, the the.,\n\n. the the.\n,. the the.,. the the,.\n\n\n,, the\n,... the., the. the\n\n the the,.\n, to the the the\n the\n the\n. the.. the and,..., the\n.,..,.,,., the, the..\n\n\n the., the, the,., the,,\n\n the,.,, the.,\n the the., the the, the. the the,,,, the,\n\n,, the. the.\n the,\n, the..\n\n the the the,\n\n\n the.., the.\n,.,, and the, to. the,.\n,\n\n. the. of. the the the the the,,\n.\n\n\n\n\n the\n\n\n the the.,, the..\n\n the the the,,\n the. the\n the..,.,\n. the\n\n.\n\n the. a the\n the,\n the the\n in the the.\n\n\n,.. to the the,\n the.,\n.\n\n\n\n\n\n.,,.\n.. the, the.. the\n.,,..\n the. to to the the the,,,,\n., the, of. the..\n\n,, the..\n,\n.\n.,. the\n\n a, the the..\n\n... the, the\n\n<|endoftext|>", + "token_ids": [ + 50256, + 11, + 13, + 11, + 262, + 13, + 262, + 13, + 262, + 13, + 198, + 13, + 11, + 11, + 262, + 198, + 198, + 13, + 286, + 262, + 262, + 284, + 13, + 13, + 262, + 198, + 198, + 198, + 198, + 13, + 198, + 198, + 284, + 262, + 198, + 262, + 13, + 13, + 11, + 11, + 284, + 11, + 198, + 11, + 13, + 13, + 262, + 262, + 11, + 262, + 11, + 11, + 262, + 198, + 198, + 11, + 13, + 198, + 11, + 11, + 11, + 11, + 11, + 262, + 262, + 262, + 11, + 11, + 13, + 13, + 13, + 262, + 13, + 198, + 198, + 198, + 262, + 262, + 284, + 13, + 13, + 262, + 284, + 13, + 11, + 13, + 262, + 262, + 11, + 262, + 262, + 13, + 13, + 198, + 198, + 13, + 11, + 13, + 198, + 198, + 13, + 262, + 198, + 198, + 13, + 198, + 198, + 13, + 11, + 262, + 262, + 262, + 13, + 198, + 198, + 11, + 13, + 262, + 11, + 198, + 262, + 11, + 198, + 198, + 13, + 11, + 13, + 13, + 11, + 11, + 262, + 262, + 262, + 198, + 262, + 326, + 262, + 11, + 11, + 13, + 284, + 262, + 262, + 13, + 262, + 13, + 11, + 11, + 13, + 13, + 198, + 11, + 290, + 198, + 11, + 13, + 198, + 286, + 284, + 11, + 13, + 198, + 262, + 13, + 11, + 198, + 262, + 262, + 262, + 13, + 284, + 13, + 198, + 13, + 198, + 198, + 13, + 11, + 284, + 11, + 11, + 262, + 262, + 13, + 284, + 13, + 198, + 11, + 262, + 11, + 198, + 262, + 262, + 13, + 262, + 198, + 11, + 290, + 13, + 13, + 198, + 11, + 13, + 198, + 198, + 262, + 11, + 198, + 13, + 198, + 198, + 198, + 13, + 13, + 284, + 262, + 198, + 11, + 13, + 198, + 198, + 262, + 262, + 198, + 198, + 262, + 262, + 11, + 13, + 11, + 262, + 13, + 262, + 198, + 262, + 11, + 11, + 198, + 198, + 13, + 13, + 13, + 198, + 11, + 11, + 13, + 262, + 290, + 11, + 11, + 198, + 262, + 262, + 262, + 13, + 11, + 11, + 13, + 262, + 262, + 11, + 262, + 13, + 13, + 11, + 198, + 198, + 11, + 262, + 262, + 262, + 11, + 11, + 262, + 13, + 262, + 13, + 11, + 13, + 262, + 13, + 11, + 198, + 198, + 11, + 198, + 13, + 262, + 262, + 262, + 284, + 11, + 262, + 290, + 262, + 13, + 262, + 11, + 11, + 13, + 262, + 262, + 11, + 262, + 13, + 198, + 198, + 198, + 198, + 13, + 13, + 13, + 262, + 286, + 11, + 11, + 13, + 13, + 262, + 262, + 13, + 11, + 13, + 262, + 11, + 13, + 11, + 13, + 11, + 13, + 11, + 13, + 198, + 13, + 11, + 284, + 262, + 262, + 198, + 198, + 11, + 13, + 262, + 198, + 11, + 13, + 13, + 262, + 13, + 262, + 262, + 11, + 11, + 11, + 262, + 13, + 262, + 11, + 262, + 262, + 11, + 198, + 198, + 262, + 198, + 198, + 262, + 11, + 11, + 13, + 13, + 262, + 11, + 11, + 13, + 262, + 262, + 11, + 13, + 13, + 262, + 198, + 198, + 198, + 262, + 198, + 198, + 198, + 198, + 13, + 13, + 13, + 198, + 13, + 198, + 11, + 262, + 11, + 11, + 11, + 11, + 262, + 13, + 198, + 198, + 198, + 198, + 262, + 11, + 262, + 198, + 262, + 13, + 198, + 11, + 198, + 13, + 11, + 11, + 11, + 11, + 13, + 11, + 262, + 262, + 198, + 13, + 262, + 262, + 11, + 284, + 198, + 262, + 11, + 11, + 262, + 11, + 13, + 198, + 198, + 11, + 13, + 262, + 262, + 284, + 11, + 262, + 13, + 11, + 11, + 11, + 262, + 11, + 13, + 262, + 198, + 13, + 13, + 13, + 11, + 198, + 13, + 198, + 284, + 13, + 262, + 198, + 11, + 198, + 262, + 11, + 11, + 262, + 11, + 11, + 262, + 198, + 11, + 262, + 11, + 13, + 198, + 13, + 13, + 262, + 198, + 198, + 11, + 11, + 262, + 11, + 11, + 11, + 13, + 11, + 13, + 198, + 262, + 11, + 11, + 11, + 13, + 198, + 198, + 11, + 262, + 11, + 286, + 198, + 284, + 11, + 11, + 11, + 262, + 13, + 13, + 11, + 11, + 13, + 262, + 262, + 262, + 284, + 13, + 198, + 262, + 11, + 11, + 262, + 13, + 13, + 11, + 11, + 262, + 13, + 198, + 13, + 262, + 262, + 262, + 262, + 262, + 13, + 13, + 198, + 198, + 262, + 284, + 13, + 198, + 290, + 11, + 11, + 262, + 262, + 262, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 286, + 13, + 13, + 11, + 198, + 262, + 286, + 262, + 13, + 13, + 13, + 13, + 11, + 11, + 11, + 11, + 13, + 262, + 11, + 11, + 262, + 262, + 198, + 262, + 262, + 13, + 198, + 198, + 198, + 198, + 198, + 11, + 11, + 262, + 11, + 290, + 262, + 262, + 13, + 198, + 198, + 198, + 11, + 286, + 11, + 13, + 198, + 198, + 198, + 13, + 13, + 262, + 11, + 262, + 11, + 198, + 262, + 13, + 262, + 11, + 198, + 198, + 11, + 262, + 11, + 13, + 198, + 11, + 262, + 11, + 290, + 286, + 198, + 198, + 262, + 13, + 198, + 198, + 11, + 13, + 13, + 198, + 11, + 13, + 262, + 13, + 262, + 198, + 11, + 13, + 198, + 198, + 11, + 11, + 284, + 262, + 13, + 262, + 198, + 11, + 286, + 262, + 13, + 262, + 262, + 11, + 13, + 286, + 262, + 11, + 13, + 198, + 11, + 262, + 11, + 11, + 262, + 262, + 13, + 11, + 198, + 198, + 13, + 262, + 262, + 13, + 198, + 11, + 13, + 262, + 262, + 13, + 11, + 13, + 262, + 262, + 11, + 13, + 198, + 198, + 198, + 11, + 11, + 262, + 198, + 11, + 13, + 13, + 13, + 262, + 13, + 11, + 262, + 13, + 262, + 198, + 198, + 262, + 262, + 11, + 13, + 198, + 11, + 284, + 262, + 262, + 262, + 198, + 262, + 198, + 262, + 198, + 13, + 262, + 13, + 13, + 262, + 290, + 11, + 13, + 13, + 13, + 11, + 262, + 198, + 13, + 11, + 13, + 13, + 11, + 13, + 11, + 11, + 13, + 11, + 262, + 11, + 262, + 13, + 13, + 198, + 198, + 198, + 262, + 13, + 11, + 262, + 11, + 262, + 11, + 13, + 11, + 262, + 11, + 11, + 198, + 198, + 262, + 11, + 13, + 11, + 11, + 262, + 13, + 11, + 198, + 262, + 262, + 13, + 11, + 262, + 262, + 11, + 262, + 13, + 262, + 262, + 11, + 11, + 11, + 11, + 262, + 11, + 198, + 198, + 11, + 11, + 262, + 13, + 262, + 13, + 198, + 262, + 11, + 198, + 11, + 262, + 13, + 13, + 198, + 198, + 262, + 262, + 262, + 11, + 198, + 198, + 198, + 262, + 13, + 13, + 11, + 262, + 13, + 198, + 11, + 13, + 11, + 11, + 290, + 262, + 11, + 284, + 13, + 262, + 11, + 13, + 198, + 11, + 198, + 198, + 13, + 262, + 13, + 286, + 13, + 262, + 262, + 262, + 262, + 262, + 11, + 11, + 198, + 13, + 198, + 198, + 198, + 198, + 198, + 262, + 198, + 198, + 198, + 262, + 262, + 13, + 11, + 11, + 262, + 13, + 13, + 198, + 198, + 262, + 262, + 262, + 11, + 11, + 198, + 262, + 13, + 262, + 198, + 262, + 13, + 13, + 11, + 13, + 11, + 198, + 13, + 262, + 198, + 198, + 13, + 198, + 198, + 262, + 13, + 257, + 262, + 198, + 262, + 11, + 198, + 262, + 262, + 198, + 287, + 262, + 262, + 13, + 198, + 198, + 198, + 11, + 13, + 13, + 284, + 262, + 262, + 11, + 198, + 262, + 13, + 11, + 198, + 13, + 198, + 198, + 198, + 198, + 198, + 198, + 13, + 11, + 11, + 13, + 198, + 13, + 13, + 262, + 11, + 262, + 13, + 13, + 262, + 198, + 13, + 11, + 11, + 13, + 13, + 198, + 262, + 13, + 284, + 284, + 262, + 262, + 262, + 11, + 11, + 11, + 11, + 198, + 13, + 11, + 262, + 11, + 286, + 13, + 262, + 13, + 13, + 198, + 198, + 11, + 11, + 262, + 13, + 13, + 198, + 11, + 198, + 13, + 198, + 13, + 11, + 13, + 262, + 198, + 198, + 257, + 11, + 262, + 262, + 13, + 13, + 198, + 198, + 13, + 13, + 13, + 262, + 11, + 262, + 198, + 198, + 50256 + ] + }, + { + "sample_id": 10, + "text": "<|endoftext|>\n,..\n.,\n., the the\n\n,,,. the....\n\n the of the, the,., the.....,,. the\n, and, the. the\n... and the the the the\n.\n\n\n the, the the\n,,\n\n. of..\n. the the, the\n\n, the the,\n the\n\n, the,., the the the\n,, the\n., the the the of, the.,. the,,,... to the.\n. the,, the\n, of to, the\n the, the. the., the the., the, the.\n\n. of,,, the to\n,., the,\n.\n\n.\n the. the.\n\n. the.,\n\n\n the the.,, the.\n., the., the,.. the\n\n the, the,.\n\n, the the,,., the.. a\n\n the.\n\n,\n.\n\n\n the the to.,\n\n\n\n the the to,..\n\n\n,,,, the the\n,, the\n\n..,,. the.,\n,.\n, the\n.. the\n\n,.\n,.. the, the the to\n\n,. the the\n.. the,, the, the the\n..,,,., the the the the,\n,\n. the,, the the..,\n\n.\n, the.\n\n\n the... the to, the\n\n. the,... the, the the the,\n\n., the.\n\n,..\n the., the the,, the\n,., to.,. the.,\n\n the, the,\n,,,.\n the,,\n\n,. the\n, the...\n.\n the the..\n the.,,,\n.,.,. the the. the\n,, the the the.\n,,.\n\n\n\n the\n, the the the, the\n,. the the the,\n\n. the the the the. the\n,,, the\n. the,... the the the.,\n\n,. the\n\n.,, the... the\n..\n. the the\n.,\n the,. the\n. the\n of,.\n\n\n. the\n,, the the the. of the,,. the the the the,..\n,,.,,,..\n\n the the the the\n the\n, the,,\n\n the\n\n..\n.,,, the to..,\n\n.,., the. the, the and the\n.,\n,,\n the\n.. the. the\n,, the\n the.,\n.. the\n., the,\n\n..\n,.,\n the the, to. the.\n the,, the. the the and, the\n.\n..\n,,.,\n, the.,.\n\n,, to the,,,.., to the the the,,\n.\n\n\n., the\n\n\n..\n the the. the the the the, the\n, the the,, the the.,\n,.\n the,,.. the,.., the, the\n.\n\n to, the, to to.\n.\n the the the.. the,. the., the the..,\n the,,. to. the the,,, the and,. the of\n,,.\n\n,\n the,. the. of, the,. the\n the the., the.\n the,, the.,.\n\n the. the.. the the. to..,,, the the., the the, the..,, the,., the, the., the.. of the.,. the,\n the the., of the., the, the,,, the the the the, of.\n\n, the\n\n\n.. the,. the, the. the,,. the the, the the,.\n\n,\n, the the,.\n,..,..\n, the,.\n. the..\n the the. to to the\n... the., the,\n,... the\n,. the the. the\n,., of the the\n\n\n the the.\n,,,.\n,,,, the.<|endoftext|>", + "token_ids": [ + 50256, + 198, + 11, + 13, + 13, + 198, + 13, + 11, + 198, + 13, + 11, + 262, + 262, + 198, + 198, + 11, + 11, + 11, + 13, + 262, + 13, + 13, + 13, + 13, + 198, + 198, + 262, + 286, + 262, + 11, + 262, + 11, + 13, + 11, + 262, + 13, + 13, + 13, + 13, + 13, + 11, + 11, + 13, + 262, + 198, + 11, + 290, + 11, + 262, + 13, + 262, + 198, + 13, + 13, + 13, + 290, + 262, + 262, + 262, + 262, + 198, + 13, + 198, + 198, + 198, + 262, + 11, + 262, + 262, + 198, + 11, + 11, + 198, + 198, + 13, + 286, + 13, + 13, + 198, + 13, + 262, + 262, + 11, + 262, + 198, + 198, + 11, + 262, + 262, + 11, + 198, + 262, + 198, + 198, + 11, + 262, + 11, + 13, + 11, + 262, + 262, + 262, + 198, + 11, + 11, + 262, + 198, + 13, + 11, + 262, + 262, + 262, + 286, + 11, + 262, + 13, + 11, + 13, + 262, + 11, + 11, + 11, + 13, + 13, + 13, + 284, + 262, + 13, + 198, + 13, + 262, + 11, + 11, + 262, + 198, + 11, + 286, + 284, + 11, + 262, + 198, + 262, + 11, + 262, + 13, + 262, + 13, + 11, + 262, + 262, + 13, + 11, + 262, + 11, + 262, + 13, + 198, + 198, + 13, + 286, + 11, + 11, + 11, + 262, + 284, + 198, + 11, + 13, + 11, + 262, + 11, + 198, + 13, + 198, + 198, + 13, + 198, + 262, + 13, + 262, + 13, + 198, + 198, + 13, + 262, + 13, + 11, + 198, + 198, + 198, + 262, + 262, + 13, + 11, + 11, + 262, + 13, + 198, + 13, + 11, + 262, + 13, + 11, + 262, + 11, + 13, + 13, + 262, + 198, + 198, + 262, + 11, + 262, + 11, + 13, + 198, + 198, + 11, + 262, + 262, + 11, + 11, + 13, + 11, + 262, + 13, + 13, + 257, + 198, + 198, + 262, + 13, + 198, + 198, + 11, + 198, + 13, + 198, + 198, + 198, + 262, + 262, + 284, + 13, + 11, + 198, + 198, + 198, + 198, + 262, + 262, + 284, + 11, + 13, + 13, + 198, + 198, + 198, + 11, + 11, + 11, + 11, + 262, + 262, + 198, + 11, + 11, + 262, + 198, + 198, + 13, + 13, + 11, + 11, + 13, + 262, + 13, + 11, + 198, + 11, + 13, + 198, + 11, + 262, + 198, + 13, + 13, + 262, + 198, + 198, + 11, + 13, + 198, + 11, + 13, + 13, + 262, + 11, + 262, + 262, + 284, + 198, + 198, + 11, + 13, + 262, + 262, + 198, + 13, + 13, + 262, + 11, + 11, + 262, + 11, + 262, + 262, + 198, + 13, + 13, + 11, + 11, + 11, + 13, + 11, + 262, + 262, + 262, + 262, + 11, + 198, + 11, + 198, + 13, + 262, + 11, + 11, + 262, + 262, + 13, + 13, + 11, + 198, + 198, + 13, + 198, + 11, + 262, + 13, + 198, + 198, + 198, + 262, + 13, + 13, + 13, + 262, + 284, + 11, + 262, + 198, + 198, + 13, + 262, + 11, + 13, + 13, + 13, + 262, + 11, + 262, + 262, + 262, + 11, + 198, + 198, + 13, + 11, + 262, + 13, + 198, + 198, + 11, + 13, + 13, + 198, + 262, + 13, + 11, + 262, + 262, + 11, + 11, + 262, + 198, + 11, + 13, + 11, + 284, + 13, + 11, + 13, + 262, + 13, + 11, + 198, + 198, + 262, + 11, + 262, + 11, + 198, + 11, + 11, + 11, + 13, + 198, + 262, + 11, + 11, + 198, + 198, + 11, + 13, + 262, + 198, + 11, + 262, + 13, + 13, + 13, + 198, + 13, + 198, + 262, + 262, + 13, + 13, + 198, + 262, + 13, + 11, + 11, + 11, + 198, + 13, + 11, + 13, + 11, + 13, + 262, + 262, + 13, + 262, + 198, + 11, + 11, + 262, + 262, + 262, + 13, + 198, + 11, + 11, + 13, + 198, + 198, + 198, + 198, + 262, + 198, + 11, + 262, + 262, + 262, + 11, + 262, + 198, + 11, + 13, + 262, + 262, + 262, + 11, + 198, + 198, + 13, + 262, + 262, + 262, + 262, + 13, + 262, + 198, + 11, + 11, + 11, + 262, + 198, + 13, + 262, + 11, + 13, + 13, + 13, + 262, + 262, + 262, + 13, + 11, + 198, + 198, + 11, + 13, + 262, + 198, + 198, + 13, + 11, + 11, + 262, + 13, + 13, + 13, + 262, + 198, + 13, + 13, + 198, + 13, + 262, + 262, + 198, + 13, + 11, + 198, + 262, + 11, + 13, + 262, + 198, + 13, + 262, + 198, + 286, + 11, + 13, + 198, + 198, + 198, + 13, + 262, + 198, + 11, + 11, + 262, + 262, + 262, + 13, + 286, + 262, + 11, + 11, + 13, + 262, + 262, + 262, + 262, + 11, + 13, + 13, + 198, + 11, + 11, + 13, + 11, + 11, + 11, + 13, + 13, + 198, + 198, + 262, + 262, + 262, + 262, + 198, + 262, + 198, + 11, + 262, + 11, + 11, + 198, + 198, + 262, + 198, + 198, + 13, + 13, + 198, + 13, + 11, + 11, + 11, + 262, + 284, + 13, + 13, + 11, + 198, + 198, + 13, + 11, + 13, + 11, + 262, + 13, + 262, + 11, + 262, + 290, + 262, + 198, + 13, + 11, + 198, + 11, + 11, + 198, + 262, + 198, + 13, + 13, + 262, + 13, + 262, + 198, + 11, + 11, + 262, + 198, + 262, + 13, + 11, + 198, + 13, + 13, + 262, + 198, + 13, + 11, + 262, + 11, + 198, + 198, + 13, + 13, + 198, + 11, + 13, + 11, + 198, + 262, + 262, + 11, + 284, + 13, + 262, + 13, + 198, + 262, + 11, + 11, + 262, + 13, + 262, + 262, + 290, + 11, + 262, + 198, + 13, + 198, + 13, + 13, + 198, + 11, + 11, + 13, + 11, + 198, + 11, + 262, + 13, + 11, + 13, + 198, + 198, + 11, + 11, + 284, + 262, + 11, + 11, + 11, + 13, + 13, + 11, + 284, + 262, + 262, + 262, + 11, + 11, + 198, + 13, + 198, + 198, + 198, + 13, + 11, + 262, + 198, + 198, + 198, + 13, + 13, + 198, + 262, + 262, + 13, + 262, + 262, + 262, + 262, + 11, + 262, + 198, + 11, + 262, + 262, + 11, + 11, + 262, + 262, + 13, + 11, + 198, + 11, + 13, + 198, + 262, + 11, + 11, + 13, + 13, + 262, + 11, + 13, + 13, + 11, + 262, + 11, + 262, + 198, + 13, + 198, + 198, + 284, + 11, + 262, + 11, + 284, + 284, + 13, + 198, + 13, + 198, + 262, + 262, + 262, + 13, + 13, + 262, + 11, + 13, + 262, + 13, + 11, + 262, + 262, + 13, + 13, + 11, + 198, + 262, + 11, + 11, + 13, + 284, + 13, + 262, + 262, + 11, + 11, + 11, + 262, + 290, + 11, + 13, + 262, + 286, + 198, + 11, + 11, + 13, + 198, + 198, + 11, + 198, + 262, + 11, + 13, + 262, + 13, + 286, + 11, + 262, + 11, + 13, + 262, + 198, + 262, + 262, + 13, + 11, + 262, + 13, + 198, + 262, + 11, + 11, + 262, + 13, + 11, + 13, + 198, + 198, + 262, + 13, + 262, + 13, + 13, + 262, + 262, + 13, + 284, + 13, + 13, + 11, + 11, + 11, + 262, + 262, + 13, + 11, + 262, + 262, + 11, + 262, + 13, + 13, + 11, + 11, + 262, + 11, + 13, + 11, + 262, + 11, + 262, + 13, + 11, + 262, + 13, + 13, + 286, + 262, + 13, + 11, + 13, + 262, + 11, + 198, + 262, + 262, + 13, + 11, + 286, + 262, + 13, + 11, + 262, + 11, + 262, + 11, + 11, + 11, + 262, + 262, + 262, + 262, + 11, + 286, + 13, + 198, + 198, + 11, + 262, + 198, + 198, + 198, + 13, + 13, + 262, + 11, + 13, + 262, + 11, + 262, + 13, + 262, + 11, + 11, + 13, + 262, + 262, + 11, + 262, + 262, + 11, + 13, + 198, + 198, + 11, + 198, + 11, + 262, + 262, + 11, + 13, + 198, + 11, + 13, + 13, + 11, + 13, + 13, + 198, + 11, + 262, + 11, + 13, + 198, + 13, + 262, + 13, + 13, + 198, + 262, + 262, + 13, + 284, + 284, + 262, + 198, + 13, + 13, + 13, + 262, + 13, + 11, + 262, + 11, + 198, + 11, + 13, + 13, + 13, + 262, + 198, + 11, + 13, + 262, + 262, + 13, + 262, + 198, + 11, + 13, + 11, + 286, + 262, + 262, + 198, + 198, + 198, + 262, + 262, + 13, + 198, + 11, + 11, + 11, + 13, + 198, + 11, + 11, + 11, + 11, + 262, + 13, + 50256 + ] + }, + { + "sample_id": 11, + "text": ".. to,. the, of,. of,\n., the..\n\n\n. the\n the of the the to. the. to. the...\n. the,, the\n\n the. the,\n..,,,, and. to the,,\n. a the,., the, the\n,,,, the, of,.\n\n\n. the\n.,\n,.\n.\n the the the,, the.,. to the the.\n the.\n\n,\n,, the, the..\n\n\n., to,,\n\n the..\n the,,. the. in the\n\n.\n,.,\n.\n\n.., the.,. the.\n.,\n the the the. of, the the. the the the. the. of the the,,.,,., a the\n\n\n, the. the, the. the..\n, to\n\n.. the the of,, the the.\n\n,,,. the the,,.,\n\n\n. the the the,,,. the.. the,,,\n\n the.,.\n the,. the the\n\n\n,.\n\n the,.. the.\n, the,. the the, the the the. the.,.\n\n.\n.\n.\n the\n, the.\n\n the,, the, of,.,, the,\n to the.,.,, the..\n,, the,... the the, the., the the,,., and,,\n the the,,. the the, to,..,.\n\n, the,,, the, the the the,\n the,, the, the the the the, the the,,. of,.,.\n, the\n the\n the,\n\n\n a\n,. the the.\n\n the. the, the the, the the.,, the\n the,,,,. the the,\n,\n,.,,,, the.. the the.\n, the the. to the,...\n the, the.,,.\n, the, the\n\n. the\n the the\n. the, the. the the the\n.\n\n the\n. the..\n,, the the,,\n\n of the,,. the\n\n\n\n.,. the the\n\n,\n, the the, the the\n,,, the\n. the\n\n, the,\n\n. the the,\n\n,,,,. and,,.. the.\n the.\n, the,\n the\n, the. the the.\n the,, the the.,,.,,..\n\n\n. the the, the.,,,\n.\n. the the the,,, the,.. the the the,, the.,,,.\n\n the,..\n\n. the the, of.,.\n,,. the,,.\n, the.\n\n the,.,\n.\n., the. the the.,..,\n.. the, the. the\n. the.,,,, the\n\n.,,, the..\n the.,. in,\n\n.., the...,,.\n the\n, the. the\n the. the\n the,\n\n,,,,,, the. the\n,,\n\n,,\n. the the the\n. the..,.\n,,,,.\n\n the..\n,, of\n\n the. the.\n\n the.\n\n the,..\n,., the, the the the, the the,. the.,. and\n\n,..,,., the the\n,\n\n\n, the.\n, the\n,,, the, the\n\n, the.\n the the the.\n\n the the..., the the the, of the,., the the\n, the.\n\n the the..\n\n,, the the.\n\n\n\n..., the, the,, a the the. the., the,.. the\n,,\n\n to the,.. the.\n,, the,., the the.,. the the\n, the.,.,. the,,,..,,,,, the\n.,.\n. the,\n,, the,,,\n,,.... the..,\n,., the,\n,,,\n the. the\n, the", + "token_ids": [ + 13, + 13, + 284, + 11, + 13, + 262, + 11, + 286, + 11, + 13, + 286, + 11, + 198, + 13, + 11, + 262, + 13, + 13, + 198, + 198, + 198, + 13, + 262, + 198, + 262, + 286, + 262, + 262, + 284, + 13, + 262, + 13, + 284, + 13, + 262, + 13, + 13, + 13, + 198, + 13, + 262, + 11, + 11, + 262, + 198, + 198, + 262, + 13, + 262, + 11, + 198, + 13, + 13, + 11, + 11, + 11, + 11, + 290, + 13, + 284, + 262, + 11, + 11, + 198, + 13, + 257, + 262, + 11, + 13, + 11, + 262, + 11, + 262, + 198, + 11, + 11, + 11, + 11, + 262, + 11, + 286, + 11, + 13, + 198, + 198, + 198, + 13, + 262, + 198, + 13, + 11, + 198, + 11, + 13, + 198, + 13, + 198, + 262, + 262, + 262, + 11, + 11, + 262, + 13, + 11, + 13, + 284, + 262, + 262, + 13, + 198, + 262, + 13, + 198, + 198, + 11, + 198, + 11, + 11, + 262, + 11, + 262, + 13, + 13, + 198, + 198, + 198, + 13, + 11, + 284, + 11, + 11, + 198, + 198, + 262, + 13, + 13, + 198, + 262, + 11, + 11, + 13, + 262, + 13, + 287, + 262, + 198, + 198, + 13, + 198, + 11, + 13, + 11, + 198, + 13, + 198, + 198, + 13, + 13, + 11, + 262, + 13, + 11, + 13, + 262, + 13, + 198, + 13, + 11, + 198, + 262, + 262, + 262, + 13, + 286, + 11, + 262, + 262, + 13, + 262, + 262, + 262, + 13, + 262, + 13, + 286, + 262, + 262, + 11, + 11, + 13, + 11, + 11, + 13, + 11, + 257, + 262, + 198, + 198, + 198, + 11, + 262, + 13, + 262, + 11, + 262, + 13, + 262, + 13, + 13, + 198, + 11, + 284, + 198, + 198, + 13, + 13, + 262, + 262, + 286, + 11, + 11, + 262, + 262, + 13, + 198, + 198, + 11, + 11, + 11, + 13, + 262, + 262, + 11, + 11, + 13, + 11, + 198, + 198, + 198, + 13, + 262, + 262, + 262, + 11, + 11, + 11, + 13, + 262, + 13, + 13, + 262, + 11, + 11, + 11, + 198, + 198, + 262, + 13, + 11, + 13, + 198, + 262, + 11, + 13, + 262, + 262, + 198, + 198, + 198, + 11, + 13, + 198, + 198, + 262, + 11, + 13, + 13, + 262, + 13, + 198, + 11, + 262, + 11, + 13, + 262, + 262, + 11, + 262, + 262, + 262, + 13, + 262, + 13, + 11, + 13, + 198, + 198, + 13, + 198, + 13, + 198, + 13, + 198, + 262, + 198, + 11, + 262, + 13, + 198, + 198, + 262, + 11, + 11, + 262, + 11, + 286, + 11, + 13, + 11, + 11, + 262, + 11, + 198, + 284, + 262, + 13, + 11, + 13, + 11, + 11, + 262, + 13, + 13, + 198, + 11, + 11, + 262, + 11, + 13, + 13, + 13, + 262, + 262, + 11, + 262, + 13, + 11, + 262, + 262, + 11, + 11, + 13, + 11, + 290, + 11, + 11, + 198, + 262, + 262, + 11, + 11, + 13, + 262, + 262, + 11, + 284, + 11, + 13, + 13, + 11, + 13, + 198, + 198, + 11, + 262, + 11, + 11, + 11, + 262, + 11, + 262, + 262, + 262, + 11, + 198, + 262, + 11, + 11, + 262, + 11, + 262, + 262, + 262, + 262, + 11, + 262, + 262, + 11, + 11, + 13, + 286, + 11, + 13, + 11, + 13, + 198, + 11, + 262, + 198, + 262, + 198, + 262, + 11, + 198, + 198, + 198, + 257, + 198, + 11, + 13, + 262, + 262, + 13, + 198, + 198, + 262, + 13, + 262, + 11, + 262, + 262, + 11, + 262, + 262, + 13, + 11, + 11, + 262, + 198, + 262, + 11, + 11, + 11, + 11, + 13, + 262, + 262, + 11, + 198, + 11, + 198, + 11, + 13, + 11, + 11, + 11, + 11, + 262, + 13, + 13, + 262, + 262, + 13, + 198, + 11, + 262, + 262, + 13, + 284, + 262, + 11, + 13, + 13, + 13, + 198, + 262, + 11, + 262, + 13, + 11, + 11, + 13, + 198, + 11, + 262, + 11, + 262, + 198, + 198, + 13, + 262, + 198, + 262, + 262, + 198, + 13, + 262, + 11, + 262, + 13, + 262, + 262, + 262, + 198, + 13, + 198, + 198, + 262, + 198, + 13, + 262, + 13, + 13, + 198, + 11, + 11, + 262, + 262, + 11, + 11, + 198, + 198, + 286, + 262, + 11, + 11, + 13, + 262, + 198, + 198, + 198, + 198, + 13, + 11, + 13, + 262, + 262, + 198, + 198, + 11, + 198, + 11, + 262, + 262, + 11, + 262, + 262, + 198, + 11, + 11, + 11, + 262, + 198, + 13, + 262, + 198, + 198, + 11, + 262, + 11, + 198, + 198, + 13, + 262, + 262, + 11, + 198, + 198, + 11, + 11, + 11, + 11, + 13, + 290, + 11, + 11, + 13, + 13, + 262, + 13, + 198, + 262, + 13, + 198, + 11, + 262, + 11, + 198, + 262, + 198, + 11, + 262, + 13, + 262, + 262, + 13, + 198, + 262, + 11, + 11, + 262, + 262, + 13, + 11, + 11, + 13, + 11, + 11, + 13, + 13, + 198, + 198, + 198, + 13, + 262, + 262, + 11, + 262, + 13, + 11, + 11, + 11, + 198, + 13, + 198, + 13, + 262, + 262, + 262, + 11, + 11, + 11, + 262, + 11, + 13, + 13, + 262, + 262, + 262, + 11, + 11, + 262, + 13, + 11, + 11, + 11, + 13, + 198, + 198, + 262, + 11, + 13, + 13, + 198, + 198, + 13, + 262, + 262, + 11, + 286, + 13, + 11, + 13, + 198, + 11, + 11, + 13, + 262, + 11, + 11, + 13, + 198, + 11, + 262, + 13, + 198, + 198, + 262, + 11, + 13, + 11, + 198, + 13, + 198, + 13, + 11, + 262, + 13, + 262, + 262, + 13, + 11, + 13, + 13, + 11, + 198, + 13, + 13, + 262, + 11, + 262, + 13, + 262, + 198, + 13, + 262, + 13, + 11, + 11, + 11, + 11, + 262, + 198, + 198, + 13, + 11, + 11, + 11, + 262, + 13, + 13, + 198, + 262, + 13, + 11, + 13, + 287, + 11, + 198, + 198, + 13, + 13, + 11, + 262, + 13, + 13, + 13, + 11, + 11, + 13, + 198, + 262, + 198, + 11, + 262, + 13, + 262, + 198, + 262, + 13, + 262, + 198, + 262, + 11, + 198, + 198, + 11, + 11, + 11, + 11, + 11, + 11, + 262, + 13, + 262, + 198, + 11, + 11, + 198, + 198, + 11, + 11, + 198, + 13, + 262, + 262, + 262, + 198, + 13, + 262, + 13, + 13, + 11, + 13, + 198, + 11, + 11, + 11, + 11, + 13, + 198, + 198, + 262, + 13, + 13, + 198, + 11, + 11, + 286, + 198, + 198, + 262, + 13, + 262, + 13, + 198, + 198, + 262, + 13, + 198, + 198, + 262, + 11, + 13, + 13, + 198, + 11, + 13, + 11, + 262, + 11, + 262, + 262, + 262, + 11, + 262, + 262, + 11, + 13, + 262, + 13, + 11, + 13, + 290, + 198, + 198, + 11, + 13, + 13, + 11, + 11, + 13, + 11, + 262, + 262, + 198, + 11, + 198, + 198, + 198, + 11, + 262, + 13, + 198, + 11, + 262, + 198, + 11, + 11, + 11, + 262, + 11, + 262, + 198, + 198, + 11, + 262, + 13, + 198, + 262, + 262, + 262, + 13, + 198, + 198, + 262, + 262, + 13, + 13, + 13, + 11, + 262, + 262, + 262, + 11, + 286, + 262, + 11, + 13, + 11, + 262, + 262, + 198, + 11, + 262, + 13, + 198, + 198, + 262, + 262, + 13, + 13, + 198, + 198, + 11, + 11, + 262, + 262, + 13, + 198, + 198, + 198, + 198, + 13, + 13, + 13, + 11, + 262, + 11, + 262, + 11, + 11, + 257, + 262, + 262, + 13, + 262, + 13, + 11, + 262, + 11, + 13, + 13, + 262, + 198, + 11, + 11, + 198, + 198, + 284, + 262, + 11, + 13, + 13, + 262, + 13, + 198, + 11, + 11, + 262, + 11, + 13, + 11, + 262, + 262, + 13, + 11, + 13, + 262, + 262, + 198, + 11, + 262, + 13, + 11, + 13, + 11, + 13, + 262, + 11, + 11, + 11, + 13, + 13, + 11, + 11, + 11, + 11, + 11, + 262, + 198, + 13, + 11, + 13, + 198, + 13, + 262, + 11, + 198, + 11, + 11, + 262, + 11, + 11, + 11, + 198, + 11, + 11, + 13, + 13, + 13, + 13, + 262, + 13, + 13, + 11, + 198, + 11, + 13, + 11, + 262, + 11, + 198, + 11, + 11, + 11, + 198, + 262, + 13, + 262, + 198, + 11, + 262 + ] + }, + { + "sample_id": 12, + "text": "<|endoftext|>.,\n,. the,,,,\n,\n,,..\n to,, the,.\n.,..\n\n the the. the, the, the the. the the, the..,.. the., the the,\n to the the the, the. the.,,,. the,,,. the. the.,,.,, the the.,, the the. to the the,\n,.., the the,. the\n..,, the the. the,,,., the the\n,, the, of., the., and the. the the,.., the.,,. the,,. the the the the the,. the.\n,.,.. the, the.\n\n.,, the\n...\n the,\n the the the.. the,\n,\n, the, the. the\n,,. the\n\n.,,\n\n\n\n..,. a., the\n\n,,,\n\n. the,..\n\n the,, the, the.,\n,, the, the,.,\n\n,, the,. the, the.,, the\n, the,\n, to the. the., the,. the,, the, of the the. the the the., the,,,, the. the the...\n\n,.,.,. the,..\n,\n the,,.,.,.\n,,\n.\n,, the. the\n,. the\n,, the,... of the,., to the the\n.,\n, the..\n\n. the,,,,, the,, the the to,,,, the..\n\n.. the\n,, and the.,., the.\n the the\n the a the\n\n.,\n the,.,,,., the, the, the. the the.\n\n, the the to,\n\n. of,...,, the,, the\n\n the..,\n\n..,. the,\n the., a the to to the,,,, the the. the,,,,, the..\n, the,.\n,., the the the, the.. the,, the....., the,\n\n, the, a,, a. the the\n and,,.\n.,,, a\n.. the the.,\n.., the a. the.\n.,,., of the the the.. the,..,,\n..,\n the the the,..\n,,, the,, the,\n\n,\n the., the,,, the the,\n, the,,.\n. the. the\n the, the\n\n\n the the the the, a,..,,.,,,,,,,\n.,\n\n, the\n, the the,...,..\n,.. the.,\n.\n,, the.., the the. the,, the the the\n.\n. the,,,,. the,.., the.,,. the\n\n the,...,\n, the the,, the the the,,\n\n., the the.\n, the, the.. the,,.,.\n\n.,, the, and the,\n... the the\n\n\n, the., the the,. the the,.,.,.\n.. the,.\n the, and, the the...\n.,,\n. the.,,. of the, the.\n. the. to,. the\n\n the,, the,,. the, the.\n\n.. the,,., the,, the.\n,\n,.,\n of the....,, the the,,,. the,\n., the.. the\n\n the... the,., the. the, the\n,\n\n of,. the the the, the the\n..... the\n the the the the the the,,, the the....\n, the the,.\n. the\n..\n the\n,,,,.., the,,..,,,, to.,.,\n the the the. the the.. the.,, the the, the\n\n.,,,, the the, to\n, the the,,..,..\n the, the,, and the<|endoftext|>", + "token_ids": [ + 50256, + 13, + 11, + 198, + 11, + 13, + 262, + 11, + 11, + 11, + 11, + 198, + 11, + 198, + 11, + 11, + 13, + 13, + 198, + 284, + 11, + 11, + 262, + 11, + 13, + 198, + 13, + 11, + 13, + 13, + 198, + 198, + 262, + 262, + 13, + 262, + 11, + 262, + 11, + 262, + 262, + 13, + 262, + 262, + 11, + 262, + 13, + 13, + 11, + 13, + 13, + 262, + 13, + 11, + 262, + 262, + 11, + 198, + 284, + 262, + 262, + 262, + 11, + 262, + 13, + 262, + 13, + 11, + 11, + 11, + 13, + 262, + 11, + 11, + 11, + 13, + 262, + 13, + 262, + 13, + 11, + 11, + 13, + 11, + 11, + 262, + 262, + 13, + 11, + 11, + 262, + 262, + 13, + 284, + 262, + 262, + 11, + 198, + 11, + 13, + 13, + 11, + 262, + 262, + 11, + 13, + 262, + 198, + 13, + 13, + 11, + 11, + 262, + 262, + 13, + 262, + 11, + 11, + 11, + 13, + 11, + 262, + 262, + 198, + 11, + 11, + 262, + 11, + 286, + 13, + 11, + 262, + 13, + 11, + 290, + 262, + 13, + 262, + 262, + 11, + 13, + 13, + 11, + 262, + 13, + 11, + 11, + 13, + 262, + 11, + 11, + 13, + 262, + 262, + 262, + 262, + 262, + 11, + 13, + 262, + 13, + 198, + 11, + 13, + 11, + 13, + 13, + 262, + 11, + 262, + 13, + 198, + 198, + 13, + 11, + 11, + 262, + 198, + 13, + 13, + 13, + 198, + 262, + 11, + 198, + 262, + 262, + 262, + 13, + 13, + 262, + 11, + 198, + 11, + 198, + 11, + 262, + 11, + 262, + 13, + 262, + 198, + 11, + 11, + 13, + 262, + 198, + 198, + 13, + 11, + 11, + 198, + 198, + 198, + 198, + 13, + 13, + 11, + 13, + 257, + 13, + 11, + 262, + 198, + 198, + 11, + 11, + 11, + 198, + 198, + 13, + 262, + 11, + 13, + 13, + 198, + 198, + 262, + 11, + 11, + 262, + 11, + 262, + 13, + 11, + 198, + 11, + 11, + 262, + 11, + 262, + 11, + 13, + 11, + 198, + 198, + 11, + 11, + 262, + 11, + 13, + 262, + 11, + 262, + 13, + 11, + 11, + 262, + 198, + 11, + 262, + 11, + 198, + 11, + 284, + 262, + 13, + 262, + 13, + 11, + 262, + 11, + 13, + 262, + 11, + 11, + 262, + 11, + 286, + 262, + 262, + 13, + 262, + 262, + 262, + 13, + 11, + 262, + 11, + 11, + 11, + 11, + 262, + 13, + 262, + 262, + 13, + 13, + 13, + 198, + 198, + 11, + 13, + 11, + 13, + 11, + 13, + 262, + 11, + 13, + 13, + 198, + 11, + 198, + 262, + 11, + 11, + 13, + 11, + 13, + 11, + 13, + 198, + 11, + 11, + 198, + 13, + 198, + 11, + 11, + 262, + 13, + 262, + 198, + 11, + 13, + 262, + 198, + 11, + 11, + 262, + 11, + 13, + 13, + 13, + 286, + 262, + 11, + 13, + 11, + 284, + 262, + 262, + 198, + 13, + 11, + 198, + 11, + 262, + 13, + 13, + 198, + 198, + 13, + 262, + 11, + 11, + 11, + 11, + 11, + 262, + 11, + 11, + 262, + 262, + 284, + 11, + 11, + 11, + 11, + 262, + 13, + 13, + 198, + 198, + 13, + 13, + 262, + 198, + 11, + 11, + 290, + 262, + 13, + 11, + 13, + 11, + 262, + 13, + 198, + 262, + 262, + 198, + 262, + 257, + 262, + 198, + 198, + 13, + 11, + 198, + 262, + 11, + 13, + 11, + 11, + 11, + 13, + 11, + 262, + 11, + 262, + 11, + 262, + 13, + 262, + 262, + 13, + 198, + 198, + 11, + 262, + 262, + 284, + 11, + 198, + 198, + 13, + 286, + 11, + 13, + 13, + 13, + 11, + 11, + 262, + 11, + 11, + 262, + 198, + 198, + 262, + 13, + 13, + 11, + 198, + 198, + 13, + 13, + 11, + 13, + 262, + 11, + 198, + 262, + 13, + 11, + 257, + 262, + 284, + 284, + 262, + 11, + 11, + 11, + 11, + 262, + 262, + 13, + 262, + 11, + 11, + 11, + 11, + 11, + 262, + 13, + 13, + 198, + 11, + 262, + 11, + 13, + 198, + 11, + 13, + 11, + 262, + 262, + 262, + 11, + 262, + 13, + 13, + 262, + 11, + 11, + 262, + 13, + 13, + 13, + 13, + 13, + 11, + 262, + 11, + 198, + 198, + 11, + 262, + 11, + 257, + 11, + 11, + 257, + 13, + 262, + 262, + 198, + 290, + 11, + 11, + 13, + 198, + 13, + 11, + 11, + 11, + 257, + 198, + 13, + 13, + 262, + 262, + 13, + 11, + 198, + 13, + 13, + 11, + 262, + 257, + 13, + 262, + 13, + 198, + 13, + 11, + 11, + 13, + 11, + 286, + 262, + 262, + 262, + 13, + 13, + 262, + 11, + 13, + 13, + 11, + 11, + 198, + 13, + 13, + 11, + 198, + 262, + 262, + 262, + 11, + 13, + 13, + 198, + 11, + 11, + 11, + 262, + 11, + 11, + 262, + 11, + 198, + 198, + 11, + 198, + 262, + 13, + 11, + 262, + 11, + 11, + 11, + 262, + 262, + 11, + 198, + 11, + 262, + 11, + 11, + 13, + 198, + 13, + 262, + 13, + 262, + 198, + 262, + 11, + 262, + 198, + 198, + 198, + 262, + 262, + 262, + 262, + 11, + 257, + 11, + 13, + 13, + 11, + 11, + 13, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 198, + 13, + 11, + 198, + 198, + 11, + 262, + 198, + 11, + 262, + 262, + 11, + 13, + 13, + 13, + 11, + 13, + 13, + 198, + 11, + 13, + 13, + 262, + 13, + 11, + 198, + 13, + 198, + 11, + 11, + 262, + 13, + 13, + 11, + 262, + 262, + 13, + 262, + 11, + 11, + 262, + 262, + 262, + 198, + 13, + 198, + 13, + 262, + 11, + 11, + 11, + 11, + 13, + 262, + 11, + 13, + 13, + 11, + 262, + 13, + 11, + 11, + 13, + 262, + 198, + 198, + 262, + 11, + 13, + 13, + 13, + 11, + 198, + 11, + 262, + 262, + 11, + 11, + 262, + 262, + 262, + 11, + 11, + 198, + 198, + 13, + 11, + 262, + 262, + 13, + 198, + 11, + 262, + 11, + 262, + 13, + 13, + 262, + 11, + 11, + 13, + 11, + 13, + 198, + 198, + 13, + 11, + 11, + 262, + 11, + 290, + 262, + 11, + 198, + 13, + 13, + 13, + 262, + 262, + 198, + 198, + 198, + 11, + 262, + 13, + 11, + 262, + 262, + 11, + 13, + 262, + 262, + 11, + 13, + 11, + 13, + 11, + 13, + 198, + 13, + 13, + 262, + 11, + 13, + 198, + 262, + 11, + 290, + 11, + 262, + 262, + 13, + 13, + 13, + 198, + 13, + 11, + 11, + 198, + 13, + 262, + 13, + 11, + 11, + 13, + 286, + 262, + 11, + 262, + 13, + 198, + 13, + 262, + 13, + 284, + 11, + 13, + 262, + 198, + 198, + 262, + 11, + 11, + 262, + 11, + 11, + 13, + 262, + 11, + 262, + 13, + 198, + 198, + 13, + 13, + 262, + 11, + 11, + 13, + 11, + 262, + 11, + 11, + 262, + 13, + 198, + 11, + 198, + 11, + 13, + 11, + 198, + 286, + 262, + 13, + 13, + 13, + 13, + 11, + 11, + 262, + 262, + 11, + 11, + 11, + 13, + 262, + 11, + 198, + 13, + 11, + 262, + 13, + 13, + 262, + 198, + 198, + 262, + 13, + 13, + 13, + 262, + 11, + 13, + 11, + 262, + 13, + 262, + 11, + 262, + 198, + 11, + 198, + 198, + 286, + 11, + 13, + 262, + 262, + 262, + 11, + 262, + 262, + 198, + 13, + 13, + 13, + 13, + 13, + 262, + 198, + 262, + 262, + 262, + 262, + 262, + 262, + 11, + 11, + 11, + 262, + 262, + 13, + 13, + 13, + 13, + 198, + 11, + 262, + 262, + 11, + 13, + 198, + 13, + 262, + 198, + 13, + 13, + 198, + 262, + 198, + 11, + 11, + 11, + 11, + 13, + 13, + 11, + 262, + 11, + 11, + 13, + 13, + 11, + 11, + 11, + 11, + 284, + 13, + 11, + 13, + 11, + 198, + 262, + 262, + 262, + 13, + 262, + 262, + 13, + 13, + 262, + 13, + 11, + 11, + 262, + 262, + 11, + 262, + 198, + 198, + 13, + 11, + 11, + 11, + 11, + 262, + 262, + 11, + 284, + 198, + 11, + 262, + 262, + 11, + 11, + 13, + 13, + 11, + 13, + 13, + 198, + 262, + 11, + 262, + 11, + 11, + 290, + 262, + 50256 + ] + }, + { + "sample_id": 13, + "text": "<|endoftext|> the\n,. the,,, the.,, the the,,.\n the the the,,,.\n\n the,,,,,,.,.,,, and..\n., the\n.\n\n\n,. the the, the the and, and the, the,...\n the the.,.\n\n,.,.,, the.,\n. the,,,., the\n,.. the the. the. and the, the the\n, of.,. the of.,,,.\n the the,,,...,,. the the\n the\n\n. the.\n..\n. the\n,,,,,,\n. the. the\n\n the, the the....\n the,. the,.. to, the,\n the, the,.\n the, the,,.,,\n the the,.,,\n,, the,, the\n\n the.\n.. the, the, the., the the.. the\n\n,,, the. the the the,, the the the, the the,,.,. the, the the, the, to. the,, the the the the, of the. to..\n the and. the, the of the the of the..,,,.\n\n\n,\n., the,\n\n the,.\n,.,., the,...\n, the. the\n the.\n\n.\n\n the,\n the.. the, the\n the..\n the. the the.,\n, the.\n\n\n,, the the, of of.,. the..\n\n\n, the\n the the,, the\n,.\n the,,\n, the. the\n the the of..\n the... the the the.\n\n. the a, the\n. the.\n., the... the,,.., the. the.\n\n,. the.\n\n the, the,\n.\n the.\n\n\n\n, the\n the a, the, and of the the.\n\n\n\n.,\n,, the to,.,.\n\n the the.,., the the. the. a.\n\n\n a the\n,, the.,.,, the,, the the,....\n\n\n the. the the,..\n.. to\n the the,,., to,. the the.,\n. the,.,\n\n the to,, the,,,\n, the,\n the, the\n. the,, and.,\n,,,. the. of... to, of,. the, the,, the.\n. the the the,,, the.,\n\n. the the.,, the, the\n\n, the\n, the the.\n\n the the, the,\n to the to the,, the the, the to\n, the.\n\n,, the and the, the\n, the\n the. to,., the, the., the.\n. the,.\n,.\n, the the and the the.. the the\n., and.,\n the the,. the\n\n. to, to the., to the the the,. the.. the,,.. the the,.\n, the,\n.,.,\n the, the the the the...\n\n.,\n,., the,,.\n the, the.... to the,,. the, the the the, the. the\n the the\n,\n the,, the the. the,, the the the..\n, the the. the...,. the the the,. the the...\n\n,.,,\n\n the, the,\n\n,.\n\n. the,. the. to\n\n.. the, the the. the,\n\n the.,, the\n\n\n,.\n the, the\n the\n the,. the.\n\n of. the the,. the.\n. the,\n to,,.,\n, the,,, the, the the the\n\n,, the. the the the.\n\n.\n,. of.\n\n., a. to to,.\n\n, the\n\n the the,,\n the the,, the, the.,.\n\n the., to\n,,\n to,,\n the,,\n the,\n,. of. the, the the\n\n,, the,.\n.,\n\n", + "token_ids": [ + 50256, + 262, + 198, + 11, + 13, + 262, + 11, + 11, + 11, + 262, + 13, + 11, + 11, + 262, + 262, + 11, + 11, + 13, + 198, + 262, + 262, + 262, + 11, + 11, + 11, + 13, + 198, + 198, + 262, + 11, + 11, + 11, + 11, + 11, + 11, + 13, + 11, + 13, + 11, + 11, + 11, + 290, + 13, + 13, + 198, + 13, + 11, + 262, + 198, + 13, + 198, + 198, + 198, + 11, + 13, + 262, + 262, + 11, + 262, + 262, + 290, + 11, + 290, + 262, + 11, + 262, + 11, + 13, + 13, + 13, + 198, + 262, + 262, + 13, + 11, + 13, + 198, + 198, + 11, + 13, + 11, + 13, + 11, + 11, + 262, + 13, + 11, + 198, + 13, + 262, + 11, + 11, + 11, + 13, + 11, + 262, + 198, + 11, + 13, + 13, + 262, + 262, + 13, + 262, + 13, + 290, + 262, + 11, + 262, + 262, + 198, + 11, + 286, + 13, + 11, + 13, + 262, + 286, + 13, + 11, + 11, + 11, + 13, + 198, + 262, + 262, + 11, + 11, + 11, + 13, + 13, + 13, + 11, + 11, + 13, + 262, + 262, + 198, + 262, + 198, + 198, + 13, + 262, + 13, + 198, + 13, + 13, + 198, + 13, + 262, + 198, + 11, + 11, + 11, + 11, + 11, + 11, + 198, + 13, + 262, + 13, + 262, + 198, + 198, + 262, + 11, + 262, + 262, + 13, + 13, + 13, + 13, + 198, + 262, + 11, + 13, + 262, + 11, + 13, + 13, + 284, + 11, + 262, + 11, + 198, + 262, + 11, + 262, + 11, + 13, + 198, + 262, + 11, + 262, + 11, + 11, + 13, + 11, + 11, + 198, + 262, + 262, + 11, + 13, + 11, + 11, + 198, + 11, + 11, + 262, + 11, + 11, + 262, + 198, + 198, + 262, + 13, + 198, + 13, + 13, + 262, + 11, + 262, + 11, + 262, + 13, + 11, + 262, + 262, + 13, + 13, + 262, + 198, + 198, + 11, + 11, + 11, + 262, + 13, + 262, + 262, + 262, + 11, + 11, + 262, + 262, + 262, + 11, + 262, + 262, + 11, + 11, + 13, + 11, + 13, + 262, + 11, + 262, + 262, + 11, + 262, + 11, + 284, + 13, + 262, + 11, + 11, + 262, + 262, + 262, + 262, + 11, + 286, + 262, + 13, + 284, + 13, + 13, + 198, + 262, + 290, + 13, + 262, + 11, + 262, + 286, + 262, + 262, + 286, + 262, + 13, + 13, + 11, + 11, + 11, + 13, + 198, + 198, + 198, + 11, + 198, + 13, + 11, + 262, + 11, + 198, + 198, + 262, + 11, + 13, + 198, + 11, + 13, + 11, + 13, + 11, + 262, + 11, + 13, + 13, + 13, + 198, + 11, + 262, + 13, + 262, + 198, + 262, + 13, + 198, + 198, + 13, + 198, + 198, + 262, + 11, + 198, + 262, + 13, + 13, + 262, + 11, + 262, + 198, + 262, + 13, + 13, + 198, + 262, + 13, + 262, + 262, + 13, + 11, + 198, + 11, + 262, + 13, + 198, + 198, + 198, + 11, + 11, + 262, + 262, + 11, + 286, + 286, + 13, + 11, + 13, + 262, + 13, + 13, + 198, + 198, + 198, + 11, + 262, + 198, + 262, + 262, + 11, + 11, + 262, + 198, + 11, + 13, + 198, + 262, + 11, + 11, + 198, + 11, + 262, + 13, + 262, + 198, + 262, + 262, + 286, + 13, + 13, + 198, + 262, + 13, + 13, + 13, + 262, + 262, + 262, + 13, + 198, + 198, + 13, + 262, + 257, + 11, + 262, + 198, + 13, + 262, + 13, + 198, + 13, + 11, + 262, + 13, + 13, + 13, + 262, + 11, + 11, + 13, + 13, + 11, + 262, + 13, + 262, + 13, + 198, + 198, + 11, + 13, + 262, + 13, + 198, + 198, + 262, + 11, + 262, + 11, + 198, + 13, + 198, + 262, + 13, + 198, + 198, + 198, + 198, + 11, + 262, + 198, + 262, + 257, + 11, + 262, + 11, + 290, + 286, + 262, + 262, + 13, + 198, + 198, + 198, + 198, + 13, + 11, + 198, + 11, + 11, + 262, + 284, + 11, + 13, + 11, + 13, + 198, + 198, + 262, + 262, + 13, + 11, + 13, + 11, + 262, + 262, + 13, + 262, + 13, + 257, + 13, + 198, + 198, + 198, + 257, + 262, + 198, + 11, + 11, + 262, + 13, + 11, + 13, + 11, + 11, + 262, + 11, + 11, + 262, + 262, + 11, + 13, + 13, + 13, + 13, + 198, + 198, + 198, + 262, + 13, + 262, + 262, + 11, + 13, + 13, + 198, + 13, + 13, + 284, + 198, + 262, + 262, + 11, + 11, + 13, + 11, + 284, + 11, + 13, + 262, + 262, + 13, + 11, + 198, + 13, + 262, + 11, + 13, + 11, + 198, + 198, + 262, + 284, + 11, + 11, + 262, + 11, + 11, + 11, + 198, + 11, + 262, + 11, + 198, + 262, + 11, + 262, + 198, + 13, + 262, + 11, + 11, + 290, + 13, + 11, + 198, + 11, + 11, + 11, + 13, + 262, + 13, + 286, + 13, + 13, + 13, + 284, + 11, + 286, + 11, + 13, + 262, + 11, + 262, + 11, + 11, + 262, + 13, + 198, + 13, + 262, + 262, + 262, + 11, + 11, + 11, + 262, + 13, + 11, + 198, + 198, + 13, + 262, + 262, + 13, + 11, + 11, + 262, + 11, + 262, + 198, + 198, + 11, + 262, + 198, + 11, + 262, + 262, + 13, + 198, + 198, + 262, + 262, + 11, + 262, + 11, + 198, + 284, + 262, + 284, + 262, + 11, + 11, + 262, + 262, + 11, + 262, + 284, + 198, + 11, + 262, + 13, + 198, + 198, + 11, + 11, + 262, + 290, + 262, + 11, + 262, + 198, + 11, + 262, + 198, + 262, + 13, + 284, + 11, + 13, + 11, + 262, + 11, + 262, + 13, + 11, + 262, + 13, + 198, + 13, + 262, + 11, + 13, + 198, + 11, + 13, + 198, + 11, + 262, + 262, + 290, + 262, + 262, + 13, + 13, + 262, + 262, + 198, + 13, + 11, + 290, + 13, + 11, + 198, + 262, + 262, + 11, + 13, + 262, + 198, + 198, + 13, + 284, + 11, + 284, + 262, + 13, + 11, + 284, + 262, + 262, + 262, + 11, + 13, + 262, + 13, + 13, + 262, + 11, + 11, + 13, + 13, + 262, + 262, + 11, + 13, + 198, + 11, + 262, + 11, + 198, + 13, + 11, + 13, + 11, + 198, + 262, + 11, + 262, + 262, + 262, + 262, + 13, + 13, + 13, + 198, + 198, + 13, + 11, + 198, + 11, + 13, + 11, + 262, + 11, + 11, + 13, + 198, + 262, + 11, + 262, + 13, + 13, + 13, + 13, + 284, + 262, + 11, + 11, + 13, + 262, + 11, + 262, + 262, + 262, + 11, + 262, + 13, + 262, + 198, + 262, + 262, + 198, + 11, + 198, + 262, + 11, + 11, + 262, + 262, + 13, + 262, + 11, + 11, + 262, + 262, + 262, + 13, + 13, + 198, + 11, + 262, + 262, + 13, + 262, + 13, + 13, + 13, + 11, + 13, + 262, + 262, + 262, + 11, + 13, + 262, + 262, + 13, + 13, + 13, + 198, + 198, + 11, + 13, + 11, + 11, + 198, + 198, + 262, + 11, + 262, + 11, + 198, + 198, + 11, + 13, + 198, + 198, + 13, + 262, + 11, + 13, + 262, + 13, + 284, + 198, + 198, + 13, + 13, + 262, + 11, + 262, + 262, + 13, + 262, + 11, + 198, + 198, + 262, + 13, + 11, + 11, + 262, + 198, + 198, + 198, + 11, + 13, + 198, + 262, + 11, + 262, + 198, + 262, + 198, + 262, + 11, + 13, + 262, + 13, + 198, + 198, + 286, + 13, + 262, + 262, + 11, + 13, + 262, + 13, + 198, + 13, + 262, + 11, + 198, + 284, + 11, + 11, + 13, + 11, + 198, + 11, + 262, + 11, + 11, + 11, + 262, + 11, + 262, + 262, + 262, + 198, + 198, + 11, + 11, + 262, + 13, + 262, + 262, + 262, + 13, + 198, + 198, + 13, + 198, + 11, + 13, + 286, + 13, + 198, + 198, + 13, + 11, + 257, + 13, + 284, + 284, + 11, + 13, + 198, + 198, + 11, + 262, + 198, + 198, + 262, + 262, + 11, + 11, + 198, + 262, + 262, + 11, + 11, + 262, + 11, + 262, + 13, + 11, + 13, + 198, + 198, + 262, + 13, + 11, + 284, + 198, + 11, + 11, + 198, + 284, + 11, + 11, + 198, + 262, + 11, + 11, + 198, + 262, + 11, + 198, + 11, + 13, + 286, + 13, + 262, + 11, + 262, + 262, + 198, + 198, + 11, + 11, + 262, + 11, + 13, + 198, + 13, + 11, + 198, + 198 + ] + }, + { + "sample_id": 14, + "text": " the the,. the.\n.,., the,,. of the,,, of the, the the\n.,,\n of and\n\n,., the to.,..\n of the,,.\n the,.,.,.,\n\n.,\n,,. the,\n the the, the the.. of the, the\n, the the the, the.... the, and the. the the the, the\n.,, the..,,,, the\n..,. the to\n,.\n,,, the the and the,., the, to.,, the, the,\n, the,\n\n,.\n\n. the, the the., the the the... the\n, the the,,,., the of the and,\n, the, the.,\n a,, the,..\n the. the,. the.,\n. the the the.\n\n,,,, the.,\n\n. to of the.,,,,, to the,..\n the,.,,,. the, to, the the the,.\n the,,.\n\n,,\n,,,,,, the.,.,,,.,.. the.\n.,.,, the. the the..\n\n. the,, the the, the the,\n\n,.,. the and,,.\n,,. of the..\n to the in the..\n., the, the the the.,.\n.,. the the the, of.. the\n, the, the the,. the,,,, the... the, the to the the...\n\n the. the and. of the.\n the.,. the,, of, the,, the the,.., the the,.\n the.\n,\n,., the the.,.,.,, to, the,,\n\n... the the the. of the,,. the..\n the. the,.\n,.. the the,\n\n\n the the., the,,,, the the.,., the,,..,\n,., the.\n the. the,,,, the, to the the..\n, to the to., the. the,\n of the, of\n.\n the. the.\n\n the the the., the the of, the\n. the.,., the the., the, the the the the the\n\n the. the the.,.,, the the, the\n, the the the\n\n,,., the.,,..\n,,. the..\n the to,,..., the the the of.\n\n\n the the.,,,, the the the, to the...,\n. the\n the,.. the the, the to, the the. the, the the\n\n.,\n\n. the the,,., the the the the,, the,.\n\n the the of the, the, of the the.,. the..\n., to., the,,.\n. the,. the the.\n the,.,,,,\n, the the,,., the the, the....\n, the,\n,.,,,,,\n., the,,,\n.\n\n the,.\n,,,, the the the,, the the\n the the.\n\n the\n,,\n\n,,\n the the.,\n,. the\n the, the the,..,, the the... the to the,,.\n the the..\n\n,, the., the the the, the,., the,,. the,. the., the the,,, the,,,\n the..\n\n,, the.\n\n the,,, the the the,,, the the,\n. the., the,,, the the,, the of.,,.\n\n the,.\n\n the.\n,\n\n,, the,\n. the the the, the\n.,,.\n., the\n the the the.\n the..,, the,\n the the,\n. the., the\n\n\n\n the the,.\n, the the the the to the the, the\n. the,.\n the..,.., the the, the.\n\n\n the the, the,, the.., the,., the. the the the\n", + "token_ids": [ + 262, + 262, + 11, + 13, + 262, + 13, + 198, + 13, + 11, + 13, + 11, + 262, + 11, + 11, + 13, + 286, + 262, + 11, + 11, + 11, + 286, + 262, + 11, + 262, + 262, + 198, + 13, + 11, + 11, + 198, + 286, + 290, + 198, + 198, + 11, + 13, + 11, + 262, + 284, + 13, + 11, + 13, + 13, + 198, + 286, + 262, + 11, + 11, + 13, + 198, + 262, + 11, + 13, + 11, + 13, + 11, + 13, + 11, + 198, + 198, + 13, + 11, + 198, + 11, + 11, + 13, + 262, + 11, + 198, + 262, + 262, + 11, + 262, + 262, + 13, + 13, + 286, + 262, + 11, + 262, + 198, + 11, + 262, + 262, + 262, + 11, + 262, + 13, + 13, + 13, + 13, + 262, + 11, + 290, + 262, + 13, + 262, + 262, + 262, + 11, + 262, + 198, + 13, + 11, + 11, + 262, + 13, + 13, + 11, + 11, + 11, + 11, + 262, + 198, + 13, + 13, + 11, + 13, + 262, + 284, + 198, + 11, + 13, + 198, + 11, + 11, + 11, + 262, + 262, + 290, + 262, + 11, + 13, + 11, + 262, + 11, + 284, + 13, + 11, + 11, + 262, + 11, + 262, + 11, + 198, + 11, + 262, + 11, + 198, + 198, + 11, + 13, + 198, + 198, + 13, + 262, + 11, + 262, + 262, + 13, + 11, + 262, + 262, + 262, + 13, + 13, + 13, + 262, + 198, + 11, + 262, + 262, + 11, + 11, + 11, + 13, + 11, + 262, + 286, + 262, + 290, + 11, + 198, + 11, + 262, + 11, + 262, + 13, + 11, + 198, + 257, + 11, + 11, + 262, + 11, + 13, + 13, + 198, + 262, + 13, + 262, + 11, + 13, + 262, + 13, + 11, + 198, + 13, + 262, + 262, + 262, + 13, + 198, + 198, + 11, + 11, + 11, + 11, + 262, + 13, + 11, + 198, + 198, + 13, + 284, + 286, + 262, + 13, + 11, + 11, + 11, + 11, + 11, + 284, + 262, + 11, + 13, + 13, + 198, + 262, + 11, + 13, + 11, + 11, + 11, + 13, + 262, + 11, + 284, + 11, + 262, + 262, + 262, + 11, + 13, + 198, + 262, + 11, + 11, + 13, + 198, + 198, + 11, + 11, + 198, + 11, + 11, + 11, + 11, + 11, + 11, + 262, + 13, + 11, + 13, + 11, + 11, + 11, + 13, + 11, + 13, + 13, + 262, + 13, + 198, + 13, + 11, + 13, + 11, + 11, + 262, + 13, + 262, + 262, + 13, + 13, + 198, + 198, + 13, + 262, + 11, + 11, + 262, + 262, + 11, + 262, + 262, + 11, + 198, + 198, + 11, + 13, + 11, + 13, + 262, + 290, + 11, + 11, + 13, + 198, + 11, + 11, + 13, + 286, + 262, + 13, + 13, + 198, + 284, + 262, + 287, + 262, + 13, + 13, + 198, + 13, + 11, + 262, + 11, + 262, + 262, + 262, + 13, + 11, + 13, + 198, + 13, + 11, + 13, + 262, + 262, + 262, + 11, + 286, + 13, + 13, + 262, + 198, + 11, + 262, + 11, + 262, + 262, + 11, + 13, + 262, + 11, + 11, + 11, + 11, + 262, + 13, + 13, + 13, + 262, + 11, + 262, + 284, + 262, + 262, + 13, + 13, + 13, + 198, + 198, + 262, + 13, + 262, + 290, + 13, + 286, + 262, + 13, + 198, + 262, + 13, + 11, + 13, + 262, + 11, + 11, + 286, + 11, + 262, + 11, + 11, + 262, + 262, + 11, + 13, + 13, + 11, + 262, + 262, + 11, + 13, + 198, + 262, + 13, + 198, + 11, + 198, + 11, + 13, + 11, + 262, + 262, + 13, + 11, + 13, + 11, + 13, + 11, + 11, + 284, + 11, + 262, + 11, + 11, + 198, + 198, + 13, + 13, + 13, + 262, + 262, + 262, + 13, + 286, + 262, + 11, + 11, + 13, + 262, + 13, + 13, + 198, + 262, + 13, + 262, + 11, + 13, + 198, + 11, + 13, + 13, + 262, + 262, + 11, + 198, + 198, + 198, + 262, + 262, + 13, + 11, + 262, + 11, + 11, + 11, + 11, + 262, + 262, + 13, + 11, + 13, + 11, + 262, + 11, + 11, + 13, + 13, + 11, + 198, + 11, + 13, + 11, + 262, + 13, + 198, + 262, + 13, + 262, + 11, + 11, + 11, + 11, + 262, + 11, + 284, + 262, + 262, + 13, + 13, + 198, + 11, + 284, + 262, + 284, + 13, + 11, + 262, + 13, + 262, + 11, + 198, + 286, + 262, + 11, + 286, + 198, + 13, + 198, + 262, + 13, + 262, + 13, + 198, + 198, + 262, + 262, + 262, + 13, + 11, + 262, + 262, + 286, + 11, + 262, + 198, + 13, + 262, + 13, + 11, + 13, + 11, + 262, + 262, + 13, + 11, + 262, + 11, + 262, + 262, + 262, + 262, + 262, + 198, + 198, + 262, + 13, + 262, + 262, + 13, + 11, + 13, + 11, + 11, + 262, + 262, + 11, + 262, + 198, + 11, + 262, + 262, + 262, + 198, + 198, + 11, + 11, + 13, + 11, + 262, + 13, + 11, + 11, + 13, + 13, + 198, + 11, + 11, + 13, + 262, + 13, + 13, + 198, + 262, + 284, + 11, + 11, + 13, + 13, + 13, + 11, + 262, + 262, + 262, + 286, + 13, + 198, + 198, + 198, + 262, + 262, + 13, + 11, + 11, + 11, + 11, + 262, + 262, + 262, + 11, + 284, + 262, + 13, + 13, + 13, + 11, + 198, + 13, + 262, + 198, + 262, + 11, + 13, + 13, + 262, + 262, + 11, + 262, + 284, + 11, + 262, + 262, + 13, + 262, + 11, + 262, + 262, + 198, + 198, + 13, + 11, + 198, + 198, + 13, + 262, + 262, + 11, + 11, + 13, + 11, + 262, + 262, + 262, + 262, + 11, + 11, + 262, + 11, + 13, + 198, + 198, + 262, + 262, + 286, + 262, + 11, + 262, + 11, + 286, + 262, + 262, + 13, + 11, + 13, + 262, + 13, + 13, + 198, + 13, + 11, + 284, + 13, + 11, + 262, + 11, + 11, + 13, + 198, + 13, + 262, + 11, + 13, + 262, + 262, + 13, + 198, + 262, + 11, + 13, + 11, + 11, + 11, + 11, + 198, + 11, + 262, + 262, + 11, + 11, + 13, + 11, + 262, + 262, + 11, + 262, + 13, + 13, + 13, + 13, + 198, + 11, + 262, + 11, + 198, + 11, + 13, + 11, + 11, + 11, + 11, + 11, + 198, + 13, + 11, + 262, + 11, + 11, + 11, + 198, + 13, + 198, + 198, + 262, + 11, + 13, + 198, + 11, + 11, + 11, + 11, + 262, + 262, + 262, + 11, + 11, + 262, + 262, + 198, + 262, + 262, + 13, + 198, + 198, + 262, + 198, + 11, + 11, + 198, + 198, + 11, + 11, + 198, + 262, + 262, + 13, + 11, + 198, + 11, + 13, + 262, + 198, + 262, + 11, + 262, + 262, + 11, + 13, + 13, + 11, + 11, + 262, + 262, + 13, + 13, + 13, + 262, + 284, + 262, + 11, + 11, + 13, + 198, + 262, + 262, + 13, + 13, + 198, + 198, + 11, + 11, + 262, + 13, + 11, + 262, + 262, + 262, + 11, + 262, + 11, + 13, + 11, + 262, + 11, + 11, + 13, + 262, + 11, + 13, + 262, + 13, + 11, + 262, + 262, + 11, + 11, + 11, + 262, + 11, + 11, + 11, + 198, + 262, + 13, + 13, + 198, + 198, + 11, + 11, + 262, + 13, + 198, + 198, + 262, + 11, + 11, + 11, + 262, + 262, + 262, + 11, + 11, + 11, + 262, + 262, + 11, + 198, + 13, + 262, + 13, + 11, + 262, + 11, + 11, + 11, + 262, + 262, + 11, + 11, + 262, + 286, + 13, + 11, + 11, + 13, + 198, + 198, + 262, + 11, + 13, + 198, + 198, + 262, + 13, + 198, + 11, + 198, + 198, + 11, + 11, + 262, + 11, + 198, + 13, + 262, + 262, + 262, + 11, + 262, + 198, + 13, + 11, + 11, + 13, + 198, + 13, + 11, + 262, + 198, + 262, + 262, + 262, + 13, + 198, + 262, + 13, + 13, + 11, + 11, + 262, + 11, + 198, + 262, + 262, + 11, + 198, + 13, + 262, + 13, + 11, + 262, + 198, + 198, + 198, + 198, + 262, + 262, + 11, + 13, + 198, + 11, + 262, + 262, + 262, + 262, + 284, + 262, + 262, + 11, + 262, + 198, + 13, + 262, + 11, + 13, + 198, + 262, + 13, + 13, + 11, + 13, + 13, + 11, + 262, + 262, + 11, + 262, + 13, + 198, + 198, + 198, + 262, + 262, + 11, + 262, + 11, + 11, + 262, + 13, + 13, + 11, + 262, + 11, + 13, + 11, + 262, + 13, + 262, + 262, + 262, + 198 + ] + }, + { + "sample_id": 15, + "text": " of.,. the,.. the\n the..\n. the..\n\n of the the the., the, of\n\n.,. the..,,. the the the, to..\n,.\n\n\n the..\n,\n\n\n. the the.,...\n the the,,,.., to., the the\n the...\n\n the,,,,, the.,\n, the a the.. the,,,... the.,.\n to,,.. the\n. the, the. the the to the, the.., the the the. the, to the,,, the. the\n, the.,,. the the the the the the.\n.\n the\n\n\n a..,\n\n the the the,,,\n the the, of the the the,. the the,.\n\n,,,..,. a,. the the a the.,\n the,\n.\n, the. of the,.\n. the.\n\n,, the the,.,,\n\n,.,\n..,\n\n the the\n the,. the\n\n\n,, the the,.\n,.,\n, to, the\n\n.\n\n\n\n.,,.�. the.. the, the of\n\n... the the.,,\n.\n.\n\n\n\n,\n,, the\n,,,. to,.. to\n the the the,, and the,.,.\n the,..\n the the, the the,,.,,\n\n the the the., the\n..\n\n,,..\n\n,,, the..\n\n,. the the the, the a..,,\n.,. the..\n.\n\n..\n\n the the of,\n.\n, the\n. to the the to,,\n\n the.,, the\n\n\n..,.\n\n the, the.\n,,,.., the,. the\n., the.,, the the.\n\n. the the the,., the the.\n., to\n the the,, the.,, the the,. the\n..\n.,\n\n.. the....,,. the the the.,\n.\n the.\n, the,, the., the,\n the.\n,,,. the\n.\n, the,,.\n, the the the.,,\n\n\n..\n\n,.,..\n the the,,, the, the the.,, the the the,,.\n\n the\n...\n.\n\n, the the.. the the., the, the..., the the.,,,, to\n. the.\n\n,.\n, the,.,, the, the the,,\n, the the in. a.,. the,\n,\n to the, the the, the, the the\n\n the the,,, the.\n\n. the, the the..\n of.\n the the.,\n,, and the the the of.,.\n\n, the, the the. the the,\n.\n,, the,,, the the, the the,,....,,\n.,, the the the the the.,\n. the,, the,\n of,,.\n\n..\n\n, the the the,, the the\n,. the the,., the the,.,\n,. the. the\n.,, the a the the., the,,... the,\n..\n.,.,.\n\n,. the the the.\n\n\n\n.\n.,.,,. the,,..,.. to the. the the..\n...\n\n\n\n the the the,, the\n,,,\n,,,, the\n,, the. the\n the.,..\n. the, the,.. the, the,. the,, the\n, the, and,.,, the the the the the, the the... the\n a\n, the a,.\n.,\n.. to. the. the the of the,. the the...,,,, the the. the\n\n the the,, the\n\n..,. the.\n\n\n,, and the, the.\n\n the,,\n\n.\n\n the. to,,, the, to a the", + "token_ids": [ + 286, + 13, + 11, + 13, + 262, + 11, + 13, + 13, + 262, + 198, + 262, + 13, + 13, + 198, + 13, + 262, + 13, + 13, + 198, + 198, + 286, + 262, + 262, + 262, + 13, + 11, + 262, + 11, + 286, + 198, + 198, + 13, + 11, + 13, + 262, + 13, + 13, + 11, + 11, + 13, + 262, + 262, + 262, + 11, + 284, + 13, + 13, + 198, + 11, + 13, + 198, + 198, + 198, + 262, + 13, + 13, + 198, + 11, + 198, + 198, + 198, + 13, + 262, + 262, + 13, + 11, + 13, + 13, + 13, + 198, + 262, + 262, + 11, + 11, + 11, + 13, + 13, + 11, + 284, + 13, + 11, + 262, + 262, + 198, + 262, + 13, + 13, + 13, + 198, + 198, + 262, + 11, + 11, + 11, + 11, + 11, + 262, + 13, + 11, + 198, + 11, + 262, + 257, + 262, + 13, + 13, + 262, + 11, + 11, + 11, + 13, + 13, + 13, + 262, + 13, + 11, + 13, + 198, + 284, + 11, + 11, + 13, + 13, + 262, + 198, + 13, + 262, + 11, + 262, + 13, + 262, + 262, + 284, + 262, + 11, + 262, + 13, + 13, + 11, + 262, + 262, + 262, + 13, + 262, + 11, + 284, + 262, + 11, + 11, + 11, + 262, + 13, + 262, + 198, + 11, + 262, + 13, + 11, + 11, + 13, + 262, + 262, + 262, + 262, + 262, + 262, + 13, + 198, + 13, + 198, + 262, + 198, + 198, + 198, + 257, + 13, + 13, + 11, + 198, + 198, + 262, + 262, + 262, + 11, + 11, + 11, + 198, + 262, + 262, + 11, + 286, + 262, + 262, + 262, + 11, + 13, + 262, + 262, + 11, + 13, + 198, + 198, + 11, + 11, + 11, + 13, + 13, + 11, + 13, + 257, + 11, + 13, + 262, + 262, + 257, + 262, + 13, + 11, + 198, + 262, + 11, + 198, + 13, + 198, + 11, + 262, + 13, + 286, + 262, + 11, + 13, + 198, + 13, + 262, + 13, + 198, + 198, + 11, + 11, + 262, + 262, + 11, + 13, + 11, + 11, + 198, + 198, + 11, + 13, + 11, + 198, + 13, + 13, + 11, + 198, + 198, + 262, + 262, + 198, + 262, + 11, + 13, + 262, + 198, + 198, + 198, + 11, + 11, + 262, + 262, + 11, + 13, + 198, + 11, + 13, + 11, + 198, + 11, + 284, + 11, + 262, + 198, + 198, + 13, + 198, + 198, + 198, + 198, + 13, + 11, + 11, + 13, + 447, + 13, + 262, + 13, + 13, + 262, + 11, + 262, + 286, + 198, + 198, + 13, + 13, + 13, + 262, + 262, + 13, + 11, + 11, + 198, + 13, + 198, + 13, + 198, + 198, + 198, + 198, + 11, + 198, + 11, + 11, + 262, + 198, + 11, + 11, + 11, + 13, + 284, + 11, + 13, + 13, + 284, + 198, + 262, + 262, + 262, + 11, + 11, + 290, + 262, + 11, + 13, + 11, + 13, + 198, + 262, + 11, + 13, + 13, + 198, + 262, + 262, + 11, + 262, + 262, + 11, + 11, + 13, + 11, + 11, + 198, + 198, + 262, + 262, + 262, + 13, + 11, + 262, + 198, + 13, + 13, + 198, + 198, + 11, + 11, + 13, + 13, + 198, + 198, + 11, + 11, + 11, + 262, + 13, + 13, + 198, + 198, + 11, + 13, + 262, + 262, + 262, + 11, + 262, + 257, + 13, + 13, + 11, + 11, + 198, + 13, + 11, + 13, + 262, + 13, + 13, + 198, + 13, + 198, + 198, + 13, + 13, + 198, + 198, + 262, + 262, + 286, + 11, + 198, + 13, + 198, + 11, + 262, + 198, + 13, + 284, + 262, + 262, + 284, + 11, + 11, + 198, + 198, + 262, + 13, + 11, + 11, + 262, + 198, + 198, + 198, + 13, + 13, + 11, + 13, + 198, + 198, + 262, + 11, + 262, + 13, + 198, + 11, + 11, + 11, + 13, + 13, + 11, + 262, + 11, + 13, + 262, + 198, + 13, + 11, + 262, + 13, + 11, + 11, + 262, + 262, + 13, + 198, + 198, + 13, + 262, + 262, + 262, + 11, + 13, + 11, + 262, + 262, + 13, + 198, + 13, + 11, + 284, + 198, + 262, + 262, + 11, + 11, + 262, + 13, + 11, + 11, + 262, + 262, + 11, + 13, + 262, + 198, + 13, + 13, + 198, + 13, + 11, + 198, + 198, + 13, + 13, + 262, + 13, + 13, + 13, + 13, + 11, + 11, + 13, + 262, + 262, + 262, + 13, + 11, + 198, + 13, + 198, + 262, + 13, + 198, + 11, + 262, + 11, + 11, + 262, + 13, + 11, + 262, + 11, + 198, + 262, + 13, + 198, + 11, + 11, + 11, + 13, + 262, + 198, + 13, + 198, + 11, + 262, + 11, + 11, + 13, + 198, + 11, + 262, + 262, + 262, + 13, + 11, + 11, + 198, + 198, + 198, + 13, + 13, + 198, + 198, + 11, + 13, + 11, + 13, + 13, + 198, + 262, + 262, + 11, + 11, + 11, + 262, + 11, + 262, + 262, + 13, + 11, + 11, + 262, + 262, + 262, + 11, + 11, + 13, + 198, + 198, + 262, + 198, + 13, + 13, + 13, + 198, + 13, + 198, + 198, + 11, + 262, + 262, + 13, + 13, + 262, + 262, + 13, + 11, + 262, + 11, + 262, + 13, + 13, + 13, + 11, + 262, + 262, + 13, + 11, + 11, + 11, + 11, + 284, + 198, + 13, + 262, + 13, + 198, + 198, + 11, + 13, + 198, + 11, + 262, + 11, + 13, + 11, + 11, + 262, + 11, + 262, + 262, + 11, + 11, + 198, + 11, + 262, + 262, + 287, + 13, + 257, + 13, + 11, + 13, + 262, + 11, + 198, + 11, + 198, + 284, + 262, + 11, + 262, + 262, + 11, + 262, + 11, + 262, + 262, + 198, + 198, + 262, + 262, + 11, + 11, + 11, + 262, + 13, + 198, + 198, + 13, + 262, + 11, + 262, + 262, + 13, + 13, + 198, + 286, + 13, + 198, + 262, + 262, + 13, + 11, + 198, + 11, + 11, + 290, + 262, + 262, + 262, + 286, + 13, + 11, + 13, + 198, + 198, + 11, + 262, + 11, + 262, + 262, + 13, + 262, + 262, + 11, + 198, + 13, + 198, + 11, + 11, + 262, + 11, + 11, + 11, + 262, + 262, + 11, + 262, + 262, + 11, + 11, + 13, + 13, + 13, + 13, + 11, + 11, + 198, + 13, + 11, + 11, + 262, + 262, + 262, + 262, + 262, + 13, + 11, + 198, + 13, + 262, + 11, + 11, + 262, + 11, + 198, + 286, + 11, + 11, + 13, + 198, + 198, + 13, + 13, + 198, + 198, + 11, + 262, + 262, + 262, + 11, + 11, + 262, + 262, + 198, + 11, + 13, + 262, + 262, + 11, + 13, + 11, + 262, + 262, + 11, + 13, + 11, + 198, + 11, + 13, + 262, + 13, + 262, + 198, + 13, + 11, + 11, + 262, + 257, + 262, + 262, + 13, + 11, + 262, + 11, + 11, + 13, + 13, + 13, + 262, + 11, + 198, + 13, + 13, + 198, + 13, + 11, + 13, + 11, + 13, + 198, + 198, + 11, + 13, + 262, + 262, + 262, + 13, + 198, + 198, + 198, + 198, + 13, + 198, + 13, + 11, + 13, + 11, + 11, + 13, + 262, + 11, + 11, + 13, + 13, + 11, + 13, + 13, + 284, + 262, + 13, + 262, + 262, + 13, + 13, + 198, + 13, + 13, + 13, + 198, + 198, + 198, + 198, + 262, + 262, + 262, + 11, + 11, + 262, + 198, + 11, + 11, + 11, + 198, + 11, + 11, + 11, + 11, + 262, + 198, + 11, + 11, + 262, + 13, + 262, + 198, + 262, + 13, + 11, + 13, + 13, + 198, + 13, + 262, + 11, + 262, + 11, + 13, + 13, + 262, + 11, + 262, + 11, + 13, + 262, + 11, + 11, + 262, + 198, + 11, + 262, + 11, + 290, + 11, + 13, + 11, + 11, + 262, + 262, + 262, + 262, + 262, + 11, + 262, + 262, + 13, + 13, + 13, + 262, + 198, + 257, + 198, + 11, + 262, + 257, + 11, + 13, + 198, + 13, + 11, + 198, + 13, + 13, + 284, + 13, + 262, + 13, + 262, + 262, + 286, + 262, + 11, + 13, + 262, + 262, + 13, + 13, + 13, + 11, + 11, + 11, + 11, + 262, + 262, + 13, + 262, + 198, + 198, + 262, + 262, + 11, + 11, + 262, + 198, + 198, + 13, + 13, + 11, + 13, + 262, + 13, + 198, + 198, + 198, + 11, + 11, + 290, + 262, + 11, + 262, + 13, + 198, + 198, + 262, + 11, + 11, + 198, + 198, + 13, + 198, + 198, + 262, + 13, + 284, + 11, + 11, + 11, + 262, + 11, + 284, + 257, + 262 + ] + } + ], + "schema": "dlb-upstream-token-capture-v1" +} diff --git a/results/smoke/samples/owt/flm/steps_1024/upstream_hydra/.hydra/hydra.yaml b/results/smoke/samples/owt/flm/steps_1024/upstream_hydra/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb36d27ed1f8b302ad5df13171db347bbf50817b --- /dev/null +++ b/results/smoke/samples/owt/flm/steps_1024/upstream_hydra/.hydra/hydra.yaml @@ -0,0 +1,182 @@ +hydra: + run: + dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_1024/upstream_hydra + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_1024/upstream_hydra + - hydra.mode=RUN + task: + - mode=sample_eval + - seed=42 + - data=openwebtext-split + - model=small + - model.length=1024 + - loader.batch_size=2 + - loader.eval_batch_size=16 + - sampling.num_sample_batches=1 + - sampling.steps=1024 + - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt + - trainer.devices=1 + - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_1024/upstream_hydra + - +wandb.offline=true + - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_1024/upstream_samples.json + - algo=flm + - algo.backbone=dit + - eval.disable_ema=False + - sampling.solver=euler + - algo.double_temb=False + - sampling.gamma=0.0 + job: + name: dlb_pinned_upstream_main + chdir: true + override_dirname: +wandb.offline=true,algo.backbone=dit,algo.double_temb=False,algo=flm,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_1024/upstream_hydra,data=openwebtext-split,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_1024/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=16,mode=sample_eval,model.length=1024,model=small,sampling.gamma=0.0,sampling.num_sample_batches=1,sampling.solver=euler,sampling.steps=1024,seed=42,trainer.devices=1 + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: /home/ruizzhao/diffusion_baseline + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/ruizzhao/diffusion_baseline/upstreams/flm/configs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_1024/upstream_hydra + choices: + algo: flm + prior: none + lr_scheduler: constant_warmup + noise: log-linear + strategy: ddp + model: small + data: openwebtext-split + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/results/smoke/samples/owt/flm/steps_32/upstream_hydra/.hydra/hydra.yaml b/results/smoke/samples/owt/flm/steps_32/upstream_hydra/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d3940ff86b3d9eb061fba9e559a68fc7c179a95 --- /dev/null +++ b/results/smoke/samples/owt/flm/steps_32/upstream_hydra/.hydra/hydra.yaml @@ -0,0 +1,182 @@ +hydra: + run: + dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_32/upstream_hydra + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_32/upstream_hydra + - hydra.mode=RUN + task: + - mode=sample_eval + - seed=42 + - data=openwebtext-split + - model=small + - model.length=1024 + - loader.batch_size=2 + - loader.eval_batch_size=16 + - sampling.num_sample_batches=1 + - sampling.steps=32 + - eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt + - trainer.devices=1 + - checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_32/upstream_hydra + - +wandb.offline=true + - eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_32/upstream_samples.json + - algo=flm + - algo.backbone=dit + - eval.disable_ema=False + - sampling.solver=euler + - algo.double_temb=False + - sampling.gamma=0.0 + job: + name: dlb_pinned_upstream_main + chdir: true + override_dirname: +wandb.offline=true,algo.backbone=dit,algo.double_temb=False,algo=flm,checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_32/upstream_hydra,data=openwebtext-split,eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/flm_ckpt/owt/owt_flm.ckpt,eval.disable_ema=False,eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_32/upstream_samples.json,loader.batch_size=2,loader.eval_batch_size=16,mode=sample_eval,model.length=1024,model=small,sampling.gamma=0.0,sampling.num_sample_batches=1,sampling.solver=euler,sampling.steps=32,seed=42,trainer.devices=1 + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: /home/ruizzhao/diffusion_baseline + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/ruizzhao/diffusion_baseline/upstreams/flm/configs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/flm/steps_32/upstream_hydra + choices: + algo: flm + prior: none + lr_scheduler: constant_warmup + noise: log-linear + strategy: ddp + model: small + data: openwebtext-split + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/results/smoke/samples/owt/fmlm/steps_16/request.json b/results/smoke/samples/owt/fmlm/steps_16/request.json new file mode 100644 index 0000000000000000000000000000000000000000..62c54834b86e608778c624c579f616687502ad08 --- /dev/null +++ b/results/smoke/samples/owt/fmlm/steps_16/request.json @@ -0,0 +1,90 @@ +{ + "command": [ + "/opt/anaconda3/envs/dlb-flm/bin/python", + "-u", + "-m", + "dlb.adapters.capture", + "--upstream-entrypoint=/home/ruizzhao/diffusion_baseline/upstreams/flm/main.py", + "--capture-path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/fmlm/steps_16/upstream_token_ids.json", + "--", + "mode=sample_eval", + "seed=42", + "data=openwebtext-split", + "model=small", + "model.length=1024", + "loader.batch_size=2", + "loader.eval_batch_size=16", + "sampling.num_sample_batches=1", + "sampling.steps=16", + "eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/official/fmlm_ckpt/owt/owt_fmlm.ckpt", + "trainer.devices=1", + "hydra.run.dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/fmlm/steps_16/upstream_hydra", + "checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/fmlm/steps_16/upstream_hydra", + "+wandb.offline=true", + "eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/fmlm/steps_16/upstream_samples.json", + "algo=fmlm", + "algo.backbone=dit", + "eval.disable_ema=False", + "sampling.solver=euler", + "algo.double_temb=True", + "algo.learnable_loss_weighting=False", + "sampling.gamma=1.0" + ], + "identity": { + "adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0", + "checkpoint_lock_id": "recipe:fmlm_owt_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0", + "checkpoint_selection": { + "output": "checkpoints/official/fmlm_ckpt/owt", + "recipe": "fmlm_owt_official_ckpt", + "sampling_checkpoint": "owt_fmlm.ckpt", + "sampling_config": null, + "source": "flm", + "source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3", + "teacher_adapter": null, + "teacher_family": "continuous_flm" + }, + "checkpoint_sha256": "a5ca77b77a26dec9746f627858d42c19803d11c1e932dcf14a3ce80f6bbbbd76", + "checkpoint_teacher_family": "continuous_flm", + "command_sha256": "d13885d4326c17c9ddd2fe9519ba33c45253f2eeb28a811ebaa44f7af1fa5632", + "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994", + "dataset_id": "owt", + "device": null, + "environment": "dlb-flm", + "model_id": "fmlm", + "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke", + "run_id": "fmlm-owt-steps-16", + "sample_count": 1, + "seed": 42, + "source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3", + "step_count": 16 + }, + "request": { + "adapter_identity": "dlb.adapters.flm:v2|dlb.adapters.flm:FLMAdapter|c3a6a4573200fecf38a47fc6e7bc78254ca362fff3b96079dc2efa9f8e27aff0", + "checkpoint_lock_id": "recipe:fmlm_owt_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0", + "checkpoint_selection": { + "output": "checkpoints/official/fmlm_ckpt/owt", + "recipe": "fmlm_owt_official_ckpt", + "sampling_checkpoint": "owt_fmlm.ckpt", + "sampling_config": null, + "source": "flm", + "source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3", + "teacher_adapter": null, + "teacher_family": "continuous_flm" + }, + "checkpoint_sha256": "a5ca77b77a26dec9746f627858d42c19803d11c1e932dcf14a3ce80f6bbbbd76", + "checkpoint_teacher_family": "continuous_flm", + "command": null, + "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994", + "dataset_id": "owt", + "device": null, + "environment": "dlb-flm", + "model_id": "fmlm", + "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke", + "run_id": "fmlm-owt-steps-16", + "sample_count": 1, + "seed": 42, + "source_sha256": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3", + "step_count": 16 + }, + "started_at": "2026-08-03T06:48:06.847669Z" +} diff --git a/results/smoke/samples/owt/fmlm/steps_4/upstream_token_ids.json b/results/smoke/samples/owt/fmlm/steps_4/upstream_token_ids.json new file mode 100644 index 0000000000000000000000000000000000000000..a5bf0c50bc9e0f0b66f8c6ab64d98ef65d61b76a --- /dev/null +++ b/results/smoke/samples/owt/fmlm/steps_4/upstream_token_ids.json @@ -0,0 +1,16485 @@ +{ + "samples": [ + { + "sample_id": 0, + "text": "<|endoftext|>, and doesn't care that his wife, nothing instead. It is simple. He alleys into won; he wants the court to record and run free as alabons.\n\nThe rest is out good, as the council members tell them.\n\nThe mayor was going to keep the record alive and, given he was led away, the day that was to take a dud. All of this has money to the right-wing old whose machine coming by most of African-Americans, namely Sanders and Clinton, and as if position from the Left about to have left thing, angry, under, even more, woman it could seem that she's been at least somewhatirful rather than, in their case, of relying on the big, water and their rights to go beyond their interests.\n\n* * Below\n\nCastro has said that near the heart of, it may be this, the most dangerous place for in America. All jobs are half afoot and the first of four sex jobs are consideredicallyous – they’d be in the middle of being the legal wage; the last cannot; become women, to sit by the side of “ino 9” or in an otherwise ungodly sex party. It amounted to a video of the mayor’s office was known as Plan M, but on social media, Emanuel's (un) pay was next to zero, in effect a supposed \"out of the whole nothing.\"\n\nThat's the definition of late still, and given that the city is still more expensive than his, as they've been hanging on and selling the land for years until the feds will take it, perhaps the housing market that is made asordable could be budgeted for a drive-thousand dollars.\n\nYes, it's a gray well that exists in now; nothing worse is the one that has always been out and the rest of people's lives lives on.\n\nBut in court, his group in all directions about who have been telling their fair, is that Cosby is getting well paid.\n\nThat for record, the same day, on the evidence, he is quoted as running up tongue-in-cheek against the abuse of power – the city to over the boundaries of no man’s land – will hopefully drive this even more and out of the game.\n\nAnd important, O knows (and he has from his book is no different) than just words do. It's what you would expect from a person that’s the wrong way over.\n\nSoro won the money to protect him (all of his family) and that today he'll come to video to help test his cause.\n\nC7 Letter from Chicago News We are unable to form the solution to some help on the outside. Unfortunately, unless he has access to the rest of the land, his we have one job offer and two kids. But, and the city is very high due to debt, and will never be sure how to do a job. Hopefully these then needs become soon.<|endoftext|>Let’s wrong with Britain to have it easier to live in The Hague or to die: We hope those with mental skills are still behind in new global need\n\nWith this, be another question: the UK will soon be in dispersal.\n\nThe global globalisation of Brexit, the UK company, “iised” in a way of dealing with China – has left Britain the job to place around a great pressing need. As far as politics, the question remains out of the blue for the British economy, and back in terms of geostrances, such as China has made its mark. And judging by the emergence of The America as Britain in 2010, we’re still a pretty nation. Who seems to arrive at bringing compared to our global neighbours?\n\nFor the Royal Museum. Organ puts it in a great look, and the multisitely international need to help the nation come up with ideas and an international sense of creative design. It’ll require a strong business deal and partnership, when you buy and you yourself value by being in middle class – given rich and everything – and that’s just there and there. The job education her has tried, worked, schools and quite, with the old front, which will allow children with learning disabilities, and others to learn things like a new educational toy, the Elderly Bear. The less old and run to don’t belong here, nor what was to be common in the 1990s, but the last bit of a generation: to being a no-brainer. People like to work with the students we have during the boom years against the backdrop of financial, and use all those resources together to keep the rest of 2013 alive as well, and pro-free, and put in a long and sustainableicallyational environment. Hillary will be in third because she doesn’t have a plan, this is especially part of an effort to keep and<|endoftext|>", + "token_ids": [ + 50256, + 11, + 290, + 1595, + 470, + 1337, + 326, + 465, + 3656, + 11, + 2147, + 2427, + 13, + 632, + 318, + 2829, + 13, + 679, + 435, + 21325, + 656, + 1839, + 26, + 339, + 3382, + 262, + 2184, + 284, + 1700, + 290, + 1057, + 1479, + 355, + 435, + 397, + 684, + 13, + 198, + 198, + 464, + 1334, + 318, + 503, + 922, + 11, + 355, + 262, + 6745, + 1866, + 1560, + 606, + 13, + 198, + 198, + 464, + 9591, + 373, + 1016, + 284, + 1394, + 262, + 1700, + 6776, + 290, + 11, + 1813, + 339, + 373, + 2957, + 1497, + 11, + 262, + 1110, + 326, + 373, + 284, + 1011, + 257, + 288, + 463, + 13, + 1439, + 286, + 428, + 468, + 1637, + 284, + 262, + 826, + 12, + 5469, + 1468, + 3025, + 4572, + 2406, + 416, + 749, + 286, + 5510, + 12, + 17636, + 11, + 14811, + 5831, + 290, + 2605, + 11, + 290, + 355, + 611, + 2292, + 422, + 262, + 9578, + 546, + 284, + 423, + 1364, + 1517, + 11, + 7954, + 11, + 739, + 11, + 772, + 517, + 11, + 2415, + 340, + 714, + 1283, + 326, + 673, + 338, + 587, + 379, + 1551, + 6454, + 343, + 913, + 2138, + 621, + 11, + 287, + 511, + 1339, + 11, + 286, + 17965, + 319, + 262, + 1263, + 11, + 1660, + 290, + 511, + 2489, + 284, + 467, + 3675, + 511, + 5353, + 13, + 198, + 198, + 9, + 1635, + 10383, + 198, + 198, + 19248, + 305, + 468, + 531, + 326, + 1474, + 262, + 2612, + 286, + 11, + 340, + 743, + 307, + 428, + 11, + 262, + 749, + 4923, + 1295, + 329, + 287, + 2253, + 13, + 1439, + 3946, + 389, + 2063, + 257, + 5898, + 290, + 262, + 717, + 286, + 1440, + 1714, + 3946, + 389, + 3177, + 1146, + 516, + 784, + 484, + 447, + 247, + 67, + 307, + 287, + 262, + 3504, + 286, + 852, + 262, + 2742, + 7699, + 26, + 262, + 938, + 2314, + 26, + 1716, + 1466, + 11, + 284, + 1650, + 416, + 262, + 1735, + 286, + 564, + 250, + 259, + 78, + 860, + 447, + 251, + 393, + 287, + 281, + 4306, + 555, + 25344, + 306, + 1714, + 2151, + 13, + 632, + 30008, + 284, + 257, + 2008, + 286, + 262, + 9591, + 447, + 247, + 82, + 2607, + 373, + 1900, + 355, + 5224, + 337, + 11, + 475, + 319, + 1919, + 2056, + 11, + 31920, + 338, + 357, + 403, + 8, + 1414, + 373, + 1306, + 284, + 6632, + 11, + 287, + 1245, + 257, + 4385, + 366, + 448, + 286, + 262, + 2187, + 2147, + 526, + 198, + 198, + 2504, + 338, + 262, + 6770, + 286, + 2739, + 991, + 11, + 290, + 1813, + 326, + 262, + 1748, + 318, + 991, + 517, + 5789, + 621, + 465, + 11, + 355, + 484, + 1053, + 587, + 10938, + 319, + 290, + 6301, + 262, + 1956, + 329, + 812, + 1566, + 262, + 43646, + 481, + 1011, + 340, + 11, + 3737, + 262, + 5627, + 1910, + 326, + 318, + 925, + 355, + 16819, + 714, + 307, + 4466, + 276, + 329, + 257, + 3708, + 12, + 400, + 29910, + 5054, + 13, + 198, + 198, + 5297, + 11, + 340, + 338, + 257, + 12768, + 880, + 326, + 7160, + 287, + 783, + 26, + 2147, + 4785, + 318, + 262, + 530, + 326, + 468, + 1464, + 587, + 503, + 290, + 262, + 1334, + 286, + 661, + 338, + 3160, + 3160, + 319, + 13, + 198, + 198, + 1537, + 287, + 2184, + 11, + 465, + 1448, + 287, + 477, + 11678, + 546, + 508, + 423, + 587, + 5149, + 511, + 3148, + 11, + 318, + 326, + 36485, + 318, + 1972, + 880, + 3432, + 13, + 198, + 198, + 2504, + 329, + 1700, + 11, + 262, + 976, + 1110, + 11, + 319, + 262, + 2370, + 11, + 339, + 318, + 10947, + 355, + 2491, + 510, + 11880, + 12, + 259, + 12, + 2395, + 988, + 1028, + 262, + 5076, + 286, + 1176, + 784, + 262, + 1748, + 284, + 625, + 262, + 13215, + 286, + 645, + 582, + 447, + 247, + 82, + 1956, + 784, + 481, + 11481, + 3708, + 428, + 772, + 517, + 290, + 503, + 286, + 262, + 983, + 13, + 198, + 198, + 1870, + 1593, + 11, + 440, + 4206, + 357, + 392, + 339, + 468, + 422, + 465, + 1492, + 318, + 645, + 1180, + 8, + 621, + 655, + 2456, + 466, + 13, + 632, + 338, + 644, + 345, + 561, + 1607, + 422, + 257, + 1048, + 326, + 447, + 247, + 82, + 262, + 2642, + 835, + 625, + 13, + 198, + 198, + 2396, + 305, + 1839, + 262, + 1637, + 284, + 1805, + 683, + 357, + 439, + 286, + 465, + 1641, + 8, + 290, + 326, + 1909, + 339, + 1183, + 1282, + 284, + 2008, + 284, + 1037, + 1332, + 465, + 2728, + 13, + 198, + 198, + 34, + 22, + 18121, + 422, + 4842, + 3000, + 775, + 389, + 5906, + 284, + 1296, + 262, + 4610, + 284, + 617, + 1037, + 319, + 262, + 2354, + 13, + 8989, + 11, + 4556, + 339, + 468, + 1895, + 284, + 262, + 1334, + 286, + 262, + 1956, + 11, + 465, + 356, + 423, + 530, + 1693, + 2897, + 290, + 734, + 3988, + 13, + 887, + 11, + 290, + 262, + 1748, + 318, + 845, + 1029, + 2233, + 284, + 5057, + 11, + 290, + 481, + 1239, + 307, + 1654, + 703, + 284, + 466, + 257, + 1693, + 13, + 19996, + 777, + 788, + 2476, + 1716, + 2582, + 13, + 50256, + 5756, + 447, + 247, + 82, + 2642, + 351, + 5491, + 284, + 423, + 340, + 4577, + 284, + 2107, + 287, + 383, + 37206, + 393, + 284, + 4656, + 25, + 775, + 2911, + 883, + 351, + 5110, + 4678, + 389, + 991, + 2157, + 287, + 649, + 3298, + 761, + 198, + 198, + 3152, + 428, + 11, + 307, + 1194, + 1808, + 25, + 262, + 3482, + 481, + 2582, + 307, + 287, + 39895, + 282, + 13, + 198, + 198, + 464, + 3298, + 3298, + 5612, + 286, + 11435, + 11, + 262, + 3482, + 1664, + 11, + 564, + 250, + 72, + 1417, + 447, + 251, + 287, + 257, + 835, + 286, + 7219, + 351, + 2807, + 784, + 468, + 1364, + 5491, + 262, + 1693, + 284, + 1295, + 1088, + 257, + 1049, + 12273, + 761, + 13, + 1081, + 1290, + 355, + 4819, + 11, + 262, + 1808, + 3793, + 503, + 286, + 262, + 4171, + 329, + 262, + 3517, + 3773, + 11, + 290, + 736, + 287, + 2846, + 286, + 4903, + 455, + 81, + 1817, + 11, + 884, + 355, + 2807, + 468, + 925, + 663, + 1317, + 13, + 843, + 22989, + 416, + 262, + 22106, + 286, + 383, + 2253, + 355, + 5491, + 287, + 3050, + 11, + 356, + 447, + 247, + 260, + 991, + 257, + 2495, + 3277, + 13, + 5338, + 2331, + 284, + 9240, + 379, + 6079, + 3688, + 284, + 674, + 3298, + 23788, + 30, + 198, + 198, + 1890, + 262, + 8111, + 9594, + 13, + 7221, + 7584, + 340, + 287, + 257, + 1049, + 804, + 11, + 290, + 262, + 1963, + 271, + 3973, + 3230, + 761, + 284, + 1037, + 262, + 3277, + 1282, + 510, + 351, + 4213, + 290, + 281, + 3230, + 2565, + 286, + 7325, + 1486, + 13, + 632, + 447, + 247, + 297, + 2421, + 257, + 1913, + 1597, + 1730, + 290, + 10413, + 11, + 618, + 345, + 2822, + 290, + 345, + 3511, + 1988, + 416, + 852, + 287, + 3504, + 1398, + 784, + 1813, + 5527, + 290, + 2279, + 784, + 290, + 326, + 447, + 247, + 82, + 655, + 612, + 290, + 612, + 13, + 383, + 1693, + 3707, + 607, + 468, + 3088, + 11, + 3111, + 11, + 4266, + 290, + 2407, + 11, + 351, + 262, + 1468, + 2166, + 11, + 543, + 481, + 1249, + 1751, + 351, + 4673, + 19358, + 11, + 290, + 1854, + 284, + 2193, + 1243, + 588, + 257, + 649, + 9856, + 13373, + 11, + 262, + 15624, + 306, + 14732, + 13, + 383, + 1342, + 1468, + 290, + 1057, + 284, + 836, + 447, + 247, + 83, + 5594, + 994, + 11, + 4249, + 644, + 373, + 284, + 307, + 2219, + 287, + 262, + 6303, + 82, + 11, + 475, + 262, + 938, + 1643, + 286, + 257, + 5270, + 25, + 284, + 852, + 257, + 645, + 12, + 49334, + 13, + 4380, + 588, + 284, + 670, + 351, + 262, + 2444, + 356, + 423, + 1141, + 262, + 14166, + 812, + 1028, + 262, + 26373, + 286, + 3176, + 11, + 290, + 779, + 477, + 883, + 4133, + 1978, + 284, + 1394, + 262, + 1334, + 286, + 2211, + 6776, + 355, + 880, + 11, + 290, + 386, + 12, + 5787, + 11, + 290, + 1234, + 287, + 257, + 890, + 290, + 13347, + 1146, + 864, + 2858, + 13, + 5073, + 481, + 307, + 287, + 2368, + 780, + 673, + 1595, + 447, + 247, + 83, + 423, + 257, + 1410, + 11, + 428, + 318, + 2592, + 636, + 286, + 281, + 3626, + 284, + 1394, + 290, + 50256 + ] + }, + { + "sample_id": 1, + "text": "<|endoftext|> hard it is just trying to read into the words written by a oncoming business? Part of the night of Bells’ …\n\nThe truth is that we arrived at the Rulist. Not so was it young and to feel like a winged American symor.\n\nAnd so do their pre-hour day—ating— Rites on and for the stage—all of us are being invited to pieces and for the line national to be played on a lot.\n\nWhile we see more original than the usual of early a million dollars sold and certainly isn’t an easy; not, when our friend from Llamburses the prize $100,000, the day so so as to cover the audience in three dimensions and shows how well the orchestra reflects the great work of the people from Saintville’s law school in the early twenties, the nineteenth century, and beyond. The truth is that some of the things to get inspired by in this year’s Rhapsas or also were the works of our Ex pianist Society (above) to be made off auctioning. That well drew us in; that one can hear everyone songs from the other.es with equal works on all different sides. In a sense, we think that musical songs are gleaned from the music.\n\nWhy does the artist like the art, anyway:\n\nOver the years, the work on the arts in the southern United States (now particularly in New York) was lost by the artist during Takers. This is followed by tedious work, weekends, and long hours days, as in Chicago today; an important reason to be paid to continue are late so that money has passed. Between this use in government; and the death required to win on the community are down; the poorors will often play sell out, as do the public, New Year’s new owners will begin years after that some of the pieces, some in color, were done to sell, or to add to the real estate, by showing up, including some short pieces of old bigt...y that they could be done in by mail, and to all the other places. But we got another 30 to about 30,000 and then used in June.\n\nThese ideas, even when good ones, need to play out in the American World. I remember many musicians, who were involved in many parts of the class, and fall into off-by-town with an art debt that our family has to bear. But they are not the ones benefiting from it. That’s our type of ofine thats to let after the big. We need them. Let’s try to go beyond public education and came out of the blue and beyond. It was air only as umb orchestras got into the power room; in early March, some folks, whose went into the air on a bodyguard meant for the year of their music, yesterday’s band joined in on another, larger personal act as a kind of symbol. Look out into here and ask for more art and use in art.\n\nSo little. As anyone to through D. Barnontas, that’s happened to me in many ways ways.\n\n* * * (PDF)<|endoftext|> vs. New-Navy Man’s History\n\nMost of us am stories of working with their higher names, playing with something that works and works. The country we met and lost to ourselves, on stage, in March was T Roosevelt. On stage he “I had a Dream,” recezes, and has described his courage, courage and soul as an accomplished immigrant who lost to the point of two wars with cancer. Some contributions to the Anti-Rome as political and personal Americans include Giffen’s growing problem on the front with our national-t laws, Generallessness, and love to deny U.S. citizens a final power.\n\nThen, from before the young people are out it and have 15 memories of our greats with the year, there is the matter of the movement. Our millions of people aren’t the long-term question, the 2014-, blue-loving U.S. Federalists. Seek out more than one hundred line from out-of-state Americans in their preamations, go back as the Americans, and enjoy the nation. We are living as a nation-based nation. Washington shall take law and everything else goes to decide who shall be. Look for nothing in case—- one word, something in a good America forever.”\n\nI�gIn the effort to promote peace and liberty, focus on states with and like a New-NixonAs as by the will of God over law, or common thereof, etc. All� vs.”\n\nThe movement began by establishingizing will, among American patriians as a home of all nations, raised and raised in society by its founders. It was<|endoftext|>", + "token_ids": [ + 50256, + 1327, + 340, + 318, + 655, + 2111, + 284, + 1100, + 656, + 262, + 2456, + 3194, + 416, + 257, + 319, + 4976, + 1597, + 30, + 2142, + 286, + 262, + 1755, + 286, + 7459, + 82, + 447, + 247, + 3926, + 198, + 198, + 464, + 3872, + 318, + 326, + 356, + 5284, + 379, + 262, + 371, + 377, + 396, + 13, + 1892, + 523, + 373, + 340, + 1862, + 290, + 284, + 1254, + 588, + 257, + 8539, + 276, + 1605, + 5659, + 273, + 13, + 198, + 198, + 1870, + 523, + 466, + 511, + 662, + 12, + 9769, + 1110, + 960, + 803, + 960, + 371, + 2737, + 319, + 290, + 329, + 262, + 3800, + 960, + 439, + 286, + 514, + 389, + 852, + 9392, + 284, + 5207, + 290, + 329, + 262, + 1627, + 2260, + 284, + 307, + 2826, + 319, + 257, + 1256, + 13, + 198, + 198, + 3633, + 356, + 766, + 517, + 2656, + 621, + 262, + 6678, + 286, + 1903, + 257, + 1510, + 5054, + 2702, + 290, + 3729, + 2125, + 447, + 247, + 83, + 281, + 2562, + 26, + 407, + 11, + 618, + 674, + 1545, + 422, + 406, + 2543, + 65, + 46998, + 262, + 11596, + 720, + 3064, + 11, + 830, + 11, + 262, + 1110, + 523, + 523, + 355, + 284, + 3002, + 262, + 5386, + 287, + 1115, + 15225, + 290, + 2523, + 703, + 880, + 262, + 40095, + 12497, + 262, + 1049, + 670, + 286, + 262, + 661, + 422, + 9281, + 4244, + 447, + 247, + 82, + 1099, + 1524, + 287, + 262, + 1903, + 44471, + 11, + 262, + 29666, + 4289, + 11, + 290, + 3675, + 13, + 383, + 3872, + 318, + 326, + 617, + 286, + 262, + 1243, + 284, + 651, + 7867, + 416, + 287, + 428, + 614, + 447, + 247, + 82, + 371, + 2772, + 292, + 393, + 635, + 547, + 262, + 2499, + 286, + 674, + 1475, + 43923, + 396, + 7023, + 357, + 29370, + 8, + 284, + 307, + 925, + 572, + 14389, + 278, + 13, + 1320, + 880, + 9859, + 514, + 287, + 26, + 326, + 530, + 460, + 3285, + 2506, + 7259, + 422, + 262, + 584, + 13, + 274, + 351, + 4961, + 2499, + 319, + 477, + 1180, + 5389, + 13, + 554, + 257, + 2565, + 11, + 356, + 892, + 326, + 10530, + 7259, + 389, + 41881, + 276, + 422, + 262, + 2647, + 13, + 198, + 198, + 5195, + 857, + 262, + 6802, + 588, + 262, + 1242, + 11, + 6949, + 25, + 198, + 198, + 5886, + 262, + 812, + 11, + 262, + 670, + 319, + 262, + 10848, + 287, + 262, + 8372, + 1578, + 1829, + 357, + 2197, + 3573, + 287, + 968, + 1971, + 8, + 373, + 2626, + 416, + 262, + 6802, + 1141, + 309, + 3979, + 13, + 770, + 318, + 3940, + 416, + 32460, + 670, + 11, + 21511, + 11, + 290, + 890, + 2250, + 1528, + 11, + 355, + 287, + 4842, + 1909, + 26, + 281, + 1593, + 1738, + 284, + 307, + 3432, + 284, + 2555, + 389, + 2739, + 523, + 326, + 1637, + 468, + 3804, + 13, + 14307, + 428, + 779, + 287, + 1230, + 26, + 290, + 262, + 1918, + 2672, + 284, + 1592, + 319, + 262, + 2055, + 389, + 866, + 26, + 262, + 3595, + 669, + 481, + 1690, + 711, + 3677, + 503, + 11, + 355, + 466, + 262, + 1171, + 11, + 968, + 6280, + 447, + 247, + 82, + 649, + 4393, + 481, + 2221, + 812, + 706, + 326, + 617, + 286, + 262, + 5207, + 11, + 617, + 287, + 3124, + 11, + 547, + 1760, + 284, + 3677, + 11, + 393, + 284, + 751, + 284, + 262, + 1103, + 7964, + 11, + 416, + 4478, + 510, + 11, + 1390, + 617, + 1790, + 5207, + 286, + 1468, + 1263, + 83, + 986, + 88, + 326, + 484, + 714, + 307, + 1760, + 287, + 416, + 6920, + 11, + 290, + 284, + 477, + 262, + 584, + 4113, + 13, + 887, + 356, + 1392, + 1194, + 1542, + 284, + 546, + 1542, + 11, + 830, + 290, + 788, + 973, + 287, + 2795, + 13, + 198, + 198, + 4711, + 4213, + 11, + 772, + 618, + 922, + 3392, + 11, + 761, + 284, + 711, + 503, + 287, + 262, + 1605, + 2159, + 13, + 314, + 3505, + 867, + 17245, + 11, + 508, + 547, + 2950, + 287, + 867, + 3354, + 286, + 262, + 1398, + 11, + 290, + 2121, + 656, + 572, + 12, + 1525, + 12, + 12735, + 351, + 281, + 1242, + 5057, + 326, + 674, + 1641, + 468, + 284, + 6842, + 13, + 887, + 484, + 389, + 407, + 262, + 3392, + 39125, + 422, + 340, + 13, + 1320, + 447, + 247, + 82, + 674, + 2099, + 286, + 286, + 500, + 326, + 82, + 284, + 1309, + 706, + 262, + 1263, + 13, + 775, + 761, + 606, + 13, + 3914, + 447, + 247, + 82, + 1949, + 284, + 467, + 3675, + 1171, + 3707, + 290, + 1625, + 503, + 286, + 262, + 4171, + 290, + 3675, + 13, + 632, + 373, + 1633, + 691, + 355, + 20810, + 28127, + 292, + 1392, + 656, + 262, + 1176, + 2119, + 26, + 287, + 1903, + 2805, + 11, + 617, + 7974, + 11, + 3025, + 1816, + 656, + 262, + 1633, + 319, + 257, + 1767, + 14864, + 4001, + 329, + 262, + 614, + 286, + 511, + 2647, + 11, + 7415, + 447, + 247, + 82, + 4097, + 5399, + 287, + 319, + 1194, + 11, + 4025, + 2614, + 719, + 355, + 257, + 1611, + 286, + 6194, + 13, + 6803, + 503, + 656, + 994, + 290, + 1265, + 329, + 517, + 1242, + 290, + 779, + 287, + 1242, + 13, + 198, + 198, + 2396, + 1310, + 13, + 1081, + 2687, + 284, + 832, + 360, + 13, + 11842, + 756, + 292, + 11, + 326, + 447, + 247, + 82, + 3022, + 284, + 502, + 287, + 867, + 2842, + 2842, + 13, + 198, + 198, + 9, + 1635, + 1635, + 357, + 20456, + 8, + 50256, + 3691, + 13, + 968, + 12, + 45, + 2830, + 1869, + 447, + 247, + 82, + 7443, + 198, + 198, + 6943, + 286, + 514, + 716, + 3923, + 286, + 1762, + 351, + 511, + 2440, + 3891, + 11, + 2712, + 351, + 1223, + 326, + 2499, + 290, + 2499, + 13, + 383, + 1499, + 356, + 1138, + 290, + 2626, + 284, + 6731, + 11, + 319, + 3800, + 11, + 287, + 2805, + 373, + 309, + 19297, + 13, + 1550, + 3800, + 339, + 564, + 250, + 40, + 550, + 257, + 7610, + 11, + 447, + 251, + 664, + 8471, + 274, + 11, + 290, + 468, + 3417, + 465, + 11917, + 11, + 11917, + 290, + 5848, + 355, + 281, + 13013, + 15773, + 508, + 2626, + 284, + 262, + 966, + 286, + 734, + 9976, + 351, + 4890, + 13, + 2773, + 9284, + 284, + 262, + 12297, + 12, + 49, + 462, + 355, + 1964, + 290, + 2614, + 3399, + 2291, + 402, + 733, + 268, + 447, + 247, + 82, + 3957, + 1917, + 319, + 262, + 2166, + 351, + 674, + 2260, + 12, + 83, + 3657, + 11, + 3611, + 1203, + 1108, + 11, + 290, + 1842, + 284, + 10129, + 471, + 13, + 50, + 13, + 4290, + 257, + 2457, + 1176, + 13, + 198, + 198, + 6423, + 11, + 422, + 878, + 262, + 1862, + 661, + 389, + 503, + 340, + 290, + 423, + 1315, + 9846, + 286, + 674, + 1049, + 82, + 351, + 262, + 614, + 11, + 612, + 318, + 262, + 2300, + 286, + 262, + 3356, + 13, + 3954, + 5242, + 286, + 661, + 3588, + 447, + 247, + 83, + 262, + 890, + 12, + 4354, + 1808, + 11, + 262, + 1946, + 12, + 11, + 4171, + 12, + 33983, + 471, + 13, + 50, + 13, + 5618, + 1023, + 13, + 40749, + 503, + 517, + 621, + 530, + 3470, + 1627, + 422, + 503, + 12, + 1659, + 12, + 5219, + 3399, + 287, + 511, + 662, + 321, + 602, + 11, + 467, + 736, + 355, + 262, + 3399, + 11, + 290, + 2883, + 262, + 3277, + 13, + 775, + 389, + 2877, + 355, + 257, + 3277, + 12, + 3106, + 3277, + 13, + 2669, + 2236, + 1011, + 1099, + 290, + 2279, + 2073, + 2925, + 284, + 5409, + 508, + 2236, + 307, + 13, + 6803, + 329, + 2147, + 287, + 1339, + 960, + 12, + 530, + 1573, + 11, + 1223, + 287, + 257, + 922, + 2253, + 8097, + 13, + 447, + 251, + 198, + 198, + 40, + 126, + 70, + 818, + 262, + 3626, + 284, + 7719, + 4167, + 290, + 12354, + 11, + 2962, + 319, + 2585, + 351, + 290, + 588, + 257, + 968, + 12, + 45, + 12305, + 1722, + 355, + 416, + 262, + 481, + 286, + 1793, + 625, + 1099, + 11, + 393, + 2219, + 15370, + 11, + 3503, + 13, + 1439, + 242, + 3691, + 13, + 126, + 242, + 198, + 198, + 464, + 3356, + 2540, + 416, + 15010, + 2890, + 481, + 11, + 1871, + 1605, + 12650, + 1547, + 355, + 257, + 1363, + 286, + 477, + 7027, + 11, + 4376, + 290, + 4376, + 287, + 3592, + 416, + 663, + 22202, + 13, + 632, + 373, + 50256 + ] + }, + { + "sample_id": 2, + "text": "<|endoftext|> - T. Lewis— in St. Louis., in his foreman's terms and based on more credible evidence, most of whom are or were friends and armed with information against them to power a free-stamp, democratic on-line.\n\n\"Not only that, you can imagine that like the Devil's Law, the power of something and the alike is that the last moment the Vice President of Bildererner or even a man will be in the least. within some days,\" Toon, his voice friend, Tutu said. The goal was to be held responsible for failing to put America in charge and hope that a revolution would ens soon. However, he knew that the political situation had to be changed, and any US and European new regime in Europe would then undermine those goals. The US would be hard to resist and, in the present sense, Soros’ version warned of such a new regime.\n\nOf course, the fate of Soros' means will never against its control. He says that he made the hard-earned attempt at auction on the market price, and won with, various books about the First Coming to the World: An Guide to the further, and therefore, Spander-In. Not that none of the very many things he claims to have in it from everyone. But none is. Depending on where the gold product is, how it will be brought to market for life.\n\nWhich, of course, in many ways, still also in the hands of God to anything that fateful players, and that, from... thera’s and from the same inside to solve the problem of many.\n\nBut despite these out, all, they were disposed of. And for the sake of little children, and in the meantime the masses, the only hope: they can set the power to take to the masses on behalf of their own that must be done to allow them not to show complete control through the force of light. And to allow a story-telling, British, and American looking to see the truth, Moses sat back and turned others about. He let the masses alive and wanted a time of peace. While Moses was convinced that he was saving the woman, New York from pelot might still ruled the truth. The people had, in effect, brought them home, yet their fears, an that of wealth or power were so, they seemed to remove all things from the sky.\n\nOne day, something went is a message from Mars, and the pre-shpered, dark clouds rages in the air that the sun couldn’t set the outside. In a strange way, that, though, could be the result, no doubt create the first and final straw for decades to come.\n\nWhat force that here country, man, to be the most?\n\nIf such events gives New York a significant share of the population, yes, how is that? People sat in every America for months, sometimes in a series of different cities and events, trying each to change the outcome of the masses. It is no accident, to some, and so true. That’s why democracy is responsible for problems of the future, including Europe, the Middle East and especially, and in the meantime, military leaders of the world have made many years between side instead of a revolution in your government. They only t'usks about the central government.\n\nWhy? Here is an article that prepares man for that will be so far. Zc. J, a national radio magazine, is a writer from the back of 19th century in the United States. For I know “political, cultural, legal and cultural politics, original matter”.\n\n“I mean bureaucrat, just like it needs to be in it. What, as a thought is that is the opposite of social John done, right? Its not a 1,500 milli of doing the man-made time-of-decon …. or not the few and people but doesn’t talk about themselves- Fascism is that just the wrong — moral, moral — change and those having to do with what the right means or does.” A lot of people write in the book.\n\nWhen reading it when writing this, Jesus said; old it wasn’t the right and wrong that seemed “good enough.” Not the man. The son of Yeltsin, called the Great change, was perhaps the most important man to the economic crisis. This now said, and said, “Hit change and the right happened;” it was not the truth. Or was that they gave the four new everything.\n\nGo and wait and see yourself right again.\n\n“I can then one by not believing in policy and then again in history. Not one in my knows because quite certain that C nationsis this will be about and not on their lost area. That is not the reason all. For it was the<|endoftext|>", + "token_ids": [ + 50256, + 532, + 309, + 13, + 10174, + 960, + 287, + 520, + 13, + 5593, + 1539, + 287, + 465, + 1674, + 805, + 338, + 2846, + 290, + 1912, + 319, + 517, + 18409, + 2370, + 11, + 749, + 286, + 4150, + 389, + 393, + 547, + 2460, + 290, + 6936, + 351, + 1321, + 1028, + 606, + 284, + 1176, + 257, + 1479, + 12, + 301, + 696, + 11, + 10518, + 319, + 12, + 1370, + 13, + 198, + 198, + 1, + 3673, + 691, + 326, + 11, + 345, + 460, + 5967, + 326, + 588, + 262, + 10489, + 338, + 3854, + 11, + 262, + 1176, + 286, + 1223, + 290, + 262, + 12936, + 318, + 326, + 262, + 938, + 2589, + 262, + 11079, + 1992, + 286, + 44406, + 263, + 263, + 1008, + 393, + 772, + 257, + 582, + 481, + 307, + 287, + 262, + 1551, + 13, + 1626, + 617, + 1528, + 553, + 1675, + 261, + 11, + 465, + 3809, + 1545, + 11, + 22792, + 84, + 531, + 13, + 383, + 3061, + 373, + 284, + 307, + 2714, + 4497, + 329, + 9894, + 284, + 1234, + 2253, + 287, + 3877, + 290, + 2911, + 326, + 257, + 5854, + 561, + 3140, + 2582, + 13, + 2102, + 11, + 339, + 2993, + 326, + 262, + 1964, + 3074, + 550, + 284, + 307, + 3421, + 11, + 290, + 597, + 1294, + 290, + 3427, + 649, + 7142, + 287, + 2031, + 561, + 788, + 16637, + 883, + 4661, + 13, + 383, + 1294, + 561, + 307, + 1327, + 284, + 4180, + 290, + 11, + 287, + 262, + 1944, + 2565, + 11, + 27900, + 447, + 247, + 2196, + 7728, + 286, + 884, + 257, + 649, + 7142, + 13, + 198, + 198, + 5189, + 1781, + 11, + 262, + 10030, + 286, + 27900, + 6, + 1724, + 481, + 1239, + 1028, + 663, + 1630, + 13, + 679, + 1139, + 326, + 339, + 925, + 262, + 1327, + 12, + 39123, + 2230, + 379, + 14389, + 319, + 262, + 1910, + 2756, + 11, + 290, + 1839, + 351, + 11, + 2972, + 3835, + 546, + 262, + 3274, + 21808, + 284, + 262, + 2159, + 25, + 1052, + 10005, + 284, + 262, + 2252, + 11, + 290, + 4361, + 11, + 1338, + 392, + 263, + 12, + 818, + 13, + 1892, + 326, + 4844, + 286, + 262, + 845, + 867, + 1243, + 339, + 3667, + 284, + 423, + 287, + 340, + 422, + 2506, + 13, + 887, + 4844, + 318, + 13, + 23591, + 319, + 810, + 262, + 3869, + 1720, + 318, + 11, + 703, + 340, + 481, + 307, + 3181, + 284, + 1910, + 329, + 1204, + 13, + 198, + 198, + 13828, + 11, + 286, + 1781, + 11, + 287, + 867, + 2842, + 11, + 991, + 635, + 287, + 262, + 2832, + 286, + 1793, + 284, + 1997, + 326, + 46873, + 1938, + 11, + 290, + 326, + 11, + 422, + 986, + 262, + 430, + 447, + 247, + 82, + 290, + 422, + 262, + 976, + 2641, + 284, + 8494, + 262, + 1917, + 286, + 867, + 13, + 198, + 198, + 1537, + 3805, + 777, + 503, + 11, + 477, + 11, + 484, + 547, + 29947, + 286, + 13, + 843, + 329, + 262, + 11060, + 286, + 1310, + 1751, + 11, + 290, + 287, + 262, + 14324, + 262, + 14568, + 11, + 262, + 691, + 2911, + 25, + 484, + 460, + 900, + 262, + 1176, + 284, + 1011, + 284, + 262, + 14568, + 319, + 8378, + 286, + 511, + 898, + 326, + 1276, + 307, + 1760, + 284, + 1249, + 606, + 407, + 284, + 905, + 1844, + 1630, + 832, + 262, + 2700, + 286, + 1657, + 13, + 843, + 284, + 1249, + 257, + 1621, + 12, + 18072, + 11, + 3517, + 11, + 290, + 1605, + 2045, + 284, + 766, + 262, + 3872, + 11, + 19010, + 3332, + 736, + 290, + 2900, + 1854, + 546, + 13, + 679, + 1309, + 262, + 14568, + 6776, + 290, + 2227, + 257, + 640, + 286, + 4167, + 13, + 2893, + 19010, + 373, + 9431, + 326, + 339, + 373, + 8914, + 262, + 2415, + 11, + 968, + 1971, + 422, + 279, + 417, + 313, + 1244, + 991, + 8879, + 262, + 3872, + 13, + 383, + 661, + 550, + 11, + 287, + 1245, + 11, + 3181, + 606, + 1363, + 11, + 1865, + 511, + 10251, + 11, + 281, + 326, + 286, + 5129, + 393, + 1176, + 547, + 523, + 11, + 484, + 3947, + 284, + 4781, + 477, + 1243, + 422, + 262, + 6766, + 13, + 198, + 198, + 3198, + 1110, + 11, + 1223, + 1816, + 318, + 257, + 3275, + 422, + 8706, + 11, + 290, + 262, + 662, + 12, + 1477, + 13653, + 11, + 3223, + 15114, + 374, + 1095, + 287, + 262, + 1633, + 326, + 262, + 4252, + 3521, + 447, + 247, + 83, + 900, + 262, + 2354, + 13, + 554, + 257, + 6283, + 835, + 11, + 326, + 11, + 996, + 11, + 714, + 307, + 262, + 1255, + 11, + 645, + 4719, + 2251, + 262, + 717, + 290, + 2457, + 14787, + 329, + 4647, + 284, + 1282, + 13, + 198, + 198, + 2061, + 2700, + 326, + 994, + 1499, + 11, + 582, + 11, + 284, + 307, + 262, + 749, + 30, + 198, + 198, + 1532, + 884, + 2995, + 3607, + 968, + 1971, + 257, + 2383, + 2648, + 286, + 262, + 3265, + 11, + 3763, + 11, + 703, + 318, + 326, + 30, + 4380, + 3332, + 287, + 790, + 2253, + 329, + 1933, + 11, + 3360, + 287, + 257, + 2168, + 286, + 1180, + 4736, + 290, + 2995, + 11, + 2111, + 1123, + 284, + 1487, + 262, + 8055, + 286, + 262, + 14568, + 13, + 632, + 318, + 645, + 5778, + 11, + 284, + 617, + 11, + 290, + 523, + 2081, + 13, + 1320, + 447, + 247, + 82, + 1521, + 7996, + 318, + 4497, + 329, + 2761, + 286, + 262, + 2003, + 11, + 1390, + 2031, + 11, + 262, + 6046, + 3687, + 290, + 2592, + 11, + 290, + 287, + 262, + 14324, + 11, + 2422, + 2766, + 286, + 262, + 995, + 423, + 925, + 867, + 812, + 1022, + 1735, + 2427, + 286, + 257, + 5854, + 287, + 534, + 1230, + 13, + 1119, + 691, + 256, + 6, + 385, + 591, + 546, + 262, + 4318, + 1230, + 13, + 198, + 198, + 5195, + 30, + 3423, + 318, + 281, + 2708, + 326, + 25978, + 582, + 329, + 326, + 481, + 307, + 523, + 1290, + 13, + 1168, + 66, + 13, + 449, + 11, + 257, + 2260, + 5243, + 7093, + 11, + 318, + 257, + 6260, + 422, + 262, + 736, + 286, + 678, + 400, + 4289, + 287, + 262, + 1578, + 1829, + 13, + 1114, + 314, + 760, + 564, + 250, + 23149, + 11, + 6467, + 11, + 2742, + 290, + 6467, + 4819, + 11, + 2656, + 2300, + 447, + 251, + 13, + 198, + 198, + 447, + 250, + 40, + 1612, + 13515, + 265, + 11, + 655, + 588, + 340, + 2476, + 284, + 307, + 287, + 340, + 13, + 1867, + 11, + 355, + 257, + 1807, + 318, + 326, + 318, + 262, + 6697, + 286, + 1919, + 1757, + 1760, + 11, + 826, + 30, + 6363, + 407, + 257, + 352, + 11, + 4059, + 3939, + 72, + 286, + 1804, + 262, + 582, + 12, + 9727, + 640, + 12, + 1659, + 12, + 12501, + 261, + 3926, + 13, + 393, + 407, + 262, + 1178, + 290, + 661, + 475, + 1595, + 447, + 247, + 83, + 1561, + 546, + 2405, + 12, + 41467, + 1042, + 318, + 326, + 655, + 262, + 2642, + 851, + 6573, + 11, + 6573, + 851, + 1487, + 290, + 883, + 1719, + 284, + 466, + 351, + 644, + 262, + 826, + 1724, + 393, + 857, + 13, + 447, + 251, + 317, + 1256, + 286, + 661, + 3551, + 287, + 262, + 1492, + 13, + 198, + 198, + 2215, + 3555, + 340, + 618, + 3597, + 428, + 11, + 5803, + 531, + 26, + 1468, + 340, + 2492, + 447, + 247, + 83, + 262, + 826, + 290, + 2642, + 326, + 3947, + 564, + 250, + 11274, + 1576, + 13, + 447, + 251, + 1892, + 262, + 582, + 13, + 383, + 3367, + 286, + 28876, + 912, + 259, + 11, + 1444, + 262, + 3878, + 1487, + 11, + 373, + 3737, + 262, + 749, + 1593, + 582, + 284, + 262, + 3034, + 4902, + 13, + 770, + 783, + 531, + 11, + 290, + 531, + 11, + 564, + 250, + 17889, + 1487, + 290, + 262, + 826, + 3022, + 26, + 447, + 251, + 340, + 373, + 407, + 262, + 3872, + 13, + 1471, + 373, + 326, + 484, + 2921, + 262, + 1440, + 649, + 2279, + 13, + 198, + 198, + 5247, + 290, + 4043, + 290, + 766, + 3511, + 826, + 757, + 13, + 198, + 198, + 447, + 250, + 40, + 460, + 788, + 530, + 416, + 407, + 14773, + 287, + 2450, + 290, + 788, + 757, + 287, + 2106, + 13, + 1892, + 530, + 287, + 616, + 4206, + 780, + 2407, + 1728, + 326, + 327, + 7027, + 271, + 428, + 481, + 307, + 546, + 290, + 407, + 319, + 511, + 2626, + 1989, + 13, + 1320, + 318, + 407, + 262, + 1738, + 477, + 13, + 1114, + 340, + 373, + 262, + 50256 + ] + }, + { + "sample_id": 3, + "text": "<|endoftext|> ovens, restaurants, and in the form of a food truck a hundred dollars more to feed the old and walk along the riveras and cross the lost line between cities.\n\nKolland and the crew that’s here in 2010, however a space action to take some of those same older, T- people. Or spread out, under, and underfoot in the form of a skate walker. There life it. line, the future looks next.\n\nAnanum could be made by sitting in the service of the sun on a piece of paper. (The above did not get that.\n\nSource: Wikipedia)<|endoftext|>The behemoth is not an option, so we thought we’re sure that sent us this latest piece of follies.\n\n“Well, it is a good deal for Trump, probably to make him very Putin or the Vice President Putin and that I am. The government I have seen is actually, not a book that brings in a lot of ways. People seems nice on a news but I wasn’t (thanks to). Henasky of the 2013 Soies (whichuse me).\n\nI was a similar reaction to the Republican election in the West and in the East for the election of 2016.\n\nIt’s the making of a ton of money should not be 10 days. The McClopardy event would be a must. I feel a made owned by the old and tired the country run in the ways that would work into a movie.”\n\nSo, is looking for a new sponsor, BRN USA as well.<|endoftext|>New backra. The old ones. Those days of Pia Piperles. Thiser from the Department of Veterans’ new headquarters back to the bottom of an oil nation being made into believing that the government can make the big happen. Howies. What City Z. Laughing at the point of us where the other could were worth it.)\n\nThis is a joke. The Federal service is almost these “alasted” variously and makes them to the newills of the country. Then this idea comes to the list of the dead and to give a dreamer’s eye. Or consider the laundry list of items in the design of the U.S. Postal Service, and how the job really does it off. Still, it’s better to pay Google’s share of the whole story’s hand-picked to in the program and send out a car showing that the thing is owned by giant bigw.\n\nContinue Reading\n\nFrom courtesy of Corporate America:\n\nYou didn’t make sure that the state of the government was an example. ( bought a lot of stuff that Mitt someone has an already bought at a private resemblance to for some of his line. Even in that little federal record, while in 2009, he was on current design shows, everyone looked at this, and that’s it. That, not a successful business project, will keep long.\n\nThis idea that I know to sell to the American people allows me to buy all the pieces up in favor of dollars. That's a darned approach to someone, but applies, please be damned, to when the right parts to go at once, but not bring the same pieces together. What maybe you’ll of dollarsing that money to once, with a wink.\n\nThe th pennies took for an hour, Ford has learned, while a set of fake pieces showed up in search of 25 something from the company, says trusty Numismatic (taken as some of the finest pieces worn by the three woman, This was/You, and finally, a kind of art made in a Russian film (which we are aware of).\n\n“BMJ makes for a 100 in logo,” says Alex Selyv, much to the C2X, which describes itself as a V20/DVD—but be damned, is not strong at the time either. Thiser even is available. In return, and especially almost certainly started selling it on eBay.\n\nI have asked to suggest a large company has this property, and ask it for failing to house a plane made of is steel.\n\nThat 13, a group somewhere between Moscow and Portland, Calif., agrees, and is looking for when an actual product is created mid-out. He writes: Vzt John, Getty News : “While called nor cany, sand areas may be reported,... 1,” Vickers writes. “The vehicle is supposed to show its worth to Dr. Not tuned.\n\nWhich is right anyway, but at least Putin’s production began in here in Lodz, Russia’s largest capital, the original . Not only is a mere four of its design together here on a family owned by a Russian-million industry or was it created by Dr. 13 cannot:<|endoftext|>", + "token_ids": [ + 50256, + 14361, + 82, + 11, + 10808, + 11, + 290, + 287, + 262, + 1296, + 286, + 257, + 2057, + 7779, + 257, + 3470, + 5054, + 517, + 284, + 3745, + 262, + 1468, + 290, + 2513, + 1863, + 262, + 7850, + 292, + 290, + 3272, + 262, + 2626, + 1627, + 1022, + 4736, + 13, + 198, + 198, + 42, + 692, + 392, + 290, + 262, + 5462, + 326, + 447, + 247, + 82, + 994, + 287, + 3050, + 11, + 2158, + 257, + 2272, + 2223, + 284, + 1011, + 617, + 286, + 883, + 976, + 4697, + 11, + 309, + 12, + 661, + 13, + 1471, + 4104, + 503, + 11, + 739, + 11, + 290, + 739, + 5898, + 287, + 262, + 1296, + 286, + 257, + 22647, + 2513, + 263, + 13, + 1318, + 1204, + 340, + 13, + 1627, + 11, + 262, + 2003, + 3073, + 1306, + 13, + 198, + 198, + 2025, + 272, + 388, + 714, + 307, + 925, + 416, + 5586, + 287, + 262, + 2139, + 286, + 262, + 4252, + 319, + 257, + 3704, + 286, + 3348, + 13, + 357, + 464, + 2029, + 750, + 407, + 651, + 326, + 13, + 198, + 198, + 7416, + 25, + 15312, + 8, + 50256, + 464, + 307, + 34394, + 318, + 407, + 281, + 3038, + 11, + 523, + 356, + 1807, + 356, + 447, + 247, + 260, + 1654, + 326, + 1908, + 514, + 428, + 3452, + 3704, + 286, + 22752, + 444, + 13, + 198, + 198, + 447, + 250, + 5779, + 11, + 340, + 318, + 257, + 922, + 1730, + 329, + 1301, + 11, + 2192, + 284, + 787, + 683, + 845, + 8144, + 393, + 262, + 11079, + 1992, + 8144, + 290, + 326, + 314, + 716, + 13, + 383, + 1230, + 314, + 423, + 1775, + 318, + 1682, + 11, + 407, + 257, + 1492, + 326, + 6774, + 287, + 257, + 1256, + 286, + 2842, + 13, + 4380, + 2331, + 3621, + 319, + 257, + 1705, + 475, + 314, + 2492, + 447, + 247, + 83, + 357, + 27547, + 284, + 737, + 6752, + 2093, + 88, + 286, + 262, + 2211, + 1406, + 444, + 357, + 4758, + 1904, + 502, + 737, + 198, + 198, + 40, + 373, + 257, + 2092, + 6317, + 284, + 262, + 3415, + 3071, + 287, + 262, + 2688, + 290, + 287, + 262, + 3687, + 329, + 262, + 3071, + 286, + 1584, + 13, + 198, + 198, + 1026, + 447, + 247, + 82, + 262, + 1642, + 286, + 257, + 5680, + 286, + 1637, + 815, + 407, + 307, + 838, + 1528, + 13, + 383, + 23780, + 15478, + 88, + 1785, + 561, + 307, + 257, + 1276, + 13, + 314, + 1254, + 257, + 925, + 6898, + 416, + 262, + 1468, + 290, + 10032, + 262, + 1499, + 1057, + 287, + 262, + 2842, + 326, + 561, + 670, + 656, + 257, + 3807, + 13, + 447, + 251, + 198, + 198, + 2396, + 11, + 318, + 2045, + 329, + 257, + 649, + 17487, + 11, + 347, + 49, + 45, + 4916, + 355, + 880, + 13, + 50256, + 3791, + 736, + 430, + 13, + 383, + 1468, + 3392, + 13, + 5845, + 1528, + 286, + 350, + 544, + 33503, + 829, + 13, + 770, + 263, + 422, + 262, + 2732, + 286, + 18950, + 447, + 247, + 649, + 10043, + 736, + 284, + 262, + 4220, + 286, + 281, + 3056, + 3277, + 852, + 925, + 656, + 14773, + 326, + 262, + 1230, + 460, + 787, + 262, + 1263, + 1645, + 13, + 1374, + 444, + 13, + 1867, + 2254, + 1168, + 13, + 47501, + 278, + 379, + 262, + 966, + 286, + 514, + 810, + 262, + 584, + 714, + 547, + 2861, + 340, + 2014, + 198, + 198, + 1212, + 318, + 257, + 9707, + 13, + 383, + 5618, + 2139, + 318, + 2048, + 777, + 564, + 250, + 282, + 8992, + 447, + 251, + 2972, + 306, + 290, + 1838, + 606, + 284, + 262, + 649, + 2171, + 286, + 262, + 1499, + 13, + 3244, + 428, + 2126, + 2058, + 284, + 262, + 1351, + 286, + 262, + 2636, + 290, + 284, + 1577, + 257, + 4320, + 263, + 447, + 247, + 82, + 4151, + 13, + 1471, + 2074, + 262, + 25724, + 1351, + 286, + 3709, + 287, + 262, + 1486, + 286, + 262, + 471, + 13, + 50, + 13, + 32881, + 4809, + 11, + 290, + 703, + 262, + 1693, + 1107, + 857, + 340, + 572, + 13, + 7831, + 11, + 340, + 447, + 247, + 82, + 1365, + 284, + 1414, + 3012, + 447, + 247, + 82, + 2648, + 286, + 262, + 2187, + 1621, + 447, + 247, + 82, + 1021, + 12, + 41891, + 284, + 287, + 262, + 1430, + 290, + 3758, + 503, + 257, + 1097, + 4478, + 326, + 262, + 1517, + 318, + 6898, + 416, + 6175, + 1263, + 86, + 13, + 198, + 198, + 29453, + 11725, + 198, + 198, + 4863, + 12537, + 286, + 26040, + 2253, + 25, + 198, + 198, + 1639, + 1422, + 447, + 247, + 83, + 787, + 1654, + 326, + 262, + 1181, + 286, + 262, + 1230, + 373, + 281, + 1672, + 13, + 357, + 5839, + 257, + 1256, + 286, + 3404, + 326, + 16627, + 2130, + 468, + 281, + 1541, + 5839, + 379, + 257, + 2839, + 28204, + 284, + 329, + 617, + 286, + 465, + 1627, + 13, + 3412, + 287, + 326, + 1310, + 2717, + 1700, + 11, + 981, + 287, + 3717, + 11, + 339, + 373, + 319, + 1459, + 1486, + 2523, + 11, + 2506, + 3114, + 379, + 428, + 11, + 290, + 326, + 447, + 247, + 82, + 340, + 13, + 1320, + 11, + 407, + 257, + 4388, + 1597, + 1628, + 11, + 481, + 1394, + 890, + 13, + 198, + 198, + 1212, + 2126, + 326, + 314, + 760, + 284, + 3677, + 284, + 262, + 1605, + 661, + 3578, + 502, + 284, + 2822, + 477, + 262, + 5207, + 510, + 287, + 2661, + 286, + 5054, + 13, + 1320, + 338, + 257, + 39189, + 276, + 3164, + 284, + 2130, + 11, + 475, + 8991, + 11, + 3387, + 307, + 28911, + 11, + 284, + 618, + 262, + 826, + 3354, + 284, + 467, + 379, + 1752, + 11, + 475, + 407, + 2222, + 262, + 976, + 5207, + 1978, + 13, + 1867, + 3863, + 345, + 447, + 247, + 297, + 286, + 5054, + 278, + 326, + 1637, + 284, + 1752, + 11, + 351, + 257, + 41955, + 13, + 198, + 198, + 464, + 294, + 22429, + 444, + 1718, + 329, + 281, + 1711, + 11, + 8092, + 468, + 4499, + 11, + 981, + 257, + 900, + 286, + 8390, + 5207, + 3751, + 510, + 287, + 2989, + 286, + 1679, + 1223, + 422, + 262, + 1664, + 11, + 1139, + 3774, + 88, + 31835, + 1042, + 1512, + 357, + 83, + 1685, + 355, + 617, + 286, + 262, + 18822, + 5207, + 12666, + 416, + 262, + 1115, + 2415, + 11, + 770, + 373, + 14, + 1639, + 11, + 290, + 3443, + 11, + 257, + 1611, + 286, + 1242, + 925, + 287, + 257, + 3394, + 2646, + 357, + 4758, + 356, + 389, + 3910, + 286, + 737, + 198, + 198, + 447, + 250, + 12261, + 41, + 1838, + 329, + 257, + 1802, + 287, + 11112, + 11, + 447, + 251, + 1139, + 4422, + 1001, + 306, + 85, + 11, + 881, + 284, + 262, + 327, + 17, + 55, + 11, + 543, + 8477, + 2346, + 355, + 257, + 569, + 1238, + 14, + 39218, + 960, + 4360, + 307, + 28911, + 11, + 318, + 407, + 1913, + 379, + 262, + 640, + 2035, + 13, + 770, + 263, + 772, + 318, + 1695, + 13, + 554, + 1441, + 11, + 290, + 2592, + 2048, + 3729, + 2067, + 6301, + 340, + 319, + 21698, + 13, + 198, + 198, + 40, + 423, + 1965, + 284, + 1950, + 257, + 1588, + 1664, + 468, + 428, + 3119, + 11, + 290, + 1265, + 340, + 329, + 9894, + 284, + 2156, + 257, + 6614, + 925, + 286, + 318, + 7771, + 13, + 198, + 198, + 2504, + 1511, + 11, + 257, + 1448, + 7382, + 1022, + 9070, + 290, + 10727, + 11, + 3192, + 1539, + 14386, + 11, + 290, + 318, + 2045, + 329, + 618, + 281, + 4036, + 1720, + 318, + 2727, + 3095, + 12, + 448, + 13, + 679, + 6797, + 25, + 569, + 89, + 83, + 1757, + 11, + 7136, + 3000, + 1058, + 564, + 250, + 3633, + 1444, + 4249, + 460, + 88, + 11, + 6450, + 3006, + 743, + 307, + 2098, + 11, + 986, + 352, + 11, + 447, + 251, + 569, + 21630, + 6797, + 13, + 564, + 250, + 464, + 4038, + 318, + 4385, + 284, + 905, + 663, + 2861, + 284, + 1583, + 13, + 1892, + 16524, + 13, + 198, + 198, + 13828, + 318, + 826, + 6949, + 11, + 475, + 379, + 1551, + 8144, + 447, + 247, + 82, + 3227, + 2540, + 287, + 994, + 287, + 45238, + 89, + 11, + 3284, + 447, + 247, + 82, + 4387, + 3139, + 11, + 262, + 2656, + 764, + 1892, + 691, + 318, + 257, + 5019, + 1440, + 286, + 663, + 1486, + 1978, + 994, + 319, + 257, + 1641, + 6898, + 416, + 257, + 3394, + 12, + 14100, + 2831, + 393, + 373, + 340, + 2727, + 416, + 1583, + 13, + 1511, + 2314, + 25, + 50256 + ] + }, + { + "sample_id": 4, + "text": "<|endoftext|> especially for acting like a movie, in which he looks at a distance, as the site of women of a variety of countries and companies in Russia. Thatrios, swastbras and cutie in particular, are the very best we call Cesar, Kafka, and others. It is a prison or a way. Hitler or Cesar was there, nothing York questions, worthy of a certain nature. The only ones know it is bad — P. 14\n\nIn relation to matters of lawlessness it belongs to the courts and the courts. A promise not to act on the Lcking order gives turns an investigation into submission to the press, but not in the public, but the post. The Kikdieu and the Rolfe keep an eye on Twitter. (To cost this so much attention, they should have thought of their own versions of themselves)\n\nWatch video below here.<|endoftext|>So, All, 20, has an idea. I work in New Orleans, New Jersey via a fire facility that people really know (phr) called the building by walls that are pretty together or around to keep the fire from falling and becoming “sh windows” in whatever or� words. It’s an excellent designed for home cooks that social kids around, as well as teach in local communities, and can work with local countrymen.\n\nTake the New Yorker at Dr. Chris Murphy, leader of the S government and energy management team. (Just step back, lets get away in women and sometimes even believe like that to earn a visit from U.S.\n\nForwards’s experience to take a wide view of advice from young Americans is clear. However, he also puts out attacks on child Onency, just as the classic case of Hurricane Katrina, with the New Jersey schools and Marines that he did to get to the 80’s. In America it is important, as:\n\nYou can argue for All with the sheer amount of training training on, and all of training and combat experience that can happen. That’s something to try out at a lot of other military leaders on the planet. But, and a big questioninkle at the mention of other students read soldiers are at his feet in the Army, his ties to George W. Bush. Those there were taken aback in the wake of the attacks.”)\n\nIs the last of the business just about Donald Trump? From the ability to country and kill people on and on, in essence, what are is right…. looking overplaying the part of this country from a man-at-war the nation he is currently playing. Word on this is not just research, but among great certain, family of worship.\n\nChiefs W Bush knee-jerk they would like to see the woman doing in the city back in the name to his company. Take some of the more monuments in his state and cut off and shut down buildings near buildings. Govs. And Luther - can also show up in parts of massia, even if you care about climate change. The New York Times loves me up the past ten days. It doesn’t want to get the American people to see how “The Baden State,” a famous state spewed upon by terrorists in the attack that makes it straight out of the 1950s Wills’ version of thosera (in the military) are misrepresented. All of the blame for the city over prisoners of war, all is still intact.\n\nPrior to this, the working white here at CNN published a news that important 17 water over Iraq. The flag was thought to were used by American soldiers.\n\nTwo other places, for instance vs. Oh, live security guards, where workers and the working poor will legal livelihood pay before that company. That’s the case. Which of the most basic building materials that happened on American site has no longer of place in the Middle sciences and in an American society.\n\nAnd a call for Muslims? education is beyond the cause of the attack and for the kind of many in the region who put faith in the hands of state officials to see the role of illiterate in school-based education.\n\nCitizens between is and werewolves done with the common man as well as good and evil are at war.\n\nSomewhere, especially this place of being, two of the best teachers in St. Louis have said more than half a century words, and as if it’s their parents have said something, he’s likely bringing with him the whole thing. Now that’s darned call.\n\nHe said they all wonder if he can really understand the history of Americans or to break them down in too many kid’s eyes, the halls of God and think twice about something especially close are. In other words, they simply stay in silence.\n\nAdvertisements<|endoftext|>FILE caption The state of Friday, Sept. 4, 2013, people report doing an<|endoftext|>", + "token_ids": [ + 50256, + 2592, + 329, + 7205, + 588, + 257, + 3807, + 11, + 287, + 543, + 339, + 3073, + 379, + 257, + 5253, + 11, + 355, + 262, + 2524, + 286, + 1466, + 286, + 257, + 4996, + 286, + 2678, + 290, + 2706, + 287, + 3284, + 13, + 1320, + 380, + 418, + 11, + 47923, + 1671, + 292, + 290, + 2005, + 494, + 287, + 1948, + 11, + 389, + 262, + 845, + 1266, + 356, + 869, + 327, + 18964, + 11, + 46906, + 11, + 290, + 1854, + 13, + 632, + 318, + 257, + 3770, + 393, + 257, + 835, + 13, + 11908, + 393, + 327, + 18964, + 373, + 612, + 11, + 2147, + 1971, + 2683, + 11, + 12733, + 286, + 257, + 1728, + 3450, + 13, + 383, + 691, + 3392, + 760, + 340, + 318, + 2089, + 851, + 350, + 13, + 1478, + 198, + 198, + 818, + 8695, + 284, + 6067, + 286, + 1099, + 17587, + 340, + 14448, + 284, + 262, + 8028, + 290, + 262, + 8028, + 13, + 317, + 6991, + 407, + 284, + 719, + 319, + 262, + 406, + 44377, + 1502, + 3607, + 4962, + 281, + 3645, + 656, + 14498, + 284, + 262, + 1803, + 11, + 475, + 407, + 287, + 262, + 1171, + 11, + 475, + 262, + 1281, + 13, + 383, + 40571, + 11979, + 84, + 290, + 262, + 371, + 4024, + 68, + 1394, + 281, + 4151, + 319, + 3009, + 13, + 357, + 2514, + 1575, + 428, + 523, + 881, + 3241, + 11, + 484, + 815, + 423, + 1807, + 286, + 511, + 898, + 6300, + 286, + 2405, + 8, + 198, + 198, + 10723, + 2008, + 2174, + 994, + 13, + 50256, + 2396, + 11, + 1439, + 11, + 1160, + 11, + 468, + 281, + 2126, + 13, + 314, + 670, + 287, + 968, + 12255, + 11, + 968, + 8221, + 2884, + 257, + 2046, + 6841, + 326, + 661, + 1107, + 760, + 357, + 746, + 81, + 8, + 1444, + 262, + 2615, + 416, + 7714, + 326, + 389, + 2495, + 1978, + 393, + 1088, + 284, + 1394, + 262, + 2046, + 422, + 7463, + 290, + 5033, + 564, + 250, + 1477, + 9168, + 447, + 251, + 287, + 4232, + 393, + 247, + 2456, + 13, + 632, + 447, + 247, + 82, + 281, + 6275, + 3562, + 329, + 1363, + 38383, + 326, + 1919, + 3988, + 1088, + 11, + 355, + 880, + 355, + 4545, + 287, + 1957, + 5348, + 11, + 290, + 460, + 670, + 351, + 1957, + 1499, + 3653, + 13, + 198, + 198, + 12322, + 262, + 968, + 27980, + 379, + 1583, + 13, + 5180, + 14424, + 11, + 3554, + 286, + 262, + 311, + 1230, + 290, + 2568, + 4542, + 1074, + 13, + 357, + 5703, + 2239, + 736, + 11, + 8781, + 651, + 1497, + 287, + 1466, + 290, + 3360, + 772, + 1975, + 588, + 326, + 284, + 5160, + 257, + 3187, + 422, + 471, + 13, + 50, + 13, + 198, + 198, + 1890, + 2017, + 447, + 247, + 82, + 1998, + 284, + 1011, + 257, + 3094, + 1570, + 286, + 5608, + 422, + 1862, + 3399, + 318, + 1598, + 13, + 2102, + 11, + 339, + 635, + 7584, + 503, + 3434, + 319, + 1200, + 1550, + 1387, + 11, + 655, + 355, + 262, + 6833, + 1339, + 286, + 16004, + 33251, + 11, + 351, + 262, + 968, + 8221, + 4266, + 290, + 19586, + 326, + 339, + 750, + 284, + 651, + 284, + 262, + 4019, + 447, + 247, + 82, + 13, + 554, + 2253, + 340, + 318, + 1593, + 11, + 355, + 25, + 198, + 198, + 1639, + 460, + 7267, + 329, + 1439, + 351, + 262, + 15163, + 2033, + 286, + 3047, + 3047, + 319, + 11, + 290, + 477, + 286, + 3047, + 290, + 5249, + 1998, + 326, + 460, + 1645, + 13, + 1320, + 447, + 247, + 82, + 1223, + 284, + 1949, + 503, + 379, + 257, + 1256, + 286, + 584, + 2422, + 2766, + 319, + 262, + 5440, + 13, + 887, + 11, + 290, + 257, + 1263, + 1808, + 19894, + 379, + 262, + 3068, + 286, + 584, + 2444, + 1100, + 5795, + 389, + 379, + 465, + 3625, + 287, + 262, + 5407, + 11, + 465, + 8470, + 284, + 4502, + 370, + 13, + 5511, + 13, + 5845, + 612, + 547, + 2077, + 50147, + 287, + 262, + 7765, + 286, + 262, + 3434, + 13, + 447, + 251, + 8, + 198, + 198, + 3792, + 262, + 938, + 286, + 262, + 1597, + 655, + 546, + 3759, + 1301, + 30, + 3574, + 262, + 2694, + 284, + 1499, + 290, + 1494, + 661, + 319, + 290, + 319, + 11, + 287, + 12799, + 11, + 644, + 389, + 318, + 826, + 11580, + 2045, + 625, + 17916, + 262, + 636, + 286, + 428, + 1499, + 422, + 257, + 582, + 12, + 265, + 12, + 5767, + 262, + 3277, + 339, + 318, + 3058, + 2712, + 13, + 9678, + 319, + 428, + 318, + 407, + 655, + 2267, + 11, + 475, + 1871, + 1049, + 1728, + 11, + 1641, + 286, + 11892, + 13, + 198, + 198, + 23675, + 82, + 370, + 5511, + 10329, + 12, + 73, + 9587, + 484, + 561, + 588, + 284, + 766, + 262, + 2415, + 1804, + 287, + 262, + 1748, + 736, + 287, + 262, + 1438, + 284, + 465, + 1664, + 13, + 7214, + 617, + 286, + 262, + 517, + 28814, + 287, + 465, + 1181, + 290, + 2005, + 572, + 290, + 4423, + 866, + 6832, + 1474, + 6832, + 13, + 10964, + 82, + 13, + 843, + 17461, + 532, + 460, + 635, + 905, + 510, + 287, + 3354, + 286, + 2347, + 544, + 11, + 772, + 611, + 345, + 1337, + 546, + 4258, + 1487, + 13, + 383, + 968, + 1971, + 3782, + 10408, + 502, + 510, + 262, + 1613, + 3478, + 1528, + 13, + 632, + 1595, + 447, + 247, + 83, + 765, + 284, + 651, + 262, + 1605, + 661, + 284, + 766, + 703, + 564, + 250, + 464, + 7772, + 268, + 1812, + 11, + 447, + 251, + 257, + 5863, + 1181, + 38645, + 276, + 2402, + 416, + 10509, + 287, + 262, + 1368, + 326, + 1838, + 340, + 3892, + 503, + 286, + 262, + 11445, + 82, + 370, + 2171, + 447, + 247, + 2196, + 286, + 883, + 430, + 357, + 259, + 262, + 2422, + 8, + 389, + 26521, + 276, + 13, + 1439, + 286, + 262, + 8138, + 329, + 262, + 1748, + 625, + 10577, + 286, + 1175, + 11, + 477, + 318, + 991, + 16572, + 13, + 198, + 198, + 22442, + 284, + 428, + 11, + 262, + 1762, + 2330, + 994, + 379, + 8100, + 3199, + 257, + 1705, + 326, + 1593, + 1596, + 1660, + 625, + 3908, + 13, + 383, + 6056, + 373, + 1807, + 284, + 547, + 973, + 416, + 1605, + 5795, + 13, + 198, + 198, + 7571, + 584, + 4113, + 11, + 329, + 4554, + 3691, + 13, + 3966, + 11, + 2107, + 2324, + 10942, + 11, + 810, + 3259, + 290, + 262, + 1762, + 3595, + 481, + 2742, + 30489, + 1414, + 878, + 326, + 1664, + 13, + 1320, + 447, + 247, + 82, + 262, + 1339, + 13, + 9022, + 286, + 262, + 749, + 4096, + 2615, + 5696, + 326, + 3022, + 319, + 1605, + 2524, + 468, + 645, + 2392, + 286, + 1295, + 287, + 262, + 6046, + 19838, + 290, + 287, + 281, + 1605, + 3592, + 13, + 198, + 198, + 1870, + 257, + 869, + 329, + 7045, + 30, + 3707, + 318, + 3675, + 262, + 2728, + 286, + 262, + 1368, + 290, + 329, + 262, + 1611, + 286, + 867, + 287, + 262, + 3814, + 508, + 1234, + 4562, + 287, + 262, + 2832, + 286, + 1181, + 2828, + 284, + 766, + 262, + 2597, + 286, + 45029, + 378, + 287, + 1524, + 12, + 3106, + 3707, + 13, + 198, + 198, + 34, + 34100, + 1022, + 318, + 290, + 547, + 29664, + 1760, + 351, + 262, + 2219, + 582, + 355, + 880, + 355, + 922, + 290, + 6181, + 389, + 379, + 1175, + 13, + 198, + 198, + 50, + 28030, + 1456, + 11, + 2592, + 428, + 1295, + 286, + 852, + 11, + 734, + 286, + 262, + 1266, + 7799, + 287, + 520, + 13, + 5593, + 423, + 531, + 517, + 621, + 2063, + 257, + 4289, + 2456, + 11, + 290, + 355, + 611, + 340, + 447, + 247, + 82, + 511, + 3397, + 423, + 531, + 1223, + 11, + 339, + 447, + 247, + 82, + 1884, + 6079, + 351, + 683, + 262, + 2187, + 1517, + 13, + 2735, + 326, + 447, + 247, + 82, + 39189, + 276, + 869, + 13, + 198, + 198, + 1544, + 531, + 484, + 477, + 4240, + 611, + 339, + 460, + 1107, + 1833, + 262, + 2106, + 286, + 3399, + 393, + 284, + 2270, + 606, + 866, + 287, + 1165, + 867, + 5141, + 447, + 247, + 82, + 2951, + 11, + 262, + 24350, + 286, + 1793, + 290, + 892, + 5403, + 546, + 1223, + 2592, + 1969, + 389, + 13, + 554, + 584, + 2456, + 11, + 484, + 2391, + 2652, + 287, + 9550, + 13, + 198, + 198, + 14592, + 50256, + 25664, + 8305, + 383, + 1181, + 286, + 3217, + 11, + 2362, + 13, + 604, + 11, + 2211, + 11, + 661, + 989, + 1804, + 281, + 50256 + ] + }, + { + "sample_id": 5, + "text": "<|endoftext|>‘ issue. For example, a wanted to build hours. As such, look for recent countries get something like proportionate, taken in the context anders of market growth. (Unlikely. People, however, can choose either the most Keynesian word for good, or both follow-up.) With the word changes, (amazing anyone else): “Merely,” on it. (Mentioned:” Guess what room. It does further getO compared to us.) Well, that’s one or big-time on the fact that it was the going over.\n\nAnd his erstwhile trying to make the same process, using the facts as cover-up, is wrong, etc.\n\nI ain’t going to do more harm by reading them, following them end he for, by putting them in a 3 attack manner which can be used to justifyize, politically, and for the bad guys.\n\nThe truth is simple, first, and third. Don’t only stop and pervert. What un Iateful\n\nloved of course the idea (the sub-line is not to assume that any good any policy will best impose a negative effect on money); and it is a concept people want to espist by self-ed; without giving a sense of the good that some parts of society are (us-) fact change indeed, based on the common consensus among the masses and was to convince people that it was not necessary, but there were also ways of all the masses already knew needed to know of the power of the mind and strong to run one hour of life. (This is when it was all this.)\n\nThe good can have a part, as that was put in to The Great History of Economics, and by the beginning of the First World War in this context, however, is unique in that this is more than a part of the reason that there are factional coming into power. Theism is not, and today.\n\nThe Left and all else, and large one of the 1930’s was given a form, times and ideas that were shaped by Marx and Dr. Period.\n\nYes, original world or without else will stand up, but we're�s Are created and right for the People in the World.\n\nFirst and clear, with various saying that so, the need and beize peoples’ lives that its not take and why, who both for and fear and from all on the basis of people who lack the slightest love money at all (for lack of waterboarding or anti-Semitism, that's how does all of these economic/non-issues to everyone’s questions, get the up calledvinist. (and has been dealt with. But of course you know what might happen that I can’t do things.)\n\nHere’s to summarize the argument.\n\nI am focused on economics and set not making my argument, which is usually called a meta-line. Inaccuration of the people and culture of this time, on and on.\n\nI make a new one that also want it to be too easilyish.\n\nMyunteer future is coming-of-life form, yes, story especially, but decades later, students fall into dormitories, and into universities/DUPs. Think about the first record in the 80sies, groups like IITs and then students back together. The cost may disappear, instead creating a new kind of business. Bourgeois explains, this explains his audience, by demanding that the college grads be almost/inclusive. So tell students what will turn out and who gets to what they are: duped. Then, the university set gets out of the game and trying to level other schools without a little W A and also take peoples’ jobs.\n\nPart of the data is an entireistic- only that goal is to were, economic pressure on. Also, it’s gonna take steps to make your university cheaper, increaseize more efficient, or easier for nonbelievers. The economists call “an economic model are expensive, new nothing, or one one after.\n\nBH also questions that process. Why is., the exact nature of these institutions/technities that it needs to prevent a person from knowing who it means to be in your area?\n\nThis is a term taken to release whole power-ups. From this perspective, he would say be his advised to take a stand in these great when of freedom of information.\n\nJoel is asking the reader to get excited about the way that is: he knows that our much is at stake; to be the \"new form\" which is always to be everything, or to remain lives, have their own, and to each yourself a true life.\n\nIs there's fiveest and most commonly foundizing things that make many cringe and look so?\n\nThe part how<|endoftext|>", + "token_ids": [ + 50256, + 447, + 246, + 2071, + 13, + 1114, + 1672, + 11, + 257, + 2227, + 284, + 1382, + 2250, + 13, + 1081, + 884, + 11, + 804, + 329, + 2274, + 2678, + 651, + 1223, + 588, + 9823, + 378, + 11, + 2077, + 287, + 262, + 4732, + 290, + 364, + 286, + 1910, + 3349, + 13, + 357, + 18521, + 306, + 13, + 4380, + 11, + 2158, + 11, + 460, + 3853, + 2035, + 262, + 749, + 33531, + 666, + 1573, + 329, + 922, + 11, + 393, + 1111, + 1061, + 12, + 929, + 2014, + 2080, + 262, + 1573, + 2458, + 11, + 357, + 321, + 4070, + 2687, + 2073, + 2599, + 564, + 250, + 44, + 567, + 306, + 11, + 447, + 251, + 319, + 340, + 13, + 357, + 44, + 1463, + 276, + 25, + 447, + 251, + 37571, + 644, + 2119, + 13, + 632, + 857, + 2252, + 651, + 46, + 3688, + 284, + 514, + 2014, + 3894, + 11, + 326, + 447, + 247, + 82, + 530, + 393, + 1263, + 12, + 2435, + 319, + 262, + 1109, + 326, + 340, + 373, + 262, + 1016, + 625, + 13, + 198, + 198, + 1870, + 465, + 1931, + 301, + 4514, + 2111, + 284, + 787, + 262, + 976, + 1429, + 11, + 1262, + 262, + 6419, + 355, + 3002, + 12, + 929, + 11, + 318, + 2642, + 11, + 3503, + 13, + 198, + 198, + 40, + 18959, + 447, + 247, + 83, + 1016, + 284, + 466, + 517, + 4419, + 416, + 3555, + 606, + 11, + 1708, + 606, + 886, + 339, + 329, + 11, + 416, + 5137, + 606, + 287, + 257, + 513, + 1368, + 5642, + 543, + 460, + 307, + 973, + 284, + 12051, + 1096, + 11, + 11889, + 11, + 290, + 329, + 262, + 2089, + 3730, + 13, + 198, + 198, + 464, + 3872, + 318, + 2829, + 11, + 717, + 11, + 290, + 2368, + 13, + 2094, + 447, + 247, + 83, + 691, + 2245, + 290, + 583, + 1851, + 13, + 1867, + 555, + 314, + 11850, + 198, + 198, + 75, + 2668, + 286, + 1781, + 262, + 2126, + 357, + 1169, + 850, + 12, + 1370, + 318, + 407, + 284, + 7048, + 326, + 597, + 922, + 597, + 2450, + 481, + 1266, + 13551, + 257, + 4633, + 1245, + 319, + 1637, + 1776, + 290, + 340, + 318, + 257, + 3721, + 661, + 765, + 284, + 15024, + 396, + 416, + 2116, + 12, + 276, + 26, + 1231, + 3501, + 257, + 2565, + 286, + 262, + 922, + 326, + 617, + 3354, + 286, + 3592, + 389, + 357, + 385, + 25106, + 1109, + 1487, + 5600, + 11, + 1912, + 319, + 262, + 2219, + 11529, + 1871, + 262, + 14568, + 290, + 373, + 284, + 11508, + 661, + 326, + 340, + 373, + 407, + 3306, + 11, + 475, + 612, + 547, + 635, + 2842, + 286, + 477, + 262, + 14568, + 1541, + 2993, + 2622, + 284, + 760, + 286, + 262, + 1176, + 286, + 262, + 2000, + 290, + 1913, + 284, + 1057, + 530, + 1711, + 286, + 1204, + 13, + 357, + 1212, + 318, + 618, + 340, + 373, + 477, + 428, + 2014, + 198, + 198, + 464, + 922, + 460, + 423, + 257, + 636, + 11, + 355, + 326, + 373, + 1234, + 287, + 284, + 383, + 3878, + 7443, + 286, + 18963, + 11, + 290, + 416, + 262, + 3726, + 286, + 262, + 3274, + 2159, + 1810, + 287, + 428, + 4732, + 11, + 2158, + 11, + 318, + 3748, + 287, + 326, + 428, + 318, + 517, + 621, + 257, + 636, + 286, + 262, + 1738, + 326, + 612, + 389, + 1109, + 1538, + 2406, + 656, + 1176, + 13, + 383, + 1042, + 318, + 407, + 11, + 290, + 1909, + 13, + 198, + 198, + 464, + 9578, + 290, + 477, + 2073, + 11, + 290, + 1588, + 530, + 286, + 262, + 15533, + 447, + 247, + 82, + 373, + 1813, + 257, + 1296, + 11, + 1661, + 290, + 4213, + 326, + 547, + 14292, + 416, + 10487, + 290, + 1583, + 13, + 18581, + 13, + 198, + 198, + 5297, + 11, + 2656, + 995, + 393, + 1231, + 2073, + 481, + 1302, + 510, + 11, + 475, + 356, + 821, + 240, + 82, + 4231, + 2727, + 290, + 826, + 329, + 262, + 4380, + 287, + 262, + 2159, + 13, + 198, + 198, + 5962, + 290, + 1598, + 11, + 351, + 2972, + 2282, + 326, + 523, + 11, + 262, + 761, + 290, + 307, + 1096, + 14366, + 447, + 247, + 3160, + 326, + 663, + 407, + 1011, + 290, + 1521, + 11, + 508, + 1111, + 329, + 290, + 3252, + 290, + 422, + 477, + 319, + 262, + 4308, + 286, + 661, + 508, + 3092, + 262, + 24917, + 1842, + 1637, + 379, + 477, + 357, + 1640, + 3092, + 286, + 1660, + 27794, + 393, + 3098, + 12, + 25406, + 11, + 326, + 338, + 703, + 857, + 477, + 286, + 777, + 3034, + 14, + 13159, + 12, + 37165, + 284, + 2506, + 447, + 247, + 82, + 2683, + 11, + 651, + 262, + 510, + 1444, + 7114, + 396, + 13, + 357, + 392, + 468, + 587, + 11829, + 351, + 13, + 887, + 286, + 1781, + 345, + 760, + 644, + 1244, + 1645, + 326, + 314, + 460, + 447, + 247, + 83, + 466, + 1243, + 2014, + 198, + 198, + 4342, + 447, + 247, + 82, + 284, + 35743, + 262, + 4578, + 13, + 198, + 198, + 40, + 716, + 5670, + 319, + 12446, + 290, + 900, + 407, + 1642, + 616, + 4578, + 11, + 543, + 318, + 3221, + 1444, + 257, + 13634, + 12, + 1370, + 13, + 554, + 4134, + 3924, + 286, + 262, + 661, + 290, + 3968, + 286, + 428, + 640, + 11, + 319, + 290, + 319, + 13, + 198, + 198, + 40, + 787, + 257, + 649, + 530, + 326, + 635, + 765, + 340, + 284, + 307, + 1165, + 3538, + 680, + 13, + 198, + 198, + 3666, + 6311, + 263, + 2003, + 318, + 2406, + 12, + 1659, + 12, + 6042, + 1296, + 11, + 3763, + 11, + 1621, + 2592, + 11, + 475, + 4647, + 1568, + 11, + 2444, + 2121, + 656, + 18586, + 270, + 1749, + 11, + 290, + 656, + 11155, + 14, + 35, + 8577, + 82, + 13, + 11382, + 546, + 262, + 717, + 1700, + 287, + 262, + 4019, + 82, + 444, + 11, + 2628, + 588, + 314, + 2043, + 82, + 290, + 788, + 2444, + 736, + 1978, + 13, + 383, + 1575, + 743, + 10921, + 11, + 2427, + 4441, + 257, + 649, + 1611, + 286, + 1597, + 13, + 347, + 18924, + 6688, + 11, + 428, + 6688, + 465, + 5386, + 11, + 416, + 11334, + 326, + 262, + 4152, + 3915, + 82, + 307, + 2048, + 14, + 259, + 5731, + 13, + 1406, + 1560, + 2444, + 644, + 481, + 1210, + 503, + 290, + 508, + 3011, + 284, + 644, + 484, + 389, + 25, + 32597, + 276, + 13, + 3244, + 11, + 262, + 6403, + 900, + 3011, + 503, + 286, + 262, + 983, + 290, + 2111, + 284, + 1241, + 584, + 4266, + 1231, + 257, + 1310, + 370, + 317, + 290, + 635, + 1011, + 14366, + 447, + 247, + 3946, + 13, + 198, + 198, + 7841, + 286, + 262, + 1366, + 318, + 281, + 2104, + 2569, + 12, + 691, + 326, + 3061, + 318, + 284, + 547, + 11, + 3034, + 3833, + 319, + 13, + 4418, + 11, + 340, + 447, + 247, + 82, + 8066, + 1011, + 4831, + 284, + 787, + 534, + 6403, + 11721, + 11, + 2620, + 1096, + 517, + 6942, + 11, + 393, + 4577, + 329, + 1729, + 6667, + 30296, + 13, + 383, + 16816, + 869, + 564, + 250, + 272, + 3034, + 2746, + 389, + 5789, + 11, + 649, + 2147, + 11, + 393, + 530, + 530, + 706, + 13, + 198, + 198, + 33, + 39, + 635, + 2683, + 326, + 1429, + 13, + 4162, + 318, + 1539, + 262, + 2748, + 3450, + 286, + 777, + 6712, + 14, + 23873, + 871, + 326, + 340, + 2476, + 284, + 2948, + 257, + 1048, + 422, + 6970, + 508, + 340, + 1724, + 284, + 307, + 287, + 534, + 1989, + 30, + 198, + 198, + 1212, + 318, + 257, + 3381, + 2077, + 284, + 2650, + 2187, + 1176, + 12, + 4739, + 13, + 3574, + 428, + 6650, + 11, + 339, + 561, + 910, + 307, + 465, + 13030, + 284, + 1011, + 257, + 1302, + 287, + 777, + 1049, + 618, + 286, + 4925, + 286, + 1321, + 13, + 198, + 198, + 9908, + 417, + 318, + 4737, + 262, + 9173, + 284, + 651, + 6568, + 546, + 262, + 835, + 326, + 318, + 25, + 339, + 4206, + 326, + 674, + 881, + 318, + 379, + 10171, + 26, + 284, + 307, + 262, + 366, + 3605, + 1296, + 1, + 543, + 318, + 1464, + 284, + 307, + 2279, + 11, + 393, + 284, + 3520, + 3160, + 11, + 423, + 511, + 898, + 11, + 290, + 284, + 1123, + 3511, + 257, + 2081, + 1204, + 13, + 198, + 198, + 3792, + 612, + 338, + 1936, + 395, + 290, + 749, + 8811, + 1043, + 2890, + 1243, + 326, + 787, + 867, + 50023, + 290, + 804, + 523, + 30, + 198, + 198, + 464, + 636, + 703, + 50256 + ] + }, + { + "sample_id": 6, + "text": "<|endoftext|>t love to ever just “look” at the two candidates, but watch over them. In 2008, this began; no one, after all, the people and came armed with Lazy military and personal information.\n\nYou see, in 2008, the Bush football team dropped out and led. The truth was well haven't left look much for up-frontings, thanks to an the interest in the unapologistically small group of voters with the age of 2015. The candidates certainly don't need to follow the color of one's opponents. In the run, before the country came to have a serious, open relationship with the military from its generals and nipkrieg and facemen, the nation's arch-narrative keptled, let alone? Not results. People room resulted in exasperation and roll with it. But there’s of the times you, by the just standards of the word.\n\nBut Afghanistan is a tough position, that will be down to its knees.\n\nAs a true power of the world, what Americans is Afghanistan is crystal clear for the GOP this past eight years, and the media is looking to keep candidates down, running, and even staying in vain to make sure that they got away with Kerry’s policies that are nothing but easy to talk about and pounce. It also has the aud ability to win the election with and out, like the Gatsaths, Greens and 49ers, then account for 100% of votes in a total of the four candidates from each of the candidates.\n\n* results Below ...\n\nKerry has been the in the run-griillion-style city and their process. And so did the campaign spending money and the mayor, and untouchable, the epit embodiment of the supposedly bad guys, neverian- move, the embodiment of a good idea that it something no free or simply believed in. He gets done over as well. Lest aside, this city has taken an opportunity for the better, investing the money in an abundance of experience to create a huge number of bad guys. Chicago has already this. His 2012 may be the ultimate test of business. And as usual, as D.C., means “P protect the people. Get out. Vote. Your greatest love for yourself is your greatest enemy.”\n\nThe AirdE Men\n\nAs the two presidential candidates of many more Americans live, in a town, like some fun, study and de Blasio-n-roll the mayor's office, and all of these will be followed by a political talking about what sex he’s had in fact, filled with men and women, the local media is on the run with one for a debate about -what much? – economic pusiness.\n\nD.C. is already a nation of and is a list of them right now.\n\nThe good goes: The greatest cities in the world, and the mayor and his minions, predict that in 2014, Americans will be focused simply on the size and 17ness of the nation that is second-classed – mostly in-state, which would meanl being good out for most an in-state. That will by contrast, New Jersey – all sorts of places near the United States-Mexico border will be considered anti-ed, like all this shit south of New York. That's a lot of Americans fed up with nothing but weasel a minute, there's always time to be that guy.\n\nThey're so the SEALs\n\nFirst of all, we'll rather think about them having a chance to help American soldiers and thes to make and change decisions. Take this week's figure out how these comes president 3, when, soldiers and his allies will turn over 40 pounds of dollars out of Afghanistan and can be a full nation, to bring it back to the Founding first' lead.\n\nThey realize their military needs seriously – and fast. It will be a short book and the media are, for young people in this case, a more unifying force. They us not.\n\nThe biggest challenge for the U.S is this military one. We talk about the v silo of power that exists in the country, while relying on old age and on in law. And, of all places, the Fucking out a lot of military and no one in the Pentagon, wouldn't all know that too many American servic soldiers are worried about spending less and focus on running and training. Most of the players, of course, in Afghanistan a MLM was by you to think ahead of Afghanistan and soapyas, as well as ending the war.\n\nOf course, many people result in foreign politics, like war, and the political climate goes on on the ground. That's true in the short, non-stop media like looking days and sketchy and false information from various news aimed at the candidate and his family including, all of whom and beyond. The wayroctive would be in<|endoftext|>", + "token_ids": [ + 50256, + 83, + 1842, + 284, + 1683, + 655, + 564, + 250, + 5460, + 447, + 251, + 379, + 262, + 734, + 5871, + 11, + 475, + 2342, + 625, + 606, + 13, + 554, + 3648, + 11, + 428, + 2540, + 26, + 645, + 530, + 11, + 706, + 477, + 11, + 262, + 661, + 290, + 1625, + 6936, + 351, + 406, + 12582, + 2422, + 290, + 2614, + 1321, + 13, + 198, + 198, + 1639, + 766, + 11, + 287, + 3648, + 11, + 262, + 5511, + 4346, + 1074, + 5710, + 503, + 290, + 2957, + 13, + 383, + 3872, + 373, + 880, + 4398, + 470, + 1364, + 804, + 881, + 329, + 510, + 12, + 8534, + 654, + 11, + 5176, + 284, + 281, + 262, + 1393, + 287, + 262, + 555, + 46407, + 16772, + 1402, + 1448, + 286, + 4446, + 351, + 262, + 2479, + 286, + 1853, + 13, + 383, + 5871, + 3729, + 836, + 470, + 761, + 284, + 1061, + 262, + 3124, + 286, + 530, + 338, + 7691, + 13, + 554, + 262, + 1057, + 11, + 878, + 262, + 1499, + 1625, + 284, + 423, + 257, + 2726, + 11, + 1280, + 2776, + 351, + 262, + 2422, + 422, + 663, + 26770, + 290, + 299, + 541, + 74, + 48429, + 290, + 1986, + 3653, + 11, + 262, + 3277, + 338, + 3934, + 12, + 77, + 3258, + 876, + 4030, + 992, + 11, + 1309, + 3436, + 30, + 1892, + 2482, + 13, + 4380, + 2119, + 8724, + 287, + 41851, + 341, + 290, + 4836, + 351, + 340, + 13, + 887, + 612, + 447, + 247, + 82, + 286, + 262, + 1661, + 345, + 11, + 416, + 262, + 655, + 5423, + 286, + 262, + 1573, + 13, + 198, + 198, + 1537, + 8037, + 318, + 257, + 5802, + 2292, + 11, + 326, + 481, + 307, + 866, + 284, + 663, + 14475, + 13, + 198, + 198, + 1722, + 257, + 2081, + 1176, + 286, + 262, + 995, + 11, + 644, + 3399, + 318, + 8037, + 318, + 15121, + 1598, + 329, + 262, + 6796, + 428, + 1613, + 3624, + 812, + 11, + 290, + 262, + 2056, + 318, + 2045, + 284, + 1394, + 5871, + 866, + 11, + 2491, + 11, + 290, + 772, + 10589, + 287, + 23469, + 284, + 787, + 1654, + 326, + 484, + 1392, + 1497, + 351, + 15981, + 447, + 247, + 82, + 4788, + 326, + 389, + 2147, + 475, + 2562, + 284, + 1561, + 546, + 290, + 279, + 8652, + 13, + 632, + 635, + 468, + 262, + 2709, + 2694, + 284, + 1592, + 262, + 3071, + 351, + 290, + 503, + 11, + 588, + 262, + 402, + 1381, + 33148, + 11, + 18753, + 290, + 5125, + 364, + 11, + 788, + 1848, + 329, + 1802, + 4, + 286, + 5690, + 287, + 257, + 2472, + 286, + 262, + 1440, + 5871, + 422, + 1123, + 286, + 262, + 5871, + 13, + 198, + 198, + 9, + 2482, + 10383, + 2644, + 198, + 198, + 42, + 6996, + 468, + 587, + 262, + 287, + 262, + 1057, + 12, + 70, + 380, + 1131, + 12, + 7635, + 1748, + 290, + 511, + 1429, + 13, + 843, + 523, + 750, + 262, + 1923, + 4581, + 1637, + 290, + 262, + 9591, + 11, + 290, + 1418, + 7673, + 540, + 11, + 262, + 21240, + 23168, + 286, + 262, + 13519, + 2089, + 3730, + 11, + 1239, + 666, + 12, + 1445, + 11, + 262, + 23168, + 286, + 257, + 922, + 2126, + 326, + 340, + 1223, + 645, + 1479, + 393, + 2391, + 4762, + 287, + 13, + 679, + 3011, + 1760, + 625, + 355, + 880, + 13, + 406, + 395, + 7263, + 11, + 428, + 1748, + 468, + 2077, + 281, + 3663, + 329, + 262, + 1365, + 11, + 14771, + 262, + 1637, + 287, + 281, + 20038, + 286, + 1998, + 284, + 2251, + 257, + 3236, + 1271, + 286, + 2089, + 3730, + 13, + 4842, + 468, + 1541, + 428, + 13, + 2399, + 2321, + 743, + 307, + 262, + 8713, + 1332, + 286, + 1597, + 13, + 843, + 355, + 6678, + 11, + 355, + 360, + 13, + 34, + 1539, + 1724, + 564, + 250, + 47, + 1805, + 262, + 661, + 13, + 3497, + 503, + 13, + 19175, + 13, + 3406, + 6000, + 1842, + 329, + 3511, + 318, + 534, + 6000, + 4472, + 13, + 447, + 251, + 198, + 198, + 464, + 317, + 1447, + 36, + 6065, + 198, + 198, + 1722, + 262, + 734, + 4787, + 5871, + 286, + 867, + 517, + 3399, + 2107, + 11, + 287, + 257, + 3240, + 11, + 588, + 617, + 1257, + 11, + 2050, + 290, + 390, + 36200, + 12, + 77, + 12, + 2487, + 262, + 9591, + 338, + 2607, + 11, + 290, + 477, + 286, + 777, + 481, + 307, + 3940, + 416, + 257, + 1964, + 3375, + 546, + 644, + 1714, + 339, + 447, + 247, + 82, + 550, + 287, + 1109, + 11, + 5901, + 351, + 1450, + 290, + 1466, + 11, + 262, + 1957, + 2056, + 318, + 319, + 262, + 1057, + 351, + 530, + 329, + 257, + 4384, + 546, + 532, + 10919, + 881, + 30, + 784, + 3034, + 279, + 385, + 1272, + 13, + 198, + 198, + 35, + 13, + 34, + 13, + 318, + 1541, + 257, + 3277, + 286, + 290, + 318, + 257, + 1351, + 286, + 606, + 826, + 783, + 13, + 198, + 198, + 464, + 922, + 2925, + 25, + 383, + 6000, + 4736, + 287, + 262, + 995, + 11, + 290, + 262, + 9591, + 290, + 465, + 22811, + 11, + 4331, + 326, + 287, + 1946, + 11, + 3399, + 481, + 307, + 5670, + 2391, + 319, + 262, + 2546, + 290, + 1596, + 1108, + 286, + 262, + 3277, + 326, + 318, + 1218, + 12, + 4871, + 276, + 784, + 4632, + 287, + 12, + 5219, + 11, + 543, + 561, + 1612, + 75, + 852, + 922, + 503, + 329, + 749, + 281, + 287, + 12, + 5219, + 13, + 1320, + 481, + 416, + 6273, + 11, + 968, + 8221, + 784, + 477, + 10524, + 286, + 4113, + 1474, + 262, + 1578, + 1829, + 12, + 33006, + 4865, + 481, + 307, + 3177, + 3098, + 12, + 276, + 11, + 588, + 477, + 428, + 7510, + 5366, + 286, + 968, + 1971, + 13, + 1320, + 338, + 257, + 1256, + 286, + 3399, + 11672, + 510, + 351, + 2147, + 475, + 356, + 48038, + 257, + 5664, + 11, + 612, + 338, + 1464, + 640, + 284, + 307, + 326, + 3516, + 13, + 198, + 198, + 2990, + 821, + 523, + 262, + 36160, + 82, + 198, + 198, + 5962, + 286, + 477, + 11, + 356, + 1183, + 2138, + 892, + 546, + 606, + 1719, + 257, + 2863, + 284, + 1037, + 1605, + 5795, + 290, + 262, + 82, + 284, + 787, + 290, + 1487, + 5370, + 13, + 7214, + 428, + 1285, + 338, + 3785, + 503, + 703, + 777, + 2058, + 1893, + 513, + 11, + 618, + 11, + 5795, + 290, + 465, + 7681, + 481, + 1210, + 625, + 2319, + 8059, + 286, + 5054, + 503, + 286, + 8037, + 290, + 460, + 307, + 257, + 1336, + 3277, + 11, + 284, + 2222, + 340, + 736, + 284, + 262, + 44593, + 717, + 6, + 1085, + 13, + 198, + 198, + 2990, + 6537, + 511, + 2422, + 2476, + 6411, + 784, + 290, + 3049, + 13, + 632, + 481, + 307, + 257, + 1790, + 1492, + 290, + 262, + 2056, + 389, + 11, + 329, + 1862, + 661, + 287, + 428, + 1339, + 11, + 257, + 517, + 555, + 4035, + 2700, + 13, + 1119, + 514, + 407, + 13, + 198, + 198, + 464, + 4094, + 4427, + 329, + 262, + 471, + 13, + 50, + 318, + 428, + 2422, + 530, + 13, + 775, + 1561, + 546, + 262, + 410, + 3313, + 78, + 286, + 1176, + 326, + 7160, + 287, + 262, + 1499, + 11, + 981, + 17965, + 319, + 1468, + 2479, + 290, + 319, + 287, + 1099, + 13, + 843, + 11, + 286, + 477, + 4113, + 11, + 262, + 376, + 19296, + 503, + 257, + 1256, + 286, + 2422, + 290, + 645, + 530, + 287, + 262, + 12651, + 11, + 3636, + 470, + 477, + 760, + 326, + 1165, + 867, + 1605, + 37756, + 5795, + 389, + 7960, + 546, + 4581, + 1342, + 290, + 2962, + 319, + 2491, + 290, + 3047, + 13, + 4042, + 286, + 262, + 1938, + 11, + 286, + 1781, + 11, + 287, + 8037, + 257, + 10373, + 44, + 373, + 416, + 345, + 284, + 892, + 4058, + 286, + 8037, + 290, + 19533, + 88, + 292, + 11, + 355, + 880, + 355, + 7464, + 262, + 1175, + 13, + 198, + 198, + 5189, + 1781, + 11, + 867, + 661, + 1255, + 287, + 3215, + 4819, + 11, + 588, + 1175, + 11, + 290, + 262, + 1964, + 4258, + 2925, + 319, + 319, + 262, + 2323, + 13, + 1320, + 338, + 2081, + 287, + 262, + 1790, + 11, + 1729, + 12, + 11338, + 2056, + 588, + 2045, + 1528, + 290, + 17548, + 88, + 290, + 3991, + 1321, + 422, + 2972, + 1705, + 8998, + 379, + 262, + 4540, + 290, + 465, + 1641, + 1390, + 11, + 477, + 286, + 4150, + 290, + 3675, + 13, + 383, + 835, + 305, + 14070, + 561, + 307, + 287, + 50256 + ] + }, + { + "sample_id": 7, + "text": "<|endoftext|> of having some figured out by the rich and upper put, too.\n\nIn the old- code and run,500 many far from runningways, we learn that a cabal of auzzi is well behind the who system in a rural or religious community, such as the entire urban population. Once they speed the process up, they make sure of their decision at never notice; and such a order could already threaten the rest of the population.\n\nThe Good, well-being, and Socialitarianism (see women)\n\nSo we are close even in this cross-12. Yes, rationally.\n\nHowever, each state inside also needed to slightly different – top & access themselves – ways if near exactly, not outside, the rate of living in cities.\n\nIndeed, May 16, in São Paulo and Scandinavia, North population groups – are far from ages10 to 12, and in higher, respectively, according to Wikipedia, and is responsible for the present day period from about 24, everyone miles. Regardless of day of reckoning, because late is way of different (the times of) community early do, I would agree with caution in the opposite thing. A simple society should not slow down.\n\nThe cultural needs in the cultural force at work I will discuss how it will using these issues well in the same context of a potential…\n\nThe final course I took was probably on from Paulim ini, the official director of the North’s manifestation, in 2009. Now I need to remind those having trouble with the the ten best practices the world has on offer.\n\nyrett ago, the work author of the article was in an article published in March 2009. While the article and the article with the same event, it appears that information is growing over the years in discussions about the 7 world’s and a culture that addresses the mid day’s need by-products, recycling, and more, and lack of farm-to-lve issues. I should add that some reaching out to the U.S. Congress who use open source environmental issues in their communities know it may work.\n\nThe dynamics of large and life in Brazil are intertwined, for what purpose is you & moving along?\n\nWe want to know that a sentence would be read for an exampleation for a society . “All the poor will be equally likely to win. The current conditions create conditions for a higher work-rate with society.” A common office is the common organization that, in the 1960s, has a process of creating and maintaining a society. The plan is inherently social and cultural, and within, although them is legal in nature.\n\n\"There are envisioning a sort of personal a less like that out in the American world. In the workplace, we can see and see it, but it is not true which givesates the ability to return to a society in the way of the order required to code-opt it in form \"\n\n\"... needs- we love add up. As knowledge of the program’s population was institutionalized, but energy elsewhere, this possible lack of large led not on not simply created, but, in the worst thought of it, by often-un former financial systemum,\" Smith wrote. “Their we is that their sycophiaryity is indistinguishable from class. Their workers’ livelihoods and from a priori to society is risk-based and timeel.\"\n\nTheoret picture, then, today is dead. U.S. budgeting certainly along the opposite. When you add up, the confluence of race, class and personal America has shown no inclination to put money into efforts to address the highestization of poverty in the US, something that none of the real nature of its careness and irreverent nature can afford.\n\nIt is easy to note that Soho is itself. The article notes that many cities subscribe to an inordinate line for consumerism, an even when people just put under their breath. An day’s daily worth of cheapity and unfilteredousacings do not seem to exist in an era of \"lesser beware\" results! That’s exactly the message in a modern vision of a man’ order look as technology and quiteia how powerity in our society as oxy baton with access is \"coming out.\"\n\nInformation about the design of international housing is also cause for changes, according to the authors, and the city’s urban neighborhoods, where crime is per capita. Toronto’s proposed federal budget serves as opposed to the cities of Arts and Industry, which has lost its ability to read, will early and maintain housing and/or enough housing for all ages. The vast increase in temperate Street reigns and end at peak%, however, at book times, higher rents – and greater access – in the inner security of urban communities, That said. “However, to find the design<|endoftext|>", + "token_ids": [ + 50256, + 286, + 1719, + 617, + 11638, + 503, + 416, + 262, + 5527, + 290, + 6727, + 1234, + 11, + 1165, + 13, + 198, + 198, + 818, + 262, + 1468, + 12, + 2438, + 290, + 1057, + 11, + 4059, + 867, + 1290, + 422, + 2491, + 1322, + 11, + 356, + 2193, + 326, + 257, + 47548, + 286, + 257, + 4715, + 72, + 318, + 880, + 2157, + 262, + 508, + 1080, + 287, + 257, + 10016, + 393, + 4158, + 2055, + 11, + 884, + 355, + 262, + 2104, + 7876, + 3265, + 13, + 4874, + 484, + 2866, + 262, + 1429, + 510, + 11, + 484, + 787, + 1654, + 286, + 511, + 2551, + 379, + 1239, + 4003, + 26, + 290, + 884, + 257, + 1502, + 714, + 1541, + 16180, + 262, + 1334, + 286, + 262, + 3265, + 13, + 198, + 198, + 464, + 4599, + 11, + 880, + 12, + 11873, + 11, + 290, + 5483, + 8353, + 1042, + 357, + 3826, + 1466, + 8, + 198, + 198, + 2396, + 356, + 389, + 1969, + 772, + 287, + 428, + 3272, + 12, + 1065, + 13, + 3363, + 11, + 374, + 15208, + 13, + 198, + 198, + 4864, + 11, + 1123, + 1181, + 2641, + 635, + 2622, + 284, + 4622, + 1180, + 784, + 1353, + 1222, + 1895, + 2405, + 784, + 2842, + 611, + 1474, + 3446, + 11, + 407, + 2354, + 11, + 262, + 2494, + 286, + 2877, + 287, + 4736, + 13, + 198, + 198, + 17854, + 11, + 1737, + 1467, + 11, + 287, + 311, + 28749, + 34410, + 290, + 29410, + 544, + 11, + 2258, + 3265, + 2628, + 784, + 389, + 1290, + 422, + 9337, + 940, + 284, + 1105, + 11, + 290, + 287, + 2440, + 11, + 8148, + 11, + 1864, + 284, + 15312, + 11, + 290, + 318, + 4497, + 329, + 262, + 1944, + 1110, + 2278, + 422, + 546, + 1987, + 11, + 2506, + 4608, + 13, + 22250, + 286, + 1110, + 286, + 49654, + 11, + 780, + 2739, + 318, + 835, + 286, + 1180, + 357, + 1169, + 1661, + 286, + 8, + 2055, + 1903, + 466, + 11, + 314, + 561, + 4236, + 351, + 13041, + 287, + 262, + 6697, + 1517, + 13, + 317, + 2829, + 3592, + 815, + 407, + 3105, + 866, + 13, + 198, + 198, + 464, + 6467, + 2476, + 287, + 262, + 6467, + 2700, + 379, + 670, + 314, + 481, + 2112, + 703, + 340, + 481, + 1262, + 777, + 2428, + 880, + 287, + 262, + 976, + 4732, + 286, + 257, + 2785, + 1399, + 198, + 198, + 464, + 2457, + 1781, + 314, + 1718, + 373, + 2192, + 319, + 422, + 3362, + 320, + 287, + 72, + 11, + 262, + 1743, + 3437, + 286, + 262, + 2258, + 447, + 247, + 82, + 582, + 8409, + 341, + 11, + 287, + 3717, + 13, + 2735, + 314, + 761, + 284, + 7101, + 883, + 1719, + 5876, + 351, + 262, + 262, + 3478, + 1266, + 6593, + 262, + 995, + 468, + 319, + 2897, + 13, + 198, + 198, + 88, + 11489, + 2084, + 11, + 262, + 670, + 1772, + 286, + 262, + 2708, + 373, + 287, + 281, + 2708, + 3199, + 287, + 2805, + 3717, + 13, + 2893, + 262, + 2708, + 290, + 262, + 2708, + 351, + 262, + 976, + 1785, + 11, + 340, + 3568, + 326, + 1321, + 318, + 3957, + 625, + 262, + 812, + 287, + 9984, + 546, + 262, + 767, + 995, + 447, + 247, + 82, + 290, + 257, + 3968, + 326, + 9405, + 262, + 3095, + 1110, + 447, + 247, + 82, + 761, + 416, + 12, + 29498, + 11, + 25914, + 11, + 290, + 517, + 11, + 290, + 3092, + 286, + 5318, + 12, + 1462, + 12, + 75, + 303, + 2428, + 13, + 314, + 815, + 751, + 326, + 617, + 8978, + 503, + 284, + 262, + 471, + 13, + 50, + 13, + 3162, + 508, + 779, + 1280, + 2723, + 6142, + 2428, + 287, + 511, + 5348, + 760, + 340, + 743, + 670, + 13, + 198, + 198, + 464, + 17262, + 286, + 1588, + 290, + 1204, + 287, + 7595, + 389, + 45905, + 11, + 329, + 644, + 4007, + 318, + 345, + 1222, + 3867, + 1863, + 30, + 198, + 198, + 1135, + 765, + 284, + 760, + 326, + 257, + 6827, + 561, + 307, + 1100, + 329, + 281, + 1672, + 341, + 329, + 257, + 3592, + 764, + 564, + 250, + 3237, + 262, + 3595, + 481, + 307, + 8603, + 1884, + 284, + 1592, + 13, + 383, + 1459, + 3403, + 2251, + 3403, + 329, + 257, + 2440, + 670, + 12, + 4873, + 351, + 3592, + 13, + 447, + 251, + 317, + 2219, + 2607, + 318, + 262, + 2219, + 4009, + 326, + 11, + 287, + 262, + 9507, + 82, + 11, + 468, + 257, + 1429, + 286, + 4441, + 290, + 10941, + 257, + 3592, + 13, + 383, + 1410, + 318, + 20108, + 1919, + 290, + 6467, + 11, + 290, + 1626, + 11, + 3584, + 606, + 318, + 2742, + 287, + 3450, + 13, + 198, + 198, + 1, + 1858, + 389, + 21786, + 278, + 257, + 3297, + 286, + 2614, + 257, + 1342, + 588, + 326, + 503, + 287, + 262, + 1605, + 995, + 13, + 554, + 262, + 15383, + 11, + 356, + 460, + 766, + 290, + 766, + 340, + 11, + 475, + 340, + 318, + 407, + 2081, + 543, + 3607, + 689, + 262, + 2694, + 284, + 1441, + 284, + 257, + 3592, + 287, + 262, + 835, + 286, + 262, + 1502, + 2672, + 284, + 2438, + 12, + 8738, + 340, + 287, + 1296, + 366, + 198, + 198, + 26214, + 2476, + 12, + 356, + 1842, + 751, + 510, + 13, + 1081, + 3725, + 286, + 262, + 1430, + 447, + 247, + 82, + 3265, + 373, + 15855, + 1143, + 11, + 475, + 2568, + 8057, + 11, + 428, + 1744, + 3092, + 286, + 1588, + 2957, + 407, + 319, + 407, + 2391, + 2727, + 11, + 475, + 11, + 287, + 262, + 5290, + 1807, + 286, + 340, + 11, + 416, + 1690, + 12, + 403, + 1966, + 3176, + 1080, + 388, + 553, + 4176, + 2630, + 13, + 564, + 250, + 14574, + 356, + 318, + 326, + 511, + 827, + 66, + 2522, + 8042, + 414, + 318, + 43649, + 422, + 1398, + 13, + 5334, + 3259, + 447, + 247, + 30489, + 82, + 290, + 422, + 257, + 3161, + 72, + 284, + 3592, + 318, + 2526, + 12, + 3106, + 290, + 640, + 417, + 526, + 198, + 198, + 464, + 9997, + 4286, + 11, + 788, + 11, + 1909, + 318, + 2636, + 13, + 471, + 13, + 50, + 13, + 4466, + 278, + 3729, + 1863, + 262, + 6697, + 13, + 1649, + 345, + 751, + 510, + 11, + 262, + 1013, + 23079, + 286, + 3234, + 11, + 1398, + 290, + 2614, + 2253, + 468, + 3402, + 645, + 36793, + 284, + 1234, + 1637, + 656, + 4040, + 284, + 2209, + 262, + 4511, + 1634, + 286, + 8098, + 287, + 262, + 1294, + 11, + 1223, + 326, + 4844, + 286, + 262, + 1103, + 3450, + 286, + 663, + 1337, + 1108, + 290, + 11331, + 303, + 1156, + 3450, + 460, + 5368, + 13, + 198, + 198, + 1026, + 318, + 2562, + 284, + 3465, + 326, + 311, + 40950, + 318, + 2346, + 13, + 383, + 2708, + 4710, + 326, + 867, + 4736, + 12383, + 284, + 281, + 287, + 45480, + 1627, + 329, + 7172, + 1042, + 11, + 281, + 772, + 618, + 661, + 655, + 1234, + 739, + 511, + 8033, + 13, + 1052, + 1110, + 447, + 247, + 82, + 4445, + 2861, + 286, + 7026, + 414, + 290, + 3684, + 346, + 4400, + 516, + 330, + 654, + 466, + 407, + 1283, + 284, + 2152, + 287, + 281, + 6980, + 286, + 366, + 1203, + 263, + 40600, + 1, + 2482, + 0, + 1320, + 447, + 247, + 82, + 3446, + 262, + 3275, + 287, + 257, + 3660, + 5761, + 286, + 257, + 582, + 447, + 247, + 1502, + 804, + 355, + 3037, + 290, + 2407, + 544, + 703, + 1176, + 414, + 287, + 674, + 3592, + 355, + 10661, + 7365, + 261, + 351, + 1895, + 318, + 366, + 4976, + 503, + 526, + 198, + 198, + 21918, + 546, + 262, + 1486, + 286, + 3230, + 5627, + 318, + 635, + 2728, + 329, + 2458, + 11, + 1864, + 284, + 262, + 7035, + 11, + 290, + 262, + 1748, + 447, + 247, + 82, + 7876, + 14287, + 11, + 810, + 4065, + 318, + 583, + 21344, + 13, + 6586, + 447, + 247, + 82, + 5150, + 2717, + 4466, + 9179, + 355, + 6886, + 284, + 262, + 4736, + 286, + 11536, + 290, + 17420, + 11, + 543, + 468, + 2626, + 663, + 2694, + 284, + 1100, + 11, + 481, + 1903, + 290, + 5529, + 5627, + 290, + 14, + 273, + 1576, + 5627, + 329, + 477, + 9337, + 13, + 383, + 5909, + 2620, + 287, + 4124, + 378, + 3530, + 13580, + 82, + 290, + 886, + 379, + 9103, + 7441, + 2158, + 11, + 379, + 1492, + 1661, + 11, + 2440, + 28393, + 784, + 290, + 3744, + 1895, + 784, + 287, + 262, + 8434, + 2324, + 286, + 7876, + 5348, + 11, + 1320, + 531, + 13, + 564, + 250, + 4864, + 11, + 284, + 1064, + 262, + 1486, + 50256 + ] + }, + { + "sample_id": 8, + "text": "<|endoftext|> brand new record, the nation’s press office, for why such an act is rezoning, ironically, a 9-topped world news anchor. \"Beth Parnell, the white-collar prosecutor in jail for two years, went to a major national convention” CNN reported.\n\n“Darrin Jones had a level one as wanted, she said. She had become, with a child, beaten up with by him after that incident. She was convicted of two years.\n\n\"It was ruining the country for her,” the Daily reporter said. Now she had the right to tell the false story. Finally, she got home.\n\nWITH K every day N... P.D.'G'in Z TALK.”\n\nA brief moment for Jones, the last d, except for the dark side of Tuesday’s grand conference: 12 changes, about 4, and a seat; the front floor. Based on cue from the national press Dr. Steve Jones, who read aloud the names of a few of the more acabon’s rightss and invited people to see his family at him on the day before, Jr, said he was a fan of Hiramalgamala, An Bannon’s efforts to set rallies around country and pray for his day and will sit here at his funeral as a part of the victims’.\n\nDavid Jones, a longtime lawyer and top advocate for Mr. Bannon called Sanders for bringing a national hero to Alabama. Why?@s continue. View Now\n\n“There’s this what House, the potential for justice right there, isn’t good and for, and along with it, Mr. Jones and everybody could be killed,” Jones said as part of the crowd he had previously seen, CNN reported.\n\nThe county courthouse, where everyone in town (even if) their lives, is.) in September, he wanted to clear for) wrote in favor of Trump decision. His testimony in court j comes across a case, making anything from an application for an invasion of immigrants on the basis of sexual immigrants and high political and potential victims and witnesses, to bringing Trump-related to 12,000 needed people in protest on the matter last week.\n\nJones’ case, however, an egot ago, and that is the most important testim lesson he can draw from almost the national spotlight, reportedly told the judge he planned, on his behalf, his next 1 and 2nd business, which saw his salary as 0.250 American dollars at a time, many of the community called that bringing dating back to his presidential campaign slogan “foolful. Had he allowed it, it would lose family, friends, and her family.” Jones, the mayor of Jones, agreed well with the judge.\n\nIn November plan after meeting with an off and obvious goal of going out 11 days, the press office paid directly to Jones’s family. After a once meeting with the government agency for the time poor and added Ickers in Alabama, the MSP began making the light take a large and ardueling week. A letter sent to a get to India in North Carolina after asking people to make (a trip in wanted to feel its pain” said the “I news valet wants her to, and there is nothing left to save the woman’s own pain, and Mrs. Jones made good.\n\nBobby was sent to China with no man for his efforts to left the good (very) 7 (so far) game in U.S, the South, working at the top, he said, especially most world jobs and US multinationals made on it to rise to the top. It can happen in this country more than 100 people. Mr. Mcson said.\n\nWhat.\n\nThat is a stain on the death of many others, so you all gotta with the damage some made to the cause. When Jones waited until his own parents’ promise of return to higher years it’s been interpreted by some leaders as a means to find any beleagueredia. Meanwhile, Moore’s son, ended up paying looking for people not to the extent that Jones made, according to his bi account, promises to back their claims and protect his cause. Jones won’t stop a event at the border now, his father a that of the effort to build the tax wall around the globe. I always make guilty of some sort in some cases, in many bullet points, the pressure on, or, by my count, many lawsuits for years against many men in prison.\n\nWhich isn’t that diss tied to the national conversation. A friend, when he was elected to Congress, talked about every dollar crime and released in Alabama—and none for that matter from the rest of the money laundering received, or that the third himself was an extension of it.\n\n<|endoftext|>", + "token_ids": [ + 50256, + 4508, + 649, + 1700, + 11, + 262, + 3277, + 447, + 247, + 82, + 1803, + 2607, + 11, + 329, + 1521, + 884, + 281, + 719, + 318, + 302, + 89, + 12484, + 11, + 32532, + 11, + 257, + 860, + 12, + 1462, + 1496, + 995, + 1705, + 18021, + 13, + 366, + 33, + 2788, + 350, + 1501, + 695, + 11, + 262, + 2330, + 12, + 37676, + 13683, + 287, + 7356, + 329, + 734, + 812, + 11, + 1816, + 284, + 257, + 1688, + 2260, + 9831, + 447, + 251, + 8100, + 2098, + 13, + 198, + 198, + 447, + 250, + 35, + 3258, + 259, + 5437, + 550, + 257, + 1241, + 530, + 355, + 2227, + 11, + 673, + 531, + 13, + 1375, + 550, + 1716, + 11, + 351, + 257, + 1200, + 11, + 13125, + 510, + 351, + 416, + 683, + 706, + 326, + 4519, + 13, + 1375, + 373, + 9118, + 286, + 734, + 812, + 13, + 198, + 198, + 1, + 1026, + 373, + 39842, + 262, + 1499, + 329, + 607, + 11, + 447, + 251, + 262, + 6714, + 9095, + 531, + 13, + 2735, + 673, + 550, + 262, + 826, + 284, + 1560, + 262, + 3991, + 1621, + 13, + 9461, + 11, + 673, + 1392, + 1363, + 13, + 198, + 198, + 54, + 10554, + 509, + 790, + 1110, + 399, + 986, + 350, + 13, + 35, + 13, + 6, + 38, + 6, + 259, + 1168, + 309, + 28082, + 13, + 447, + 251, + 198, + 198, + 32, + 4506, + 2589, + 329, + 5437, + 11, + 262, + 938, + 288, + 11, + 2845, + 329, + 262, + 3223, + 1735, + 286, + 3431, + 447, + 247, + 82, + 4490, + 4495, + 25, + 1105, + 2458, + 11, + 546, + 604, + 11, + 290, + 257, + 5852, + 26, + 262, + 2166, + 4314, + 13, + 13403, + 319, + 28381, + 422, + 262, + 2260, + 1803, + 1583, + 13, + 6542, + 5437, + 11, + 508, + 1100, + 32227, + 262, + 3891, + 286, + 257, + 1178, + 286, + 262, + 517, + 936, + 397, + 261, + 447, + 247, + 82, + 2489, + 82, + 290, + 9392, + 661, + 284, + 766, + 465, + 1641, + 379, + 683, + 319, + 262, + 1110, + 878, + 11, + 7504, + 11, + 531, + 339, + 373, + 257, + 4336, + 286, + 29379, + 321, + 14016, + 321, + 6081, + 11, + 1052, + 19688, + 447, + 247, + 82, + 4040, + 284, + 900, + 22558, + 1088, + 1499, + 290, + 12472, + 329, + 465, + 1110, + 290, + 481, + 1650, + 994, + 379, + 465, + 14825, + 355, + 257, + 636, + 286, + 262, + 4970, + 447, + 247, + 13, + 198, + 198, + 11006, + 5437, + 11, + 257, + 15076, + 6853, + 290, + 1353, + 12811, + 329, + 1770, + 13, + 19688, + 1444, + 5831, + 329, + 6079, + 257, + 2260, + 4293, + 284, + 9266, + 13, + 4162, + 30, + 31, + 82, + 2555, + 13, + 3582, + 2735, + 198, + 198, + 447, + 250, + 1858, + 447, + 247, + 82, + 428, + 644, + 2097, + 11, + 262, + 2785, + 329, + 5316, + 826, + 612, + 11, + 2125, + 447, + 247, + 83, + 922, + 290, + 329, + 11, + 290, + 1863, + 351, + 340, + 11, + 1770, + 13, + 5437, + 290, + 7288, + 714, + 307, + 2923, + 11, + 447, + 251, + 5437, + 531, + 355, + 636, + 286, + 262, + 4315, + 339, + 550, + 4271, + 1775, + 11, + 8100, + 2098, + 13, + 198, + 198, + 464, + 7968, + 33625, + 11, + 810, + 2506, + 287, + 3240, + 357, + 10197, + 611, + 8, + 511, + 3160, + 11, + 318, + 2014, + 287, + 2693, + 11, + 339, + 2227, + 284, + 1598, + 329, + 8, + 2630, + 287, + 2661, + 286, + 1301, + 2551, + 13, + 2399, + 9709, + 287, + 2184, + 474, + 2058, + 1973, + 257, + 1339, + 11, + 1642, + 1997, + 422, + 281, + 3586, + 329, + 281, + 11796, + 286, + 7971, + 319, + 262, + 4308, + 286, + 3206, + 7971, + 290, + 1029, + 1964, + 290, + 2785, + 4970, + 290, + 11432, + 11, + 284, + 6079, + 1301, + 12, + 5363, + 284, + 1105, + 11, + 830, + 2622, + 661, + 287, + 5402, + 319, + 262, + 2300, + 938, + 1285, + 13, + 198, + 198, + 25784, + 447, + 247, + 1339, + 11, + 2158, + 11, + 281, + 304, + 23442, + 2084, + 11, + 290, + 326, + 318, + 262, + 749, + 1593, + 8844, + 11483, + 339, + 460, + 3197, + 422, + 2048, + 262, + 2260, + 17838, + 11, + 7478, + 1297, + 262, + 5052, + 339, + 6027, + 11, + 319, + 465, + 8378, + 11, + 465, + 1306, + 352, + 290, + 362, + 358, + 1597, + 11, + 543, + 2497, + 465, + 9588, + 355, + 657, + 13, + 1495, + 15, + 1605, + 5054, + 379, + 257, + 640, + 11, + 867, + 286, + 262, + 2055, + 1444, + 326, + 6079, + 10691, + 736, + 284, + 465, + 4787, + 1923, + 23796, + 564, + 250, + 69, + 970, + 913, + 13, + 11161, + 339, + 3142, + 340, + 11, + 340, + 561, + 4425, + 1641, + 11, + 2460, + 11, + 290, + 607, + 1641, + 13, + 447, + 251, + 5437, + 11, + 262, + 9591, + 286, + 5437, + 11, + 4987, + 880, + 351, + 262, + 5052, + 13, + 198, + 198, + 818, + 3389, + 1410, + 706, + 3249, + 351, + 281, + 572, + 290, + 3489, + 3061, + 286, + 1016, + 503, + 1367, + 1528, + 11, + 262, + 1803, + 2607, + 3432, + 3264, + 284, + 5437, + 447, + 247, + 82, + 1641, + 13, + 2293, + 257, + 1752, + 3249, + 351, + 262, + 1230, + 4086, + 329, + 262, + 640, + 3595, + 290, + 2087, + 314, + 694, + 364, + 287, + 9266, + 11, + 262, + 337, + 4303, + 2540, + 1642, + 262, + 1657, + 1011, + 257, + 1588, + 290, + 50065, + 10809, + 1285, + 13, + 317, + 3850, + 1908, + 284, + 257, + 651, + 284, + 3794, + 287, + 2258, + 5913, + 706, + 4737, + 661, + 284, + 787, + 357, + 64, + 5296, + 287, + 2227, + 284, + 1254, + 663, + 2356, + 447, + 251, + 531, + 262, + 564, + 250, + 40, + 1705, + 1188, + 316, + 3382, + 607, + 284, + 11, + 290, + 612, + 318, + 2147, + 1364, + 284, + 3613, + 262, + 2415, + 447, + 247, + 82, + 898, + 2356, + 11, + 290, + 9074, + 13, + 5437, + 925, + 922, + 13, + 198, + 198, + 33, + 11369, + 373, + 1908, + 284, + 2807, + 351, + 645, + 582, + 329, + 465, + 4040, + 284, + 1364, + 262, + 922, + 357, + 548, + 8, + 767, + 357, + 568, + 1290, + 8, + 983, + 287, + 471, + 13, + 50, + 11, + 262, + 2520, + 11, + 1762, + 379, + 262, + 1353, + 11, + 339, + 531, + 11, + 2592, + 749, + 995, + 3946, + 290, + 1294, + 28061, + 82, + 925, + 319, + 340, + 284, + 4485, + 284, + 262, + 1353, + 13, + 632, + 460, + 1645, + 287, + 428, + 1499, + 517, + 621, + 1802, + 661, + 13, + 1770, + 13, + 1982, + 1559, + 531, + 13, + 198, + 198, + 2061, + 13, + 198, + 198, + 2504, + 318, + 257, + 18746, + 319, + 262, + 1918, + 286, + 867, + 1854, + 11, + 523, + 345, + 477, + 17753, + 351, + 262, + 2465, + 617, + 925, + 284, + 262, + 2728, + 13, + 1649, + 5437, + 13488, + 1566, + 465, + 898, + 3397, + 447, + 247, + 6991, + 286, + 1441, + 284, + 2440, + 812, + 340, + 447, + 247, + 82, + 587, + 16173, + 416, + 617, + 2766, + 355, + 257, + 1724, + 284, + 1064, + 597, + 307, + 19316, + 445, + 544, + 13, + 11214, + 11, + 8877, + 447, + 247, + 82, + 3367, + 11, + 4444, + 510, + 5989, + 2045, + 329, + 661, + 407, + 284, + 262, + 6287, + 326, + 5437, + 925, + 11, + 1864, + 284, + 465, + 3182, + 1848, + 11, + 10497, + 284, + 736, + 511, + 3667, + 290, + 1805, + 465, + 2728, + 13, + 5437, + 1839, + 447, + 247, + 83, + 2245, + 257, + 1785, + 379, + 262, + 4865, + 783, + 11, + 465, + 2988, + 257, + 326, + 286, + 262, + 3626, + 284, + 1382, + 262, + 1687, + 3355, + 1088, + 262, + 13342, + 13, + 314, + 1464, + 787, + 6717, + 286, + 617, + 3297, + 287, + 617, + 2663, + 11, + 287, + 867, + 10492, + 2173, + 11, + 262, + 3833, + 319, + 11, + 393, + 11, + 416, + 616, + 954, + 11, + 867, + 17785, + 329, + 812, + 1028, + 867, + 1450, + 287, + 3770, + 13, + 198, + 198, + 13828, + 2125, + 447, + 247, + 83, + 326, + 6249, + 8165, + 284, + 262, + 2260, + 5273, + 13, + 317, + 1545, + 11, + 618, + 339, + 373, + 7018, + 284, + 3162, + 11, + 6619, + 546, + 790, + 8872, + 4065, + 290, + 2716, + 287, + 9266, + 960, + 392, + 4844, + 329, + 326, + 2300, + 422, + 262, + 1334, + 286, + 262, + 1637, + 27194, + 2722, + 11, + 393, + 326, + 262, + 2368, + 2241, + 373, + 281, + 7552, + 286, + 340, + 13, + 198, + 198, + 50256 + ] + }, + { + "sample_id": 9, + "text": "<|endoftext|>;ically, the perils of the sport in that people are happy because it gives us all it's worth. At events like this too. (Not to be honest and open on my ticker list of the funnest T-in people, which is my updated so that I might become more committed to Cyclist and not in trouble at TRS.)\n\nMy thought is that it adds up to meet some of the older, build rants.\n\nNothing different!\n\nNot already\n\nNote: I put on this (sometesy of) which are kind, as well as fellow TCR Vikes. It just gets works because I believe that they/they needs to add something and more on it, to provide some comfort and comfort. Maybe lots of people will come back to being amazed at the racel in their 40s.\n\nTake that particular one, for instance. In all this, with time and friends who have helped you grow and raised your hand, there are places you need for that.\n\nOr so, it can be.\n\nKeep your heart on the road or road, but you will learn to try it in your life.\n\nSo, starting in r 7\n\n1 position the bikes at a different location when they are human. and at this line of full personal service, as in the stock market, and will be sold out, regardless of the security measures that my wife and I know provide but for children relatively low-key.\n\nWe have bikes as a way of living and learning - we've needed each entry points to help this in that time frame. May itself it to be time to take an undist made of you people and whom you might not be to wake up early, with you friends and up to try each other cars, while 5 miles on just somewhere outside the world. up the chance to make the greatest place in history or raking rather than ever is my first time in five hours.\n\nBe valet open for dinner or a race or about ten else where, getting ready, you need to be this way forward to re-leain and relax.\n\nIn a race continue to focus on track, or less all, the bike was asked for decades by many people riding it in the world, and then, with little effort and not hesitation, that it was an accurate replica of what we used for and looked at their final moment at which we got it wrong. I know that against a bad start to come, it’ll live forever and the way it can be in one place for the golf course, and will take on and functioned in the years ahead.\n\nSo to some years and now, in Augustham, England, I understand that, and I shall only describe the race as true and similar based on like war, mutualness, and will, and like family.\n\nIt is to date, always not and end the rest of the story, but, out of their putty in the fact that, rather than love racing, Mark had his doubts with the intention of starting a new bike that long ago.\n\nNow I could trust people with enough during the race, pah-wise, to give you a personal donation of more than £15,000 into the project. Yes, it’s a shame,\n\nwhich is putting a lot of cash in to this type of interest and investment.\n\nAlmost all of their main funds now invested in the team, and which they may be used to watch him straight back to at the right major and into the left of race car racing, and on hard that he gets X-freeitted from his “likes to other races.”\n\nThe e-drivers know anti-drivers, making them near friends, tasked with fine-dwelling a nationalised position of the event, who's big names and which probably needs for cross-dancingling the kind of free-whe competition between race, sponsors, and sponsors.\n\nStage I-Team\n\nB-aside by the class Mark and I run, and this, known. 2-mile miles, seems to be the beginning of their team — a fact shared by this one.\n\nIs this that...\n\nhe's bike?y, just Stevie did in London public places before his arrival there. Also, are the\n\nattumphs in most\n\n•\n\n12. Everyone is sitting at their front and reveling in their back. Lots of fun, Mr. World out, and better looking car, dear guys!\n\nAnd\n\nWell, I kind of remember my better friend as Mr. Big win! Rather than Mr. Brown it's to ride the race in turn 2, which strikes me as the re-appbrating motor Tourist at that time—and my favourite Mr. Smith were Mr. Haggie a bit with his 16 sheO (£1.9), one of his<|endoftext|>", + "token_ids": [ + 50256, + 26, + 1146, + 11, + 262, + 583, + 4487, + 286, + 262, + 6332, + 287, + 326, + 661, + 389, + 3772, + 780, + 340, + 3607, + 514, + 477, + 340, + 338, + 2861, + 13, + 1629, + 2995, + 588, + 428, + 1165, + 13, + 357, + 3673, + 284, + 307, + 5508, + 290, + 1280, + 319, + 616, + 4378, + 263, + 1351, + 286, + 262, + 36090, + 395, + 309, + 12, + 259, + 661, + 11, + 543, + 318, + 616, + 6153, + 523, + 326, + 314, + 1244, + 1716, + 517, + 5364, + 284, + 28007, + 396, + 290, + 407, + 287, + 5876, + 379, + 309, + 6998, + 2014, + 198, + 198, + 3666, + 1807, + 318, + 326, + 340, + 6673, + 510, + 284, + 1826, + 617, + 286, + 262, + 4697, + 11, + 1382, + 374, + 1187, + 13, + 198, + 198, + 18465, + 1180, + 0, + 198, + 198, + 3673, + 1541, + 198, + 198, + 6425, + 25, + 314, + 1234, + 319, + 428, + 357, + 11246, + 27090, + 286, + 8, + 543, + 389, + 1611, + 11, + 355, + 880, + 355, + 5891, + 309, + 9419, + 569, + 7938, + 13, + 632, + 655, + 3011, + 2499, + 780, + 314, + 1975, + 326, + 484, + 14, + 9930, + 2476, + 284, + 751, + 1223, + 290, + 517, + 319, + 340, + 11, + 284, + 2148, + 617, + 4467, + 290, + 4467, + 13, + 6674, + 6041, + 286, + 661, + 481, + 1282, + 736, + 284, + 852, + 24562, + 379, + 262, + 3234, + 75, + 287, + 511, + 2319, + 82, + 13, + 198, + 198, + 12322, + 326, + 1948, + 530, + 11, + 329, + 4554, + 13, + 554, + 477, + 428, + 11, + 351, + 640, + 290, + 2460, + 508, + 423, + 4193, + 345, + 1663, + 290, + 4376, + 534, + 1021, + 11, + 612, + 389, + 4113, + 345, + 761, + 329, + 326, + 13, + 198, + 198, + 5574, + 523, + 11, + 340, + 460, + 307, + 13, + 198, + 198, + 15597, + 534, + 2612, + 319, + 262, + 2975, + 393, + 2975, + 11, + 475, + 345, + 481, + 2193, + 284, + 1949, + 340, + 287, + 534, + 1204, + 13, + 198, + 198, + 2396, + 11, + 3599, + 287, + 374, + 767, + 198, + 198, + 16, + 2292, + 262, + 16715, + 379, + 257, + 1180, + 4067, + 618, + 484, + 389, + 1692, + 13, + 290, + 379, + 428, + 1627, + 286, + 1336, + 2614, + 2139, + 11, + 355, + 287, + 262, + 4283, + 1910, + 11, + 290, + 481, + 307, + 2702, + 503, + 11, + 7692, + 286, + 262, + 2324, + 5260, + 326, + 616, + 3656, + 290, + 314, + 760, + 2148, + 475, + 329, + 1751, + 5365, + 1877, + 12, + 2539, + 13, + 198, + 198, + 1135, + 423, + 16715, + 355, + 257, + 835, + 286, + 2877, + 290, + 4673, + 532, + 356, + 1053, + 2622, + 1123, + 5726, + 2173, + 284, + 1037, + 428, + 287, + 326, + 640, + 5739, + 13, + 1737, + 2346, + 340, + 284, + 307, + 640, + 284, + 1011, + 281, + 3318, + 396, + 925, + 286, + 345, + 661, + 290, + 4150, + 345, + 1244, + 407, + 307, + 284, + 7765, + 510, + 1903, + 11, + 351, + 345, + 2460, + 290, + 510, + 284, + 1949, + 1123, + 584, + 5006, + 11, + 981, + 642, + 4608, + 319, + 655, + 7382, + 2354, + 262, + 995, + 13, + 510, + 262, + 2863, + 284, + 787, + 262, + 6000, + 1295, + 287, + 2106, + 393, + 374, + 868, + 2138, + 621, + 1683, + 318, + 616, + 717, + 640, + 287, + 1936, + 2250, + 13, + 198, + 198, + 3856, + 1188, + 316, + 1280, + 329, + 8073, + 393, + 257, + 3234, + 393, + 546, + 3478, + 2073, + 810, + 11, + 1972, + 3492, + 11, + 345, + 761, + 284, + 307, + 428, + 835, + 2651, + 284, + 302, + 12, + 293, + 391, + 290, + 8960, + 13, + 198, + 198, + 818, + 257, + 3234, + 2555, + 284, + 2962, + 319, + 2610, + 11, + 393, + 1342, + 477, + 11, + 262, + 7161, + 373, + 1965, + 329, + 4647, + 416, + 867, + 661, + 10311, + 340, + 287, + 262, + 995, + 11, + 290, + 788, + 11, + 351, + 1310, + 3626, + 290, + 407, + 29592, + 11, + 326, + 340, + 373, + 281, + 7187, + 30069, + 286, + 644, + 356, + 973, + 329, + 290, + 3114, + 379, + 511, + 2457, + 2589, + 379, + 543, + 356, + 1392, + 340, + 2642, + 13, + 314, + 760, + 326, + 1028, + 257, + 2089, + 923, + 284, + 1282, + 11, + 340, + 447, + 247, + 297, + 2107, + 8097, + 290, + 262, + 835, + 340, + 460, + 307, + 287, + 530, + 1295, + 329, + 262, + 13126, + 1781, + 11, + 290, + 481, + 1011, + 319, + 290, + 2163, + 276, + 287, + 262, + 812, + 4058, + 13, + 198, + 198, + 2396, + 284, + 617, + 812, + 290, + 783, + 11, + 287, + 2932, + 2763, + 11, + 4492, + 11, + 314, + 1833, + 326, + 11, + 290, + 314, + 2236, + 691, + 6901, + 262, + 3234, + 355, + 2081, + 290, + 2092, + 1912, + 319, + 588, + 1175, + 11, + 13584, + 1108, + 11, + 290, + 481, + 11, + 290, + 588, + 1641, + 13, + 198, + 198, + 1026, + 318, + 284, + 3128, + 11, + 1464, + 407, + 290, + 886, + 262, + 1334, + 286, + 262, + 1621, + 11, + 475, + 11, + 503, + 286, + 511, + 1234, + 774, + 287, + 262, + 1109, + 326, + 11, + 2138, + 621, + 1842, + 11717, + 11, + 2940, + 550, + 465, + 17188, + 351, + 262, + 6778, + 286, + 3599, + 257, + 649, + 7161, + 326, + 890, + 2084, + 13, + 198, + 198, + 3844, + 314, + 714, + 3774, + 661, + 351, + 1576, + 1141, + 262, + 3234, + 11, + 279, + 993, + 12, + 3083, + 11, + 284, + 1577, + 345, + 257, + 2614, + 13784, + 286, + 517, + 621, + 4248, + 1314, + 11, + 830, + 656, + 262, + 1628, + 13, + 3363, + 11, + 340, + 447, + 247, + 82, + 257, + 10195, + 11, + 198, + 198, + 4758, + 318, + 5137, + 257, + 1256, + 286, + 5003, + 287, + 284, + 428, + 2099, + 286, + 1393, + 290, + 4896, + 13, + 198, + 198, + 23379, + 477, + 286, + 511, + 1388, + 5153, + 783, + 13356, + 287, + 262, + 1074, + 11, + 290, + 543, + 484, + 743, + 307, + 973, + 284, + 2342, + 683, + 3892, + 736, + 284, + 379, + 262, + 826, + 1688, + 290, + 656, + 262, + 1364, + 286, + 3234, + 1097, + 11717, + 11, + 290, + 319, + 1327, + 326, + 339, + 3011, + 1395, + 12, + 5787, + 2175, + 422, + 465, + 564, + 250, + 75, + 7938, + 284, + 584, + 9558, + 13, + 447, + 251, + 198, + 198, + 464, + 304, + 12, + 36702, + 760, + 3098, + 12, + 36702, + 11, + 1642, + 606, + 1474, + 2460, + 11, + 23052, + 351, + 3734, + 12, + 67, + 4053, + 278, + 257, + 2260, + 1417, + 2292, + 286, + 262, + 1785, + 11, + 508, + 338, + 1263, + 3891, + 290, + 543, + 2192, + 2476, + 329, + 3272, + 12, + 67, + 5077, + 1359, + 262, + 1611, + 286, + 1479, + 12, + 12491, + 5449, + 1022, + 3234, + 11, + 18073, + 11, + 290, + 18073, + 13, + 198, + 198, + 29391, + 314, + 12, + 15592, + 198, + 198, + 33, + 12, + 292, + 485, + 416, + 262, + 1398, + 2940, + 290, + 314, + 1057, + 11, + 290, + 428, + 11, + 1900, + 13, + 362, + 12, + 18085, + 4608, + 11, + 2331, + 284, + 307, + 262, + 3726, + 286, + 511, + 1074, + 851, + 257, + 1109, + 4888, + 416, + 428, + 530, + 13, + 198, + 198, + 3792, + 428, + 326, + 986, + 198, + 198, + 258, + 338, + 7161, + 30, + 88, + 11, + 655, + 42786, + 494, + 750, + 287, + 3576, + 1171, + 4113, + 878, + 465, + 10325, + 612, + 13, + 4418, + 11, + 389, + 262, + 198, + 198, + 1078, + 12875, + 82, + 287, + 749, + 198, + 198, + 3581, + 198, + 198, + 1065, + 13, + 11075, + 318, + 5586, + 379, + 511, + 2166, + 290, + 9890, + 278, + 287, + 511, + 736, + 13, + 25024, + 286, + 1257, + 11, + 1770, + 13, + 2159, + 503, + 11, + 290, + 1365, + 2045, + 1097, + 11, + 13674, + 3730, + 0, + 198, + 198, + 1870, + 198, + 198, + 5779, + 11, + 314, + 1611, + 286, + 3505, + 616, + 1365, + 1545, + 355, + 1770, + 13, + 4403, + 1592, + 0, + 11317, + 621, + 1770, + 13, + 4373, + 340, + 338, + 284, + 6594, + 262, + 3234, + 287, + 1210, + 362, + 11, + 543, + 8956, + 502, + 355, + 262, + 302, + 12, + 1324, + 1671, + 803, + 5584, + 9852, + 396, + 379, + 326, + 640, + 960, + 392, + 616, + 12507, + 1770, + 13, + 4176, + 547, + 1770, + 13, + 367, + 9460, + 494, + 257, + 1643, + 351, + 465, + 1467, + 673, + 46, + 357, + 14988, + 16, + 13, + 24, + 828, + 530, + 286, + 465, + 50256 + ] + }, + { + "sample_id": 10, + "text": "<|endoftext|> Russian government. Instead of rightsizing was situation, nothing can go off, from perspective of Russia’s borders, as members of the Russian military has. That will be for them and their families, as the oil company, however the group will be responsible for what this campaign has been fighting for.\n\n* As special these Russianations, freedom of believe and other related that will be political prisoners, will be involved in this case – then however and why it is a well deserved support for the oil company. We will conduct an extremely large amount of work through the creation of a lawyer,a and with some success i.e. in person, this includes our ability to enter the oil business. We hope the company owner’s choice (or whoever – the latter, one with the things necessary to follow).<|endoftext|>Sometimes the candidate for president of the USA is bad news indeed. Not good for the small companies after the recession or the great backlash against the massive research programs of Facebook and Facebook, all building with those made to be an objective fact. After all, the idea we can’t hit by a few unelectedoked; between case lines.\n\nHopefully things will end that this and science is now the way to public well.\n\nOne or another, too politically either andless, the Democrats will embrace that right-wing crowd. Their candidates will be needed, and they got some at-risk votes. For a moment on on, we wanted to boil ourselves down to the building of, in turn, a kind different on other things. We gave by S.D the chance to spread schismus, sexism, and ideas between us. Once they run on Linux, the Democrats find a means to make sure they cannot released from the true needed to be just more useful for the content and content you get fired up from your computer.\n\nThis was plan that did not try to the left-wing crowd’s wishes. They tried to think of what is the ultimate right to research right itself, but maybe the Democrats will follow that goal.\n\nIt resulted in be saying 1,000 signatures people a letter to the FCC, one Marchc and how Koch, the two candidates, or a $100 million tax infusion from scratch by 2020.\n\nNot that the plan requires the rest of the Internet or 5.5 to take advantage of many projects in the election year. Millions of dollars were tried and won and failed, especially for one that seemed to have a \"get out of the blue,\" room prior to that candidate's job in 2010.\n\nHow the campaign platform plan to help Americans prepare for the future in the face Read more with!\n\nSo to take Obama in the epicirre of post-election politics this year is an understatement. This article happened in one, but on 4.<|endoftext|>On the program – there are some exceptions, though. When Sanders said later – several issues there of certain Sanders added a lot to her.\n\nZedish Johnson, her running mate, gets all the attention. Now he’s the fact housing candidate. Johnson-Boulder is the space for housing and housing. But he getsides over it, but not that thing is gentr, it’s hit face paint. At least 10 council cities found their own done in the week of their release. Oh, doing community, and the BZ program. “It did not happen in a city a night,” the local mayorquirer.\n\nSanders is also another involved in other cities and places that money (known as the program) first that made the Sanders campaign early on history.\n\nthis was known by the term “un livable housing,” helped feel, housing and rights, and allows states to give housing and housing to individuals and someone on the market with the cost of their own homes.\n\nSanders said it also helped in the footsteps of housing, housing and violence at the Davos when, for those interested in politics, it was the first of many meetings at Tent City that said the reason more members got into trouble after implementing the housing plan.\n\nhint is the mayor has a plan to push — stand up against thero for on everyone else, and they just now began the door. In terms of housing he thinks it is 2 of the possible housing7. People that exists inside and out of the Mt. There, they live in outside of the walls with marijuana beds.\n\nNot that National children from the drug use not allowed to shoot someone inside the way, and that the space for legal marijuana use in schools, and when the drug will be legal, as is still allowed by Seattle, San Francisco or Chongqing.\n\nThe government wants to create anything from scratch on land that can sell out, later on itself. Sanders made a different housing proposal in Vancouver in March, and said, the building is not small, and you will have one-bedroomrooms, yes, a woman has<|endoftext|>", + "token_ids": [ + 50256, + 3394, + 1230, + 13, + 5455, + 286, + 2489, + 2890, + 373, + 3074, + 11, + 2147, + 460, + 467, + 572, + 11, + 422, + 6650, + 286, + 3284, + 447, + 247, + 82, + 11637, + 11, + 355, + 1866, + 286, + 262, + 3394, + 2422, + 468, + 13, + 1320, + 481, + 307, + 329, + 606, + 290, + 511, + 4172, + 11, + 355, + 262, + 3056, + 1664, + 11, + 2158, + 262, + 1448, + 481, + 307, + 4497, + 329, + 644, + 428, + 1923, + 468, + 587, + 4330, + 329, + 13, + 198, + 198, + 9, + 1081, + 2041, + 777, + 3394, + 602, + 11, + 4925, + 286, + 1975, + 290, + 584, + 3519, + 326, + 481, + 307, + 1964, + 10577, + 11, + 481, + 307, + 2950, + 287, + 428, + 1339, + 784, + 788, + 2158, + 290, + 1521, + 340, + 318, + 257, + 880, + 20143, + 1104, + 329, + 262, + 3056, + 1664, + 13, + 775, + 481, + 3189, + 281, + 4457, + 1588, + 2033, + 286, + 670, + 832, + 262, + 6282, + 286, + 257, + 6853, + 11, + 64, + 290, + 351, + 617, + 1943, + 1312, + 13, + 68, + 13, + 287, + 1048, + 11, + 428, + 3407, + 674, + 2694, + 284, + 3802, + 262, + 3056, + 1597, + 13, + 775, + 2911, + 262, + 1664, + 4870, + 447, + 247, + 82, + 3572, + 357, + 273, + 16958, + 784, + 262, + 6846, + 11, + 530, + 351, + 262, + 1243, + 3306, + 284, + 1061, + 737, + 50256, + 15468, + 262, + 4540, + 329, + 1893, + 286, + 262, + 4916, + 318, + 2089, + 1705, + 5600, + 13, + 1892, + 922, + 329, + 262, + 1402, + 2706, + 706, + 262, + 16457, + 393, + 262, + 1049, + 19135, + 1028, + 262, + 4858, + 2267, + 4056, + 286, + 3203, + 290, + 3203, + 11, + 477, + 2615, + 351, + 883, + 925, + 284, + 307, + 281, + 9432, + 1109, + 13, + 2293, + 477, + 11, + 262, + 2126, + 356, + 460, + 447, + 247, + 83, + 2277, + 416, + 257, + 1178, + 555, + 28604, + 6545, + 26, + 1022, + 1339, + 3951, + 13, + 198, + 198, + 32365, + 1243, + 481, + 886, + 326, + 428, + 290, + 3783, + 318, + 783, + 262, + 835, + 284, + 1171, + 880, + 13, + 198, + 198, + 3198, + 393, + 1194, + 11, + 1165, + 11889, + 2035, + 290, + 1203, + 11, + 262, + 4956, + 481, + 12553, + 326, + 826, + 12, + 5469, + 4315, + 13, + 5334, + 5871, + 481, + 307, + 2622, + 11, + 290, + 484, + 1392, + 617, + 379, + 12, + 19121, + 5690, + 13, + 1114, + 257, + 2589, + 319, + 319, + 11, + 356, + 2227, + 284, + 20667, + 6731, + 866, + 284, + 262, + 2615, + 286, + 11, + 287, + 1210, + 11, + 257, + 1611, + 1180, + 319, + 584, + 1243, + 13, + 775, + 2921, + 416, + 311, + 13, + 35, + 262, + 2863, + 284, + 4104, + 5513, + 1042, + 84, + 82, + 11, + 24953, + 11, + 290, + 4213, + 1022, + 514, + 13, + 4874, + 484, + 1057, + 319, + 7020, + 11, + 262, + 4956, + 1064, + 257, + 1724, + 284, + 787, + 1654, + 484, + 2314, + 2716, + 422, + 262, + 2081, + 2622, + 284, + 307, + 655, + 517, + 4465, + 329, + 262, + 2695, + 290, + 2695, + 345, + 651, + 6294, + 510, + 422, + 534, + 3644, + 13, + 198, + 198, + 1212, + 373, + 1410, + 326, + 750, + 407, + 1949, + 284, + 262, + 1364, + 12, + 5469, + 4315, + 447, + 247, + 82, + 12802, + 13, + 1119, + 3088, + 284, + 892, + 286, + 644, + 318, + 262, + 8713, + 826, + 284, + 2267, + 826, + 2346, + 11, + 475, + 3863, + 262, + 4956, + 481, + 1061, + 326, + 3061, + 13, + 198, + 198, + 1026, + 8724, + 287, + 307, + 2282, + 352, + 11, + 830, + 17239, + 661, + 257, + 3850, + 284, + 262, + 15915, + 11, + 530, + 2805, + 66, + 290, + 703, + 17009, + 11, + 262, + 734, + 5871, + 11, + 393, + 257, + 720, + 3064, + 1510, + 1687, + 35547, + 422, + 12692, + 416, + 12131, + 13, + 198, + 198, + 3673, + 326, + 262, + 1410, + 4433, + 262, + 1334, + 286, + 262, + 4455, + 393, + 642, + 13, + 20, + 284, + 1011, + 4621, + 286, + 867, + 4493, + 287, + 262, + 3071, + 614, + 13, + 39830, + 286, + 5054, + 547, + 3088, + 290, + 1839, + 290, + 4054, + 11, + 2592, + 329, + 530, + 326, + 3947, + 284, + 423, + 257, + 366, + 1136, + 503, + 286, + 262, + 4171, + 553, + 2119, + 3161, + 284, + 326, + 4540, + 338, + 1693, + 287, + 3050, + 13, + 198, + 198, + 2437, + 262, + 1923, + 3859, + 1410, + 284, + 1037, + 3399, + 8335, + 329, + 262, + 2003, + 287, + 262, + 1986, + 4149, + 517, + 351, + 0, + 198, + 198, + 2396, + 284, + 1011, + 2486, + 287, + 262, + 12191, + 343, + 260, + 286, + 1281, + 12, + 14300, + 4819, + 428, + 614, + 318, + 281, + 46196, + 13, + 770, + 2708, + 3022, + 287, + 530, + 11, + 475, + 319, + 604, + 13, + 50256, + 2202, + 262, + 1430, + 784, + 612, + 389, + 617, + 13269, + 11, + 996, + 13, + 1649, + 5831, + 531, + 1568, + 784, + 1811, + 2428, + 612, + 286, + 1728, + 5831, + 2087, + 257, + 1256, + 284, + 607, + 13, + 198, + 198, + 57, + 276, + 680, + 5030, + 11, + 607, + 2491, + 16133, + 11, + 3011, + 477, + 262, + 3241, + 13, + 2735, + 339, + 447, + 247, + 82, + 262, + 1109, + 5627, + 4540, + 13, + 5030, + 12, + 33, + 17601, + 318, + 262, + 2272, + 329, + 5627, + 290, + 5627, + 13, + 887, + 339, + 3011, + 1460, + 625, + 340, + 11, + 475, + 407, + 326, + 1517, + 318, + 25049, + 81, + 11, + 340, + 447, + 247, + 82, + 2277, + 1986, + 7521, + 13, + 1629, + 1551, + 838, + 6745, + 4736, + 1043, + 511, + 898, + 1760, + 287, + 262, + 1285, + 286, + 511, + 2650, + 13, + 3966, + 11, + 1804, + 2055, + 11, + 290, + 262, + 347, + 57, + 1430, + 13, + 564, + 250, + 1026, + 750, + 407, + 1645, + 287, + 257, + 1748, + 257, + 1755, + 11, + 447, + 251, + 262, + 1957, + 9591, + 29782, + 81, + 13, + 198, + 198, + 26747, + 318, + 635, + 1194, + 2950, + 287, + 584, + 4736, + 290, + 4113, + 326, + 1637, + 357, + 4002, + 355, + 262, + 1430, + 8, + 717, + 326, + 925, + 262, + 5831, + 1923, + 1903, + 319, + 2106, + 13, + 198, + 198, + 5661, + 373, + 1900, + 416, + 262, + 3381, + 564, + 250, + 403, + 17717, + 540, + 5627, + 11, + 447, + 251, + 4193, + 1254, + 11, + 5627, + 290, + 2489, + 11, + 290, + 3578, + 2585, + 284, + 1577, + 5627, + 290, + 5627, + 284, + 3925, + 290, + 2130, + 319, + 262, + 1910, + 351, + 262, + 1575, + 286, + 511, + 898, + 5682, + 13, + 198, + 198, + 26747, + 531, + 340, + 635, + 4193, + 287, + 262, + 27146, + 286, + 5627, + 11, + 5627, + 290, + 3685, + 379, + 262, + 2544, + 418, + 618, + 11, + 329, + 883, + 4609, + 287, + 4819, + 11, + 340, + 373, + 262, + 717, + 286, + 867, + 8292, + 379, + 30765, + 2254, + 326, + 531, + 262, + 1738, + 517, + 1866, + 1392, + 656, + 5876, + 706, + 15427, + 262, + 5627, + 1410, + 13, + 198, + 198, + 71, + 600, + 318, + 262, + 9591, + 468, + 257, + 1410, + 284, + 4574, + 851, + 1302, + 510, + 1028, + 262, + 305, + 329, + 319, + 2506, + 2073, + 11, + 290, + 484, + 655, + 783, + 2540, + 262, + 3420, + 13, + 554, + 2846, + 286, + 5627, + 339, + 6834, + 340, + 318, + 362, + 286, + 262, + 1744, + 5627, + 22, + 13, + 4380, + 326, + 7160, + 2641, + 290, + 503, + 286, + 262, + 18632, + 13, + 1318, + 11, + 484, + 2107, + 287, + 2354, + 286, + 262, + 7714, + 351, + 5727, + 20237, + 13, + 198, + 198, + 3673, + 326, + 2351, + 1751, + 422, + 262, + 2563, + 779, + 407, + 3142, + 284, + 2686, + 2130, + 2641, + 262, + 835, + 11, + 290, + 326, + 262, + 2272, + 329, + 2742, + 5727, + 779, + 287, + 4266, + 11, + 290, + 618, + 262, + 2563, + 481, + 307, + 2742, + 11, + 355, + 318, + 991, + 3142, + 416, + 7312, + 11, + 2986, + 6033, + 393, + 46892, + 80, + 278, + 13, + 198, + 198, + 464, + 1230, + 3382, + 284, + 2251, + 1997, + 422, + 12692, + 319, + 1956, + 326, + 460, + 3677, + 503, + 11, + 1568, + 319, + 2346, + 13, + 5831, + 925, + 257, + 1180, + 5627, + 6961, + 287, + 10930, + 287, + 2805, + 11, + 290, + 531, + 11, + 262, + 2615, + 318, + 407, + 1402, + 11, + 290, + 345, + 481, + 423, + 530, + 12, + 36269, + 9649, + 11, + 3763, + 11, + 257, + 2415, + 468, + 50256 + ] + }, + { + "sample_id": 11, + "text": "<|endoftext|>. All information about the nature of Islam. O anarchy. The low ground.\n\nThe Democratic Party has enough with the police to stop this “revolution”. The Democratic Party there is to be accused of “politicalophobia”, dutifully, and forced to portray his world community as a war zone of different races, not racist, racist, hard-liners against humanity.\n\nThis is truly an important, tiny, small movement. This is the most powerful, the people, working to rule, and return power.\n\nRecently a man with the political accent set me off on a road trip to get to a city to hear what people believe in U.S. politics or comment on it on TV.\n\nNancy Mandela’s death was in a neighborhood that has shown that for the whole of the state, the mayor is the arbiter. A sign of the election, as Obama announced his visit to a Democraticstill on Tuesday, having not last week simply letating back into his campaign to be an ally with the city’s last mayor.\n\nAfter Obama’s election, police refuse to meet with and set the party’s people at the table on such violence and violence. They get me to sit on the back of a truck.\n\nThey takes us over the new day – the historic, two-day race between the two candidates from Iraq and Syria. There is much up in U.N. politics and Jill Hillary’s transition from final to the greatestest of all will be needed, North to mention State,… and/ read. “Truth, peace, and justice” in general terms are what drives peace, xen people, and lies, things which you have a copy of Clinton and TheBlClinton.com. The story is rooted in policy, the desire for the truth to be reallyified, although the Left doesn’t understand and then by many of its citizens who’s the source of peace and order.\n\nMore importantly, Hillary is key to build the wall upon George W. Bush, the corporate media, and the nation’s highest office. His presidency was released by a lack of thousands due to millions of life. His pastures are held in the name of 5,000 lives. Which of our favorite heroes, G. According to Life, who is a story of when brave young people breaking through in breaking down an otherwise hard form when her back to her parents a shared if and live vision for the world, is remembered as the great secretary of state?\n\nYet it’s only one key factor to keep the nation down by force that is familiar to all Americans on the ballot in 2014.\n\nWithM turnout in the late States’s both Trump and worse still it is high and one more US police, police and fire African-police thugs. Instead of a people through andling over the Hillary Clinton’s disregard for workers’ rights for 90 days, we witness this series of U.S. actions by young activists, activists and activists from action as the American goes grows and in our streets grows.\n\n* * * *\n\nHere’s remember Franklin D. Roosevelt: “Diversity is L’ Theres ,” he tells The Guardian.\n\nThe new Blackra There will be bigoted, a black and brown organization whose there are plenty of African-American problems involved, which keeps them racist, ignored, others not heard and understanding.<|endoftext|>Playing with 7\n\n1 while, to hear from me is short. We both talk to and tell looking more perplexed T soon she’s going to die. Toss around to get a little \"minor\" ready; a true, puts him before the steamy post. L- ah, a friend told us.\n\nand one point here is why he would shut up about him talking to him. Really.\n\n3.1 When \"X\" opens his eyes and talking about looking. Ooh and that yelah below, my last-minute of mine, T '' caught up in his eyes and I was interested more in awe.\n\n* followed by \"X\" etc.\n\nPart 2.4 with 2-X am, with Jack, the 5-X girl, followed by of G (WV. X's wife, an older friend).\n\nLet 'bout. the path of him: right/right/right way, and pass it down through him, and then back to the tk of us. At and through that, he has to be taken (and always the opposite of you.3 or 10); build the old as they are the things that is needed to get a main that’s strong enough, but those less powerful and fall far into his heart. Make him home better, faster, faster, on something else, pre- simply before his own am 2013<|endoftext|>", + "token_ids": [ + 50256, + 13, + 1439, + 1321, + 546, + 262, + 3450, + 286, + 3449, + 13, + 440, + 281, + 9282, + 13, + 383, + 1877, + 2323, + 13, + 198, + 198, + 464, + 4390, + 3615, + 468, + 1576, + 351, + 262, + 1644, + 284, + 2245, + 428, + 564, + 250, + 32243, + 447, + 251, + 13, + 383, + 4390, + 3615, + 612, + 318, + 284, + 307, + 5371, + 286, + 564, + 250, + 23149, + 19851, + 447, + 251, + 11, + 288, + 315, + 17049, + 11, + 290, + 4137, + 284, + 10993, + 465, + 995, + 2055, + 355, + 257, + 1175, + 6516, + 286, + 1180, + 9558, + 11, + 407, + 9811, + 11, + 9811, + 11, + 1327, + 12, + 34380, + 1028, + 9265, + 13, + 198, + 198, + 1212, + 318, + 4988, + 281, + 1593, + 11, + 7009, + 11, + 1402, + 3356, + 13, + 770, + 318, + 262, + 749, + 3665, + 11, + 262, + 661, + 11, + 1762, + 284, + 3896, + 11, + 290, + 1441, + 1176, + 13, + 198, + 198, + 24661, + 257, + 582, + 351, + 262, + 1964, + 18702, + 900, + 502, + 572, + 319, + 257, + 2975, + 5296, + 284, + 651, + 284, + 257, + 1748, + 284, + 3285, + 644, + 661, + 1975, + 287, + 471, + 13, + 50, + 13, + 4819, + 393, + 2912, + 319, + 340, + 319, + 3195, + 13, + 198, + 198, + 45, + 3883, + 40233, + 447, + 247, + 82, + 1918, + 373, + 287, + 257, + 6232, + 326, + 468, + 3402, + 326, + 329, + 262, + 2187, + 286, + 262, + 1181, + 11, + 262, + 9591, + 318, + 262, + 9277, + 263, + 13, + 317, + 1051, + 286, + 262, + 3071, + 11, + 355, + 2486, + 3414, + 465, + 3187, + 284, + 257, + 4390, + 24219, + 319, + 3431, + 11, + 1719, + 407, + 938, + 1285, + 2391, + 1309, + 803, + 736, + 656, + 465, + 1923, + 284, + 307, + 281, + 12525, + 351, + 262, + 1748, + 447, + 247, + 82, + 938, + 9591, + 13, + 198, + 198, + 3260, + 2486, + 447, + 247, + 82, + 3071, + 11, + 1644, + 11148, + 284, + 1826, + 351, + 290, + 900, + 262, + 2151, + 447, + 247, + 82, + 661, + 379, + 262, + 3084, + 319, + 884, + 3685, + 290, + 3685, + 13, + 1119, + 651, + 502, + 284, + 1650, + 319, + 262, + 736, + 286, + 257, + 7779, + 13, + 198, + 198, + 2990, + 2753, + 514, + 625, + 262, + 649, + 1110, + 784, + 262, + 9566, + 11, + 734, + 12, + 820, + 3234, + 1022, + 262, + 734, + 5871, + 422, + 3908, + 290, + 4392, + 13, + 1318, + 318, + 881, + 510, + 287, + 471, + 13, + 45, + 13, + 4819, + 290, + 22772, + 5073, + 447, + 247, + 82, + 6801, + 422, + 2457, + 284, + 262, + 6000, + 395, + 286, + 477, + 481, + 307, + 2622, + 11, + 2258, + 284, + 3068, + 1812, + 11, + 1399, + 290, + 14, + 1100, + 13, + 564, + 250, + 38782, + 11, + 4167, + 11, + 290, + 5316, + 447, + 251, + 287, + 2276, + 2846, + 389, + 644, + 10182, + 4167, + 11, + 27132, + 661, + 11, + 290, + 7363, + 11, + 1243, + 543, + 345, + 423, + 257, + 4866, + 286, + 2605, + 290, + 383, + 3629, + 16549, + 13, + 785, + 13, + 383, + 1621, + 318, + 19459, + 287, + 2450, + 11, + 262, + 6227, + 329, + 262, + 3872, + 284, + 307, + 1107, + 1431, + 11, + 3584, + 262, + 9578, + 1595, + 447, + 247, + 83, + 1833, + 290, + 788, + 416, + 867, + 286, + 663, + 4290, + 508, + 447, + 247, + 82, + 262, + 2723, + 286, + 4167, + 290, + 1502, + 13, + 198, + 198, + 5167, + 11003, + 11, + 5073, + 318, + 1994, + 284, + 1382, + 262, + 3355, + 2402, + 4502, + 370, + 13, + 5511, + 11, + 262, + 6355, + 2056, + 11, + 290, + 262, + 3277, + 447, + 247, + 82, + 4511, + 2607, + 13, + 2399, + 12112, + 373, + 2716, + 416, + 257, + 3092, + 286, + 4138, + 2233, + 284, + 5242, + 286, + 1204, + 13, + 2399, + 1613, + 942, + 389, + 2714, + 287, + 262, + 1438, + 286, + 642, + 11, + 830, + 3160, + 13, + 9022, + 286, + 674, + 4004, + 10281, + 11, + 402, + 13, + 4784, + 284, + 5155, + 11, + 508, + 318, + 257, + 1621, + 286, + 618, + 14802, + 1862, + 661, + 7163, + 832, + 287, + 7163, + 866, + 281, + 4306, + 1327, + 1296, + 618, + 607, + 736, + 284, + 607, + 3397, + 257, + 4888, + 611, + 290, + 2107, + 5761, + 329, + 262, + 995, + 11, + 318, + 12086, + 355, + 262, + 1049, + 7705, + 286, + 1181, + 30, + 198, + 198, + 11486, + 340, + 447, + 247, + 82, + 691, + 530, + 1994, + 5766, + 284, + 1394, + 262, + 3277, + 866, + 416, + 2700, + 326, + 318, + 5385, + 284, + 477, + 3399, + 319, + 262, + 11100, + 287, + 1946, + 13, + 198, + 198, + 3152, + 44, + 21666, + 287, + 262, + 2739, + 1829, + 447, + 247, + 82, + 1111, + 1301, + 290, + 4785, + 991, + 340, + 318, + 1029, + 290, + 530, + 517, + 1294, + 1644, + 11, + 1644, + 290, + 2046, + 5510, + 12, + 38191, + 35052, + 13, + 5455, + 286, + 257, + 661, + 832, + 290, + 1359, + 625, + 262, + 5073, + 2605, + 447, + 247, + 82, + 25070, + 329, + 3259, + 447, + 247, + 2489, + 329, + 4101, + 1528, + 11, + 356, + 4973, + 428, + 2168, + 286, + 471, + 13, + 50, + 13, + 4028, + 416, + 1862, + 7941, + 11, + 7941, + 290, + 7941, + 422, + 2223, + 355, + 262, + 1605, + 2925, + 13676, + 290, + 287, + 674, + 6483, + 13676, + 13, + 198, + 198, + 9, + 1635, + 1635, + 1635, + 198, + 198, + 4342, + 447, + 247, + 82, + 3505, + 14021, + 360, + 13, + 19297, + 25, + 564, + 250, + 35, + 1608, + 318, + 406, + 447, + 247, + 383, + 411, + 837, + 447, + 251, + 339, + 4952, + 383, + 8283, + 13, + 198, + 198, + 464, + 649, + 2619, + 430, + 1318, + 481, + 307, + 1263, + 5191, + 11, + 257, + 2042, + 290, + 7586, + 4009, + 3025, + 612, + 389, + 6088, + 286, + 5510, + 12, + 7437, + 2761, + 2950, + 11, + 543, + 7622, + 606, + 9811, + 11, + 9514, + 11, + 1854, + 407, + 2982, + 290, + 4547, + 13, + 50256, + 36530, + 351, + 767, + 198, + 198, + 16, + 981, + 11, + 284, + 3285, + 422, + 502, + 318, + 1790, + 13, + 775, + 1111, + 1561, + 284, + 290, + 1560, + 2045, + 517, + 35682, + 276, + 309, + 2582, + 673, + 447, + 247, + 82, + 1016, + 284, + 4656, + 13, + 309, + 793, + 1088, + 284, + 651, + 257, + 1310, + 366, + 1084, + 273, + 1, + 3492, + 26, + 257, + 2081, + 11, + 7584, + 683, + 878, + 262, + 13324, + 88, + 1281, + 13, + 406, + 12, + 257, + 71, + 11, + 257, + 1545, + 1297, + 514, + 13, + 198, + 198, + 392, + 530, + 966, + 994, + 318, + 1521, + 339, + 561, + 4423, + 510, + 546, + 683, + 3375, + 284, + 683, + 13, + 16123, + 13, + 198, + 198, + 18, + 13, + 16, + 1649, + 366, + 55, + 1, + 9808, + 465, + 2951, + 290, + 3375, + 546, + 2045, + 13, + 440, + 1219, + 290, + 326, + 331, + 417, + 993, + 2174, + 11, + 616, + 938, + 12, + 11374, + 286, + 6164, + 11, + 309, + 705, + 6, + 4978, + 510, + 287, + 465, + 2951, + 290, + 314, + 373, + 4609, + 517, + 287, + 25030, + 13, + 198, + 198, + 9, + 3940, + 416, + 366, + 55, + 1, + 3503, + 13, + 198, + 198, + 7841, + 362, + 13, + 19, + 351, + 362, + 12, + 55, + 716, + 11, + 351, + 3619, + 11, + 262, + 642, + 12, + 55, + 2576, + 11, + 3940, + 416, + 286, + 402, + 357, + 54, + 53, + 13, + 1395, + 338, + 3656, + 11, + 281, + 4697, + 1545, + 737, + 198, + 198, + 5756, + 705, + 65, + 448, + 13, + 262, + 3108, + 286, + 683, + 25, + 826, + 14, + 3506, + 14, + 3506, + 835, + 11, + 290, + 1208, + 340, + 866, + 832, + 683, + 11, + 290, + 788, + 736, + 284, + 262, + 256, + 74, + 286, + 514, + 13, + 1629, + 290, + 832, + 326, + 11, + 339, + 468, + 284, + 307, + 2077, + 357, + 392, + 1464, + 262, + 6697, + 286, + 345, + 13, + 18, + 393, + 838, + 1776, + 1382, + 262, + 1468, + 355, + 484, + 389, + 262, + 1243, + 326, + 318, + 2622, + 284, + 651, + 257, + 1388, + 326, + 447, + 247, + 82, + 1913, + 1576, + 11, + 475, + 883, + 1342, + 3665, + 290, + 2121, + 1290, + 656, + 465, + 2612, + 13, + 6889, + 683, + 1363, + 1365, + 11, + 5443, + 11, + 5443, + 11, + 319, + 1223, + 2073, + 11, + 662, + 12, + 2391, + 878, + 465, + 898, + 716, + 2211, + 50256 + ] + }, + { + "sample_id": 12, + "text": "<|endoftext|> should have tried to keep every law intact long-term, and no longer a people willing to vote for another, short-term special election.\n\nIn a fool's 1 situation, for one that actually class, probably, character, bought into the record, and money too soon to clear his agenda was a let failure. The new president and among Democrats had time to push back the case for Operation North Korea, but Mr. Obama, in a bid to appease the recalem, alien party, didn no need to kill him aside. Yes, the FBI was about to fire two men he knew out of business and it was doing. During the game, and the time he said it being a fair, lost, and gentlemanly process, Bush was the most powerful he'd ever had been (one less, anyone important since Bill Clinton with the DoJ) in Washington and up has been everywhere, hecticled in right ways. The few examples of these de-v events are well above average average.**\n\n*L below\n\n* Dr. O-JY . MayD wrote on Twitter that most of our readers are dead and cannot be under the radar. We some issues remain, and it turns out to most Americans to talk about this place, or both. Much to the left that was killed and two others killed on fire in the summer of 2011, all of this matters, but the rest of the public has the value and soul of this class nation on their hands. Like the sort of 30 was when H.A. was still around, Mr. Bush is moving on, and angry, and him. But please matters first.\n\nIf the United States wants to release the lives of people again, not a part of this group. By turning the world body into a nation state, terrorism is not a civil war, and being independent of the left in the future, it must stay at the market.\n\nThe CIA runs through various countries and sends their same go-out for all nations and all countries that the good will soon be overth upon, for us, in Iraq, yet now picking up the pressure on the left and it is very clear to get a major dose of mass support.\n\nThis has been going step-by-year. All-Egyptian Day, a recent day in rioting, has brought well-bogg good people for the whole city of 50,000 people, the rest of the world actually lasts as long as expected, and the only ones who will move on the streets in New forever. Such as such that tomorrow was going to be between a long and all day for the poor and culminating in the one major international conflict, a first time in the 1960 century, of any kind between the Israeli and U.S. government in certain, in the region for most of the 1980s. Instead, he gave us not stay there, because of whatever believe we were facing at the time but the world in general, except in religious and non-interetary ways and by example and the unconditional love, and your life.So\n together in its way while in the dark and because of a P. Morgan, and his true service, national, single, and un its building us, the entire force of the local House.Meanwhile, in 7:00 p.m. against morning, Israel, and the government Union have taken God's money-makingic love in Utopia or whatever that represents the physical parts of the body. When forces have taken 10,000 people directly from the building caused by death, torture, and the tectonic det animus, powerful words that are again—. God's position because it is easy to use and that can bring, as for Mandela, the South African nation's m leader, to Rosie O'Donnell on September 7, along with Israel and Israel come in and out at war. In Europe and the world dealing with a vengeance and the Internet and financial surrounding this service areim images of a massive building building and thousands of life. c. Except for the conference itself, and the footage is available on both the two sites before and was published following the event. In this article, we had the difficulty in using�.'s month, not year, and decades, though the organizers have seem to be working overtime to win in time to provide an excuse to keep the focus on those groups.For example, one of the most common conspiracy facts laid out in series of articles in the press, is that the P. Morgan Icardiaceae is the species that has historically been killed, toil P. Lovecraft's largest c, and thus its de lead. During the year 2000 Icardiote that had 13 business-years-long, and 13 twice as long as national and America that has little, was responsible for the Nobel Olympics. Apart from this year's conference, was held and China, as part of $1,000 they designed to bring men to last. Note that at rioting increased the number of such groups<|endoftext|>", + "token_ids": [ + 50256, + 815, + 423, + 3088, + 284, + 1394, + 790, + 1099, + 16572, + 890, + 12, + 4354, + 11, + 290, + 645, + 2392, + 257, + 661, + 4684, + 284, + 3015, + 329, + 1194, + 11, + 1790, + 12, + 4354, + 2041, + 3071, + 13, + 198, + 198, + 818, + 257, + 9192, + 338, + 352, + 3074, + 11, + 329, + 530, + 326, + 1682, + 1398, + 11, + 2192, + 11, + 2095, + 11, + 5839, + 656, + 262, + 1700, + 11, + 290, + 1637, + 1165, + 2582, + 284, + 1598, + 465, + 8666, + 373, + 257, + 1309, + 5287, + 13, + 383, + 649, + 1893, + 290, + 1871, + 4956, + 550, + 640, + 284, + 4574, + 736, + 262, + 1339, + 329, + 14680, + 2258, + 4969, + 11, + 475, + 1770, + 13, + 2486, + 11, + 287, + 257, + 8406, + 284, + 47316, + 262, + 42653, + 368, + 11, + 8756, + 2151, + 11, + 1422, + 645, + 761, + 284, + 1494, + 683, + 7263, + 13, + 3363, + 11, + 262, + 5349, + 373, + 546, + 284, + 2046, + 734, + 1450, + 339, + 2993, + 503, + 286, + 1597, + 290, + 340, + 373, + 1804, + 13, + 5856, + 262, + 983, + 11, + 290, + 262, + 640, + 339, + 531, + 340, + 852, + 257, + 3148, + 11, + 2626, + 11, + 290, + 22427, + 306, + 1429, + 11, + 5511, + 373, + 262, + 749, + 3665, + 339, + 1549, + 1683, + 550, + 587, + 357, + 505, + 1342, + 11, + 2687, + 1593, + 1201, + 3941, + 2605, + 351, + 262, + 2141, + 41, + 8, + 287, + 2669, + 290, + 510, + 468, + 587, + 8347, + 11, + 339, + 11048, + 992, + 287, + 826, + 2842, + 13, + 383, + 1178, + 6096, + 286, + 777, + 390, + 12, + 85, + 2995, + 389, + 880, + 2029, + 2811, + 2811, + 13, + 1174, + 198, + 198, + 9, + 43, + 2174, + 198, + 198, + 9, + 1583, + 13, + 440, + 12, + 41, + 56, + 764, + 1737, + 35, + 2630, + 319, + 3009, + 326, + 749, + 286, + 674, + 7183, + 389, + 2636, + 290, + 2314, + 307, + 739, + 262, + 13428, + 13, + 775, + 617, + 2428, + 3520, + 11, + 290, + 340, + 4962, + 503, + 284, + 749, + 3399, + 284, + 1561, + 546, + 428, + 1295, + 11, + 393, + 1111, + 13, + 13111, + 284, + 262, + 1364, + 326, + 373, + 2923, + 290, + 734, + 1854, + 2923, + 319, + 2046, + 287, + 262, + 3931, + 286, + 2813, + 11, + 477, + 286, + 428, + 6067, + 11, + 475, + 262, + 1334, + 286, + 262, + 1171, + 468, + 262, + 1988, + 290, + 5848, + 286, + 428, + 1398, + 3277, + 319, + 511, + 2832, + 13, + 4525, + 262, + 3297, + 286, + 1542, + 373, + 618, + 367, + 13, + 32, + 13, + 373, + 991, + 1088, + 11, + 1770, + 13, + 5511, + 318, + 3867, + 319, + 11, + 290, + 7954, + 11, + 290, + 683, + 13, + 887, + 3387, + 6067, + 717, + 13, + 198, + 198, + 1532, + 262, + 1578, + 1829, + 3382, + 284, + 2650, + 262, + 3160, + 286, + 661, + 757, + 11, + 407, + 257, + 636, + 286, + 428, + 1448, + 13, + 2750, + 6225, + 262, + 995, + 1767, + 656, + 257, + 3277, + 1181, + 11, + 8649, + 318, + 407, + 257, + 3026, + 1175, + 11, + 290, + 852, + 4795, + 286, + 262, + 1364, + 287, + 262, + 2003, + 11, + 340, + 1276, + 2652, + 379, + 262, + 1910, + 13, + 198, + 198, + 464, + 7688, + 4539, + 832, + 2972, + 2678, + 290, + 12800, + 511, + 976, + 467, + 12, + 448, + 329, + 477, + 7027, + 290, + 477, + 2678, + 326, + 262, + 922, + 481, + 2582, + 307, + 34514, + 2402, + 11, + 329, + 514, + 11, + 287, + 3908, + 11, + 1865, + 783, + 10868, + 510, + 262, + 3833, + 319, + 262, + 1364, + 290, + 340, + 318, + 845, + 1598, + 284, + 651, + 257, + 1688, + 10742, + 286, + 2347, + 1104, + 13, + 198, + 198, + 1212, + 468, + 587, + 1016, + 2239, + 12, + 1525, + 12, + 1941, + 13, + 1439, + 12, + 39299, + 666, + 3596, + 11, + 257, + 2274, + 1110, + 287, + 16352, + 278, + 11, + 468, + 3181, + 880, + 12, + 65, + 10332, + 922, + 661, + 329, + 262, + 2187, + 1748, + 286, + 2026, + 11, + 830, + 661, + 11, + 262, + 1334, + 286, + 262, + 995, + 1682, + 20374, + 355, + 890, + 355, + 2938, + 11, + 290, + 262, + 691, + 3392, + 508, + 481, + 1445, + 319, + 262, + 6483, + 287, + 968, + 8097, + 13, + 8013, + 355, + 884, + 326, + 9439, + 373, + 1016, + 284, + 307, + 1022, + 257, + 890, + 290, + 477, + 1110, + 329, + 262, + 3595, + 290, + 43896, + 287, + 262, + 530, + 1688, + 3230, + 5358, + 11, + 257, + 717, + 640, + 287, + 262, + 9507, + 4289, + 11, + 286, + 597, + 1611, + 1022, + 262, + 6085, + 290, + 471, + 13, + 50, + 13, + 1230, + 287, + 1728, + 11, + 287, + 262, + 3814, + 329, + 749, + 286, + 262, + 7169, + 82, + 13, + 5455, + 11, + 339, + 2921, + 514, + 407, + 2652, + 612, + 11, + 780, + 286, + 4232, + 1975, + 356, + 547, + 6476, + 379, + 262, + 640, + 475, + 262, + 995, + 287, + 2276, + 11, + 2845, + 287, + 4158, + 290, + 1729, + 12, + 3849, + 8527, + 2842, + 290, + 416, + 1672, + 290, + 262, + 42423, + 1842, + 11, + 290, + 534, + 1204, + 13, + 2396, + 198, + 1978, + 287, + 663, + 835, + 981, + 287, + 262, + 3223, + 290, + 780, + 286, + 257, + 350, + 13, + 10805, + 11, + 290, + 465, + 2081, + 2139, + 11, + 2260, + 11, + 2060, + 11, + 290, + 555, + 663, + 2615, + 514, + 11, + 262, + 2104, + 2700, + 286, + 262, + 1957, + 2097, + 13, + 10294, + 11, + 287, + 767, + 25, + 405, + 279, + 13, + 76, + 13, + 1028, + 3329, + 11, + 2692, + 11, + 290, + 262, + 1230, + 4479, + 423, + 2077, + 1793, + 338, + 1637, + 12, + 8601, + 291, + 1842, + 287, + 471, + 46575, + 393, + 4232, + 326, + 6870, + 262, + 3518, + 3354, + 286, + 262, + 1767, + 13, + 1649, + 3386, + 423, + 2077, + 838, + 11, + 830, + 661, + 3264, + 422, + 262, + 2615, + 4073, + 416, + 1918, + 11, + 11543, + 11, + 290, + 262, + 256, + 478, + 9229, + 1062, + 2355, + 385, + 11, + 3665, + 2456, + 326, + 389, + 757, + 960, + 13, + 1793, + 338, + 2292, + 780, + 340, + 318, + 2562, + 284, + 779, + 290, + 326, + 460, + 2222, + 11, + 355, + 329, + 40233, + 11, + 262, + 2520, + 5510, + 3277, + 338, + 285, + 3554, + 11, + 284, + 10018, + 494, + 440, + 6, + 24853, + 319, + 2693, + 767, + 11, + 1863, + 351, + 2692, + 290, + 2692, + 1282, + 287, + 290, + 503, + 379, + 1175, + 13, + 554, + 2031, + 290, + 262, + 995, + 7219, + 351, + 257, + 30657, + 290, + 262, + 4455, + 290, + 3176, + 7346, + 428, + 2139, + 389, + 320, + 4263, + 286, + 257, + 4858, + 2615, + 2615, + 290, + 4138, + 286, + 1204, + 13, + 269, + 13, + 18181, + 329, + 262, + 4495, + 2346, + 11, + 290, + 262, + 9640, + 318, + 1695, + 319, + 1111, + 262, + 734, + 5043, + 878, + 290, + 373, + 3199, + 1708, + 262, + 1785, + 13, + 554, + 428, + 2708, + 11, + 356, + 550, + 262, + 8722, + 287, + 1262, + 126, + 2637, + 82, + 1227, + 11, + 407, + 614, + 11, + 290, + 4647, + 11, + 996, + 262, + 19043, + 423, + 1283, + 284, + 307, + 1762, + 17820, + 284, + 1592, + 287, + 640, + 284, + 2148, + 281, + 12226, + 284, + 1394, + 262, + 2962, + 319, + 883, + 2628, + 13, + 1890, + 1672, + 11, + 530, + 286, + 262, + 749, + 2219, + 10086, + 6419, + 8104, + 503, + 287, + 2168, + 286, + 6685, + 287, + 262, + 1803, + 11, + 318, + 326, + 262, + 350, + 13, + 10805, + 314, + 9517, + 72, + 48319, + 318, + 262, + 4693, + 326, + 468, + 15074, + 587, + 2923, + 11, + 284, + 346, + 350, + 13, + 37655, + 338, + 4387, + 269, + 11, + 290, + 4145, + 663, + 390, + 1085, + 13, + 5856, + 262, + 614, + 4751, + 314, + 9517, + 72, + 1258, + 326, + 550, + 1511, + 1597, + 12, + 19002, + 12, + 6511, + 11, + 290, + 1511, + 5403, + 355, + 890, + 355, + 2260, + 290, + 2253, + 326, + 468, + 1310, + 11, + 373, + 4497, + 329, + 262, + 20715, + 14935, + 13, + 22596, + 422, + 428, + 614, + 338, + 4495, + 11, + 373, + 2714, + 290, + 2807, + 11, + 355, + 636, + 286, + 720, + 16, + 11, + 830, + 484, + 3562, + 284, + 2222, + 1450, + 284, + 938, + 13, + 5740, + 326, + 379, + 16352, + 278, + 3220, + 262, + 1271, + 286, + 884, + 2628, + 50256 + ] + }, + { + "sample_id": 13, + "text": "<|endoftext|> up all efforts by her energy up against the wall is out and doing, it depends on what is coming to it soon.\n\nNow, she'll always wants to be with it over again, and knows that her hatred of the government will still be left her by the kind in the canary, and we'll see her naked and in a canary, I'll tell them, in that May or whatever she.\n\nThe cost to personalize it- and you sell off the books, just for use and possible, the money has a monopoly on them for all of these things, right.\n\nNo, not all these canals pissed off that the FBI has to throw away look like Trump's enemies.”\n\n\"Just give your every to walk through at times the worst of the press that would be any government inside America - you wouldn’t be to see it talking about the time and into a process to make war down, once’s like Watergate, my, and now worse than the National Anthem.\"\n\nAnchah O'Reilly, like of the \"Meet the Press,\" took the other government seriously by having to watch what is said publicly, from whom the terrorists are talking about, include need greatx, and oftenize them and process all that they are doing. Their own form of a shield to keep from that site and their loved old out of that life's misery. They're an asset and not overen, making to anyone the government has to take it.\n\nAnd in preparation for the Boston Women's terrorist attack, the Times gave a different definition of it.\n\n\"Even though R milit millen 17 in any case just don't want to plan entry-paying job to anyone, the stakes are high, an increasing number of lowless joblessness as a means.\"\n\nFor the women public mentioned, sexual status as a sex woman, a temporary one, is my war in the form of system or whatever, and they can make it to be a one they, a thousand degree short of the border on average than expected (!)%) of short, by the two-thirds ratio there to a available phone number.\n\nIt seems the young Obama out his fair record of terror at a woman deal or the terrorist attack in a matter of weeks ago. Those women, myself included, were beautiful, and to be around for. One of the luckyur-year-olds that was this for a court men is in a bad mood as soon as a divorce. No one would dare to hear that, U.S.\n\nAs parents were intent on may all their children’s death or any court evidence. Mind you, if in the White House not wearing a similar 4.\n\nThe signs were made for some that, yes, the Constitution is won, the sort of fear, of thoughtness, that some would feel about the cost of living before as mere months, with to include Nanny, hasies and died. The State’s view of such life as its meaning from the recent, it’s evident in the m fact that those standing in front of the process, and their coming to act in short, as the result, ought to use their force.\n\n“Most above all, the idea that only all this was lies and ends,” CNN and its version.\n\nAs it was born into, it gained new power and moved on. The editorial intercede: \"Picking with their hands in filling this trash can is, indeed, what comes closest to turning against the government.” The booty for (screaming) continued while a child in silence from the amateurs groped for the upper hand at that moment.\n\nThe. felt getting by Hillary Clinton, and her allies in taunting and control in order to throw this in Trump’s form. In the end the Court first intimated, who would not take on the Hillary Clinton with, that personal Trump and Donald Trump was really non-president vs. Oh, a brilliant former Marine, and the South. That’s not for sure, but per se at all, the next week.\n\nThe supreme major ruling is in to have begun taking the stand that drove Trump’s decision in the country, going from a get-the-vote out, hand-me-downs and hard times.\n\nTheyervatives want it to make the real estate course that was about to happen earlier this summer, and to realize that under some, the extreme right now had to be anymed in, leading to a more self-socialist government. Jonathan Turley, who is rumored to have a gazillion personal column in the Washington Post, said the ULC is to be expected. It is not that it will amount to little to allow it to happen. In effect, when there is. In a moment of the nation’s size or key, the news may be well-received<|endoftext|>", + "token_ids": [ + 50256, + 510, + 477, + 4040, + 416, + 607, + 2568, + 510, + 1028, + 262, + 3355, + 318, + 503, + 290, + 1804, + 11, + 340, + 8338, + 319, + 644, + 318, + 2406, + 284, + 340, + 2582, + 13, + 198, + 198, + 3844, + 11, + 673, + 1183, + 1464, + 3382, + 284, + 307, + 351, + 340, + 625, + 757, + 11, + 290, + 4206, + 326, + 607, + 13763, + 286, + 262, + 1230, + 481, + 991, + 307, + 1364, + 607, + 416, + 262, + 1611, + 287, + 262, + 460, + 560, + 11, + 290, + 356, + 1183, + 766, + 607, + 12105, + 290, + 287, + 257, + 460, + 560, + 11, + 314, + 1183, + 1560, + 606, + 11, + 287, + 326, + 1737, + 393, + 4232, + 673, + 13, + 198, + 198, + 464, + 1575, + 284, + 2614, + 1096, + 340, + 12, + 290, + 345, + 3677, + 572, + 262, + 3835, + 11, + 655, + 329, + 779, + 290, + 1744, + 11, + 262, + 1637, + 468, + 257, + 22036, + 319, + 606, + 329, + 477, + 286, + 777, + 1243, + 11, + 826, + 13, + 198, + 198, + 2949, + 11, + 407, + 477, + 777, + 460, + 874, + 28674, + 572, + 326, + 262, + 5349, + 468, + 284, + 3714, + 1497, + 804, + 588, + 1301, + 338, + 5775, + 13, + 447, + 251, + 198, + 198, + 1, + 5703, + 1577, + 534, + 790, + 284, + 2513, + 832, + 379, + 1661, + 262, + 5290, + 286, + 262, + 1803, + 326, + 561, + 307, + 597, + 1230, + 2641, + 2253, + 532, + 345, + 3636, + 447, + 247, + 83, + 307, + 284, + 766, + 340, + 3375, + 546, + 262, + 640, + 290, + 656, + 257, + 1429, + 284, + 787, + 1175, + 866, + 11, + 1752, + 447, + 247, + 82, + 588, + 42128, + 11, + 616, + 11, + 290, + 783, + 4785, + 621, + 262, + 2351, + 43192, + 526, + 198, + 198, + 2025, + 354, + 993, + 440, + 6, + 25819, + 11, + 588, + 286, + 262, + 366, + 29318, + 262, + 4332, + 553, + 1718, + 262, + 584, + 1230, + 6411, + 416, + 1719, + 284, + 2342, + 644, + 318, + 531, + 7271, + 11, + 422, + 4150, + 262, + 10509, + 389, + 3375, + 546, + 11, + 2291, + 761, + 1049, + 87, + 11, + 290, + 1690, + 1096, + 606, + 290, + 1429, + 477, + 326, + 484, + 389, + 1804, + 13, + 5334, + 898, + 1296, + 286, + 257, + 7614, + 284, + 1394, + 422, + 326, + 2524, + 290, + 511, + 6151, + 1468, + 503, + 286, + 326, + 1204, + 338, + 24672, + 13, + 1119, + 821, + 281, + 11171, + 290, + 407, + 625, + 268, + 11, + 1642, + 284, + 2687, + 262, + 1230, + 468, + 284, + 1011, + 340, + 13, + 198, + 198, + 1870, + 287, + 11824, + 329, + 262, + 6182, + 6926, + 338, + 7417, + 1368, + 11, + 262, + 3782, + 2921, + 257, + 1180, + 6770, + 286, + 340, + 13, + 198, + 198, + 1, + 6104, + 996, + 371, + 2036, + 3939, + 268, + 1596, + 287, + 597, + 1339, + 655, + 836, + 470, + 765, + 284, + 1410, + 5726, + 12, + 32629, + 1693, + 284, + 2687, + 11, + 262, + 21147, + 389, + 1029, + 11, + 281, + 3649, + 1271, + 286, + 1877, + 1203, + 1693, + 17587, + 355, + 257, + 1724, + 526, + 198, + 198, + 1890, + 262, + 1466, + 1171, + 4750, + 11, + 3206, + 3722, + 355, + 257, + 1714, + 2415, + 11, + 257, + 8584, + 530, + 11, + 318, + 616, + 1175, + 287, + 262, + 1296, + 286, + 1080, + 393, + 4232, + 11, + 290, + 484, + 460, + 787, + 340, + 284, + 307, + 257, + 530, + 484, + 11, + 257, + 7319, + 4922, + 1790, + 286, + 262, + 4865, + 319, + 2811, + 621, + 2938, + 357, + 8133, + 4407, + 286, + 1790, + 11, + 416, + 262, + 734, + 12, + 17936, + 8064, + 612, + 284, + 257, + 1695, + 3072, + 1271, + 13, + 198, + 198, + 1026, + 2331, + 262, + 1862, + 2486, + 503, + 465, + 3148, + 1700, + 286, + 3246, + 379, + 257, + 2415, + 1730, + 393, + 262, + 7417, + 1368, + 287, + 257, + 2300, + 286, + 2745, + 2084, + 13, + 5845, + 1466, + 11, + 3589, + 3017, + 11, + 547, + 4950, + 11, + 290, + 284, + 307, + 1088, + 329, + 13, + 1881, + 286, + 262, + 9670, + 333, + 12, + 1941, + 12, + 10119, + 326, + 373, + 428, + 329, + 257, + 2184, + 1450, + 318, + 287, + 257, + 2089, + 10038, + 355, + 2582, + 355, + 257, + 13609, + 13, + 1400, + 530, + 561, + 16498, + 284, + 3285, + 326, + 11, + 471, + 13, + 50, + 13, + 198, + 198, + 1722, + 3397, + 547, + 6824, + 319, + 743, + 477, + 511, + 1751, + 447, + 247, + 82, + 1918, + 393, + 597, + 2184, + 2370, + 13, + 10175, + 345, + 11, + 611, + 287, + 262, + 2635, + 2097, + 407, + 5762, + 257, + 2092, + 604, + 13, + 198, + 198, + 464, + 5895, + 547, + 925, + 329, + 617, + 326, + 11, + 3763, + 11, + 262, + 7965, + 318, + 1839, + 11, + 262, + 3297, + 286, + 3252, + 11, + 286, + 1807, + 1108, + 11, + 326, + 617, + 561, + 1254, + 546, + 262, + 1575, + 286, + 2877, + 878, + 355, + 5019, + 1933, + 11, + 351, + 284, + 2291, + 399, + 7737, + 11, + 468, + 444, + 290, + 3724, + 13, + 383, + 1812, + 447, + 247, + 82, + 1570, + 286, + 884, + 1204, + 355, + 663, + 3616, + 422, + 262, + 2274, + 11, + 340, + 447, + 247, + 82, + 10678, + 287, + 262, + 285, + 1109, + 326, + 883, + 5055, + 287, + 2166, + 286, + 262, + 1429, + 11, + 290, + 511, + 2406, + 284, + 719, + 287, + 1790, + 11, + 355, + 262, + 1255, + 11, + 10783, + 284, + 779, + 511, + 2700, + 13, + 198, + 198, + 447, + 250, + 6943, + 2029, + 477, + 11, + 262, + 2126, + 326, + 691, + 477, + 428, + 373, + 7363, + 290, + 5645, + 11, + 447, + 251, + 8100, + 290, + 663, + 2196, + 13, + 198, + 198, + 1722, + 340, + 373, + 4642, + 656, + 11, + 340, + 8618, + 649, + 1176, + 290, + 3888, + 319, + 13, + 383, + 13684, + 987, + 771, + 68, + 25, + 366, + 47, + 7958, + 351, + 511, + 2832, + 287, + 12591, + 428, + 13913, + 460, + 318, + 11, + 5600, + 11, + 644, + 2058, + 11706, + 284, + 6225, + 1028, + 262, + 1230, + 13, + 447, + 251, + 383, + 6297, + 88, + 329, + 357, + 1416, + 1476, + 278, + 8, + 3767, + 981, + 257, + 1200, + 287, + 9550, + 422, + 262, + 716, + 45211, + 39136, + 276, + 329, + 262, + 6727, + 1021, + 379, + 326, + 2589, + 13, + 198, + 198, + 464, + 13, + 2936, + 1972, + 416, + 5073, + 2605, + 11, + 290, + 607, + 7681, + 287, + 256, + 20706, + 290, + 1630, + 287, + 1502, + 284, + 3714, + 428, + 287, + 1301, + 447, + 247, + 82, + 1296, + 13, + 554, + 262, + 886, + 262, + 3078, + 717, + 10683, + 515, + 11, + 508, + 561, + 407, + 1011, + 319, + 262, + 5073, + 2605, + 351, + 11, + 326, + 2614, + 1301, + 290, + 3759, + 1301, + 373, + 1107, + 1729, + 12, + 22540, + 3691, + 13, + 3966, + 11, + 257, + 10457, + 1966, + 11000, + 11, + 290, + 262, + 2520, + 13, + 1320, + 447, + 247, + 82, + 407, + 329, + 1654, + 11, + 475, + 583, + 384, + 379, + 477, + 11, + 262, + 1306, + 1285, + 13, + 198, + 198, + 464, + 17700, + 1688, + 6922, + 318, + 287, + 284, + 423, + 9258, + 2263, + 262, + 1302, + 326, + 10357, + 1301, + 447, + 247, + 82, + 2551, + 287, + 262, + 1499, + 11, + 1016, + 422, + 257, + 651, + 12, + 1169, + 12, + 27257, + 503, + 11, + 1021, + 12, + 1326, + 12, + 30371, + 290, + 1327, + 1661, + 13, + 198, + 198, + 2990, + 35291, + 765, + 340, + 284, + 787, + 262, + 1103, + 7964, + 1781, + 326, + 373, + 546, + 284, + 1645, + 2961, + 428, + 3931, + 11, + 290, + 284, + 6537, + 326, + 739, + 617, + 11, + 262, + 3257, + 826, + 783, + 550, + 284, + 307, + 597, + 1150, + 287, + 11, + 3756, + 284, + 257, + 517, + 2116, + 12, + 14557, + 396, + 1230, + 13, + 11232, + 3831, + 1636, + 11, + 508, + 318, + 29586, + 284, + 423, + 257, + 308, + 1031, + 1131, + 2614, + 5721, + 287, + 262, + 2669, + 2947, + 11, + 531, + 262, + 471, + 5639, + 318, + 284, + 307, + 2938, + 13, + 632, + 318, + 407, + 326, + 340, + 481, + 2033, + 284, + 1310, + 284, + 1249, + 340, + 284, + 1645, + 13, + 554, + 1245, + 11, + 618, + 612, + 318, + 13, + 554, + 257, + 2589, + 286, + 262, + 3277, + 447, + 247, + 82, + 2546, + 393, + 1994, + 11, + 262, + 1705, + 743, + 307, + 880, + 12, + 47844, + 50256 + ] + }, + { + "sample_id": 14, + "text": "<|endoftext|>, and whole return of two hundred representatives of the top-five percent of society, in front of public money to pay rent and, in favor of a sort of land, in addition to one that killed and consisted of those in prison that were off workers. The ones that called for a big strike (which they knew who got this or were different from work.) This is of note of the role cities are in because they already poverty and violence. Racism and racism that are rooted in India all depend heavily on better place conditions. The organizers explained that the majority of workers in Detroit decided to have a place of worship in three or two and gave them permission to try to buy another, in which to have an “act with percent of members.” At the prisoners always placed one on the two-point list. The national media would be able to get rid of all of these details and learn even more about the situation. In a 20-by-longne society, that were more like riots. A source briefed on the matter told The Times of Higherf that protesters were guilty of similar bad use of force, getting good conditions, and conditions that, as imposed on the human person, caused all of the people to show the court as well.\n\nBut 2 (19 day) There was a circulation in October of members of 1,000, 16,000,000. That year Stalin received one hundred, but the full author had written on the side of his hand over as governor. He wrote a speech on why and we working in November 2011, below that included all kinds of events. You begin to find the text:\n\nBetween the riots and the Clintonirties, I expressed support for right and workers in times of crisis since the holy grail of the Soviet War. The protests broke out on big days in each week, and when the workers opened and live leaders engaged in political groups and acts of protest. During the election, he,/Secretary of State elections and then I did press building. Two years ago, we release all new photos. Today, anyone attention to one’s image is of a political party or political organization in particular. You can have a known down in the street and display it in front of everybody. That type of protest was always - now still at least as union workers, as police officers and unionists, and people who organized themselves in the labor movement. Not as many news There were indeed very real and informed people of this, but yes, it was extremely important. It was brought back to those around him and the spirit of a society of that time.\n\n*This half of this we see the issues that are of concern for the whole of Wisconsin - now to be the political community in the state of Michigan, as, than in the state of Wisconsin. When changes and places like these are where examples of the importance of social development is themselves and in the struggle that played out for months, you should expect yourself of such a space in this case alone, thanks to the kind of -\n\nand the best leadists, public clear or that it took to do more. We have also over the experience of large living in the state of Michigan with another person’s party and on upon its actions.\n\nI think the rest of Indian history has witnessed such events, twice a day.\n\nThe House of Cards is the story of having value to the masses, and to Heavens (t2) the land of God once upon one another. Doing so has already been told by theseD, women written using pencils, humanmen, books, and books. The book is other, simple, without, done, and many games, co-wise, especially of this nature.\n\nAnd we, readers, make sense of these events at this, by these books and that.\n\nThe We’re supposed to make reference to create capitalism, which is where the concept of capitalism stands for its own sake, is located and is needed to serve as bargaining power and influence for around the world. Power is being used to empower the ruling individual who is able to have complete control over all their rights and from the monarchs in order to be, therefore, to whom rule by name.\n\nThe power struggle was good in the slave states, to the common people, necessary for higher development in a future that is given a different kind of class struggle, more freedom and freedom and more near and public service.\n\nWhat seems to be the case of the USSR, what happened before, is because power/ations were too much the size of the population and proportional to the number of ag colonized, free important in the hands of American’s far north.\n\nThe evidence was not founded on; something that gave thought to the world, in particular through the lens of propaganda and propaganda. We saw this in the Second World War.\n\nWhy? Because of the early book movements. The movie Theological War began as a post-war<|endoftext|>", + "token_ids": [ + 50256, + 11, + 290, + 2187, + 1441, + 286, + 734, + 3470, + 10826, + 286, + 262, + 1353, + 12, + 13261, + 1411, + 286, + 3592, + 11, + 287, + 2166, + 286, + 1171, + 1637, + 284, + 1414, + 5602, + 290, + 11, + 287, + 2661, + 286, + 257, + 3297, + 286, + 1956, + 11, + 287, + 3090, + 284, + 530, + 326, + 2923, + 290, + 19954, + 286, + 883, + 287, + 3770, + 326, + 547, + 572, + 3259, + 13, + 383, + 3392, + 326, + 1444, + 329, + 257, + 1263, + 5587, + 357, + 4758, + 484, + 2993, + 508, + 1392, + 428, + 393, + 547, + 1180, + 422, + 670, + 2014, + 770, + 318, + 286, + 3465, + 286, + 262, + 2597, + 4736, + 389, + 287, + 780, + 484, + 1541, + 8098, + 290, + 3685, + 13, + 24746, + 1042, + 290, + 10713, + 326, + 389, + 19459, + 287, + 3794, + 477, + 4745, + 7272, + 319, + 1365, + 1295, + 3403, + 13, + 383, + 19043, + 4893, + 326, + 262, + 3741, + 286, + 3259, + 287, + 8488, + 3066, + 284, + 423, + 257, + 1295, + 286, + 11892, + 287, + 1115, + 393, + 734, + 290, + 2921, + 606, + 7170, + 284, + 1949, + 284, + 2822, + 1194, + 11, + 287, + 543, + 284, + 423, + 281, + 564, + 250, + 529, + 351, + 1411, + 286, + 1866, + 13, + 447, + 251, + 1629, + 262, + 10577, + 1464, + 4624, + 530, + 319, + 262, + 734, + 12, + 4122, + 1351, + 13, + 383, + 2260, + 2056, + 561, + 307, + 1498, + 284, + 651, + 5755, + 286, + 477, + 286, + 777, + 3307, + 290, + 2193, + 772, + 517, + 546, + 262, + 3074, + 13, + 554, + 257, + 1160, + 12, + 1525, + 12, + 6511, + 710, + 3592, + 11, + 326, + 547, + 517, + 588, + 23766, + 13, + 317, + 2723, + 32664, + 319, + 262, + 2300, + 1297, + 383, + 3782, + 286, + 16038, + 69, + 326, + 9584, + 547, + 6717, + 286, + 2092, + 2089, + 779, + 286, + 2700, + 11, + 1972, + 922, + 3403, + 11, + 290, + 3403, + 326, + 11, + 355, + 10893, + 319, + 262, + 1692, + 1048, + 11, + 4073, + 477, + 286, + 262, + 661, + 284, + 905, + 262, + 2184, + 355, + 880, + 13, + 198, + 198, + 1537, + 362, + 357, + 1129, + 1110, + 8, + 1318, + 373, + 257, + 19133, + 287, + 3267, + 286, + 1866, + 286, + 352, + 11, + 830, + 11, + 1467, + 11, + 830, + 11, + 830, + 13, + 1320, + 614, + 17482, + 2722, + 530, + 3470, + 11, + 475, + 262, + 1336, + 1772, + 550, + 3194, + 319, + 262, + 1735, + 286, + 465, + 1021, + 625, + 355, + 8153, + 13, + 679, + 2630, + 257, + 4046, + 319, + 1521, + 290, + 356, + 1762, + 287, + 3389, + 2813, + 11, + 2174, + 326, + 3017, + 477, + 6982, + 286, + 2995, + 13, + 921, + 2221, + 284, + 1064, + 262, + 2420, + 25, + 198, + 198, + 25262, + 262, + 23766, + 290, + 262, + 2605, + 2265, + 444, + 11, + 314, + 6241, + 1104, + 329, + 826, + 290, + 3259, + 287, + 1661, + 286, + 4902, + 1201, + 262, + 11386, + 7933, + 346, + 286, + 262, + 7570, + 1810, + 13, + 383, + 8536, + 6265, + 503, + 319, + 1263, + 1528, + 287, + 1123, + 1285, + 11, + 290, + 618, + 262, + 3259, + 4721, + 290, + 2107, + 2766, + 7953, + 287, + 1964, + 2628, + 290, + 6529, + 286, + 5402, + 13, + 5856, + 262, + 3071, + 11, + 339, + 11, + 14, + 38541, + 286, + 1812, + 7024, + 290, + 788, + 314, + 750, + 1803, + 2615, + 13, + 4930, + 812, + 2084, + 11, + 356, + 2650, + 477, + 649, + 5205, + 13, + 6288, + 11, + 2687, + 3241, + 284, + 530, + 447, + 247, + 82, + 2939, + 318, + 286, + 257, + 1964, + 2151, + 393, + 1964, + 4009, + 287, + 1948, + 13, + 921, + 460, + 423, + 257, + 1900, + 866, + 287, + 262, + 4675, + 290, + 3359, + 340, + 287, + 2166, + 286, + 7288, + 13, + 1320, + 2099, + 286, + 5402, + 373, + 1464, + 532, + 783, + 991, + 379, + 1551, + 355, + 6441, + 3259, + 11, + 355, + 1644, + 3790, + 290, + 6441, + 1023, + 11, + 290, + 661, + 508, + 8389, + 2405, + 287, + 262, + 4827, + 3356, + 13, + 1892, + 355, + 867, + 1705, + 1318, + 547, + 5600, + 845, + 1103, + 290, + 7981, + 661, + 286, + 428, + 11, + 475, + 3763, + 11, + 340, + 373, + 4457, + 1593, + 13, + 632, + 373, + 3181, + 736, + 284, + 883, + 1088, + 683, + 290, + 262, + 4437, + 286, + 257, + 3592, + 286, + 326, + 640, + 13, + 198, + 198, + 9, + 1212, + 2063, + 286, + 428, + 356, + 766, + 262, + 2428, + 326, + 389, + 286, + 2328, + 329, + 262, + 2187, + 286, + 9279, + 532, + 783, + 284, + 307, + 262, + 1964, + 2055, + 287, + 262, + 1181, + 286, + 7055, + 11, + 355, + 11, + 621, + 287, + 262, + 1181, + 286, + 9279, + 13, + 1649, + 2458, + 290, + 4113, + 588, + 777, + 389, + 810, + 6096, + 286, + 262, + 6817, + 286, + 1919, + 2478, + 318, + 2405, + 290, + 287, + 262, + 6531, + 326, + 2826, + 503, + 329, + 1933, + 11, + 345, + 815, + 1607, + 3511, + 286, + 884, + 257, + 2272, + 287, + 428, + 1339, + 3436, + 11, + 5176, + 284, + 262, + 1611, + 286, + 532, + 198, + 198, + 392, + 262, + 1266, + 1085, + 1023, + 11, + 1171, + 1598, + 393, + 326, + 340, + 1718, + 284, + 466, + 517, + 13, + 775, + 423, + 635, + 625, + 262, + 1998, + 286, + 1588, + 2877, + 287, + 262, + 1181, + 286, + 7055, + 351, + 1194, + 1048, + 447, + 247, + 82, + 2151, + 290, + 319, + 2402, + 663, + 4028, + 13, + 198, + 198, + 40, + 892, + 262, + 1334, + 286, + 3942, + 2106, + 468, + 13923, + 884, + 2995, + 11, + 5403, + 257, + 1110, + 13, + 198, + 198, + 464, + 2097, + 286, + 15824, + 318, + 262, + 1621, + 286, + 1719, + 1988, + 284, + 262, + 14568, + 11, + 290, + 284, + 11225, + 82, + 357, + 83, + 17, + 8, + 262, + 1956, + 286, + 1793, + 1752, + 2402, + 530, + 1194, + 13, + 25327, + 523, + 468, + 1541, + 587, + 1297, + 416, + 777, + 35, + 11, + 1466, + 3194, + 1262, + 21613, + 82, + 11, + 1692, + 3653, + 11, + 3835, + 11, + 290, + 3835, + 13, + 383, + 1492, + 318, + 584, + 11, + 2829, + 11, + 1231, + 11, + 1760, + 11, + 290, + 867, + 1830, + 11, + 763, + 12, + 3083, + 11, + 2592, + 286, + 428, + 3450, + 13, + 198, + 198, + 1870, + 356, + 11, + 7183, + 11, + 787, + 2565, + 286, + 777, + 2995, + 379, + 428, + 11, + 416, + 777, + 3835, + 290, + 326, + 13, + 198, + 198, + 464, + 775, + 447, + 247, + 260, + 4385, + 284, + 787, + 4941, + 284, + 2251, + 12129, + 11, + 543, + 318, + 810, + 262, + 3721, + 286, + 12129, + 6296, + 329, + 663, + 898, + 11060, + 11, + 318, + 5140, + 290, + 318, + 2622, + 284, + 4691, + 355, + 23189, + 1176, + 290, + 4588, + 329, + 1088, + 262, + 995, + 13, + 4333, + 318, + 852, + 973, + 284, + 17549, + 262, + 6922, + 1981, + 508, + 318, + 1498, + 284, + 423, + 1844, + 1630, + 625, + 477, + 511, + 2489, + 290, + 422, + 262, + 26464, + 82, + 287, + 1502, + 284, + 307, + 11, + 4361, + 11, + 284, + 4150, + 3896, + 416, + 1438, + 13, + 198, + 198, + 464, + 1176, + 6531, + 373, + 922, + 287, + 262, + 11778, + 2585, + 11, + 284, + 262, + 2219, + 661, + 11, + 3306, + 329, + 2440, + 2478, + 287, + 257, + 2003, + 326, + 318, + 1813, + 257, + 1180, + 1611, + 286, + 1398, + 6531, + 11, + 517, + 4925, + 290, + 4925, + 290, + 517, + 1474, + 290, + 1171, + 2139, + 13, + 198, + 198, + 2061, + 2331, + 284, + 307, + 262, + 1339, + 286, + 262, + 28905, + 11, + 644, + 3022, + 878, + 11, + 318, + 780, + 1176, + 14, + 602, + 547, + 1165, + 881, + 262, + 2546, + 286, + 262, + 3265, + 290, + 27111, + 284, + 262, + 1271, + 286, + 556, + 7633, + 1143, + 11, + 1479, + 1593, + 287, + 262, + 2832, + 286, + 1605, + 447, + 247, + 82, + 1290, + 5093, + 13, + 198, + 198, + 464, + 2370, + 373, + 407, + 9393, + 319, + 26, + 1223, + 326, + 2921, + 1807, + 284, + 262, + 995, + 11, + 287, + 1948, + 832, + 262, + 10317, + 286, + 11613, + 290, + 11613, + 13, + 775, + 2497, + 428, + 287, + 262, + 5498, + 2159, + 1810, + 13, + 198, + 198, + 5195, + 30, + 4362, + 286, + 262, + 1903, + 1492, + 8650, + 13, + 383, + 3807, + 383, + 2770, + 1810, + 2540, + 355, + 257, + 1281, + 12, + 5767, + 50256 + ] + }, + { + "sample_id": 15, + "text": "<|endoftext|> it’s death was hisidental. No questioner days. Every other American countries knows of what he’s OK’d: food, his money, corruption, and his system to bring down. No. People here at a high education level that have been showing support for Snowden. Other leaders have shown expressed support for Snowden – as well, including those that are actually going about is the violence that is happening.\n\nThatb, like talking to the N gaming community, has over allowedad Sessions to be appointed U.S. Senator. He also’s corrupt. Any public can make all out of whatever the up to be there from scratch. And no games would show the guy to death, to certain points.\n\nInera Smith has all but his allegiance to the past and stop. People grew up together in a her age, a senior U.S. senator, and then the members of the band of trying. Yes! Anti-oh! The moral and political points here is a departure from that's here point: Here is the art of death metal, the destruction of the flow of music and people.But mass media can’t actually work.\n\nGood things, folks! They need all of a sort of shit [emphasisquote]\n\nThat’s cool. Well, don’t worry, people’s is self-serving — and this community’s has no respect for power of opinion on others. But problems like all of site version of power.\n\n• W A full blown list of board games and their connection to Duty: Washington is different take on the stealthy and sci-fi stuff. No neesh. Got the point on? Either you read it up again or into the background — that on piece of 2016.\n\n“The Democrats’ message is shrunken. We want to pretend war is imminent and President Bush Bush’s agenda is dead, so yeah.” [sic] said.\n\nIf he's all my genitalia, the guy wants to have his head on key positions in the Senate, and the federal government, he'll beeding in the state legislature.\n\nIt'll stop thinking of the design of the office name aroundu. But all the same sort of another. For close to 40 years, the Internet is slowly turning out to be glamorized by the growing ever-increasing power of Google corporations and the hard Internet.\n\nAbout 80% of Internetn in today involve likehing across the floodgia. And air going viral. This suggests that the Internet is building on create infrastructure. Well-fi, a vast and old fashioned series of new technologies. Just look at the sky is clear.\n\nWhile West down last weekend, I attended a game that went to school kids, and most even good ones. He's includingating TV, in games, and shaking my hand. Not true - that won the story. When he's office about this month of dark, before ahead of the Earth, he is willing to part with that.\n\nClaims can be made by low-level folks (as side of the vast Institute of Technology) and a guy well in his 40s, and that the number are focused on average, where nextian in the sky, not in California.\n\nSchools is any idea. Thef brothers and I should be over from online chat groups and politicos, of all kinds. Going from coast to town on an airplane to the Statehouse.\n\nJohn B. said … tech companies like the CIA should pay social networks to research, encouraging every word, love, the Internet (I mean social media) to come at the end of cost and peace and support of America.\n\nPeljal’s an avid member of the UPA and out of course, but using thana. You would like to seen hang out in the dark corners of the food East. Or, yes, new ones. \"The business of evil.\n\n*-huh huh?>\n\nSet up a war hero. I think we did not forget ourselves: He's referring to this event, did. as George Lincoln said. “He who fights for all and for our freedom and on whose land is free, people and freedom, is a greatest threat to us.” Remember him. He set its record. Forget the fact that digital technology for us brought the world to collapse, and then rise up, and eventually shape the world’s future. Not that such a moment, as a result.\n\nWe case technology continues to exist here in the end of our world that the Brits were created to. However, not really sure why just do they live, you will keep toying with this whole life.\n\nThe 90s and Neo-ins.\n\n* * *\n\nDisg From the get-ups, my parents have had a<|endoftext|>", + "token_ids": [ + 50256, + 340, + 447, + 247, + 82, + 1918, + 373, + 465, + 35182, + 13, + 1400, + 1808, + 263, + 1528, + 13, + 3887, + 584, + 1605, + 2678, + 4206, + 286, + 644, + 339, + 447, + 247, + 82, + 7477, + 447, + 247, + 67, + 25, + 2057, + 11, + 465, + 1637, + 11, + 9253, + 11, + 290, + 465, + 1080, + 284, + 2222, + 866, + 13, + 1400, + 13, + 4380, + 994, + 379, + 257, + 1029, + 3707, + 1241, + 326, + 423, + 587, + 4478, + 1104, + 329, + 15015, + 13, + 3819, + 2766, + 423, + 3402, + 6241, + 1104, + 329, + 15015, + 784, + 355, + 880, + 11, + 1390, + 883, + 326, + 389, + 1682, + 1016, + 546, + 318, + 262, + 3685, + 326, + 318, + 5836, + 13, + 198, + 198, + 2504, + 65, + 11, + 588, + 3375, + 284, + 262, + 399, + 7776, + 2055, + 11, + 468, + 625, + 3142, + 324, + 15184, + 284, + 307, + 9899, + 471, + 13, + 50, + 13, + 8962, + 13, + 679, + 635, + 447, + 247, + 82, + 10622, + 13, + 4377, + 1171, + 460, + 787, + 477, + 503, + 286, + 4232, + 262, + 510, + 284, + 307, + 612, + 422, + 12692, + 13, + 843, + 645, + 1830, + 561, + 905, + 262, + 3516, + 284, + 1918, + 11, + 284, + 1728, + 2173, + 13, + 198, + 198, + 40, + 710, + 430, + 4176, + 468, + 477, + 475, + 465, + 25696, + 284, + 262, + 1613, + 290, + 2245, + 13, + 4380, + 6348, + 510, + 1978, + 287, + 257, + 607, + 2479, + 11, + 257, + 4664, + 471, + 13, + 50, + 13, + 12329, + 11, + 290, + 788, + 262, + 1866, + 286, + 262, + 4097, + 286, + 2111, + 13, + 3363, + 0, + 12297, + 12, + 1219, + 0, + 383, + 6573, + 290, + 1964, + 2173, + 994, + 318, + 257, + 12928, + 422, + 326, + 338, + 994, + 966, + 25, + 3423, + 318, + 262, + 1242, + 286, + 1918, + 6147, + 11, + 262, + 8166, + 286, + 262, + 5202, + 286, + 2647, + 290, + 661, + 13, + 1537, + 2347, + 2056, + 460, + 447, + 247, + 83, + 1682, + 670, + 13, + 198, + 198, + 10248, + 1243, + 11, + 7974, + 0, + 1119, + 761, + 477, + 286, + 257, + 3297, + 286, + 7510, + 685, + 36663, + 22708, + 60, + 198, + 198, + 2504, + 447, + 247, + 82, + 3608, + 13, + 3894, + 11, + 836, + 447, + 247, + 83, + 5490, + 11, + 661, + 447, + 247, + 82, + 318, + 2116, + 12, + 31293, + 851, + 290, + 428, + 2055, + 447, + 247, + 82, + 468, + 645, + 2461, + 329, + 1176, + 286, + 4459, + 319, + 1854, + 13, + 887, + 2761, + 588, + 477, + 286, + 2524, + 2196, + 286, + 1176, + 13, + 198, + 198, + 3581, + 370, + 317, + 1336, + 16318, + 1351, + 286, + 3096, + 1830, + 290, + 511, + 4637, + 284, + 18104, + 25, + 2669, + 318, + 1180, + 1011, + 319, + 262, + 19159, + 88, + 290, + 20681, + 12, + 12463, + 3404, + 13, + 1400, + 299, + 2841, + 71, + 13, + 11853, + 262, + 966, + 319, + 30, + 15467, + 345, + 1100, + 340, + 510, + 757, + 393, + 656, + 262, + 4469, + 851, + 326, + 319, + 3704, + 286, + 1584, + 13, + 198, + 198, + 447, + 250, + 464, + 4956, + 447, + 247, + 3275, + 318, + 38900, + 268, + 13, + 775, + 765, + 284, + 16614, + 1175, + 318, + 21161, + 290, + 1992, + 5511, + 5511, + 447, + 247, + 82, + 8666, + 318, + 2636, + 11, + 523, + 10194, + 13, + 447, + 251, + 685, + 21383, + 60, + 531, + 13, + 198, + 198, + 1532, + 339, + 338, + 477, + 616, + 24678, + 544, + 11, + 262, + 3516, + 3382, + 284, + 423, + 465, + 1182, + 319, + 1994, + 6116, + 287, + 262, + 3845, + 11, + 290, + 262, + 2717, + 1230, + 11, + 339, + 1183, + 307, + 276, + 278, + 287, + 262, + 1181, + 15928, + 13, + 198, + 198, + 1026, + 1183, + 2245, + 3612, + 286, + 262, + 1486, + 286, + 262, + 2607, + 1438, + 1088, + 84, + 13, + 887, + 477, + 262, + 976, + 3297, + 286, + 1194, + 13, + 1114, + 1969, + 284, + 2319, + 812, + 11, + 262, + 4455, + 318, + 6364, + 6225, + 503, + 284, + 307, + 27240, + 273, + 1143, + 416, + 262, + 3957, + 1683, + 12, + 42647, + 1176, + 286, + 3012, + 10225, + 290, + 262, + 1327, + 4455, + 13, + 198, + 198, + 8585, + 4019, + 4, + 286, + 4455, + 77, + 287, + 1909, + 6211, + 588, + 71, + 278, + 1973, + 262, + 6947, + 70, + 544, + 13, + 843, + 1633, + 1016, + 14416, + 13, + 770, + 5644, + 326, + 262, + 4455, + 318, + 2615, + 319, + 2251, + 6884, + 13, + 3894, + 12, + 12463, + 11, + 257, + 5909, + 290, + 1468, + 35458, + 2168, + 286, + 649, + 8514, + 13, + 2329, + 804, + 379, + 262, + 6766, + 318, + 1598, + 13, + 198, + 198, + 3633, + 2688, + 866, + 938, + 5041, + 11, + 314, + 9141, + 257, + 983, + 326, + 1816, + 284, + 1524, + 3988, + 11, + 290, + 749, + 772, + 922, + 3392, + 13, + 679, + 338, + 1390, + 803, + 3195, + 11, + 287, + 1830, + 11, + 290, + 17275, + 616, + 1021, + 13, + 1892, + 2081, + 532, + 326, + 1839, + 262, + 1621, + 13, + 1649, + 339, + 338, + 2607, + 546, + 428, + 1227, + 286, + 3223, + 11, + 878, + 4058, + 286, + 262, + 3668, + 11, + 339, + 318, + 4684, + 284, + 636, + 351, + 326, + 13, + 198, + 198, + 44819, + 82, + 460, + 307, + 925, + 416, + 1877, + 12, + 5715, + 7974, + 357, + 292, + 1735, + 286, + 262, + 5909, + 5136, + 286, + 8987, + 8, + 290, + 257, + 3516, + 880, + 287, + 465, + 2319, + 82, + 11, + 290, + 326, + 262, + 1271, + 389, + 5670, + 319, + 2811, + 11, + 810, + 1306, + 666, + 287, + 262, + 6766, + 11, + 407, + 287, + 3442, + 13, + 198, + 198, + 26130, + 82, + 318, + 597, + 2126, + 13, + 383, + 69, + 9397, + 290, + 314, + 815, + 307, + 625, + 422, + 2691, + 8537, + 2628, + 290, + 31723, + 418, + 11, + 286, + 477, + 6982, + 13, + 19219, + 422, + 7051, + 284, + 3240, + 319, + 281, + 19401, + 284, + 262, + 1812, + 4803, + 13, + 198, + 198, + 7554, + 347, + 13, + 531, + 3926, + 7261, + 2706, + 588, + 262, + 7688, + 815, + 1414, + 1919, + 7686, + 284, + 2267, + 11, + 12577, + 790, + 1573, + 11, + 1842, + 11, + 262, + 4455, + 357, + 40, + 1612, + 1919, + 2056, + 8, + 284, + 1282, + 379, + 262, + 886, + 286, + 1575, + 290, + 4167, + 290, + 1104, + 286, + 2253, + 13, + 198, + 198, + 47, + 417, + 73, + 282, + 447, + 247, + 82, + 281, + 32450, + 2888, + 286, + 262, + 471, + 4537, + 290, + 503, + 286, + 1781, + 11, + 475, + 1262, + 621, + 64, + 13, + 921, + 561, + 588, + 284, + 766, + 77, + 8181, + 503, + 287, + 262, + 3223, + 14371, + 286, + 262, + 2057, + 3687, + 13, + 1471, + 11, + 3763, + 11, + 649, + 3392, + 13, + 366, + 464, + 1597, + 286, + 6181, + 13, + 198, + 198, + 9, + 12, + 71, + 7456, + 24926, + 30, + 29, + 198, + 198, + 7248, + 510, + 257, + 1175, + 4293, + 13, + 314, + 892, + 356, + 750, + 407, + 6044, + 6731, + 25, + 679, + 338, + 9759, + 284, + 428, + 1785, + 11, + 750, + 13, + 355, + 4502, + 12406, + 531, + 13, + 564, + 250, + 1544, + 508, + 11418, + 329, + 477, + 290, + 329, + 674, + 4925, + 290, + 319, + 3025, + 1956, + 318, + 1479, + 11, + 661, + 290, + 4925, + 11, + 318, + 257, + 6000, + 2372, + 284, + 514, + 13, + 447, + 251, + 11436, + 683, + 13, + 679, + 900, + 663, + 1700, + 13, + 29624, + 262, + 1109, + 326, + 4875, + 3037, + 329, + 514, + 3181, + 262, + 995, + 284, + 9807, + 11, + 290, + 788, + 4485, + 510, + 11, + 290, + 4191, + 5485, + 262, + 995, + 447, + 247, + 82, + 2003, + 13, + 1892, + 326, + 884, + 257, + 2589, + 11, + 355, + 257, + 1255, + 13, + 198, + 198, + 1135, + 1339, + 3037, + 4477, + 284, + 2152, + 994, + 287, + 262, + 886, + 286, + 674, + 995, + 326, + 262, + 2490, + 82, + 547, + 2727, + 284, + 13, + 2102, + 11, + 407, + 1107, + 1654, + 1521, + 655, + 466, + 484, + 2107, + 11, + 345, + 481, + 1394, + 284, + 1112, + 351, + 428, + 2187, + 1204, + 13, + 198, + 198, + 464, + 4101, + 82, + 290, + 21227, + 12, + 1040, + 13, + 198, + 198, + 9, + 1635, + 1635, + 198, + 198, + 7279, + 70, + 3574, + 262, + 651, + 12, + 4739, + 11, + 616, + 3397, + 423, + 550, + 257, + 50256 + ] + } + ], + "schema": "dlb-upstream-token-capture-v1" +} diff --git a/results/smoke/samples/owt/fmlm/steps_8/conversion_metadata.json b/results/smoke/samples/owt/fmlm/steps_8/conversion_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d225c6d32d5695197637b215ee15482a2219e6a0 --- /dev/null +++ b/results/smoke/samples/owt/fmlm/steps_8/conversion_metadata.json @@ -0,0 +1,31 @@ +{ + "checkpoint_lock_id": "recipe:fmlm_owt_official_ckpt:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0", + "checkpoint_selection": { + "output": "checkpoints/official/fmlm_ckpt/owt", + "recipe": "fmlm_owt_official_ckpt", + "sampling_checkpoint": "owt_fmlm.ckpt", + "sampling_config": null, + "source": "flm", + "source_commit": "a1918d5164e5038e37d0b7a4fb2010ce75b863b3", + "teacher_adapter": null, + "teacher_family": "continuous_flm" + }, + "checkpoint_sha256": "a5ca77b77a26dec9746f627858d42c19803d11c1e932dcf14a3ce80f6bbbbd76", + "exclude_from_latency": true, + "format": "upstream-generated-seqs-v1", + "generated_samples": 16, + "generation_seconds_sentinel": 0.0, + "generation_seconds_source": "unavailable_excluded_sentinel", + "requested_samples": 1, + "teacher_family": "continuous_flm", + "token_ids_source": "upstream", + "token_ids_transformation": { + "max_length": 1024, + "operation": "validated_exact_length" + }, + "tokenizer": "gpt2", + "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", + "trim_policy": "stable_prefix", + "trimmed_samples": 15, + "upstream_output": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/fmlm/steps_8/upstream_samples.json" +} diff --git a/results/smoke/samples/owt/langflow/steps_2/stdout.log b/results/smoke/samples/owt/langflow/steps_2/stdout.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/smoke/samples/owt/langflow/steps_32/stdout.log b/results/smoke/samples/owt/langflow/steps_32/stdout.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/smoke/samples/owt/langflow/steps_4/stdout.log b/results/smoke/samples/owt/langflow/steps_4/stdout.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/smoke/samples/owt/langflow/steps_8/stdout.log b/results/smoke/samples/owt/langflow/steps_8/stdout.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/smoke/samples/owt/mdlm_di4c/steps_4/request.json b/results/smoke/samples/owt/mdlm_di4c/steps_4/request.json new file mode 100644 index 0000000000000000000000000000000000000000..c5cc06a6630adef015b84e87b1bffddcad474c64 --- /dev/null +++ b/results/smoke/samples/owt/mdlm_di4c/steps_4/request.json @@ -0,0 +1,99 @@ +{ + "command": [ + "/opt/anaconda3/envs/dlb-di4c/bin/python", + "-B", + "-u", + "/home/ruizzhao/diffusion_baseline/adapters/sample_di4c.py", + "--upstream-root", + "/home/ruizzhao/diffusion_baseline/upstreams/di4c/sdtt", + "--checkpoint", + "/home/ruizzhao/diffusion_baseline/checkpoints/official/mdlm_di4c/owt/sdtt7-di4c2.ckpt", + "--config", + "/home/ruizzhao/diffusion_baseline/configs/sampling/di4c_mdlm_owt.yaml", + "--checkpoint-sha256", + "4af61572dc82fe012ce395a2e4e4be0585ea09076597ba1f7241ca196b97ecb0", + "--config-sha256", + "cf627f89b8dede9b25ac8e10407c3b7de203a76c4ba84100723287e571dcec34", + "--data-config", + "/home/ruizzhao/diffusion_baseline/artifacts/data.yaml", + "--downloads-manifest", + "/home/ruizzhao/diffusion_baseline/data/manifests/downloads.json", + "--tokenizer-snapshot", + "/home/ruizzhao/diffusion_baseline/data/raw/huggingface/hub/models--gpt2/snapshots/607a30d783dfa663caf39e06633721c8d4cfcd7e", + "--output", + "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/mdlm_di4c/steps_4/upstream_token_ids.json", + "--sample-count", + "1", + "--batch-size", + "1", + "--num-steps", + "4", + "--seq-len", + "1024", + "--seed", + "42", + "--sampler", + "ancestral", + "--teacher-family", + "masked_mdlm", + "--dataset", + "owt", + "--offline", + "true" + ], + "identity": { + "adapter_identity": "dlb.adapters.di4c:v3|dlb.adapters.di4c:Di4CAdapter|ba289424e1babd29c99b4bb95c008a934d000e1ddb9e76140f3963398bf02111", + "checkpoint_lock_id": "di4c_mdlm_owt_zenodo:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all", + "checkpoint_selection": { + "path": null, + "resource": "di4c_mdlm_owt_zenodo", + "sampling_config": "configs/sampling/di4c_mdlm_owt.yaml", + "sampling_config_sha256": "cf627f89b8dede9b25ac8e10407c3b7de203a76c4ba84100723287e571dcec34", + "sampling_config_source": "project", + "sampling_config_source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3", + "teacher_family": null + }, + "checkpoint_sha256": "e51269cd095bb1ed1b8771e4b5c8470d22b01fa7fa0652dce8eff18fcb064f8c", + "checkpoint_teacher_family": "masked_mdlm", + "command_sha256": "4448eae936eaa6891928f885fffd3c66db53aee4444dc9cf07ae6e2139d1863b", + "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994", + "dataset_id": "owt", + "device": null, + "environment": "dlb-di4c", + "model_id": "mdlm_di4c", + "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke", + "run_id": "mdlm_di4c-owt-steps-4", + "sample_count": 1, + "seed": 42, + "source_sha256": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3", + "step_count": 4 + }, + "request": { + "adapter_identity": "dlb.adapters.di4c:v3|dlb.adapters.di4c:Di4CAdapter|ba289424e1babd29c99b4bb95c008a934d000e1ddb9e76140f3963398bf02111", + "checkpoint_lock_id": "di4c_mdlm_owt_zenodo:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all", + "checkpoint_selection": { + "path": null, + "resource": "di4c_mdlm_owt_zenodo", + "sampling_config": "configs/sampling/di4c_mdlm_owt.yaml", + "sampling_config_sha256": "cf627f89b8dede9b25ac8e10407c3b7de203a76c4ba84100723287e571dcec34", + "sampling_config_source": "project", + "sampling_config_source_commit": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3", + "teacher_family": null + }, + "checkpoint_sha256": "e51269cd095bb1ed1b8771e4b5c8470d22b01fa7fa0652dce8eff18fcb064f8c", + "checkpoint_teacher_family": "masked_mdlm", + "command": null, + "config_sha256": "206d75b3b7cf3fb35844c053ae552537d7471b93bf29756dafc1ac616b096994", + "dataset_id": "owt", + "device": null, + "environment": "dlb-di4c", + "model_id": "mdlm_di4c", + "results_root": "/home/ruizzhao/diffusion_baseline/results/smoke", + "run_id": "mdlm_di4c-owt-steps-4", + "sample_count": 1, + "seed": 42, + "source_sha256": "ac61ff9fe8e85120f9e1d2a8c5a332f8b8353dd3", + "step_count": 4 + }, + "started_at": "2026-08-03T06:48:33.478108Z" +} diff --git a/results/smoke/samples/owt/mdlm_di4c/steps_4/stderr.log b/results/smoke/samples/owt/mdlm_di4c/steps_4/stderr.log new file mode 100644 index 0000000000000000000000000000000000000000..c5f893f8039ebbf8011f377fbb0992024dc17c88 --- /dev/null +++ b/results/smoke/samples/owt/mdlm_di4c/steps_4/stderr.log @@ -0,0 +1,9 @@ +/opt/anaconda3/envs/dlb-di4c/lib/python3.10/site-packages/lightning/fabric/__init__.py:41: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. +Traceback (most recent call last): + File "/home/ruizzhao/diffusion_baseline/adapters/sample_di4c.py", line 124, in + raise SystemExit(main()) + File "/home/ruizzhao/diffusion_baseline/adapters/sample_di4c.py", line 85, in main + validate_embedded_config(config, embedded_config) + File "/home/ruizzhao/diffusion_baseline/adapters/_distilled_runtime.py", line 246, in validate_embedded_config + raise ValueError("Lightning checkpoint does not embed its training config") +ValueError: Lightning checkpoint does not embed its training config diff --git a/results/smoke/samples/owt/mdlm_sdtt/steps_1/conversion_metadata.json b/results/smoke/samples/owt/mdlm_sdtt/steps_1/conversion_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0267e7fdfc436d954350c441ae7caf65e642308a --- /dev/null +++ b/results/smoke/samples/owt/mdlm_sdtt/steps_1/conversion_metadata.json @@ -0,0 +1,30 @@ +{ + "checkpoint_lock_id": "sdtt_owt_hf:3549a2eff7c0b0ccceca8d69a22baf7795fd9843fa9fcad41a8ddc186b9892f0:all", + "checkpoint_selection": { + "path": null, + "resource": "sdtt_owt_hf", + "sampling_config": "config.json", + "sampling_config_sha256": null, + "sampling_config_source": "resource", + "sampling_config_source_commit": null, + "teacher_family": null + }, + "checkpoint_sha256": "79eda51b8c8e8877b750b24a2276e240eeb2fd4230f369c99d71fc33bb113628", + "exclude_from_latency": true, + "format": "dlb-upstream-token-capture-v1", + "generated_samples": 1, + "generation_seconds_sentinel": 0.0, + "generation_seconds_source": "unavailable_excluded_sentinel", + "requested_samples": 1, + "teacher_family": "masked_mdlm", + "token_ids_source": "upstream", + "token_ids_transformation": { + "max_length": 1024, + "operation": "validated_exact_length" + }, + "tokenizer": "gpt2", + "tokenizer_revision": "607a30d783dfa663caf39e06633721c8d4cfcd7e", + "trim_policy": "none_exact_count", + "trimmed_samples": 0, + "upstream_output": "/home/ruizzhao/diffusion_baseline/results/smoke/samples/owt/mdlm_sdtt/steps_1/upstream_token_ids.json" +} diff --git a/results/smoke/samples/owt/mdlm_sdtt/steps_1/stderr.log b/results/smoke/samples/owt/mdlm_sdtt/steps_1/stderr.log new file mode 100644 index 0000000000000000000000000000000000000000..d820c98d50554603355ba39adb49ce3c593aa008 --- /dev/null +++ b/results/smoke/samples/owt/mdlm_sdtt/steps_1/stderr.log @@ -0,0 +1 @@ +/opt/anaconda3/envs/dlb-sdtt/lib/python3.10/site-packages/lightning/fabric/__init__.py:41: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. diff --git a/results/smoke/samples/owt/mdlm_sdtt/steps_32/samples.jsonl b/results/smoke/samples/owt/mdlm_sdtt/steps_32/samples.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..119fdfcb7e9b4ee824774bff07748f686c3cf863 --- /dev/null +++ b/results/smoke/samples/owt/mdlm_sdtt/steps_32/samples.jsonl @@ -0,0 +1 @@ +{"generation_seconds":0.0,"sample_id":0,"seed":42,"text":"<|endoftext|> to be no guarantee that we will get it when we’ve done our research. My hope is that it will work, and we can eventually coax all people in this scheme into quitting,” he says. “By then, a young family was living in Mexico, in California, and after they found a cannabis camp in Tierra, I wanted to start helping. I had worked at an agency that runs the camps on bikes, so I wanted to be based in this area to help these young people, in turn, get into work for it, and ultimately make it available to people who are not in their family in the 18th grade or too late.”\n\nJonathan Hector, 23, who lives in San Francisco, says:\n\nFor years, he’s struggled to cope now. He’s been on a study tour, and soon discovered that in many different ways, it is difficult to find a time to reflect on my personal life choices, and to try to moderate the amount thoughts I have to take in relation to my medications. Also, a lot of people find it extremely difficult to watch their choices. There is a very large group of people who are most frustrated, they want to reduce the anxiety. In particular, they are not afraid to wait for a long time before they take any medication, and they are determined to avoid taking [their] medicine. They are determined to save money and try also to minimise the anxiety. Also, because these people are living here, they’re not taking recreational activities; they are taking family-oriented activities.\n\nSometime after he was commissioned, Hector received standard marijuana treatment and he decided to stop this treatment, for the last two years, because the doctors had ordered him he was not going to because he didn’t take his medication, or this would question the duration of the treatment. But he was satisfied, and so decided to take cannabis, only to do so again in two different groups of people.\n\nThe internet class was organised, and by then, he was travelling to Mexico, where a friend of mine offered to pay him for cannabis worth of medicinal cannabis, and for the extra money he wouldn’t take traditional, herbal, or even both, cannabis. The medicinal cannabis course was planned to start once, and this involved a range of different groups of respondents, and a small number of existing interviewees, but the group was not very distinct. Instead, there were a groups made of 17 randomly named people selected by one of their ethnic groups, where the participants stayed, and adapted their personal experiences. There were some Chinese participants who preferred traditional medicinal cannabis, some Japanese preferred cannabis, and by then there was a similar pool.\n\nFrom there, 37 key respondents were asked about their participation in the course. Participants were randomly allocated to proceed as if they were male to medical cannabis, according to those by nature were the most interested in cannabis, and in the early stages of treatment, tended to take their own medications at a lower rate.However, there is on ongoing research showing that older people – most often those who are unable to take cannabinoids – spend often a lot more time in the CBD. There are also a number of people who smoked while taking cannabis, and still have high rates of the disease, and considering more of those even smoked for their own benefit than the rest. However, there is only the possibility of transitioning an older patient to cannabinoid therapy, there is also the possibility of developing new therapies for cannabinoids that patients use as well.\n\n to the United States where Turkish drug minister Akbar Barış is expected to wait for guests for the use of his CBD at Istanbul Airport, Turkey, June 16, 2015. file photo.\n\nIf these findings are published, they can be applied to a variety of other diseases, some of the most important childhood neurological diseases. Thus, they will be able to develop substantially more effective and effective cannabinoid cures and therapy for both children and adolescents, and in their lives. Cannabis is also strongly associated with people who use them. Pakistan currently shows a very important proportion of those, and Iran, the largest segment of these, and this trend will most accelerate in the UK, as it will reduce language barriers and making it easier for cannabis treatment specialists. Researchers will to find a new way in reducing all access to drugs, and I’m hoping with this study, they can do that in other time.<|endoftext|>Joshua Shippuzzi’s Peeled Plant and Litter Girl\n\nThe saffron comes in varieties of edible plants and a variety of Each of the many cre the small plant can grow. The tart strawberry is larger than any plant, about three to four meters. Passing through the ripening process and into the leaves, surrounding the roots, energy is passed to the green and lemon. The berries are produced by alternating light and dark sources, such as green colours, sunlight, and wind. In the leaves, air can form sulfuric acid, rings<|endoftext|>","token_ids":[50256,284,307,645,9149,326,356,481,651,340,618,356,447,247,303,1760,674,2267,13,2011,2911,318,326,340,481,670,11,290,356,460,4191,37700,477,661,287,428,7791,656,34161,11,447,251,339,1139,13,564,250,3886,788,11,257,1862,1641,373,2877,287,5828,11,287,3442,11,290,706,484,1043,257,9499,1413,287,15917,430,11,314,2227,284,923,5742,13,314,550,3111,379,281,4086,326,4539,262,11665,319,16715,11,523,314,2227,284,307,1912,287,428,1989,284,1037,777,1862,661,11,287,1210,11,651,656,670,329,340,11,290,6165,787,340,1695,284,661,508,389,407,287,511,1641,287,262,1248,400,9559,393,1165,2739,13,447,251,198,198,30365,42621,11,2242,11,508,3160,287,2986,6033,11,1139,25,198,198,1890,812,11,339,447,247,82,11615,284,19271,783,13,679,447,247,82,587,319,257,2050,4205,11,290,2582,5071,326,287,867,1180,2842,11,340,318,2408,284,1064,257,640,284,4079,319,616,2614,1204,7747,11,290,284,1949,284,10768,262,2033,6066,314,423,284,1011,287,8695,284,616,17638,13,4418,11,257,1256,286,661,1064,340,4457,2408,284,2342,511,7747,13,1318,318,257,845,1588,1448,286,661,508,389,749,14718,11,484,765,284,4646,262,9751,13,554,1948,11,484,389,407,7787,284,4043,329,257,890,640,878,484,1011,597,14103,11,290,484,389,5295,284,3368,2263,685,24571,60,9007,13,1119,389,5295,284,3613,1637,290,1949,635,284,10356,786,262,9751,13,4418,11,780,777,661,389,2877,994,11,484,447,247,260,407,2263,18136,4568,26,484,389,2263,1641,12,17107,4568,13,198,198,50,47056,706,339,373,20462,11,42621,2722,3210,5727,3513,290,339,3066,284,2245,428,3513,11,329,262,938,734,812,11,780,262,7519,550,6149,683,339,373,407,1016,284,780,339,1422,447,247,83,1011,465,14103,11,393,428,561,1808,262,9478,286,262,3513,13,887,339,373,11378,11,290,523,3066,284,1011,9499,11,691,284,466,523,757,287,734,1180,2628,286,661,13,198,198,464,5230,1398,373,20325,11,290,416,788,11,339,373,16574,284,5828,11,810,257,1545,286,6164,4438,284,1414,683,329,9499,2861,286,30381,9499,11,290,329,262,3131,1637,339,3636,447,247,83,1011,4569,11,35867,11,393,772,1111,11,9499,13,383,30381,9499,1781,373,6027,284,923,1752,11,290,428,2950,257,2837,286,1180,2628,286,14502,11,290,257,1402,1271,286,4683,2720,2841,11,475,262,1448,373,407,845,7310,13,5455,11,612,547,257,2628,925,286,1596,15456,3706,661,6163,416,530,286,511,9450,2628,11,810,262,6809,9658,11,290,16573,511,2614,6461,13,1318,547,617,3999,6809,508,9871,4569,30381,9499,11,617,4960,9871,9499,11,290,416,788,612,373,257,2092,5933,13,198,198,4863,612,11,5214,1994,14502,547,1965,546,511,10270,287,262,1781,13,26122,547,15456,19171,284,5120,355,611,484,547,4257,284,3315,9499,11,1864,284,883,416,3450,547,262,749,4609,287,9499,11,290,287,262,1903,9539,286,3513,11,19960,284,1011,511,898,17638,379,257,2793,2494,13,4864,11,612,318,319,7044,2267,4478,326,4697,661,784,749,1690,883,508,389,5906,284,1011,46830,784,4341,1690,257,1256,517,640,287,262,27147,13,1318,389,635,257,1271,286,661,508,21603,981,2263,9499,11,290,991,423,1029,3965,286,262,4369,11,290,6402,517,286,883,772,21603,329,511,898,4414,621,262,1334,13,2102,11,612,318,691,262,5885,286,37005,281,4697,5827,284,48175,9102,11,612,318,635,262,5885,286,5922,649,29596,329,46830,326,3871,779,355,880,13,198,198,284,262,1578,1829,810,9663,2563,5342,9084,5657,2409,30102,46481,318,2938,284,4043,329,10650,329,262,779,286,465,27147,379,25299,12690,11,7137,11,2795,1467,11,1853,13,2393,4590,13,198,198,1532,777,6373,389,3199,11,484,460,307,5625,284,257,4996,286,584,10040,11,617,286,262,749,1593,9963,30232,10040,13,6660,11,484,481,307,1498,284,1205,13788,517,4050,290,4050,48175,50019,290,9102,329,1111,1751,290,25447,11,290,287,511,3160,13,28838,318,635,7634,3917,351,661,508,779,606,13,7648,3058,2523,257,845,1593,9823,286,883,11,290,4068,11,262,4387,10618,286,777,11,290,428,5182,481,749,22636,287,262,3482,11,355,340,481,4646,3303,14725,290,1642,340,4577,329,9499,3513,22447,13,26685,481,284,1064,257,649,835,287,8868,477,1895,284,5010,11,290,314,447,247,76,7725,351,428,2050,11,484,460,466,326,287,584,640,13,50256,47740,911,3974,4715,72,447,247,82,2631,18449,16561,290,406,1967,7430,198,198,464,473,487,1313,2058,287,20328,286,35988,6134,290,257,4996,286,5501,286,262,867,1126,262,1402,4618,460,1663,13,383,35842,41236,318,4025,621,597,4618,11,546,1115,284,1440,10700,13,46389,832,262,18344,3101,1429,290,656,262,5667,11,7346,262,11135,11,2568,318,3804,284,262,4077,290,18873,13,383,36322,389,4635,416,39623,1657,290,3223,4237,11,884,355,4077,18915,11,19606,11,290,2344,13,554,262,5667,11,1633,460,1296,37402,291,7408,11,13917,50256]} diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/.hydra/config.yaml b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63b8fab2e2c5c04ce80ca9149579ca26727e884f --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/.hydra/config.yaml @@ -0,0 +1,151 @@ +mode: sample_eval +seed: 42 +loader: + global_batch_size: 512 + eval_global_batch_size: ${.global_batch_size} + batch_size: 2 + eval_batch_size: 1 + num_workers: ${eval:"len(__import__('os').sched_getaffinity(0))"} + pin_memory: true +sampling: + predictor: ancestral + steps: 1 + noise_removal: ancestral + use_float64: true + p_nucleus: 1.0 + num_sample_batches: 1 + num_sample_log: 2 + semi_ar: false + stride_length: 1 + num_strides: 1 + guid_weight: null + psi: + time_profile: linear + high_mode: pure-posterior + middle_mode: pure-posterior + low_mode: pure-posterior + high_frac: 0.2 + middle_frac: 0.6 +training: + ema: 0.9999 + antithetic_sampling: true + importance_sampling: false + sampling_eps: 0.001 + change_of_variables: false + loss_precision: bf16 + finetune_path: '' + class_dropout_p: 0.1 +eval: + checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt + disable_ema: false + compute_generative_perplexity: false + perplexity_batch_size: 8 + compute_perplexity_on_sanity: false + gen_ppl_eval_model_name_or_path: gpt2-large + generate_samples: true + generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_samples.json +optim: + weight_decay: 0 + lr: 0.0003 + beta1: 0.9 + beta2: 0.999 + eps: 1.0e-08 +trainer: + _target_: lightning.Trainer + accelerator: cuda + num_nodes: 1 + devices: 1 + accumulate_grad_batches: ${div_up:${loader.global_batch_size}, ${eval:${trainer.devices} + * ${loader.batch_size} * ${trainer.num_nodes}}} + gradient_clip_val: 1.0 + precision: bf16 + num_sanity_val_steps: 2 + max_steps: 1000000 + log_every_n_steps: 100 + limit_train_batches: 1.0 + limit_val_batches: 1.0 + val_check_interval: 5000 + check_val_every_n_epoch: null +wandb: + project: duo + group: null + job_type: null + name: null + id: ${.name}_${seed} + tags: + - ${noise.type} + - ${data.train} + - ${data.valid} + - ${algo.name} + offline: true +checkpointing: + save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra + resume_from_ckpt: true + resume_ckpt_path: ${.save_dir}/checkpoints/last.ckpt +callbacks: + checkpoint_every_n_steps: + _target_: lightning.pytorch.callbacks.ModelCheckpoint + save_top_k: -1 + save_last: true + dirpath: ${checkpointing.save_dir}/checkpoints + verbose: true + auto_insert_metric_name: false + every_n_train_steps: 500 + checkpoint_monitor: + _target_: lightning.pytorch.callbacks.ModelCheckpoint + monitor: val/nll + mode: min + save_top_k: 1 + save_last: false + dirpath: ${checkpointing.save_dir}/checkpoints + filename: best + auto_insert_metric_name: false + verbose: true + learning_rate_monitor: + _target_: lightning.pytorch.callbacks.LearningRateMonitor + logging_interval: step +data: + train: lm1b + valid: lm1b + modality: text + tokenizer_name_or_path: bert-base-uncased + cache_dir: /share/kuleshov/ssahoo/textdiffusion/data + wrap: true + streaming: false + insert_train_eos: true + insert_valid_eos: true +model: + name: small + type: ddit + hidden_size: 768 + cond_dim: 128 + length: 128 + n_blocks: 12 + n_heads: 12 + scale_by_sigma: true + dropout: 0.1 + tie_word_embeddings: false + vocab_lookup: true +strategy: + _target_: lightning.pytorch.strategies.DDPStrategy + find_unused_parameters: false +noise: + type: log-linear + eps: 0.001 +lr_scheduler: + _target_: transformers.get_constant_schedule_with_warmup + num_warmup_steps: 2500 +prior: + type: none + latent_width: 0 + latent_height: 0 +algo: + name: duo_base + backbone: dit + parameterization: mean + time_conditioning: true + T: 0 + subs_masking: false + causal_attention: false + ignore_bos: false + loss_type: elbo diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/config_tree.txt b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/config_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..385ae50a2841f7e06ba815a3a13b03f67920fc84 --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/config_tree.txt @@ -0,0 +1,169 @@ +CONFIG +├── mode +│ └── sample_eval +├── seed +│ └── 42 +├── loader +│ └── global_batch_size: 512 +│ eval_global_batch_size: 512 +│ batch_size: 2 +│ eval_batch_size: 1 +│ num_workers: 512 +│ pin_memory: true +│ +├── sampling +│ └── predictor: ancestral +│ steps: 1 +│ noise_removal: ancestral +│ use_float64: true +│ p_nucleus: 1.0 +│ num_sample_batches: 1 +│ num_sample_log: 2 +│ semi_ar: false +│ stride_length: 1 +│ num_strides: 1 +│ guid_weight: null +│ psi: +│ time_profile: linear +│ high_mode: pure-posterior +│ middle_mode: pure-posterior +│ low_mode: pure-posterior +│ high_frac: 0.2 +│ middle_frac: 0.6 +│ +├── training +│ └── ema: 0.9999 +│ antithetic_sampling: true +│ importance_sampling: false +│ sampling_eps: 0.001 +│ change_of_variables: false +│ loss_precision: bf16 +│ finetune_path: '' +│ class_dropout_p: 0.1 +│ +├── eval +│ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt +│ disable_ema: false +│ compute_generative_perplexity: false +│ perplexity_batch_size: 8 +│ compute_perplexity_on_sanity: false +│ gen_ppl_eval_model_name_or_path: gpt2-large +│ generate_samples: true +│ generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/s +│ +├── optim +│ └── weight_decay: 0 +│ lr: 0.0003 +│ beta1: 0.9 +│ beta2: 0.999 +│ eps: 1.0e-08 +│ +├── trainer +│ └── _target_: lightning.Trainer +│ accelerator: cuda +│ num_nodes: 1 +│ devices: 1 +│ accumulate_grad_batches: 256 +│ gradient_clip_val: 1.0 +│ precision: bf16 +│ num_sanity_val_steps: 2 +│ max_steps: 1000000 +│ log_every_n_steps: 100 +│ limit_train_batches: 1.0 +│ limit_val_batches: 1.0 +│ val_check_interval: 5000 +│ check_val_every_n_epoch: null +│ +├── wandb +│ └── project: duo +│ group: null +│ job_type: null +│ name: null +│ id: None_42 +│ tags: +│ - log-linear +│ - lm1b +│ - lm1b +│ - duo_base +│ offline: true +│ +├── checkpointing +│ └── save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstrea +│ resume_from_ckpt: true +│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1 +│ +├── callbacks +│ └── checkpoint_every_n_steps: +│ _target_: lightning.pytorch.callbacks.ModelCheckpoint +│ save_top_k: -1 +│ save_last: true +│ dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstre +│ verbose: true +│ auto_insert_metric_name: false +│ every_n_train_steps: 500 +│ checkpoint_monitor: +│ _target_: lightning.pytorch.callbacks.ModelCheckpoint +│ monitor: val/nll +│ mode: min +│ save_top_k: 1 +│ save_last: false +│ dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstre +│ filename: best +│ auto_insert_metric_name: false +│ verbose: true +│ learning_rate_monitor: +│ _target_: lightning.pytorch.callbacks.LearningRateMonitor +│ logging_interval: step +│ +├── data +│ └── train: lm1b +│ valid: lm1b +│ modality: text +│ tokenizer_name_or_path: bert-base-uncased +│ cache_dir: /share/kuleshov/ssahoo/textdiffusion/data +│ wrap: true +│ streaming: false +│ insert_train_eos: true +│ insert_valid_eos: true +│ +├── model +│ └── name: small +│ type: ddit +│ hidden_size: 768 +│ cond_dim: 128 +│ length: 128 +│ n_blocks: 12 +│ n_heads: 12 +│ scale_by_sigma: true +│ dropout: 0.1 +│ tie_word_embeddings: false +│ vocab_lookup: true +│ +├── strategy +│ └── _target_: lightning.pytorch.strategies.DDPStrategy +│ find_unused_parameters: false +│ +├── noise +│ └── type: log-linear +│ eps: 0.001 +│ +├── lr_scheduler +│ └── _target_: transformers.get_constant_schedule_with_warmup +│ num_warmup_steps: 2500 +│ +├── prior +│ └── type: none +│ latent_width: 0 +│ latent_height: 0 +│ +└── algo + └── name: duo_base + backbone: dit + parameterization: mean + time_conditioning: true + T: 0 + subs_masking: false + causal_attention: false + ignore_bos: false + loss_type: elbo + diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/dlb_pinned_upstream_main.log b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/dlb_pinned_upstream_main.log new file mode 100644 index 0000000000000000000000000000000000000000..817e13a499a806285471924680ef4a707defdeb9 --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_hydra/dlb_pinned_upstream_main.log @@ -0,0 +1,4 @@ +[2026-08-25 10:38:05,643][dlb_pinned_upstream_main][INFO] - Starting Sample Eval. +[2026-08-25 10:38:13,066][dlb_pinned_upstream_main][INFO] - Generative perplexity: 1125.733642578125 +[2026-08-25 10:38:13,066][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.429431915283203 +[2026-08-25 10:38:13,067][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_1/upstream_samples.json diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra/.hydra/overrides.yaml b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6c290dac1083ab0aaa6a5efc62421f81475b5d2 --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra/.hydra/overrides.yaml @@ -0,0 +1,19 @@ +- mode=sample_eval +- seed=42 +- data=lm1b-wrap +- model=small +- model.length=128 +- loader.batch_size=2 +- loader.eval_batch_size=1 +- sampling.num_sample_batches=1 +- sampling.steps=2 +- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt +- trainer.devices=1 +- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_hydra +- +wandb.offline=true +- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_2/upstream_samples.json +- algo=duo_base +- algo.backbone=dit +- sampling.predictor=ancestral +- sampling.noise_removal=ancestral +- eval.disable_ema=False diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/.hydra/overrides.yaml b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b323a5fc917263b1a6c3e61b68177fedfc373a53 --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/.hydra/overrides.yaml @@ -0,0 +1,19 @@ +- mode=sample_eval +- seed=42 +- data=lm1b-wrap +- model=small +- model.length=128 +- loader.batch_size=2 +- loader.eval_batch_size=1 +- sampling.num_sample_batches=1 +- sampling.steps=32 +- eval.checkpoint_path=/home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt +- trainer.devices=1 +- checkpointing.save_dir=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra +- +wandb.offline=true +- eval.generated_samples_path=/home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_samples.json +- algo=duo_base +- algo.backbone=dit +- sampling.predictor=ancestral +- sampling.noise_removal=ancestral +- eval.disable_ema=False diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/dlb_pinned_upstream_main.log b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/dlb_pinned_upstream_main.log new file mode 100644 index 0000000000000000000000000000000000000000..74e20b271e5cee6fd0a52579abccb0d2a17188c4 --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_hydra/dlb_pinned_upstream_main.log @@ -0,0 +1,4 @@ +[2026-08-25 10:47:54,997][dlb_pinned_upstream_main][INFO] - Starting Sample Eval. +[2026-08-25 10:48:08,607][dlb_pinned_upstream_main][INFO] - Generative perplexity: 94.98780822753906 +[2026-08-25 10:48:08,608][dlb_pinned_upstream_main][INFO] - Sample entropy: 4.304296016693115 +[2026-08-25 10:48:08,608][dlb_pinned_upstream_main][INFO] - Samples saved at: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_32/upstream_samples.json diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra/config_tree.txt b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra/config_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..f4ad80e74a81979af9cee46643614a5252685c2b --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstream_hydra/config_tree.txt @@ -0,0 +1,169 @@ +CONFIG +├── mode +│ └── sample_eval +├── seed +│ └── 42 +├── loader +│ └── global_batch_size: 512 +│ eval_global_batch_size: 512 +│ batch_size: 2 +│ eval_batch_size: 1 +│ num_workers: 512 +│ pin_memory: true +│ +├── sampling +│ └── predictor: ancestral +│ steps: 4 +│ noise_removal: ancestral +│ use_float64: true +│ p_nucleus: 1.0 +│ num_sample_batches: 1 +│ num_sample_log: 2 +│ semi_ar: false +│ stride_length: 1 +│ num_strides: 1 +│ guid_weight: null +│ psi: +│ time_profile: linear +│ high_mode: pure-posterior +│ middle_mode: pure-posterior +│ low_mode: pure-posterior +│ high_frac: 0.2 +│ middle_frac: 0.6 +│ +├── training +│ └── ema: 0.9999 +│ antithetic_sampling: true +│ importance_sampling: false +│ sampling_eps: 0.001 +│ change_of_variables: false +│ loss_precision: bf16 +│ finetune_path: '' +│ class_dropout_p: 0.1 +│ +├── eval +│ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt +│ disable_ema: false +│ compute_generative_perplexity: false +│ perplexity_batch_size: 8 +│ compute_perplexity_on_sanity: false +│ gen_ppl_eval_model_name_or_path: gpt2-large +│ generate_samples: true +│ generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/s +│ +├── optim +│ └── weight_decay: 0 +│ lr: 0.0003 +│ beta1: 0.9 +│ beta2: 0.999 +│ eps: 1.0e-08 +│ +├── trainer +│ └── _target_: lightning.Trainer +│ accelerator: cuda +│ num_nodes: 1 +│ devices: 1 +│ accumulate_grad_batches: 256 +│ gradient_clip_val: 1.0 +│ precision: bf16 +│ num_sanity_val_steps: 2 +│ max_steps: 1000000 +│ log_every_n_steps: 100 +│ limit_train_batches: 1.0 +│ limit_val_batches: 1.0 +│ val_check_interval: 5000 +│ check_val_every_n_epoch: null +│ +├── wandb +│ └── project: duo +│ group: null +│ job_type: null +│ name: null +│ id: None_42 +│ tags: +│ - log-linear +│ - lm1b +│ - lm1b +│ - duo_base +│ offline: true +│ +├── checkpointing +│ └── save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstrea +│ resume_from_ckpt: true +│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4 +│ +├── callbacks +│ └── checkpoint_every_n_steps: +│ _target_: lightning.pytorch.callbacks.ModelCheckpoint +│ save_top_k: -1 +│ save_last: true +│ dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstre +│ verbose: true +│ auto_insert_metric_name: false +│ every_n_train_steps: 500 +│ checkpoint_monitor: +│ _target_: lightning.pytorch.callbacks.ModelCheckpoint +│ monitor: val/nll +│ mode: min +│ save_top_k: 1 +│ save_last: false +│ dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_4/upstre +│ filename: best +│ auto_insert_metric_name: false +│ verbose: true +│ learning_rate_monitor: +│ _target_: lightning.pytorch.callbacks.LearningRateMonitor +│ logging_interval: step +│ +├── data +│ └── train: lm1b +│ valid: lm1b +│ modality: text +│ tokenizer_name_or_path: bert-base-uncased +│ cache_dir: /share/kuleshov/ssahoo/textdiffusion/data +│ wrap: true +│ streaming: false +│ insert_train_eos: true +│ insert_valid_eos: true +│ +├── model +│ └── name: small +│ type: ddit +│ hidden_size: 768 +│ cond_dim: 128 +│ length: 128 +│ n_blocks: 12 +│ n_heads: 12 +│ scale_by_sigma: true +│ dropout: 0.1 +│ tie_word_embeddings: false +│ vocab_lookup: true +│ +├── strategy +│ └── _target_: lightning.pytorch.strategies.DDPStrategy +│ find_unused_parameters: false +│ +├── noise +│ └── type: log-linear +│ eps: 0.001 +│ +├── lr_scheduler +│ └── _target_: transformers.get_constant_schedule_with_warmup +│ num_warmup_steps: 2500 +│ +├── prior +│ └── type: none +│ latent_width: 0 +│ latent_height: 0 +│ +└── algo + └── name: duo_base + backbone: dit + parameterization: mean + time_conditioning: true + T: 0 + subs_masking: false + causal_attention: false + ignore_bos: false + loss_type: elbo + diff --git a/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra/config_tree.txt b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra/config_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..3e8d2b7d642ed73e4947652e202f9819fa207646 --- /dev/null +++ b/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstream_hydra/config_tree.txt @@ -0,0 +1,169 @@ +CONFIG +├── mode +│ └── sample_eval +├── seed +│ └── 42 +├── loader +│ └── global_batch_size: 512 +│ eval_global_batch_size: 512 +│ batch_size: 2 +│ eval_batch_size: 1 +│ num_workers: 512 +│ pin_memory: true +│ +├── sampling +│ └── predictor: ancestral +│ steps: 8 +│ noise_removal: ancestral +│ use_float64: true +│ p_nucleus: 1.0 +│ num_sample_batches: 1 +│ num_sample_log: 2 +│ semi_ar: false +│ stride_length: 1 +│ num_strides: 1 +│ guid_weight: null +│ psi: +│ time_profile: linear +│ high_mode: pure-posterior +│ middle_mode: pure-posterior +│ low_mode: pure-posterior +│ high_frac: 0.2 +│ middle_frac: 0.6 +│ +├── training +│ └── ema: 0.9999 +│ antithetic_sampling: true +│ importance_sampling: false +│ sampling_eps: 0.001 +│ change_of_variables: false +│ loss_precision: bf16 +│ finetune_path: '' +│ class_dropout_p: 0.1 +│ +├── eval +│ └── checkpoint_path: /home/ruizzhao/diffusion_baseline/checkpoints/reference_reproduction/duo_dcd/lm1b/model.ckpt +│ disable_ema: false +│ compute_generative_perplexity: false +│ perplexity_batch_size: 8 +│ compute_perplexity_on_sanity: false +│ gen_ppl_eval_model_name_or_path: gpt2-large +│ generate_samples: true +│ generated_samples_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/s +│ +├── optim +│ └── weight_decay: 0 +│ lr: 0.0003 +│ beta1: 0.9 +│ beta2: 0.999 +│ eps: 1.0e-08 +│ +├── trainer +│ └── _target_: lightning.Trainer +│ accelerator: cuda +│ num_nodes: 1 +│ devices: 1 +│ accumulate_grad_batches: 256 +│ gradient_clip_val: 1.0 +│ precision: bf16 +│ num_sanity_val_steps: 2 +│ max_steps: 1000000 +│ log_every_n_steps: 100 +│ limit_train_batches: 1.0 +│ limit_val_batches: 1.0 +│ val_check_interval: 5000 +│ check_val_every_n_epoch: null +│ +├── wandb +│ └── project: duo +│ group: null +│ job_type: null +│ name: null +│ id: None_42 +│ tags: +│ - log-linear +│ - lm1b +│ - lm1b +│ - duo_base +│ offline: true +│ +├── checkpointing +│ └── save_dir: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstrea +│ resume_from_ckpt: true +│ resume_ckpt_path: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8 +│ +├── callbacks +│ └── checkpoint_every_n_steps: +│ _target_: lightning.pytorch.callbacks.ModelCheckpoint +│ save_top_k: -1 +│ save_last: true +│ dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstre +│ verbose: true +│ auto_insert_metric_name: false +│ every_n_train_steps: 500 +│ checkpoint_monitor: +│ _target_: lightning.pytorch.callbacks.ModelCheckpoint +│ monitor: val/nll +│ mode: min +│ save_top_k: 1 +│ save_last: false +│ dirpath: /home/ruizzhao/diffusion_baseline/results_duo_dcd_ckpt_probe/best_seed42/timing_scratch/samples/lm1b/duo_dcd/steps_8/upstre +│ filename: best +│ auto_insert_metric_name: false +│ verbose: true +│ learning_rate_monitor: +│ _target_: lightning.pytorch.callbacks.LearningRateMonitor +│ logging_interval: step +│ +├── data +│ └── train: lm1b +│ valid: lm1b +│ modality: text +│ tokenizer_name_or_path: bert-base-uncased +│ cache_dir: /share/kuleshov/ssahoo/textdiffusion/data +│ wrap: true +│ streaming: false +│ insert_train_eos: true +│ insert_valid_eos: true +│ +├── model +│ └── name: small +│ type: ddit +│ hidden_size: 768 +│ cond_dim: 128 +│ length: 128 +│ n_blocks: 12 +│ n_heads: 12 +│ scale_by_sigma: true +│ dropout: 0.1 +│ tie_word_embeddings: false +│ vocab_lookup: true +│ +├── strategy +│ └── _target_: lightning.pytorch.strategies.DDPStrategy +│ find_unused_parameters: false +│ +├── noise +│ └── type: log-linear +│ eps: 0.001 +│ +├── lr_scheduler +│ └── _target_: transformers.get_constant_schedule_with_warmup +│ num_warmup_steps: 2500 +│ +├── prior +│ └── type: none +│ latent_width: 0 +│ latent_height: 0 +│ +└── algo + └── name: duo_base + backbone: dit + parameterization: mean + time_conditioning: true + T: 0 + subs_masking: false + causal_attention: false + ignore_bos: false + loss_type: elbo + diff --git a/scripts/aggregate_results.py b/scripts/aggregate_results.py new file mode 100644 index 0000000000000000000000000000000000000000..5b5c84aa73c7e08e1f390d2b22e3c273dc6f6e2c --- /dev/null +++ b/scripts/aggregate_results.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +"""Publish strict or partial baseline summary tables.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from dlb.aggregate import IncompleteMatrixError, aggregate + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument( + "--output-dir", type=Path, default=None, + help="defaults to ROOT/results/summary", + ) + parser.add_argument("--partial", action="store_true") + args = parser.parse_args(argv) + root = args.root.resolve() + output = (args.output_dir or root / "results/summary").resolve() + try: + report = aggregate( + root, + strict=not args.partial, + partial=args.partial, + output_dir=output, + ) + except IncompleteMatrixError as error: + print(str(error)) + print(f"failures={len(error.failures)}") + return 1 + print( + f"complete={str(report.complete).lower()} " + f"rows={len(report.rows)} failures={len(report.failures)} " + f"unsupported={len(report.unsupported)} output={output}" + ) + return 0 if report.complete else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_elf_inputs.py b/scripts/audit_elf_inputs.py new file mode 100644 index 0000000000000000000000000000000000000000..40a7f9e1a08cd2993ece15e79ae2370411264eca --- /dev/null +++ b/scripts/audit_elf_inputs.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Offline CPU audit of raw versus released ELF conditional validation inputs.""" +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +import json +from pathlib import Path + +from elf_common import (ROOT, asset, condition_token_ids, offline, read_jsonl, + require_server, sha256, write_json) + + +def audit_rows(raw, official, tokenizer, limit, task): + def inspect(rows): + token_rows = [condition_token_ids(row, tokenizer) for row in rows] + return token_rows, { + "rows": len(rows), + "blank_text_ids": [row["id"] for row in rows if not row["input"].strip()], + "empty_condition_ids": [row["id"] for row, ids in zip(rows, token_rows) if not ids], + "truncated_sources": sum(len(ids) > limit for ids in token_rows), + "condition_length_mean_before_cap": sum(map(len, token_rows)) / len(rows) if rows else None, + } + + raw_ids, raw_stats = inspect(raw) + official_ids, official_stats = inspect(official) + by_pair = defaultdict(list) + for row, ids in zip(official, official_ids): + by_pair[(row["input"], row["output"])].append((row, ids)) + matches = Counter() + examples = [] + for row, ids in zip(raw, raw_ids): + candidates = by_pair.get((row["input"], row["output"]), []) + if len(candidates) != 1: + matches["unmatched_text_pair" if not candidates else "ambiguous_text_pair"] += 1 + continue + author, expected = candidates[0] + matches["unique_text_pair"] += 1 + matches["exact_ids"] += int(ids == expected) + matches["exact_ids_after_source_cap"] += int(ids[:limit] == expected[:limit]) + matches["raw_plus_eos_equals_official"] += int(ids + [tokenizer.eos_token_id] == expected) + if ids[:limit] != expected[:limit] and len(examples) < 5: + examples.append({ + "raw_id": row["id"], "official_id": author["id"], + "source_excerpt": row["input"][:160], + "raw_length": len(ids), "official_length": len(expected), + "raw_first_ids": ids[:16], "raw_last_ids": ids[-16:], + "official_first_ids": expected[:16], "official_last_ids": expected[-16:], + "official_condition_decoded_excerpt": tokenizer.decode(expected[:limit], skip_special_tokens=False)[:240], + }) + corrected = [condition_token_ids(row, tokenizer, task=task) for row in raw] + # Multisets account for every duplicate's multiplicity instead of skipping + # the eight WMT rows that the unique-pair diagnostic cannot disambiguate. + def records(rows, token_rows, capped): + return [(r["input"], r["output"], tuple(ids[:limit] if capped else ids)) + for r, ids in zip(rows, token_rows)] + corrected_records = records(raw, corrected, False) + author_records = records(official, official_ids, False) + corrected_capped = records(raw, corrected, True) + author_capped = records(official, official_ids, True) + coverage = { + "raw_rows": len(raw), "official_rows": len(official), + "full_token_multisets_equal": Counter(corrected_records) == Counter(author_records), + "capped_token_multisets_equal": Counter(corrected_capped) == Counter(author_capped), + "same_order_and_full_tokens": corrected_records == author_records, + "same_order_and_capped_tokens": corrected_capped == author_capped, + "empty_condition_ids": [r["id"] for r, ids in zip(raw, corrected) if not ids], + "policy": "released_ids_or_raw_plus_eos_then_source_cap_v2", + } + coverage["ready_for_raw_validation"] = ( + bool(raw) and coverage["capped_token_multisets_equal"] and not coverage["empty_condition_ids"]) + return {"source_cap": limit, "raw": raw_stats, "official": official_stats, + "comparison_by_exact_source_and_reference_text_not_row_number": dict(matches), + "mismatch_examples": examples, + "official_conditions_nonempty": bool(official) and not official_stats["empty_condition_ids"], + "corrected_runner_comparison_including_duplicates": coverage} + + +def load_checked(path, task): + meta = json.loads(path.with_suffix(".manifest.json").read_text()) + rows = read_jsonl(path) + if (meta["sha256"] != sha256(path) or meta["task"] != task or meta["count"] != len(rows) + or len({row["id"] for row in rows}) != len(rows)): + raise ValueError(f"Input manifest/count/IDs mismatch: {path}") + return rows, {"path": str(path), "manifest": meta} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + require_server() + offline() + from transformers import AutoTokenizer + root = args.root.resolve() + tokenizer = AutoTokenizer.from_pretrained(str(asset(root, "t5")), local_files_only=True) + report = {"schema": "elf-condition-audit-v2", "tasks": {}, + "auditor_sha256": sha256(Path(__file__)), + "helper_sha256": sha256(Path(__file__).with_name("elf_common.py")), + "assets_manifest_sha256": sha256(root / "data/elf/assets.json")} + for task, limit in (("wmt14", 64), ("xsum", 1024)): + raw, raw_binding = load_checked(root / f"data/elf/{task}-validation.jsonl", task) + official, official_binding = load_checked(root / f"data/elf/{task}-official-validation.jsonl", task) + print(f"Auditing {task}: raw={len(raw)}, official={len(official)}; no GPU inference", flush=True) + result = audit_rows(raw, official, tokenizer, limit, task) + result["inputs"] = {"raw": raw_binding, "official": official_binding} + report["tasks"][task] = result + print(json.dumps({k: v for k, v in result.items() if k != "inputs"}, indent=2, ensure_ascii=False), flush=True) + write_json(args.output, report) + print(f"Saved {args.output}. Input files unchanged; compared the corrected runner's EOS policy without generating samples.") + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_4gpu.sh b/scripts/benchmark_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..ad17894b72dac788619b52099ebafcbf71699451 --- /dev/null +++ b/scripts/benchmark_4gpu.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Four-GPU local timing pass over the canonical generation matrix. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +SEED=42 +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--seed N] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.matrix \ + --root "$DLB_ROOT" --output "$MATRIX" --sample-count 1024 --seed "$SEED" || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage benchmark --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" --precision author "${DRY_RUN[@]}" diff --git a/scripts/benchmark_conditional_4gpu.sh b/scripts/benchmark_conditional_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..ed964890e8f664adb88e2fedeacaf198b1ce972a --- /dev/null +++ b/scripts/benchmark_conditional_4gpu.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Four-GPU local timing pass over the C64 conditional matrix. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_CONDITIONAL_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +SEED=42 +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--seed N] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/conditional/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.conditional_matrix \ + --root "$DLB_ROOT" --output "$MATRIX" || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage benchmark --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" --precision author "${DRY_RUN[@]}" diff --git a/scripts/benchmark_conditional_all.sh b/scripts/benchmark_conditional_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..8fda9d6c5b000788b8537e3390d0a087970b1ffa --- /dev/null +++ b/scripts/benchmark_conditional_all.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Server-side serial timing pass over the C64 conditional matrix. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_CONDITIONAL_MATRIX:-}" +SEED=42 + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --help) + printf 'Usage: %s [--root PATH] [--matrix PATH] [--seed N]\n' "$0" + exit 0 + ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/conditional/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs" +PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.conditional_matrix \ + --root "$DLB_ROOT" --output "$MATRIX" || exit $? + +failures="$DLB_ROOT/results/logs/conditional_benchmark_failures.tsv" +printf 'task_id\tmodel\tdataset\tsteps\texit_code\n' >"$failures" +status=0 +while IFS=$'\t' read -r task_id category model dataset steps sample_count seed environment adapter source provenance protocol conditioning_manifest conditioning_manifest_sha256 sample_dir metrics_path timing_path; do + [[ "$task_id" == "task_id" || -z "$task_id" ]] && continue + echo "CONDITIONAL BENCHMARK $task_id" + "$SCRIPT_DIR/benchmark_conditional_one.sh" --model "$model" --dataset "$dataset" \ + --steps "$steps" --seed "$SEED" --precision author + exit_code=$? + if ((exit_code != 0)); then + printf '%s\t%s\t%s\t%s\t%s\n' "$task_id" "$model" "$dataset" "$steps" "$exit_code" >>"$failures" + status=1 + fi +done < <(tail -n +3 "$MATRIX") +if [[ ! -s "$failures" ]]; then rm -f "$failures"; fi +exit "$status" diff --git a/scripts/benchmark_one.sh b/scripts/benchmark_one.sh new file mode 100644 index 0000000000000000000000000000000000000000..b0c9cad74c4cc90930139a15093bec1bb767a11c --- /dev/null +++ b/scripts/benchmark_one.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Run one server-side, registry-bound primary-latency benchmark. +set -u + +script_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$script_dir/.." && pwd -P)}" + +model="" +dataset="" +steps="" +seed="" +precision="" +dry_run="false" + +usage() { + echo "usage: $0 --model MODEL --dataset {lm1b,owt} --steps N --seed N --precision author [--dry-run]" >&2 +} + +while (( $# )); do + case "$1" in + --model|--dataset|--steps|--seed|--precision) + if (( $# < 2 )); then usage; exit 2; fi + case "$1" in + --model) model="$2" ;; + --dataset) dataset="$2" ;; + --steps) steps="$2" ;; + --seed) seed="$2" ;; + --precision) precision="$2" ;; + esac + shift 2 ;; + --dry-run) dry_run="true"; shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +if [[ -z "$model" || -z "$dataset" || -z "$steps" || -z "$seed" || -z "$precision" ]]; then + usage; exit 2 +fi +if [[ ! "$steps" =~ ^[1-9][0-9]*$ || ! "$seed" =~ ^-?[0-9]+$ ]]; then + echo "steps must be positive and seed must be an integer" >&2; exit 2 +fi +if [[ "$precision" != "author" ]]; then + echo "precision must be author (the pinned upstream inference policy)" >&2; exit 2 +fi + +DLB_PYTHON="${DLB_PYTHON:-python}" +module_args=(--root "$DLB_ROOT" --models "$model" --datasets "$dataset" --steps "$steps" --seed "$seed" --precision "$precision") +if [[ "$dry_run" == "true" ]]; then + exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$DLB_PYTHON" -m dlb.benchmarking "${module_args[@]}" --dry-run +fi + +if environment="$(PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$DLB_PYTHON" -m dlb.runner \ + --root "$DLB_ROOT" --model "$model" --dataset "$dataset" --steps "$steps" \ + --num-samples 1 --seed "$seed" --validate-only)"; then + : +else + exit $? +fi +if [[ ! "$environment" =~ ^[a-z][a-z0-9-]*$ ]]; then + echo "registry validation returned an unsafe environment name" >&2; exit 2 +fi + +exec "${DLB_CONDA:-conda}" run -n "$environment" env \ + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" \ + python -m dlb.benchmarking "${module_args[@]}" diff --git a/scripts/distill/di4c.sh b/scripts/distill/di4c.sh new file mode 100644 index 0000000000000000000000000000000000000000..50bfae8b488b5a8812b086791f3e3be9713c9e30 --- /dev/null +++ b/scripts/distill/di4c.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)" +export PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" +PYTHON_BIN="${DLB_PYTHON:-python}" +exec "$PYTHON_BIN" -m dlb.recipes --root "$DLB_ROOT" --recipe di4c "$@" diff --git a/scripts/distill/fmlm.sh b/scripts/distill/fmlm.sh new file mode 100644 index 0000000000000000000000000000000000000000..0d9f72182e88db262e0d784fc8434320d1e39d2d --- /dev/null +++ b/scripts/distill/fmlm.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)" +export PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" +PYTHON_BIN="${DLB_PYTHON:-python}" +exec "$PYTHON_BIN" -m dlb.recipes --root "$DLB_ROOT" --recipe fmlm "$@" diff --git a/scripts/elf_common.py b/scripts/elf_common.py new file mode 100644 index 0000000000000000000000000000000000000000..fbee4a5150d65d36e9e45a40d47b876d5401a5de --- /dev/null +++ b/scripts/elf_common.py @@ -0,0 +1,131 @@ +"""Small, independent ELF experiment helpers (no GPU imports).""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_COMMIT = "b29d8833609e9ab7f67cd9da39435ac5cea04837" + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path, data): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(data, indent=2, ensure_ascii=False, allow_nan=False) + "\n") + temporary.replace(path) + + +def read_jsonl(path): + with Path(path).open() as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def condition_token_ids(row, tokenizer, *, task=None): + """Preserve released IDs; task-trained raw sources include the released EOS. + + task=None retains plain tokenization for auditing the old input path. + Append before the model's source cap, exactly as the released Arrow inputs. + A C64 continuation is not an ended source sentence and keeps its old policy. + """ + ids = row.get("condition_input_ids") + if ids is None: + ids = tokenizer(row["input"], add_special_tokens=False, verbose=False)["input_ids"] + if task in ("wmt14", "xsum"): + eos = tokenizer.eos_token_id + if type(eos) is not int or eos < 0: + raise ValueError("Task-trained ELF conditions require a valid EOS token") + ids = ids + [eos] + vocab_size = len(tokenizer) + if not isinstance(ids, list) or any(type(t) is not int or t < 0 or t >= vocab_size for t in ids): + raise ValueError(f"Invalid condition IDs at source_id={row.get('id')}") + return ids + + +def preflight_conditions(rows, tokenizer, task, max_input_length, max_length): + """Validate the whole requested input before loading weights; never drop rows.""" + prepared, failures = [], [] + for index, row in enumerate(rows): + ids = condition_token_ids(row, tokenizer, task=task) + if task == "owt-prefix": + fits = 0 < len(ids) < max_length - 64 + else: + ids = ids[:max_input_length] + fits = bool(ids) + if not fits: + failures.append({"row_index": index, "source_id": row.get("id"), "tokens": len(ids)}) + prepared.append(ids) + if failures: + raise ValueError( + f"Invalid/empty ELF conditions: {len(failures)}/{len(rows)}; " + f"first affected rows (zero-based): {failures[:20]}. No samples were dropped. " + "Run scripts/audit_elf_inputs.py to compare raw and official inputs; " + "do not invent source text or delete rows to bypass this check." + ) + return prepared + + +def require_server(): + if sys.platform != "linux": + raise RuntimeError("ELF downloads and GPU experiments must run on the Linux server.") + + +def source_path(root): + path = Path(root) / "upstreams/elf" + commit = subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + dirty = subprocess.check_output(["git", "-C", str(path), "status", "--porcelain"], text=True) + if commit != SOURCE_COMMIT or dirty.strip(): + raise ValueError("ELF upstream must be clean and match the locked commit") + return path + + +def default_sampling(task, steps): + if steps < 1: + raise ValueError("steps must be positive") + if task in ("wmt14", "xsum"): + return {"sampling_method": "ode", "cfg": 2.0, "sc_cfg": 1.0, "gamma": 0.0} + # 8/16/32: paper Appendix D.2; 64: release configuration. + # Other budgets are explicitly an extension of our benchmark. + return {"sampling_method": "sde", "cfg": 1.0, "sc_cfg": 3.0, + "gamma": 2.0 if steps <= 16 else 1.5 if steps == 32 else 1.0} + + +def manifest(root): + path = Path(root) / "data/elf/assets.json" + if not path.is_file(): + raise FileNotFoundError( + f"ELF asset manifest is missing: {path}. " + "First complete `python scripts/prepare_elf.py assets` successfully; " + "a failed download does not publish assets.json. Do not create it manually." + ) + data = json.loads(path.read_text()) + if data["lock_sha256"] != sha256(Path(root) / "artifacts/elf_lock.json"): + raise ValueError("ELF lock changed since asset preparation") + return data + + +def asset(root, name): + record = manifest(root)["assets"][name] + directory = Path(root) / record["path"] + for relative, digest in record["sha256"].items(): + if sha256(directory / relative) != digest: + raise ValueError(f"ELF asset changed: {name}/{relative}") + return directory + + +def offline(): + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + os.environ["WANDB_MODE"] = "disabled" diff --git a/scripts/evaluate_4gpu.sh b/scripts/evaluate_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..8c4cac2cc1b54b4c6030e50ab3771f9fafa4ad75 --- /dev/null +++ b/scripts/evaluate_4gpu.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Four-GPU local quality evaluation over completed production samples. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +METRICS="gen_ppl,entropy,self_bleu" +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--metrics LIST] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --metrics) METRICS="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.matrix \ + --root "$DLB_ROOT" --output "$MATRIX" --sample-count 1024 --seed 42 || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage evaluate --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" --metrics "$METRICS" "${DRY_RUN[@]}" diff --git a/scripts/evaluate_all.sh b/scripts/evaluate_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..f4f015fefe3ed7ecf6c1bee741d0760176b23fc7 --- /dev/null +++ b/scripts/evaluate_all.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Server-side quality evaluation over completed production samples. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_EVAL_PYTHON:-${DLB_PYTHON:-python}}" +MATRIX="${DLB_MATRIX:-}" +METRICS="gen_ppl,entropy,self_bleu" + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--metrics LIST]\n' "$0" +} +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; shift 2 ;; + --metrics) METRICS="$2"; shift 2 ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/matrix/generation.tsv"; fi +export DLB_ROOT +PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.matrix \ + --root "$DLB_ROOT" --output "$MATRIX" --sample-count 1024 --seed 42 || exit $? + +mkdir -p "$DLB_ROOT/results/logs" +failures="$DLB_ROOT/results/logs/evaluation_failures.tsv" +printf 'task_id\tmodel\tdataset\tsteps\texit_code\n' >"$failures" +status=0 +while IFS=$'\t' read -r task_id category model dataset steps sample_count seed environment adapter source provenance sample_dir metrics_path timing_path; do + [[ "$task_id" == "task_id" || -z "$task_id" ]] && continue + mkdir -p "$(dirname -- "$metrics_path")" + echo "EVALUATE $task_id" + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m evaluation.evaluate \ + --root "$DLB_ROOT" --samples "$sample_dir/samples.jsonl" \ + --metrics "$METRICS" --dataset "$dataset" --output "$metrics_path" + exit_code=$? + if ((exit_code != 0)); then + printf '%s\t%s\t%s\t%s\t%s\n' "$task_id" "$model" "$dataset" "$steps" "$exit_code" >>"$failures" + status=1 + fi +done < <(tail -n +3 "$MATRIX") +if [[ ! -s "$failures" ]]; then rm -f "$failures"; fi +exit "$status" diff --git a/scripts/evaluate_conditional_4gpu.sh b/scripts/evaluate_conditional_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..213596d2838525715ef1c2073e5ab8e9953e8fb9 --- /dev/null +++ b/scripts/evaluate_conditional_4gpu.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Four-GPU local quality evaluation over completed C64 conditional samples. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_CONDITIONAL_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +METRICS="conditional_gen_ppl,mauve_suffix,entropy,self_bleu,prefix_exact_match" +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--metrics LIST] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --metrics) METRICS="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/conditional/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.conditional_matrix \ + --root "$DLB_ROOT" --output "$MATRIX" || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage evaluate --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" --metrics "$METRICS" "${DRY_RUN[@]}" diff --git a/scripts/evaluate_elf.py b/scripts/evaluate_elf.py new file mode 100644 index 0000000000000000000000000000000000000000..042792a1c6dbfa396e3a06e269ce1c937dfccf2b --- /dev/null +++ b/scripts/evaluate_elf.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Score independent ELF artifacts with pinned, offline evaluation assets.""" +from __future__ import annotations + +import argparse +from collections import defaultdict +from dataclasses import asdict +import json +import math +from pathlib import Path +import statistics +import sys + +from elf_common import ROOT, asset, manifest, offline, read_jsonl, require_server, sha256, write_json + + +def validate_records(rows, meta): + if len(rows) != meta["sample_count"]: + raise ValueError("Sample count differs from generation manifest") + n = meta["prompt_count"] + expected = [(i, 0) for i in range(n)] + if meta.get("diversity"): + expected += [(i, c) for c in range(1, 5) for i in range(min(256, n))] + actual = [(r["prompt_id"], r["completion_id"]) for r in rows] + if actual != expected or [r["id"] for r in rows] != list(range(len(rows))): + raise ValueError("Missing/duplicated/out-of-order prompt completions") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run", type=Path, required=True) + p.add_argument("--root", type=Path, default=ROOT) + p.add_argument("--batch-size", type=int, default=8) + p.add_argument("--mauve", action="store_true") + args = p.parse_args() + require_server() + offline() + root = args.root.resolve() + sys.path[:0] = [str(root), str(root / "src")] + meta = json.loads((args.run / "generation.json").read_text()) + samples = args.run / "samples.jsonl" + if sha256(samples) != meta["samples_sha256"]: + raise ValueError("Sample file changed since generation") + if sha256(root / "data/elf/assets.json") != meta["assets_manifest_sha256"]: + raise ValueError("Asset manifest changed since generation") + rows = read_jsonl(samples) + validate_records(rows, meta) + primary = [r for r in rows if r["completion_id"] == 0] + result = {"schema": "dlb-elf-metrics-v1", "task": meta["task"], + "generation_manifest_sha256": sha256(args.run / "generation.json"), + "samples_sha256": sha256(samples), "sample_count": len(primary), + "evaluator_sha256": sha256(Path(__file__)), "metrics": {}} + metrics = result["metrics"] + if meta["task"] in ("wmt14", "xsum"): + import sacrebleu + from rouge_score import rouge_scorer + hypotheses, refs = [r["generated"] for r in primary], [r["reference"] for r in primary] + # Same metric options as official utils/metrics_utils.py. + bleu = sacrebleu.metrics.BLEU(lowercase=True, effective_order=True) + metrics["bleu"] = bleu.corpus_score(hypotheses, [refs]).score + metrics["bleu_signature"] = str(bleu.get_signature()) + metrics["empty_fraction"] = sum(not x.strip() for x in hypotheses) / len(hypotheses) + scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], use_stemmer=True) + scores = [scorer.score(ref, hyp) for ref, hyp in zip(refs, hypotheses)] + for key in ("rouge1", "rouge2", "rougeL"): + values = [s[key].fmeasure * 100 for s in scores] + metrics[key] = statistics.fmean(values) + metrics[key + "_sem"] = statistics.pstdev(values) / math.sqrt(len(values)) + metrics["reference_policy"] = "full raw reference; no target truncation; ROUGE F1 stemmer=true" + else: + import torch + from transformers import AutoTokenizer, AutoModelForCausalLM + from evaluation.generative_perplexity import compute_gen_ppl, aggregate_nll + from evaluation.conditional_perplexity import PromptExcludedCausalLMScorer + from evaluation.unigram_entropy import unigram_entropy + from evaluation.self_bleu import compute_self_bleu + gpt_path = asset(root, "gpt2") + scorer_path = asset(root, "gpt2-large") + tok = AutoTokenizer.from_pretrained(str(gpt_path), local_files_only=True) + tok.pad_token = tok.eos_token + tok.padding_side = "right" + tok.truncation_side = "right" + tokenized = [tok(r["generated"], add_special_tokens=False)["input_ids"] for r in rows] + lengths = [len(ids) for ids in tokenized[:len(primary)]] + metrics["gpt2_length_mean"] = statistics.fmean(lengths) + metrics["gpt2_length_min"] = min(lengths) + metrics["empty_fraction"] = sum(n == 0 for n in lengths) / len(lengths) + metrics["below_64_tokens_fraction"] = sum(n < 64 for n in lengths) / len(lengths) + metrics["above_1024_tokens_fraction"] = sum(n > 1024 for n in lengths) / len(lengths) + limit = 64 if meta["task"] == "owt-prefix" else 1024 + tokenized = [ids[:limit] for ids in tokenized] + # Never improve apparent PPL by silently removing degenerate generations. + if any(len(ids) < 2 for ids in tokenized): + result["valid"] = False + result["reason"] = "At least one completion has fewer than two GPT-2 tokens; no samples were dropped" + write_json(args.run / "metrics.json", result) + raise ValueError(result["reason"]) + metrics["entropy_gpt2_nats"] = statistics.fmean(unigram_entropy(ids) for ids in tokenized[:len(primary)]) + result["evaluator_assets"] = {k: manifest(root)["assets"][k] for k in ("gpt2", "gpt2-large")} + model = AutoModelForCausalLM.from_pretrained(str(scorer_path), local_files_only=True).cuda().eval() + if meta["task"] == "owt": + ppl = compute_gen_ppl([r["generated"] for r in primary], model, tok, + batch_size=args.batch_size, device="cuda", + model_revision=result["evaluator_assets"]["gpt2-large"]["revision"], + tokenizer_revision=result["evaluator_assets"]["gpt2"]["revision"]) + metrics["generative_ppl"] = asdict(ppl) + else: + scorer = PromptExcludedCausalLMScorer(model, device="cuda") + def conditional_ppl(targets): + parts = [] + for start in range(0, len(primary), args.batch_size): + batch = primary[start:start + args.batch_size] + combined, loss_masks = [], [] + for j, row in enumerate(batch): + prefix = tok(row["input"], add_special_tokens=False)["input_ids"] + target = targets[start + j] + if not prefix or len(prefix) + len(target) > 1024: + raise ValueError("Conditional PPL input exceeds context or has empty prefix") + combined.append(prefix + target) + loss_masks.append([0] * len(prefix) + [1] * len(target)) + width = max(map(len, combined)) + attn = [[1] * len(x) + [0] * (width - len(x)) for x in combined] + masks = [x + [0] * (width - len(x)) for x in loss_masks] + ids = [x + [tok.pad_token_id] * (width - len(x)) for x in combined] + parts.append(scorer.score_prompt_excluded_batch(ids, attn, masks)) + return asdict(aggregate_nll(parts)) + metrics["conditional_ppl"] = conditional_ppl(tokenized[:len(primary)]) + references = [tok(r["reference"], add_special_tokens=False)["input_ids"][:64] for r in primary] + metrics["reference_conditional_ppl"] = conditional_ppl(references) + metrics["conditional_ppl_policy"] = "separate GPT-2 prefix/suffix tokenization; prefix targets masked" + metrics["entropy_all_completions_gpt2_nats"] = statistics.fmean(unigram_entropy(ids) for ids in tokenized) + if meta.get("diversity"): + groups = defaultdict(list) + for row, ids in zip(rows, tokenized): + if row["prompt_id"] < min(256, meta["prompt_count"]): + groups[row["prompt_id"]].append(" ".join(map(str, ids))) + metrics["grouped_self_bleu"] = statistics.fmean(compute_self_bleu(group).score for group in groups.values()) + if args.mauve: + import mauve + del scorer, model + torch.cuda.empty_cache() + score = mauve.compute_mauve( + p_text=[tok.decode(ids) for ids in tokenized[:len(primary)]], + q_text=[tok.decode(ids) for ids in references], featurize_model_name=str(scorer_path), + device_id=0, max_text_length=64, seed=42, verbose=False) + metrics["mauve"] = float(score.mauve) + metrics["entropy_policy"] = "mean per-sample natural-log entropy on retokenized GPT-2 suffix/text; same PPL length cap" + result["valid"] = True + write_json(args.run / "metrics.json", result) + print(json.dumps(metrics, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/evaluate_elf_owt_baseline.py b/scripts/evaluate_elf_owt_baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..6c983e686572d04801e1827b7ac551cbe04e2ee1 --- /dev/null +++ b/scripts/evaluate_elf_owt_baseline.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Rescore public ELF OWT outputs against the existing C64 baseline (no generation).""" +from __future__ import annotations + +import argparse +import csv +from dataclasses import dataclass, asdict +import json +from pathlib import Path +import statistics +import sys + +from elf_common import ROOT, asset, manifest, offline, read_jsonl, require_server, sha256, write_json +from run_elf_remaining import build_matrix, reusable_result + + +@dataclass +class C64Record: + """Token-record interface consumed by the existing baseline evaluator.""" + prompt_id: int + completion_id: int + prefix_token_ids: list[int] + continuation_token_ids: list[int] + reference_token_ids: list[int] + + +def c64_records(rows, prompts, tokenized, tokenizer, *, allow_short=False): + by_id = {p['prompt_id']: p for p in prompts} + records = [] + for row, ids in zip(rows, tokenized, strict=True): + source = by_id[row['prompt_id']] + prefix, reference = source['prefix_token_ids'], source['reference_token_ids'] + if len(prefix) != 64 or len(reference) != 64: + raise ValueError('Canonical prompt/reference must contain exactly 64 GPT-2 tokens') + # Prove the saved ELF condition/reference came from this exact prompt. + if row['input'] != tokenizer.decode(prefix, skip_special_tokens=False): + raise ValueError('Saved ELF input differs from canonical C64 prompt') + if row['reference'] != tokenizer.decode(reference, skip_special_tokens=False): + raise ValueError('Saved ELF reference differs from canonical C64 reference') + if not ids or (len(ids) < 64 and not allow_short): + raise ValueError('Short response cannot satisfy the 64-token scoring contract') + records.append(C64Record(row['prompt_id'], row['completion_id'], prefix, ids[:64], reference)) + return records + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=ROOT) + parser.add_argument('--task', choices=['owt', 'owt-prefix'], required=True) + parser.add_argument('--output', type=Path) + parser.add_argument('--batch-size', type=int, default=8) + parser.add_argument('--steps', nargs='+', type=int, choices=[1, 2, 4, 8, 16, 32, 1024]) + parser.add_argument('--short-response-policy', choices=['strict', 'observed'], default='strict', + help='observed: separately report all nonempty responses up to 64 GPT-2 tokens') + parser.add_argument('--plan', action='store_true') + args = parser.parse_args() + if args.batch_size <= 0: + parser.error('--batch-size must be positive') + observed = args.short_response_policy == 'observed' + if observed and args.task != 'owt-prefix': + parser.error('observed short-response policy is only for owt-prefix') + if args.steps and len(args.steps) != len(set(args.steps)): + parser.error('duplicate steps') + root = args.root.resolve() + sys.path[:0] = [str(root), str(root / 'src')] + output = args.output or root / 'results' / (f'elf-owt-baseline-eval-{args.task}' + ('-observed' if observed else '')) + assets = manifest(root)['assets'] + asset_hash = sha256(root / 'data/elf/assets.json') + cells, blocked = build_matrix(root, [args.task]) + if blocked: + raise ValueError(f'Invalid inputs: {blocked}') + if args.steps: + cells = [cell for cell in cells if cell.steps in args.steps] + runs = [] + for cell in cells: + found, rejected = reusable_result(root, cell, asset_hash, assets) + if not found: + raise ValueError(f'Missing verified generation: {cell.key}; first run run_elf_remaining.py; rejected={rejected}') + path = Path(found['run']) + runs.append((cell, path)) + print(f'RESCORE {cell.key}: {path}', flush=True) + if args.plan: + print(f'{len(runs)} cells; no generation, no training; output={output}') + return + require_server() + offline() + from transformers import AutoTokenizer, AutoModelForCausalLM + from evaluation.generative_perplexity import compute_gen_ppl + from evaluation.conditional_perplexity import compute_conditional_gen_ppl, conditional_texts + from evaluation.conditional_evaluate import _gpt2_tokenized_self_bleu + from evaluation.unigram_entropy import unigram_entropy + tok = AutoTokenizer.from_pretrained(str(asset(root, 'gpt2')), local_files_only=True) + tok.pad_token = tok.eos_token + tok.padding_side = tok.truncation_side = 'right' + prompts = read_jsonl(root / 'data/conditional/owt-c64/prompts.jsonl') if args.task == 'owt-prefix' else [] + dependencies = [Path(__file__), root / 'evaluation/generative_perplexity.py', + root / 'evaluation/conditional_perplexity.py', root / 'evaluation/conditional_evaluate.py', + root / 'evaluation/unigram_entropy.py', root / 'evaluation/self_bleu.py'] + evaluator = {p.name: sha256(p) for p in dependencies} + model = None + summary = [] + for cell, path in runs: + destination = output / f'steps_{cell.steps}' / 'metrics.json' + provenance = {'protocol': 'elf_owt_public_baseline_observed_v1' if observed else 'elf_owt_public_baseline_eval_v1', 'task': args.task, + 'steps': cell.steps, 'source_run': str(path), + 'samples_sha256': sha256(path / 'samples.jsonl'), + 'generation_manifest_sha256': sha256(path / 'generation.json'), + 'assets_manifest_sha256': asset_hash, 'evaluator': evaluator, + 'batch_size': args.batch_size, + 'prompts_sha256': sha256(root / 'data/conditional/owt-c64/prompts.jsonl') if prompts else None} + cached = json.loads(destination.read_text()) if destination.exists() else {} + if cached and cached.get('protocol') != provenance['protocol']: + raise ValueError(f'Different protocol already exists at {destination}; use a separate --output') + if all(cached.get(k) == v for k, v in provenance.items()) and type(cached.get('valid')) is bool: + result = cached + print(f'SKIP verified evaluation: {destination}', flush=True) + else: + rows = read_jsonl(path / 'samples.jsonl') + ids = [tok(row['generated'], add_special_tokens=False)['input_ids'] for row in rows] + lengths = list(map(len, ids)) + limit = 64 if prompts else 1024 + short = [row['id'] for row, tokens in zip(rows, ids) if len(tokens) < (64 if prompts else 2)] + invalid = [row['id'] for row, tokens in zip(rows, ids) if not tokens] if observed else short + result = {**provenance, 'sample_count': len(rows), 'prompt_count': 1024, + 'metrics': {}, 'valid': not invalid, + 'short_response_policy': args.short_response_policy, + 'invalid_sample_ids': invalid, + 'length_diagnostics': {'mean': statistics.fmean(lengths), 'min': min(lengths), + 'max': max(lengths), 'below_64_count': sum(n < 64 for n in lengths), + 'above_1024_count': sum(n > 1024 for n in lengths)}, + 'short_sample_ids': short, + 'native_tokenizer': 't5-small', 'scoring_tokenizer': 'gpt2', + 'scoring_limit': limit, 'completion_scope': 'all_saved_completions', + 'output_policy': 'released ELF EOS-trimmed decoded text; GPT-2 retokenized; no fabricated padding'} + if invalid: + result['reason'] = ('Empty outputs' if observed else 'Short outputs') + ': cannot score the requested contract; none dropped, padded, or resampled' + else: + sliced = [tokens[:limit] for tokens in ids] + result['metrics']['entropy_gpt2_nats'] = statistics.fmean(unigram_entropy(tokens) for tokens in sliced) + records = c64_records(rows, prompts, ids, tok, allow_short=observed) if prompts else None + if model is None: + model = AutoModelForCausalLM.from_pretrained(str(asset(root, 'gpt2-large')), local_files_only=True).cuda().eval() + kwargs = dict(batch_size=args.batch_size, device='cuda', + model_revision=assets['gpt2-large']['revision'], tokenizer_revision=assets['gpt2']['revision']) + if prompts: + ppl = compute_conditional_gen_ppl(records, model, tok, tok, allow_short_continuations=observed, **kwargs) + result['metrics']['conditional_ppl'] = asdict(ppl) + texts = conditional_texts(records, tok, allow_short_continuations=observed) + result['metrics']['grouped_self_bleu'] = _gpt2_tokenized_self_bleu(records, [t.generated_suffix for t in texts], tok) + result['ppl_policy'] = 'existing compute_conditional_gen_ppl; canonical prefix IDs; decode joined IDs then retokenize; prompt loss excluded' + else: + ppl = compute_gen_ppl([row['generated'] for row in rows], model, tok, **kwargs) + result['metrics']['generative_ppl'] = asdict(ppl) + if ppl.sample_count != len(rows): + raise ValueError('PPL silently changed completion count') + write_json(destination, result) + metrics = result['metrics'] + ppl = metrics.get('conditional_ppl', metrics.get('generative_ppl', {})).get('perplexity') + summary.append({'task': args.task, 'steps': cell.steps, 'protocol': result['protocol'], 'sample_count': result['sample_count'], + 'valid': result['valid'], 'ppl': ppl, 'entropy': metrics.get('entropy_gpt2_nats'), + 'short_count': len(result['short_sample_ids']), 'metrics_path': str(destination)}) + print(json.dumps(summary[-1]), flush=True) + output.mkdir(parents=True, exist_ok=True) + summary_name = 'summary' + ('-steps-' + '-'.join(str(c.steps) for c, _ in runs) if args.steps else '') + '.csv' + with (output / summary_name).open('w') as handle: + writer = csv.DictWriter(handle, fieldnames=list(summary[0])) + writer.writeheader() + writer.writerows(summary) + print(f'SUMMARY: {output / summary_name}', flush=True) + if any(not row['valid'] for row in summary): + raise SystemExit(2) + + +if __name__ == '__main__': + main() diff --git a/scripts/fetch_checkpoints.py b/scripts/fetch_checkpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..55e974966e7ad45469c17bc54d550d375ae23368 --- /dev/null +++ b/scripts/fetch_checkpoints.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Enumerate or fetch pinned public checkpoint resources.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import platform +import sys + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--config", type=Path) + parser.add_argument("--registry", type=Path) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--all-public", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + root = args.root.resolve() + project_root = Path(__file__).resolve().parents[1] + sys.path.insert(0, str(project_root / "src")) + from dlb.checkpoints import ( + fetch_all_resources, + load_checkpoint_manifest, + require_server_platform, + validate_checkpoint_coverage, + ) + from dlb.registry import load_registry + + manifest_path = (args.config or root / "artifacts" / "checkpoints.yaml").resolve() + registry_path = (args.registry or root / "configs" / "experiments.yaml").resolve() + manifest = load_checkpoint_manifest(manifest_path) + registry = load_registry(registry_path) + validate_checkpoint_coverage(registry, manifest) + if args.dry_run: + for resource_id, resource in manifest.resources.items(): + print( + f"RESOURCE {resource_id} {resource.backend} {resource.provenance} " + f"checkpoints/{resource.destination} teacher={resource.teacher_family}" + ) + for recipe_id, recipe in manifest.recipes.items(): + print( + f"RECIPE {recipe_id} {recipe.model}/{recipe.dataset} {recipe.provenance} " + f"{recipe.output} teacher={recipe.teacher_family}" + ) + return 0 + if not args.all_public: + print("ERROR: pass --all-public to acknowledge all public checkpoint downloads", file=sys.stderr) + return 2 + try: + require_server_platform(platform.system()) + except RuntimeError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 + lock = fetch_all_resources(root, manifest_path, manifest) + for resource_id, record in lock["resources"].items(): + print(f"{str(record['status']).upper()} {resource_id} {record['destination']}") + return 1 if any(record["status"] != "downloaded" for record in lock["resources"].values()) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/fetch_data.py b/scripts/fetch_data.py new file mode 100644 index 0000000000000000000000000000000000000000..3ab619e1c42733754322de2a65728cb3c63b5945 --- /dev/null +++ b/scripts/fetch_data.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Download pinned datasets and model/tokenizer snapshots with cache resume.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import os +from pathlib import Path +import shutil +import sys + +import yaml + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--config", type=Path) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--allow-low-disk", action="store_true") + return parser.parse_args() + + +def load_configuration(root: Path, config_path: Path | None) -> dict[str, object]: + path = config_path or root / "artifacts" / "data.yaml" + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def target_lines(configuration: dict[str, object]) -> list[str]: + datasets = configuration["datasets"] + models = configuration["models"] + return [ + *[ + f"DATASET {name} {datasets[name]['repo_id']}@{datasets[name]['revision']}" + for name in ("lm1b", "owt") + ], + *[ + f"MODEL {name}@{models[name]}" + for name in ("bert-base-uncased", "gpt2", "gpt2-large") + ], + ] + + +def directory_size(path: Path) -> int: + return sum(candidate.stat().st_size for candidate in path.rglob("*") if candidate.is_file()) + + +def _relative_cache_path(path: Path, root: Path) -> str: + try: + return path.resolve().relative_to(root).as_posix() + except ValueError: + return path.resolve().as_posix() + + +def main() -> int: + args = parse_args() + root = args.root.resolve() + configuration = load_configuration(root, args.config) + for line in target_lines(configuration): + print(line) + if args.dry_run: + return 0 + + hf_home = root / "data" / "raw" / "huggingface" + datasets_cache = hf_home / "datasets" + hub_cache = hf_home / "hub" + hf_home.mkdir(parents=True, exist_ok=True) + os.environ["HF_HOME"] = str(hf_home) + os.environ["HF_DATASETS_CACHE"] = str(datasets_cache) + + from datasets import load_dataset + from huggingface_hub import snapshot_download + + sys.path.insert(0, str(root / "src")) + from dlb.data import disk_preflight_evidence, validate_download_manifest + from dlb.io import atomic_json_write + + dataset_records: dict[str, object] = {} + owt_disk_preflight = None + for name in ("lm1b", "owt"): + specification = configuration["datasets"][name] + if name == "owt": + free_bytes = shutil.disk_usage(root).free + owt_disk_preflight = disk_preflight_evidence( + free_bytes, allow_low_disk=args.allow_low_disk + ) + if owt_disk_preflight["override_used"]: + print( + f"WARNING low disk override: {free_bytes} bytes free, required 55 GiB", + file=sys.stderr, + ) + if name == "lm1b": + source_metadata_snapshot = snapshot_download( + repo_id=specification["repo_id"], + repo_type="dataset", + revision=specification["revision"], + cache_dir=hub_cache, + ) + dataset_snapshot = snapshot_download( + repo_id=specification["repo_id"], + repo_type="dataset", + revision=specification["parquet_revision"], + cache_dir=hub_cache, + allow_patterns="plain_text/**/*.parquet", + ) + snapshot_path = Path(dataset_snapshot) + loaded = load_dataset( + "parquet", + data_files={ + "train": [ + str(path) + for path in sorted( + (snapshot_path / "plain_text" / "train").glob("*.parquet") + ) + ], + "test": [ + str(path) + for path in sorted( + (snapshot_path / "plain_text" / "test").glob("*.parquet") + ) + ], + }, + cache_dir=str(datasets_cache), + download_mode="reuse_dataset_if_exists", + ) + archive_specification = configuration["lm1b_archive"] + if len(loaded["train"]) != int(archive_specification["train_documents"]): + raise RuntimeError("LM1B training document count differs from pinned metadata") + if len(loaded["test"]) != int(archive_specification["test_documents"]): + raise RuntimeError("LM1B test document count differs from pinned metadata") + else: + source_metadata_snapshot = None + dataset_snapshot = None + loaded = load_dataset( + specification["repo_id"], + specification["config"], + revision=specification["revision"], + cache_dir=str(datasets_cache), + download_mode="reuse_dataset_if_exists", + ) + if len(loaded["train"]) != int(specification["documents"]): + raise RuntimeError("OpenWebText document count differs from pinned metadata") + dataset_records[name] = { + "repo_id": specification["repo_id"], + "source_revision": specification["revision"], + "cache_files": { + split: [ + _relative_cache_path(Path(entry["filename"]), root) + for entry in value.cache_files + ] + for split, value in loaded.items() + }, + "split_rows": {split: len(value) for split, value in loaded.items()}, + "dataset_snapshot": ( + _relative_cache_path(Path(dataset_snapshot), root) if dataset_snapshot else None + ), + "source_metadata_snapshot": ( + _relative_cache_path(Path(source_metadata_snapshot), root) + if source_metadata_snapshot + else None + ), + "materialization_revision": specification.get("parquet_revision"), + } + print(f"DOWNLOADED DATASET {name} {dataset_records[name]['split_rows']}") + + model_records: dict[str, object] = {} + for name in ("bert-base-uncased", "gpt2", "gpt2-large"): + revision = configuration["models"][name] + snapshot_path = Path( + snapshot_download( + repo_id=name, + revision=revision, + cache_dir=hub_cache, + ) + ) + model_records[name] = { + "repo_id": name, + "revision": revision, + "snapshot_path": snapshot_path.relative_to(root).as_posix(), + "size_bytes": directory_size(snapshot_path), + } + print(f"DOWNLOADED MODEL {name} {snapshot_path}") + + manifest = { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "hf_home": hf_home.relative_to(root).as_posix(), + "owt_disk_preflight": owt_disk_preflight, + "datasets": dataset_records, + "models": model_records, + } + for dataset_name in ("lm1b", "owt"): + validate_download_manifest(configuration, manifest, dataset_name) + atomic_json_write(root / "data" / "manifests" / "downloads.json", manifest) + print(f"RAW CACHE BYTES {directory_size(hf_home)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/inspect_elf_progress.py b/scripts/inspect_elf_progress.py new file mode 100644 index 0000000000000000000000000000000000000000..2bcdac9af316e87706d370dcb2c1de09cdc4e4eb --- /dev/null +++ b/scripts/inspect_elf_progress.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Read-only ELF quality inventory; verify saved artifacts before reporting completion.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from elf_common import ROOT, read_jsonl, sha256, write_json +from evaluate_elf import validate_records + +OWT_MAIN_STEPS = (1, 2, 4, 8, 16, 32, 1024) + + +def inspect_run(path): + row = {"run": str(path), "status": "incomplete_generation"} + try: + request = json.loads((path / "request.json").read_text()) + row.update(task=request["task"], steps=request["steps"], seed=request["seed"], + compile=request["compile"], + split=(request.get("input") or {}).get( + "split", "c64-text-t5" if request["task"] == "owt-prefix" else "unconditional"), + protocol=request.get("protocol"), + condition_policy=request.get("condition_tokenization_policy")) + gp = path / "generation.json" + if not gp.exists(): + return row + gen = json.loads(gp.read_text()) + for key in ("task", "steps", "seed", "compile"): + if gen[key] != request[key]: + raise ValueError(f"request/generation mismatch: {key}") + sample_hash = sha256(path / "samples.jsonl") + if gen["samples_sha256"] != sample_hash: + raise ValueError("samples hash mismatch") + validate_records(read_jsonl(path / "samples.jsonl"), gen) + row.update(sample_count=gen["sample_count"], prompt_count=gen["prompt_count"], + status="needs_evaluation") + mp = path / "metrics.json" + if mp.exists(): + metric = json.loads(mp.read_text()) + if (metric["generation_manifest_sha256"] != sha256(gp) + or metric["samples_sha256"] != sample_hash + or metric["task"] != gen["task"] + or metric["sample_count"] != gen["prompt_count"]): + raise ValueError("metrics provenance/count mismatch") + if type(metric.get("valid")) is not bool: + raise ValueError("missing boolean metric validity") + row.update(status="quality_valid" if metric["valid"] else "quality_invalid", + metrics=metric["metrics"], reason=metric.get("reason")) + row["owt_main_grid"] = ( + row["task"] == "owt" and row["steps"] in OWT_MAIN_STEPS + and row["prompt_count"] == 1024 and not gen.get("diversity") + and row["seed"] == 42 and not row["compile"] + ) + except (OSError, ValueError, KeyError, TypeError) as error: + row.update(status="invalid_artifact", reason=str(error)) + return row + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + paths = sorted((args.root / "results").glob("elf*/quality/*/*/steps_*/seed_*/compile_*")) + rows = [inspect_run(path) for path in paths if path.is_dir()] + for row in rows: + metrics = row.get("metrics", {}) + scores = {key: metrics[key] for key in ("bleu", "rouge1", "rouge2", "rougeL", "entropy_gpt2_nats") if key in metrics} + for key in ("generative_ppl", "conditional_ppl"): + if key in metrics: + scores[key] = metrics[key].get("perplexity") + print(json.dumps({k: v for k, v in row.items() if k != "metrics"} | {"scores": scores}, ensure_ascii=False)) + present = {row["steps"] for row in rows if row.get("owt_main_grid") and row["status"] == "quality_valid"} + missing = [step for step in OWT_MAIN_STEPS if step not in present] + print(f"OWT primary grid {list(OWT_MAIN_STEPS)}; steps without a verified valid result: {missing}") + print("Old/no-EOS conditional runs and auxiliary steps remain listed; do not merge protocols. No GPU work performed.") + if args.output: + write_json(args.output, {"schema": "elf-progress-v1", "runs": rows, + "owt_steps_without_valid_result": missing}) + + +if __name__ == "__main__": + main() diff --git a/scripts/local_runs/eval_bench_finished_distills_gpu7_rerun.sh b/scripts/local_runs/eval_bench_finished_distills_gpu7_rerun.sh new file mode 100644 index 0000000000000000000000000000000000000000..871dc6f2758919a92d3265afb91ca9fae80b9136 --- /dev/null +++ b/scripts/local_runs/eval_bench_finished_distills_gpu7_rerun.sh @@ -0,0 +1,74 @@ +set -uo pipefail +cd /home/ruizzhao/diffusion_baseline +export DLB_ROOT="$PWD" +export DLB_PYTHON=/opt/anaconda3/envs/dlb-bootstrap/bin/python +export PYTHONPATH="$PWD/src${PYTHONPATH:+:$PYTHONPATH}" + +FAILURES="results/logs/eval_bench_failures_finished_distills_gpu7_rerun.tsv" +printf 'stage\tmodel\tdataset\tsteps\tgpu\n' > "$FAILURES" + +run_eval() { + local model="$1" + local dataset="$2" + local steps="$3" + local gpu=7 + + local sample="results/samples/${dataset}/${model}/steps_${steps}/samples.jsonl" + local metrics="results/metrics/${dataset}/${model}/steps_${steps}/metrics.json" + mkdir -p "$(dirname "$metrics")" + + if [[ ! -f "$sample" ]] || [[ "$(wc -l < "$sample")" -ne 1024 ]]; then + echo "SKIP eval ${model}/${dataset} steps=${steps}: missing/incomplete samples" + printf 'eval_missing\t%s\t%s\t%s\t%s\n' "$model" "$dataset" "$steps" "$gpu" >> "$FAILURES" + return 0 + fi + + echo "[$(date)] GPU${gpu} EVAL ${model}/${dataset} steps=${steps}" + if ! CUDA_VISIBLE_DEVICES="$gpu" PYTHONPATH="$PYTHONPATH" /opt/anaconda3/envs/dlb-eval/bin/python -m evaluation.evaluate \ + --root "$PWD" \ + --samples "$sample" \ + --metrics gen_ppl,entropy,self_bleu \ + --dataset "$dataset" \ + --output "$metrics" \ + --device cuda \ + > "results/logs/eval_${model}_${dataset}_steps_${steps}_gpu${gpu}_rerun.log" 2>&1; then + printf 'eval\t%s\t%s\t%s\t%s\n' "$model" "$dataset" "$steps" "$gpu" >> "$FAILURES" + echo "FAILED eval ${model}/${dataset} steps=${steps}" + fi +} + +run_bench_if_missing() { + local model="$1" + local dataset="$2" + local steps="$3" + local gpu=7 + + # benchmark_one.sh 自己会写 results/timing 下的文件；这里用 log 是否已有成功痕迹做保守补跑。 + local log="results/logs/bench_${model}_${dataset}_steps_${steps}_gpu${gpu}_rerun.log" + echo "[$(date)] GPU${gpu} BENCH ${model}/${dataset} steps=${steps}" + if ! CUDA_VISIBLE_DEVICES="$gpu" bash scripts/benchmark_one.sh \ + --model "$model" --dataset "$dataset" --steps "$steps" \ + --seed 42 --precision author \ + > "$log" 2>&1; then + printf 'bench\t%s\t%s\t%s\t%s\n' "$model" "$dataset" "$steps" "$gpu" >> "$FAILURES" + echo "FAILED bench ${model}/${dataset} steps=${steps}" + fi +} + +for model_dataset in "duo_di4c owt" "mdlm_sdtt lm1b" "duo_dcd lm1b"; do + read -r model dataset <<< "$model_dataset" + for steps in 1 2 4 8 16 32; do + run_eval "$model" "$dataset" "$steps" + done +done + +for model_dataset in "duo_di4c owt" "mdlm_sdtt lm1b" "duo_dcd lm1b"; do + read -r model dataset <<< "$model_dataset" + for steps in 1 2 4 8 16 32; do + run_bench_if_missing "$model" "$dataset" "$steps" + done +done + +echo "===== failures =====" +cat "$FAILURES" +echo "ALL EVAL/BENCH RERUN ATTEMPTED" diff --git a/scripts/prepare_elf.py b/scripts/prepare_elf.py new file mode 100644 index 0000000000000000000000000000000000000000..cfe6677439e38991c0bfd23f2fe932bb0500955b --- /dev/null +++ b/scripts/prepare_elf.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Server-only pinned downloads and text-only evaluation inputs for ELF.""" +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +import re +import subprocess +import sys +import time +from email.utils import parsedate_to_datetime + +from elf_common import ROOT, SOURCE_COMMIT, asset, read_jsonl, require_server, sha256, source_path, write_json + + +def retryable_response(error): + """Find HTTP failures even when Hub 0.36 wraps them in a cache error.""" + seen = set() + while error is not None and id(error) not in seen: + seen.add(id(error)) + response = getattr(error, "response", None) + # requests.Response is falsey for 4xx/5xx, so do not use `if response`. + if response is not None: + if response.status_code in (429, 500, 502, 503, 504): + return response + return None # Authentication/revision errors need a fix, not retries. + error = error.__cause__ if error.__cause__ is not None else error.__context__ + return None + + +def retry_delay(response, attempt): + """Honor server cooldown headers, falling back to bounded exponential wait.""" + delay = float(min(60 * 2 ** min(attempt - 1, 3), 300)) + headers = {str(k).lower(): v for k, v in response.headers.items()} + retry_after = headers.get("retry-after", "") + try: + requested = float(retry_after) + except (TypeError, ValueError): + try: + requested = parsedate_to_datetime(retry_after).timestamp() - time.time() + except (TypeError, ValueError, OverflowError): + requested = 0 + if math.isfinite(requested): + delay = max(delay, requested) + for seconds in re.findall(r'(?:^|[;,])\s*t=(\d+)', headers.get("ratelimit", "")): + delay = max(delay, float(seconds) + 1) + return delay + + +def download_snapshot(download, *, attempts=6, sleep=time.sleep, **kwargs): + """Retry only temporary Hub HTTP failures; keep pinned revision and cache.""" + if attempts < 1: + raise ValueError("download attempts must be positive") + for attempt in range(1, attempts + 1): + try: + return download(**kwargs) + except Exception as error: + response = retryable_response(error) + if response is None: + raise + if attempt == attempts: + raise RuntimeError( + f"HF download failed after {attempts} attempts (HTTP {response.status_code}): " + f"{kwargs['repo_id']}. Local download files were retained. " + "Retry `python scripts/prepare_elf.py assets` later; do not run the data stage yet." + ) from error + seconds = retry_delay(response, attempt) + print(f"HF HTTP {response.status_code}: {kwargs['repo_id']}; " + f"waiting {seconds:.0f}s before attempt {attempt + 1}/{attempts}", flush=True) + sleep(seconds) + + +def export_rows(path, rows, provenance): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".jsonl.tmp") + count = 0 + with temporary.open("w") as handle: + for count, row in enumerate(rows, 1): + handle.write(json.dumps(row, ensure_ascii=False) + "\n") + temporary.replace(path) + write_json(path.with_suffix(".manifest.json"), { + "schema": "elf-input-v1", "count": count, "sha256": sha256(path), **provenance}) + print(f"OK: exported {count} rows to {path}", flush=True) + + +def official_arrow_rows(directory): + """Read saved-dataset IPC rows without deserializing HF Features metadata. + + The author release uses `_type: List`, absent in datasets 3.6. Its underlying + Arrow columns are ordinary lists/strings and need no HF feature conversion. + Keep the state.json shard order and leave the hashed snapshot untouched. + """ + import pyarrow as pa + + directory = Path(directory) + state = json.loads((directory / "state.json").read_text()) + shards = state.get("_data_files") + if not isinstance(shards, list) or not shards: + raise ValueError(f"Missing Arrow shard list: {directory}/state.json") + seen = set() + index = 0 + for entry in shards: + filename = entry.get("filename") if isinstance(entry, dict) else None + if not isinstance(filename, str) or not filename: + raise ValueError("Invalid Arrow shard filename in state.json") + relative = Path(filename) + if relative.is_absolute() or ".." in relative.parts or filename in seen: + raise ValueError(f"Unsafe or duplicated Arrow shard: {filename}") + seen.add(filename) + with pa.memory_map(str(directory / relative), "r") as source: + reader = pa.ipc.open_stream(source) + required = {"input", "target", "condition_input_ids"} + missing = required - set(reader.schema.names) + if missing: + raise ValueError(f"{filename}: missing raw source/reference columns: {sorted(missing)}") + for batch in reader: + for row in batch.select(sorted(required)).to_pylist(): + tokens = row["condition_input_ids"] + if not isinstance(row["input"], str) or not isinstance(row["target"], str): + raise ValueError(f"{filename}: non-text source/reference at row {index}") + if not isinstance(tokens, list) or any(type(t) is not int or t < 0 for t in tokens): + raise ValueError(f"{filename}: invalid condition token IDs at row {index}") + yield {"id": index, "input": row["input"], "output": row["target"], + "condition_input_ids": tokens} + index += 1 + + +def raw_parquet_rows(files, task): + """Read pinned raw splits locally, without a Hub/datasets builder call.""" + import pyarrow.parquet as pq + + index = 0 + for filename in files: + with pq.ParquetFile(filename) as source: + columns = ["translation"] if task == "wmt14" else ["document", "summary"] + for batch in source.iter_batches(columns=columns): + for row in batch.to_pylist(): + src, target = ((row["translation"]["de"], row["translation"]["en"]) + if task == "wmt14" else (row["document"], row["summary"])) + if not isinstance(src, str) or not isinstance(target, str): + raise ValueError(f"{filename}: non-text source/reference at row {index}") + yield {"id": index, "input": src, "output": target} + index += 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("stage", choices=["source", "assets", "data", "prefix"]) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--download-workers", type=int, default=1, + help="Concurrent snapshot file downloads (default: 1)") + parser.add_argument("--download-attempts", type=int, default=6, + help="Total attempts per snapshot for temporary Hub HTTP errors") + args = parser.parse_args() + if args.download_workers < 1 or args.download_attempts < 1: + parser.error("download-workers and download-attempts must be positive") + require_server() + root = args.root.resolve() + lock = json.loads((root / "artifacts/elf_lock.json").read_text()) + if args.stage == "source": + dst = root / "upstreams/elf" + if not dst.exists(): + dst.parent.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "clone", "--single-branch", "--branch", "pytorch_elf", + lock["source"]["url"], str(dst)], check=True) + subprocess.run(["git", "-C", str(dst), "checkout", "--detach", SOURCE_COMMIT], check=True) + print(source_path(root)) + return + if args.stage == "assets": + from huggingface_hub import snapshot_download + records = {} + for name, record in lock["assets"].items(): + dst = root / "data/elf/snapshots" / name / record["revision"] + print(f"Preparing {name}: {record['repo_id']} @ {record['revision']}", flush=True) + download_snapshot(snapshot_download, attempts=args.download_attempts, + repo_id=record["repo_id"], revision=record["revision"], + repo_type=record["repo_type"], allow_patterns=record["patterns"], + local_dir=dst, max_workers=args.download_workers) + files = {str(p.relative_to(dst)): sha256(p) for p in sorted(dst.rglob("*")) + if p.is_file() and ".cache" not in p.parts} + if not files: + raise ValueError(f"No downloaded files for {name}") + records[name] = {**record, "path": str(dst.relative_to(root)), "sha256": files} + write_json(root / "data/elf/assets.json", {"assets": records, + "source_commit": SOURCE_COMMIT, "lock_sha256": sha256(root / "artifacts/elf_lock.json")}) + print(f"OK: all {len(records)} ELF assets verified; wrote data/elf/assets.json", flush=True) + return + if args.stage == "data": + for task in ("wmt14", "xsum"): + official_name = f"{task}-official-validation" + directory = asset(root, official_name) + export_rows(root / f"data/elf/{official_name}.jsonl", + official_arrow_rows(directory), + {"task": task, "split": "official-validation", "asset": lock["assets"][official_name], + "reader": "pyarrow_ipc_v1"}) + directory = asset(root, f"{task}-raw") + for split in ("validation", "test"): + files = sorted(str(p) for p in directory.rglob(f"{split}-*.parquet")) + if not files: + raise FileNotFoundError(f"Missing {task} {split} parquet") + export_rows(root / f"data/elf/{task}-{split}.jsonl", raw_parquet_rows(files, task), + {"task": task, "split": split, "asset": lock["assets"][f"{task}-raw"], + "reader": "pyarrow_parquet_v1"}) + return + # Same held-out rows and literal prefix/reference text as the C64 benchmark; + # T5 re-encoding is a separate protocol, never reinterpret GPT-2 IDs as T5. + sys.path.insert(0, str(root / "src")) + from dlb.conditional_prompts import load_protocol, verify_prompts + from transformers import AutoTokenizer + verify_prompts(root, "owt", load_protocol(root / "configs/conditional.yaml")) + prompts = root / "data/conditional/owt-c64/prompts.jsonl" + tokenizer = AutoTokenizer.from_pretrained(str(asset(root, "gpt2")), local_files_only=True) + export_rows(root / "data/elf/owt-prefix.jsonl", + ({"id": r["prompt_id"], "input": tokenizer.decode(r["prefix_token_ids"], skip_special_tokens=False), + "output": tokenizer.decode(r["reference_token_ids"], skip_special_tokens=False), + "source_index": r["source_index"]} for r in read_jsonl(prompts)), + {"task": "owt-prefix", "protocol": "c64_text_t5_v1", "source_sha256": sha256(prompts)}) + + +if __name__ == "__main__": + main() diff --git a/scripts/print_di4c_lm1b_summary.py b/scripts/print_di4c_lm1b_summary.py new file mode 100644 index 0000000000000000000000000000000000000000..5275e65d2cb9f6d7f405b6e491709f76b1a405fe --- /dev/null +++ b/scripts/print_di4c_lm1b_summary.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +"""Print a compact LM1B Di4C summary from aggregate CSV outputs.""" + +from __future__ import annotations + +import argparse +import csv +import sys +from pathlib import Path +from typing import Iterable + + +MODELS = ("duo_di4c", "mdlm_di4c") +DATASET = "lm1b" +STEPS = (1, 2, 4, 8, 16, 32) +OUTPUT_COLUMNS = ( + "model", + "dataset", + "steps", + "ppl", + "entropy", + "self_bleu", + "seconds_per_sample", +) +FAILURE_COLUMNS = ("task_id", "model", "dataset", "steps", "reason") + + +def _read_csv(path: Path) -> list[dict[str, str]]: + if not path.is_file() or path.is_symlink(): + raise FileNotFoundError(path) + with path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def _float(value: str, digits: int) -> str: + return f"{float(value):.{digits}f}" + + +def _matching(rows: Iterable[dict[str, str]]) -> list[dict[str, str]]: + wanted = {(model, DATASET, str(step)) for model in MODELS for step in STEPS} + return [ + row + for row in rows + if (row.get("model"), row.get("dataset"), row.get("steps")) in wanted + ] + + +def _sort_key(row: dict[str, str]) -> tuple[int, int]: + return MODELS.index(row["model"]), STEPS.index(int(row["steps"])) + + +def _result_row(row: dict[str, str]) -> dict[str, str]: + return { + "model": row["model"], + "dataset": row["dataset"], + "steps": row["steps"], + "ppl": _float(row["generative_perplexity"], 2), + "entropy": _float(row["unigram_entropy"], 4), + "self_bleu": _float(row["self_bleu"], 4), + "seconds_per_sample": _float(row["generation_seconds_per_sample"], 6), + } + + +def _write_rows(rows: list[dict[str, str]]) -> None: + writer = csv.DictWriter(sys.stdout, fieldnames=OUTPUT_COLUMNS, lineterminator="\n") + writer.writeheader() + for row in sorted(rows, key=_sort_key): + writer.writerow(_result_row(row)) + + +def _write_failures(rows: list[dict[str, str]]) -> None: + failures = sorted(_matching(rows), key=_sort_key) + if not failures: + return + sys.stdout.write("\n# failures\n") + writer = csv.DictWriter(sys.stdout, fieldnames=FAILURE_COLUMNS, lineterminator="\n") + writer.writeheader() + for row in failures: + writer.writerow({column: row.get(column, "") for column in FAILURE_COLUMNS}) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--summary-dir", type=Path) + args = parser.parse_args(argv) + + root = args.root.resolve() + summary = (args.summary_dir or root / "results" / "summary").resolve() + result_rows = _matching(_read_csv(summary / "results_wide.csv")) + failure_path = summary / "failures.csv" + failure_rows = _read_csv(failure_path) if failure_path.exists() else [] + + _write_rows(result_rows) + _write_failures(failure_rows) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rescore_unconditional_ppl.py b/scripts/rescore_unconditional_ppl.py new file mode 100644 index 0000000000000000000000000000000000000000..57701da7f33e5a827c1c84b554521f5879edb6e5 --- /dev/null +++ b/scripts/rescore_unconditional_ppl.py @@ -0,0 +1,205 @@ +"""Rescore existing unconditional samples with independent causal language models. + +This leaves the canonical GPT-2 Large metrics untouched. Run from the repository +root with the dlb-eval environment and one free GPU. +""" + +from __future__ import annotations + +import argparse +from dataclasses import asdict +import json +import math +from pathlib import Path + +from dlb.io import atomic_json_write, sha256_file, validate_samples +from evaluation.generative_perplexity import compute_gen_ppl + + +ROOT = Path(__file__).resolve().parents[1] +MODELS = { + "pythia-1b": ( + "EleutherAI/pythia-1b", + "f73d7dcc545c8bd326d8559c8ef84ffe92fea6b2", + ["*.json", "model.safetensors"], + ), + "opt-1.3b": ( + "facebook/opt-1.3b", + "3f5c25d0bc631cb57ac65913f76e22c2dfb61d62", + ["*.json", "pytorch_model.bin", "merges.txt", "vocab.json"], + ), +} +DATASETS = ("lm1b", "owt") +GENERATORS = ("fmlm", "duo_dcd", "duo_di4c", "mdlm_sdtt", "mdlm_di4c") +STEPS = (1, 2, 4) +SAMPLE_COUNT = 1024 +CONTEXT_LENGTH = 1024 + + +def scorer_class(): + import torch + import torch.nn.functional as functional + + class Float32LossScorer: + def __init__(self, model): + self.model = model + + def eval(self): + self.model.eval() + return self + + def score_batch(self, input_ids, attention_mask): + ids = torch.as_tensor(input_ids, dtype=torch.long, device="cuda") + mask = torch.as_tensor(attention_mask, dtype=torch.bool, device="cuda") + with torch.inference_mode(): + logits = self.model(input_ids=ids, attention_mask=mask).logits + labels = ids[:, 1:].clone() + valid = mask[:, 1:] + labels.masked_fill_(~valid, -100) + loss = functional.cross_entropy( + logits[:, :-1, :].float().reshape(-1, logits.size(-1)), + labels.reshape(-1), + ignore_index=-100, + reduction="sum", + ) + return float(loss.item()), int(valid.sum().item()) + + return Float32LossScorer + + +def load_model(model_key: str): + import torch + from huggingface_hub import snapshot_download + from transformers import AutoModelForCausalLM, AutoTokenizer + + repo_id, revision, patterns = MODELS[model_key] + snapshot = Path( + snapshot_download(repo_id=repo_id, revision=revision, allow_patterns=patterns) + ) + tokenizer = AutoTokenizer.from_pretrained(snapshot, local_files_only=True) + tokenizer.padding_side = "right" + tokenizer.truncation_side = "right" + if tokenizer.pad_token_id is None: + tokenizer.pad_token = tokenizer.eos_token + model = AutoModelForCausalLM.from_pretrained( + snapshot, local_files_only=True, dtype=torch.float32 + ).to("cuda").eval() + return scorer_class()(model), tokenizer, snapshot + + +def sample_path(dataset: str, generator: str, steps: int) -> Path: + return ROOT / "results" / "samples" / dataset / generator / f"steps_{steps}" / "samples.jsonl" + + +def output_path(output_root: Path, model_key: str, dataset: str, generator: str, steps: int) -> Path: + return output_root / model_key / dataset / generator / f"steps_{steps}.json" + + +def valid_existing(path: Path, sample_sha: str, model_key: str) -> bool: + if not path.is_file(): + return False + try: + row = json.loads(path.read_text(encoding="utf-8")) + ppl = row["generative_perplexity"] + return ( + row["samples_sha256"] == sample_sha + and row["sample_count"] == SAMPLE_COUNT + and ppl["model_id"] == MODELS[model_key][0] + and ppl["model_revision"] == MODELS[model_key][1] + and ppl["forward_dtype"] == "float32" + and ppl["valid_token_count"] > 0 + and math.isfinite(ppl["perplexity"]) + ) + except (KeyError, TypeError, ValueError, OSError): + return False + + +def main() -> None: + import torch + import transformers + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=(*MODELS, "all"), default="all") + parser.add_argument("--output-root", type=Path, default=ROOT / "results" / "ppl_rescore_20260923") + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--smoke-samples", type=int, default=0) + args = parser.parse_args() + if args.batch_size < 1 or args.smoke_samples < 0: + parser.error("batch-size must be positive and smoke-samples must be nonnegative") + + # Validate the entire input matrix before loading a scorer or writing results. + inputs = {} + for dataset in DATASETS: + for generator in GENERATORS: + for steps in STEPS: + path = sample_path(dataset, generator, steps) + validate_samples(path, expected=SAMPLE_COUNT) + inputs[(dataset, generator, steps)] = (path, sha256_file(path)) + print(f"Validated {len(inputs)} sample files x {SAMPLE_COUNT} samples", flush=True) + + model_keys = tuple(MODELS) if args.model == "all" else (args.model,) + for model_key in model_keys: + pending = [ + (dataset, generator, steps) + for dataset in DATASETS + for generator in GENERATORS + for steps in STEPS + if args.smoke_samples or not valid_existing( + output_path(args.output_root, model_key, dataset, generator, steps), + inputs[(dataset, generator, steps)][1], model_key, + ) + ] + if not pending: + print(f"{model_key}: all {len(inputs)} outputs already validated", flush=True) + continue + scorer, tokenizer, snapshot = load_model(model_key) + print(f"{model_key}: loaded {snapshot}; pending={len(pending)}", flush=True) + for index, (dataset, generator, steps) in enumerate(pending, 1): + path, sample_sha = inputs[(dataset, generator, steps)] + with path.open(encoding="utf-8") as handle: + texts = [json.loads(line)["text"] for line in handle] + if args.smoke_samples: + texts = texts[:args.smoke_samples] + result = compute_gen_ppl( + texts, scorer, tokenizer, batch_size=args.batch_size, + max_length=CONTEXT_LENGTH, + model_revision=MODELS[model_key][1], + tokenizer_revision=MODELS[model_key][1], + ) + metric = { + **asdict(result), + "model_id": MODELS[model_key][0], + "tokenizer_id": MODELS[model_key][0], + "model_snapshot": str(snapshot), + "tokenizer_snapshot": str(snapshot), + "forward_dtype": "float32", + "loss_dtype": "float32", + "torch_version": torch.__version__, + "transformers_version": transformers.__version__, + "offline_scoring": True, + } + print( + f"{model_key} {index}/{len(pending)} {dataset}/{generator}/{steps} " + f"samples={len(texts)} tokens={result.valid_token_count} " + f"ppl={result.perplexity:.6f}", flush=True, + ) + if args.smoke_samples: + break + atomic_json_write( + output_path(args.output_root, model_key, dataset, generator, steps), + { + "schema_version": 1, + "evaluation": "independent_unconditional_ppl_rescore", + "samples": str(path), + "samples_sha256": sample_sha, + "sample_count": SAMPLE_COUNT, + "dataset": dataset, + "generator": generator, + "steps": steps, + "generative_perplexity": metric, + }, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_4gpu.sh b/scripts/run_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..b1a8ce50b88cb9833a6662dd53fddb49a860e0d9 --- /dev/null +++ b/scripts/run_4gpu.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Four-GPU local generation over the canonical matrix. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +SAMPLE_COUNT=1024 +SEED=42 +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--num-samples N] [--seed N] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --num-samples) SAMPLE_COUNT="$2"; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.matrix \ + --root "$DLB_ROOT" --output "$MATRIX" --sample-count "$SAMPLE_COUNT" --seed "$SEED" \ + || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage generate --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" "${DRY_RUN[@]}" diff --git a/scripts/run_conditional_4gpu.sh b/scripts/run_conditional_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..62e5d8d624b0993eedb0cf76dcec3d0e38099f87 --- /dev/null +++ b/scripts/run_conditional_4gpu.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Four-GPU local generation over the C64 conditional matrix. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_CONDITIONAL_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/conditional/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.conditional_matrix \ + --root "$DLB_ROOT" --output "$MATRIX" || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage generate --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" "${DRY_RUN[@]}" diff --git a/scripts/run_conditional_all.sh b/scripts/run_conditional_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..f2fd1031365c8274be51521e13e595a8b294952f --- /dev/null +++ b/scripts/run_conditional_all.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Server-side serial generation over the C64 conditional matrix. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_CONDITIONAL_MATRIX:-}" + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; shift 2 ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/conditional/matrix/generation.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs" +PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.conditional_matrix \ + --root "$DLB_ROOT" --output "$MATRIX" || exit $? + +failures="$DLB_ROOT/results/logs/conditional_generation_failures.tsv" +printf 'task_id\tmodel\tdataset\tsteps\texit_code\n' >"$failures" +status=0 +while IFS=$'\t' read -r task_id category model dataset steps sample_count seed environment adapter source provenance protocol conditioning_manifest conditioning_manifest_sha256 sample_dir metrics_path timing_path; do + [[ "$task_id" == "task_id" || -z "$task_id" ]] && continue + echo "CONDITIONAL GENERATE $task_id" + "$SCRIPT_DIR/run_conditional_one.sh" --model "$model" --dataset "$dataset" --steps "$steps" --seed "$seed" + exit_code=$? + if ((exit_code != 0)); then + printf '%s\t%s\t%s\t%s\t%s\n' "$task_id" "$model" "$dataset" "$steps" "$exit_code" >>"$failures" + status=1 + fi +done < <(tail -n +3 "$MATRIX") +if [[ ! -s "$failures" ]]; then rm -f "$failures"; fi +exit "$status" diff --git a/scripts/run_conditional_one.sh b/scripts/run_conditional_one.sh new file mode 100644 index 0000000000000000000000000000000000000000..0aabfff3cf48d5d7c3af7c1d1312e3002ae0ba4d --- /dev/null +++ b/scripts/run_conditional_one.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Run one C64 fixed-prefix conditional generation request without training. +set -u + +script_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$script_dir/.." && pwd -P)}" + +model="" +dataset="" +steps="" +seed="42" +device="" +results_root="" + +usage() { + echo "usage: $0 --model MODEL --dataset {lm1b,owt} --steps N [--seed N] [--device DEVICE] [--results-root PATH]" >&2 +} + +while (( $# )); do + case "$1" in + --model|--dataset|--steps|--seed|--device|--results-root) + if (( $# < 2 )); then usage; exit 2; fi + case "$1" in + --model) model="$2" ;; + --dataset) dataset="$2" ;; + --steps) steps="$2" ;; + --seed) seed="$2" ;; + --device) device="$2" ;; + --results-root) results_root="$2" ;; + esac + shift 2 ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +if [[ -z "$model" || -z "$dataset" || -z "$steps" ]]; then + usage; exit 2 +fi +if [[ ! "$steps" =~ ^[0-9]+$ || ! "$seed" =~ ^-?[0-9]+$ ]]; then + echo "steps and seed must be integers" >&2 + exit 2 +fi + +DLB_PYTHON="${DLB_PYTHON:-python}" +if environment="$(PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$DLB_PYTHON" -m dlb.runner \ + --root "$DLB_ROOT" --model "$model" --dataset "$dataset" --steps "$steps" \ + --num-samples 2048 --seed "$seed" --validate-only)"; then + : +else + exit $? +fi +if [[ ! "$environment" =~ ^[a-z][a-z0-9-]*$ ]]; then + echo "registry validation returned an unsafe environment name" >&2; exit 2 +fi + +runner_args=(--root "$DLB_ROOT" --model "$model" --dataset "$dataset" --steps "$steps" --seed "$seed" --generation-mode conditional_prefix) +if [[ -n "$device" ]]; then runner_args+=(--device "$device"); fi +if [[ -n "$results_root" ]]; then runner_args+=(--results-root "$results_root"); fi + +exec "${DLB_CONDA:-conda}" run -n "$environment" env \ + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" \ + python -m dlb.runner "${runner_args[@]}" diff --git a/scripts/run_elf.py b/scripts/run_elf.py new file mode 100644 index 0000000000000000000000000000000000000000..f521f8c74d18ce11feaf6e464502c4f8b18188ab --- /dev/null +++ b/scripts/run_elf.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Independent, offline ELF-B generation and synchronized CUDA timing. + +Uses the locked official model and sampler without editing upstream sources. +Outputs intentionally do not claim compatibility with the legacy DLB registry. +""" +from __future__ import annotations + +import argparse +from dataclasses import asdict +import json +import math +from pathlib import Path +import sys +import time + +from elf_common import (ROOT, SOURCE_COMMIT, asset, default_sampling, manifest, offline, + preflight_conditions, read_jsonl, require_server, sha256, source_path, write_json) + + +def parse_args(argv=None): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("mode", choices=["generate", "timing"]) + p.add_argument("--task", choices=["owt", "owt-prefix", "wmt14", "xsum"], required=True) + p.add_argument("--steps", type=int, required=True) + p.add_argument("--root", type=Path, default=ROOT) + p.add_argument("--input", type=Path) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--num-samples", type=int, default=1024, + help="Prompt count; 0 means the entire conditional input file") + p.add_argument("--batch-size", type=int, default=8) + p.add_argument("--seed", type=int, default=42) + p.add_argument("--sampler", choices=["ode", "sde"]) + p.add_argument("--cfg", type=float) + p.add_argument("--sc-cfg", type=float) + p.add_argument("--gamma", type=float) + p.add_argument("--compile", action="store_true") + p.add_argument("--diversity", action="store_true", help="C64: four extra completions for first 256 prompts") + p.add_argument("--timing-prompt-index", type=int, default=0) + p.add_argument("--warmups", type=int, default=5) + p.add_argument("--repeats", type=int, default=32) + args = p.parse_args(argv) + if args.steps < 1 or args.batch_size < 1 or args.num_samples < 0: + p.error("steps/batch-size must be positive and num-samples nonnegative") + if (args.task != "owt") != (args.input is not None): + p.error("--input is required exactly for conditional tasks") + if args.task == "owt" and args.num_samples == 0: + p.error("unconditional generation needs an explicit positive sample count") + if args.diversity and (args.task != "owt-prefix" or args.mode != "generate"): + p.error("--diversity is only for owt-prefix generation") + if args.mode == "timing" and args.batch_size != 1: + p.error("primary timing requires --batch-size 1") + if args.warmups < 1 or args.repeats < 2: + p.error("use at least one warmup and two measured repeats") + return args + + +def main(): + args = parse_args() + require_server() + offline() + root = args.root.resolve() + upstream = source_path(root) + sys.dont_write_bytecode = True + sys.path[:0] = [str(upstream / "src"), str(root / "src")] + import numpy as np + import torch + from transformers import AutoTokenizer + from configs.config import load_config_from_yaml, SamplingConfig + from modules.model import ELF_models + from modules.t5_encoder import get_encoder + from utils.encoder_utils import encode_text, build_self_attn_cond_masks + from utils.generation_utils import _generate_samples_single_batch, _dlm_decode_batch, shift_left, mask_after_eos + from utils.sampling_utils import get_sampling_steps + from dlb.timing import benchmark, cuda_runtime_metadata + + if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported(): + raise RuntimeError("A CUDA GPU supporting bf16 is required for the author precision policy") + if args.output.exists(): + raise FileExistsError(f"Use a fresh output directory: {args.output}") + device = torch.device("cuda") + torch.set_float32_matmul_precision("high") + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + generator = torch.Generator(device="cpu").manual_seed(args.seed) + checkpoint_task = "owt" if args.task == "owt-prefix" else args.task + config_task = "de-en" if checkpoint_task == "wmt14" else checkpoint_task + config_path = upstream / f"src/configs/training_configs/train_{config_task}_ELF-B.yml" + # The official loader resolves sampling YAML relative to upstream cwd. + import os + original_cwd = Path.cwd() + os.chdir(upstream) + try: + config = load_config_from_yaml(str(config_path)) + finally: + os.chdir(original_cwd) + config.use_bf16 = True + config.use_compile = args.compile + config.use_wandb = False + config.hf_repo_id = None + chosen = default_sampling(args.task, args.steps) + for key, value in (("sampling_method", args.sampler), ("cfg", args.cfg), + ("sc_cfg", args.sc_cfg), ("gamma", args.gamma)): + if value is not None: + chosen[key] = value + if args.task in ("owt", "owt-prefix") and chosen["cfg"] != 1: + raise ValueError("OWT checkpoint uses SC-CFG only; input CFG must be 1") + if any(not math.isfinite(chosen[k]) or chosen[k] < 0 for k in ("cfg", "sc_cfg", "gamma")): + raise ValueError("CFG/SC-CFG/gamma must be finite and nonnegative") + sc = SamplingConfig(sampling_method=chosen["sampling_method"], + num_sampling_steps=[args.steps], cfgs=[chosen["cfg"]], + self_cond_cfg_scales=[chosen["sc_cfg"]], time_schedule="logit_normal", + sde_gamma=chosen["gamma"]) + t5_path = asset(root, "t5") + tokenizer = AutoTokenizer.from_pretrained(str(t5_path), local_files_only=True) + rows = [] + input_manifest = None + if args.input: + args.input = args.input.resolve() + input_manifest = json.loads(args.input.with_suffix(".manifest.json").read_text()) + if input_manifest["sha256"] != sha256(args.input) or input_manifest["task"] != args.task: + raise ValueError("Input manifest does not match task/file") + rows = read_jsonl(args.input) + if not rows or len(rows) != input_manifest["count"] or len({r["id"] for r in rows}) != len(rows): + raise ValueError("Input counts/IDs do not match manifest or input is empty") + if args.num_samples > len(rows) and args.mode == "generate": + raise ValueError("Requested more prompts than input rows") + if args.mode == "timing": + if not 0 <= args.timing_prompt_index < len(rows): + raise ValueError("Timing prompt index out of range") + selected = [rows[0], rows[args.timing_prompt_index]] + else: + selected = rows[:args.num_samples] if args.num_samples else rows + print(f"Checking {len(selected)} conditions before loading ELF weights: {args.input}", flush=True) + tokens = preflight_conditions(selected, tokenizer, args.task, config.max_input_length, config.max_length) + for row, ids in zip(selected, tokens): + row["_elf_condition_ids"] = ids + args.output.mkdir(parents=True) + print(f"ELF {args.mode}: task={args.task}, steps={args.steps}, " + f"split={(input_manifest or {}).get('split', 'unconditional')}, output={args.output}", flush=True) + checkpoint_dir = asset(root, checkpoint_task) + assets = manifest(root) + checkpoint = checkpoint_dir / assets["assets"][checkpoint_task]["checkpoint"] + # Inference only: use the exact EMA state without constructing an optimizer. + payload = torch.load(checkpoint, map_location="cpu", weights_only=False) + if not payload.get("ema_params1"): + raise ValueError("The released EMA parameters are required") + model = ELF_models["ELF-B"]( + text_encoder_dim=512, max_length=config.max_length, + attn_drop=config.attn_dropout, proj_drop=config.proj_dropout, + num_time_tokens=config.num_time_tokens, num_self_cond_cfg_tokens=config.num_self_cond_cfg_tokens, + vocab_size=tokenizer.vocab_size, num_model_mode_tokens=config.num_model_mode_tokens, + bottleneck_dim=config.bottleneck_dim) + model.load_state_dict(payload["ema_params1"], strict=True) + model = model.to(device).eval().requires_grad_(False) + del payload + encoder = None + if args.input: + _, encoder = get_encoder(str(t5_path), torch.float32) + encoder = encoder.to(device).eval().requires_grad_(False) + def prepare(batch_rows): + # Encode only the visible source. References never enter the encoder. + batch = len(batch_rows) + ids = torch.full((batch, config.max_length), tokenizer.pad_token_id, device=device, dtype=torch.long) + lens = [] + for i, row in enumerate(batch_rows): + condition = row["_elf_condition_ids"] + ids[i, :len(condition)] = torch.tensor(condition, device=device) + lens.append(len(condition)) + mask = (np.arange(config.max_length)[None, :] < np.array(lens)[:, None]) + enc_mask, _, _ = build_self_attn_cond_masks(mask, mask, xp=np) + return (ids, torch.as_tensor(enc_mask, device=device), + torch.as_tensor(mask, dtype=torch.float32, device=device), + torch.tensor(lens, device=device)) + + def encode(prepared): + return encode_text(input_ids=prepared[0], attention_mask=prepared[1], encoder=encoder, + latent_mean=config.latent_mean, latent_std=config.latent_std).float() + + @torch.inference_mode() + def sample(batch, prepared=None, cached=None): + cond = cached if cached is not None else encode(prepared) if prepared is not None else None + times = get_sampling_steps(args.steps, "logit_normal", config.denoiser_p_mean, + config.denoiser_p_std, device=device, dtype=torch.float32) + z = torch.randn((batch, config.max_length, 512), device=device) * config.denoiser_noise_scale + latent = _generate_samples_single_batch(model=model, generator=generator, z=z, t_steps=times, + cond_seq=cond, cond_seq_mask=prepared[2] if prepared is not None else None, + config=config, sampling_config=sc, cfg_scale=chosen["cfg"], + self_cond_cfg_scale=chosen["sc_cfg"]) + predicted = _dlm_decode_batch(latent, model, 1.0, config, chosen["sc_cfg"]) + if prepared is not None: + # Fixed prefix is observed data, restored also in the discrete artifact. + predicted = torch.where(prepared[2].bool(), prepared[0], predicted) + predicted = shift_left(predicted, prepared[3], tokenizer.pad_token_id) + if args.task != "owt-prefix": + predicted = predicted[:, :config.max_length - config.max_input_length] + return predicted + + # A separate untimed preflight observes every real model input. Verify both + # clean prefix projection and the actual NFE, including the neural decoder. + pilot = prepare([rows[0]]) if rows else None + pilot_cond = encode(pilot) if pilot is not None else None + observed = {"calls": 0, "prefix_checked_calls": 0} + def check_forward(module, inputs, kwargs): + observed["calls"] += 1 + x = inputs[0] + if not torch.isfinite(x).all(): + raise ValueError("Non-finite ELF model input") + if pilot is not None: + mask = pilot[2].bool() + visible = x[..., :512][mask] + clean = pilot_cond[mask] + if not torch.equal(visible, clean) and not (chosen["cfg"] != 1 and torch.count_nonzero(visible) == 0): + raise ValueError("ELF prefix was not clamped before a model forward") + observed["prefix_checked_calls"] += 1 + hook = model.register_forward_pre_hook(check_forward, with_kwargs=True) + try: + sample(1, pilot, pilot_cond) + finally: + hook.remove() + expected_calls = args.steps * (1 if chosen["cfg"] == 1 else 2) + 1 + if observed["calls"] != expected_calls: + raise ValueError(f"Unexpected ELF NFE: {observed['calls']} != {expected_calls}") + # Pilot must not alter production sample assignment. + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + generator.manual_seed(args.seed) + if args.compile: + model = torch.compile(model) + + metadata = { + "schema": "dlb-elf-v1", "task": args.task, "model": "ELF-B", "source_commit": SOURCE_COMMIT, + "checkpoint_task": checkpoint_task, "checkpoint_selection": "ema_params1", + "checkpoint_sha256": sha256(checkpoint), "assets_manifest_sha256": sha256(root / "data/elf/assets.json"), + "source_config_sha256": sha256(config_path), "runner_sha256": sha256(Path(__file__)), + "helper_sha256": sha256(Path(__file__).with_name("elf_common.py")), + "lock_sha256": sha256(root / "artifacts/elf_lock.json"), + "steps": args.steps, "sampling": chosen, "seed": args.seed, "batch_size": args.batch_size, + "precision": "fp32 parameters/latents; official bf16 forward autocast; fp32 output heads", + "matmul_precision": "high", "compile": args.compile, + "model_canvas_t5_tokens": config.max_length, "input": input_manifest, + "protocol": "c64_text_t5_v1" if args.task == "owt-prefix" else "elf_native_v1", + "conditioning": "zero_shot_ood_native_projection" if args.task == "owt-prefix" else "task_trained" if rows else "none", + "condition_tokenization_policy": ( + "released_ids_or_raw_plus_eos_then_source_cap_v2" if args.task in ("wmt14", "xsum") + else "c64_raw_no_special_tokens_v1" if args.task == "owt-prefix" else "none"), + "elf_forward_calls_per_sample": args.steps * (1 if chosen["cfg"] == 1 else 2) + 1, + "condition_encoder_calls_per_sample": int(bool(rows)), + "untimed_preflight": observed, + **cuda_runtime_metadata(), + } + write_json(args.output / "request.json", metadata) + if args.mode == "timing": + if rows and not 0 <= args.timing_prompt_index < len(rows): + raise ValueError("Timing prompt index out of range") + prepared = prepare([rows[args.timing_prompt_index]]) if rows else None + cached = encode(prepared) if prepared is not None else None + results = {} + for name in (["sampler_cached_condition", "encoder_plus_sampler"] if rows else ["sampler"]): + def generate(): + return sample(1, prepared, cached if name == "sampler_cached_condition" else None) + torch.cuda.reset_peak_memory_stats() + result = benchmark(generate, torch.cuda.synchronize, warmups=args.warmups, + repeats=args.repeats, batch_size=1) + results[name] = {**asdict(result), "peak_allocated_bytes": torch.cuda.max_memory_allocated(), + "includes": ["noise_and_schedule", "denoising", "final_neural_decode", "argmax", "conditional_projection"]} + write_json(args.output / "timing.json", {**metadata, "prompt_index": args.timing_prompt_index, + "condition_length": prepared[3].tolist() if prepared is not None else [], "results": results}) + print(json.dumps(results, indent=2)) + return + count = (args.num_samples or len(rows)) if rows else args.num_samples + schedule = [(i, 0) for i in range(count)] + if args.diversity: + schedule.extend((i, completion) for completion in range(1, 5) for i in range(min(256, count))) + temporary = args.output / "samples.jsonl.tmp" + start = time.perf_counter() + with temporary.open("w") as handle: + for offset in range(0, len(schedule), args.batch_size): + shard = schedule[offset:offset + args.batch_size] + batch_rows = [rows[i] for i, _ in shard] if rows else [] + prepared = prepare(batch_rows) if rows else None + predicted = sample(len(shard), prepared) + predicted = mask_after_eos(predicted, tokenizer.eos_token_id, tokenizer.pad_token_id).cpu().tolist() + for j, ((i, completion), ids) in enumerate(zip(shard, predicted)): + if args.task == "owt-prefix": + ids = ids[:config.max_length - int(prepared[3][j].item())] + text = tokenizer.decode(ids, skip_special_tokens=True) + row = {"id": offset + j, "prompt_id": i, "completion_id": completion, + "generated": text, "token_ids": ids} + if rows: + row.update(input=rows[i]["input"], reference=rows[i]["output"], source_id=rows[i]["id"], + condition_t5_length=int(prepared[3][j].item())) + handle.write(json.dumps(row, ensure_ascii=False) + "\n") + print(f"{offset + len(shard)}/{len(schedule)} samples", flush=True) + temporary.replace(args.output / "samples.jsonl") + write_json(args.output / "generation.json", {**metadata, "sample_count": len(schedule), + "prompt_count": count, "diversity": args.diversity, + "samples_sha256": sha256(args.output / "samples.jsonl"), + "wall_seconds_including_io_not_primary_latency": time.perf_counter() - start}) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_elf_owt_c64r512.sh b/scripts/run_elf_owt_c64r512.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f1461f95f04782a016e7998bfa0823417dcdf39 --- /dev/null +++ b/scripts/run_elf_owt_c64r512.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Public ELF OWT conditional run with a text-level 64-prompt/512-response contract. +set -euo pipefail + +root="${DLB_ROOT:-$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)}" +python_bin="${ELF_PYTHON:-python}" +steps_list=(1 2 4 8 16 32 1024) +mode="${1:-generate}" +case "$mode" in generate|evaluate|timing) ;; *) echo "usage: $0 {generate|evaluate|timing}" >&2; exit 2 ;; esac + +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 WANDB_MODE=disabled PYTHONDONTWRITEBYTECODE=1 +quality_root="${ELF_R512_RESULTS:-$root/results/elf-owt-c64-r512}" +input="$root/data/elf/owt-prefix.jsonl" + +if [[ "$mode" == generate ]]; then + for steps in "${steps_list[@]}"; do + output="$quality_root/quality/owt-prefix/c64-text-t5/steps_${steps}/seed_42/compile_0" + if [[ -e "$output" ]]; then + echo "Refusing to overwrite existing output: $output" >&2 + exit 2 + fi + env ELF_PYTHON="$python_bin" DLB_ROOT="$root" \ + "$python_bin" "$root/scripts/run_elf.py" generate \ + --root "$root" --task owt-prefix --steps "$steps" --seed 42 \ + --num-samples 1024 --batch-size 8 --input "$input" \ + --output "$output" --diversity + done +elif [[ "$mode" == evaluate ]]; then + "$python_bin" "$root/scripts/evaluate_elf_owt_c64r512.py" \ + --root "$root" --input-root "$quality_root" \ + --output "$quality_root/evaluation-r512" --response-tokens 512 --batch-size 8 +else + : "${CUDA_VISIBLE_DEVICES:?Set CUDA_VISIBLE_DEVICES to one physical GPU before timing}" + timing_root="${ELF_R512_TIMING_RESULTS:-$root/results/elf-timing-c64r512-$(date -u +%Y%m%dT%H%M%S)-$$}" + for steps in "${steps_list[@]}"; do + output="$timing_root/timing/owt-prefix/c64-text-t5/steps_${steps}/seed_42/compile_0/prompt_0" + env ELF_PYTHON="$python_bin" DLB_ROOT="$root" \ + "$python_bin" "$root/scripts/run_elf.py" timing \ + --root "$root" --task owt-prefix --steps "$steps" --seed 42 \ + --batch-size 1 --warmups 5 --repeats 32 --timing-prompt-index 0 \ + --input "$input" --output "$output" + done + "$python_bin" "$root/scripts/summarize_elf.py" --results "$timing_root" \ + --output "$timing_root/summary.csv" + echo "TIMING SUMMARY: $timing_root/summary.csv" +fi diff --git a/scripts/run_elf_remaining.py b/scripts/run_elf_remaining.py new file mode 100644 index 0000000000000000000000000000000000000000..e1b47c43e68e21dddd6bac42edf073b93bd71f4b --- /dev/null +++ b/scripts/run_elf_remaining.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""Resume the ELF quality matrix on independent single-GPU workers (no timing).""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, wait +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import queue +import signal +import subprocess +import sys +import threading +import time + +from elf_common import ROOT, SOURCE_COMMIT, default_sampling, manifest, read_jsonl, sha256, write_json +from inspect_elf_progress import inspect_run, OWT_MAIN_STEPS + +EOS_POLICY = "released_ids_or_raw_plus_eos_then_source_cap_v2" +TASKS = ("owt", "owt-prefix", "wmt14", "xsum") + + +@dataclass(frozen=True) +class Cell: + task: str + split: str + steps: int + count: int + input_path: str | None = None + input_sha256: str | None = None + + @property + def key(self): + return f"{self.task}/{self.split}/steps_{self.steps}/seed_42/compile_0" + + +def build_matrix(root, tasks): + cells, blocked = [], [] + for task in tasks: + if task == "owt": + cells.extend(Cell(task, "unconditional", s, 1024) for s in OWT_MAIN_STEPS) + continue + splits = (("c64-text-t5", OWT_MAIN_STEPS, 1024),) if task == "owt-prefix" else ( + ("validation", (1, 2, 4, 8, 16, 32, 64), 3000 if task == "wmt14" else 11332), + ("test", (32, 64), 3003 if task == "wmt14" else 11334), + ) + for split, steps, expected_count in splits: + path = root / "data/elf" / ("owt-prefix.jsonl" if task == "owt-prefix" else f"{task}-{split}.jsonl") + try: + meta = json.loads(path.with_suffix(".manifest.json").read_text()) + digest = sha256(path) + rows = read_jsonl(path) + if (meta["task"] != task or meta["sha256"] != digest or meta["count"] != expected_count + or len(rows) != expected_count or len({r["id"] for r in rows}) != expected_count): + raise ValueError("input task/count/hash/IDs mismatch") + if task == "owt-prefix": + if meta.get("protocol") != "c64_text_t5_v1": + raise ValueError("prefix protocol mismatch") + source = root / "data/conditional/owt-c64/prompts.jsonl" + if meta.get("source_sha256") != sha256(source): + raise ValueError("prefix source changed; prepare prefix again") + elif meta.get("split") != split: + raise ValueError("input split mismatch") + except (OSError, KeyError, TypeError, ValueError) as error: + blocked.append({"task": task, "split": split, "status": "blocked_input", "reason": str(error)}) + continue + cells.extend(Cell(task, split, s, expected_count, str(path), digest) for s in steps) + return sorted(cells, key=lambda c: (c.steps, c.task, c.split)), blocked + + +def matches_request(request, cell, asset_hash, assets): + checkpoint_task = "owt" if cell.task == "owt-prefix" else cell.task + record = assets[checkpoint_task] + expected = { + "task": cell.task, "steps": cell.steps, "seed": 42, "compile": False, "batch_size": 8, + "source_commit": SOURCE_COMMIT, "assets_manifest_sha256": asset_hash, + "checkpoint_task": checkpoint_task, "checkpoint_selection": "ema_params1", + "checkpoint_sha256": record["sha256"][record["checkpoint"]], + "sampling": default_sampling(cell.task, cell.steps), + "model_canvas_t5_tokens": 1088 if cell.task == "xsum" else 128 if cell.task == "wmt14" else 1024, + "protocol": "c64_text_t5_v1" if cell.task == "owt-prefix" else "elf_native_v1", + } + if any(request.get(key) != value for key, value in expected.items()): + return False + if cell.task in ("wmt14", "xsum") and request.get("condition_tokenization_policy") != EOS_POLICY: + return False + return (request.get("input") or {}).get("sha256") == cell.input_sha256 + + +def reusable_result(root, cell, asset_hash, assets): + candidates, rejected = [], [] + for path in sorted((root / "results").glob(f"elf*/quality/{cell.key}")): + try: + request = json.loads((path / "request.json").read_text()) + if not matches_request(request, cell, asset_hash, assets): + continue + row = inspect_run(path) + if row["status"] not in ("quality_valid", "quality_invalid", "needs_evaluation"): + rejected.append({"run": str(path), "reason": row.get("reason", row["status"])}) + continue + gen = json.loads((path / "generation.json").read_text()) + keys = ("sampling", "input", "assets_manifest_sha256", "checkpoint_sha256", "source_commit", + "condition_tokenization_policy", "batch_size", "protocol") + if any(gen.get(k) != request.get(k) for k in keys): + raise ValueError("request/generation provenance differs") + diversity = cell.task == "owt-prefix" + total = cell.count + (min(256, cell.count) * 4 if diversity else 0) + if gen["prompt_count"] != cell.count or gen["sample_count"] != total or bool(gen.get("diversity")) != diversity: + continue # e.g. 1000-row sanity is not a 1024-row primary run. + if row["status"] == "quality_invalid": + reason = row.get("reason") or "" + if not reason.startswith("At least one completion has fewer than two GPT-2 tokens;"): + rejected.append({"run": str(path), "reason": "unrecognized invalid-quality reason"}) + continue + candidates.append(row) + except (OSError, ValueError, KeyError, TypeError) as error: + rejected.append({"run": str(path), "reason": str(error)}) + # A valid score and a recorded invalid score for one exact cell conflict; + # never silently choose the better run. + completed = [r for r in candidates if r["status"] != "needs_evaluation"] + if len({r["status"] for r in completed}) > 1: + raise ValueError(f"Conflicting completed results for {cell.key}; inspect runs manually") + return (completed or candidates or [None])[0], rejected + + +def fresh_path(output_root, cell): + for attempt in range(10000): + base = output_root if attempt == 0 else output_root.with_name(output_root.name + f"-retry{attempt}") + path = base / "quality" / cell.key + if not path.exists(): + return path + raise RuntimeError("Too many existing retry directories") + + +def commands(root, python, cell, path, generate): + result = [] + if generate: + cmd = [python, "-B", str(root / "scripts/run_elf.py"), "generate", "--root", str(root), + "--task", cell.task, "--steps", str(cell.steps), "--seed", "42", "--batch-size", "8", + "--num-samples", str(cell.count), "--output", str(path)] + if cell.input_path: + cmd += ["--input", cell.input_path] + if cell.task == "owt-prefix": + cmd += ["--diversity"] + result.append(("generate", cmd)) + result.append(("evaluate", [python, "-B", str(root / "scripts/evaluate_elf.py"), "--root", str(root), + "--run", str(path), "--batch-size", "8"])) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--output-root", type=Path) + parser.add_argument("--tasks", nargs="+", choices=TASKS, default=list(TASKS)) + parser.add_argument("--gpus", nargs="+", default=["0", "1"]) + parser.add_argument("--plan", action="store_true", help="Read-only preview; no GPU queries, writes or inference") + args = parser.parse_args() + root = args.root.resolve() + output_root = (args.output_root or root / "results/elf-gpu01").resolve() + if output_root.parent != root / "results" or not output_root.name.startswith("elf"): + parser.error("output-root must be an elf* directory directly under ROOT/results") + if len(set(args.tasks)) != len(args.tasks): + parser.error("duplicate tasks") + assets = manifest(root)["assets"] + asset_hash = sha256(root / "data/elf/assets.json") + cells, blocked = build_matrix(root, args.tasks) + jobs, reports = [], [] + for cell in cells: + existing, rejected = reusable_result(root, cell, asset_hash, assets) + action = ("skip" if existing["status"] != "needs_evaluation" else "evaluate") if existing else "generate+evaluate" + path = Path(existing["run"]) if existing else fresh_path(output_root, cell) + row = {"cell": asdict(cell), "action": action, "run": str(path), "rejected_candidates": rejected, + "status": existing["status"] if action == "skip" else "pending"} + reports.append(row) + print(f"{action:18} {cell.key} -> {path}", flush=True) + if action != "skip": + jobs.append((cell, row)) + for row in blocked: + print(f"BLOCKED {row['task']}/{row['split']}: {row['reason']}", flush=True) + print(f"Plan: {len(cells)} cells, {len(jobs)} queued, {len(blocked)} blocked input groups; no timing.", flush=True) + if args.plan: + return 2 if blocked else 0 + if sys.platform != "linux": + parser.error("GPU execution is server-only; use --plan for a read-only preview") + if not jobs: + print("Nothing to run. Existing invalid-quality results remain invalid, not successful PPL points.") + return 2 if blocked else 0 + + import fcntl + from contextlib import ExitStack + with ExitStack() as stack: + gpu_records = [] + lock_dir = root / "results/elf-queue-locks" + lock_dir.mkdir(parents=True, exist_ok=True) + for gpu in args.gpus: + line = subprocess.check_output(["nvidia-smi", "-i", gpu, "--query-gpu=uuid,name", "--format=csv,noheader"], text=True).strip() + uuid, name = [s.strip() for s in line.split(",", 1)] + if not uuid.startswith("GPU-") or any(r["uuid"] == uuid for r in gpu_records): + raise ValueError("GPU selections must resolve to distinct physical GPUs") + handle = stack.enter_context((lock_dir / f"{uuid}.lock").open("a")) + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + gpu_records.append({"physical_selection": gpu, "uuid": uuid, "name": name}) + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S") + f"-{os.getpid()}" + log_dir = output_root / "logs" / stamp + log_dir.mkdir(parents=True) + report_path = log_dir / "queue.json" + report = {"schema": "elf-quality-queue-v1", "gpus": gpu_records, "runs": reports, "blocked": blocked, + "assets_manifest_sha256": asset_hash, "timing": False} + mutex, stop = threading.Lock(), threading.Event() + processes = {} + work = queue.Queue() + for job in jobs: + work.put(job) + + def publish(): + write_json(report_path, report) + + def worker(gpu): + env = dict(os.environ, CUDA_VISIBLE_DEVICES=gpu["uuid"], PYTHONDONTWRITEBYTECODE="1", + HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", WANDB_MODE="disabled") + while not stop.is_set(): + try: + cell, row = work.get_nowait() + except queue.Empty: + break + path = Path(row["run"]) + with mutex: + row.update(status="running", gpu=gpu["uuid"]) + publish() + print(f"START GPU {gpu['physical_selection']} {cell.key}", flush=True) + try: + for stage, command in commands(root, sys.executable, cell, path, row["action"] == "generate+evaluate"): + if stop.is_set(): + raise RuntimeError("queue interrupted") + log = log_dir / (cell.key.replace("/", "__") + f"__{stage}.log") + with log.open("w") as stream: + stream.write(json.dumps(command) + "\n") + stream.flush() + with mutex: + if stop.is_set(): + raise RuntimeError("queue interrupted") + proc = subprocess.Popen(command, cwd=root, env=env, stdout=stream, + stderr=subprocess.STDOUT, start_new_session=True) + processes[gpu["uuid"]] = proc + code = proc.wait() + with mutex: + processes.pop(gpu["uuid"], None) + with mutex: + row[stage + "_log"] = str(log) + if code != 0: + checked, _ = reusable_result(root, cell, asset_hash, assets) + if stage == "evaluate" and checked and checked["run"] == str(path) and checked["status"] == "quality_invalid": + break + raise RuntimeError(f"{stage} exit={code}; see {log}") + checked, _ = reusable_result(root, cell, asset_hash, assets) + if not checked or checked["run"] != str(path) or checked["status"] not in ("quality_valid", "quality_invalid"): + raise RuntimeError("finished process did not publish a verified quality result") + with mutex: + row.update(status=checked["status"], reason=checked.get("reason")) + except Exception as error: + with mutex: + row.update(status="failed", reason=str(error)) + finally: + with mutex: + publish() + print(f"DONE GPU {gpu['physical_selection']} {cell.key}: {row['status']}", flush=True) + work.task_done() + + publish() + print(f"Logs and live queue report: {log_dir}", flush=True) + pool = ThreadPoolExecutor(max_workers=len(gpu_records)) + futures = [pool.submit(worker, gpu) for gpu in gpu_records] + interrupted = False + try: + pending = futures + while pending: + _, pending = wait(pending, timeout=30) + if pending: + with mutex: + running = [r['cell'] for r in reports if r['status'] == 'running'] + print(f"RUNNING {running}; remaining queued={work.qsize()}", flush=True) + for future in futures: + future.result() + except KeyboardInterrupt: + interrupted = True + stop.set() + with mutex: + active = list(processes.values()) + for proc in active: + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + for proc in active: + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + finally: + pool.shutdown(wait=True) + with mutex: + publish() + counts = {s: sum(r["status"] == s for r in reports) for s in sorted({r["status"] for r in reports})} + print(f"Final statuses: {counts}; blocked input groups={len(blocked)}; report={report_path}", flush=True) + return 130 if interrupted else 2 if blocked or any(r["status"] in ("failed", "pending", "running") for r in reports) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_elf_suite.sh b/scripts/run_elf_suite.sh new file mode 100644 index 0000000000000000000000000000000000000000..6c115e96841bd7ed29a6bd58ee7e6c77a6b4e2c0 --- /dev/null +++ b/scripts/run_elf_suite.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Independent ELF cells; run one process per GPU. No downloads/training here. +set -euo pipefail +root="${DLB_ROOT:-$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)}" +python_bin="${ELF_PYTHON:-python}" +stage="${1:?usage: run_elf_suite.sh STAGE TASK}" +task="${2:?task required}" +case "$stage" in generate|timing|evaluate|smoke) ;; *) exit 2 ;; esac +case "$task" in owt|owt-prefix|wmt14|xsum) ;; *) exit 2 ;; esac +seed="${ELF_SEED:-42}" +split="${ELF_SPLIT:-validation}" +compiled="${ELF_COMPILE:-0}" +steps_default="1 2 4 8 16 32 1024" +if [[ "$task" == wmt14 || "$task" == xsum ]]; then steps_default="1 2 4 8 16 32 64"; fi +read -r -a steps_list <<< "${ELF_STEPS:-$steps_default}" +input_args=() +count="${ELF_COUNT:-1024}" +if [[ "$task" == wmt14 || "$task" == xsum ]]; then + input_args=(--input "$root/data/elf/$task-$split.jsonl") + count="${ELF_COUNT:-0}" +elif [[ "$task" == owt-prefix ]]; then + input_args=(--input "$root/data/elf/owt-prefix.jsonl") + split="c64-text-t5" +else + split="unconditional" +fi +compile_args=() +if [[ "$compiled" == 1 ]]; then compile_args=(--compile); fi +output_base="${ELF_RESULTS:-$root/results/elf}" +for steps in "${steps_list[@]}"; do + stem="$task/$split/steps_$steps/seed_$seed/compile_$compiled" + case "$stage" in + generate|smoke) + kind=quality + diversity_args=() + if [[ "$task" == owt-prefix ]]; then diversity_args=(--diversity); fi + if [[ "$stage" == smoke ]]; then kind=smoke; count=2; fi + "$python_bin" "$root/scripts/run_elf.py" generate --root "$root" --task "$task" --steps "$steps" \ + --seed "$seed" --num-samples "$count" --batch-size "${ELF_BATCH_SIZE:-8}" \ + --output "$output_base/$kind/$stem" ${input_args[@]+"${input_args[@]}"} ${compile_args[@]+"${compile_args[@]}"} ${diversity_args[@]+"${diversity_args[@]}"} + ;; + timing) + "$python_bin" "$root/scripts/run_elf.py" timing --root "$root" --task "$task" --steps "$steps" \ + --seed "$seed" --batch-size 1 --timing-prompt-index "${ELF_TIMING_PROMPT:-0}" \ + --output "$output_base/timing/$stem/prompt_${ELF_TIMING_PROMPT:-0}" ${input_args[@]+"${input_args[@]}"} ${compile_args[@]+"${compile_args[@]}"} + ;; + evaluate) + "$python_bin" "$root/scripts/evaluate_elf.py" --root "$root" \ + --run "$output_base/quality/$stem" --batch-size "${ELF_EVAL_BATCH_SIZE:-8}" + ;; + esac +done diff --git a/scripts/run_elf_timing_group.sh b/scripts/run_elf_timing_group.sh new file mode 100644 index 0000000000000000000000000000000000000000..59602a9eb67bfd1fdb32ba19e29178950f27bb96 --- /dev/null +++ b/scripts/run_elf_timing_group.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# One serial timing group per physical GPU; every invocation uses fresh outputs. +set -euo pipefail +gpu="${1:?usage: run_elf_timing_group.sh GPU GROUP [--plan]}" +group="${2:?group required}" +plan="${3:-}" +[[ "$gpu" =~ ^[0-9]+$ ]] || { echo 'GPU must be a physical numeric index' >&2; exit 2; } +[[ "$group" == owt || "$group" == seq2seq ]] || { echo 'group must be owt or seq2seq' >&2; exit 2; } +[[ -z "$plan" || "$plan" == --plan ]] || exit 2 +root="${DLB_ROOT:-$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)}" +python_bin="${ELF_PYTHON:-python}" + +matrix() { + local task steps + if [[ "$group" == owt ]]; then + for task in owt owt-prefix; do + for steps in 1 2 4 8 16 32 1024; do + printf '%s validation %s\n' "$task" "$steps" + done + done + else + for task in wmt14 xsum; do + for steps in 1 2 4 8 16 32 64; do + printf '%s validation %s\n' "$task" "$steps" + done + for steps in 32 64; do + printf '%s test %s\n' "$task" "$steps" + done + done + fi +} +if [[ "$plan" == --plan ]]; then + echo "GPU=$gpu group=$group; batch=1 seed=42 compile=0 warmups=5 repeats=32 prompt=0" + echo 'OWT split is unconditional; owt-prefix split is c64-text-t5 (suite resolves these).' + matrix + exit 0 +fi +[[ "$(uname -s)" == Linux ]] || { echo 'Timing must run on the Linux GPU server' >&2; exit 2; } +uuid="$(nvidia-smi -i "$gpu" --query-gpu=uuid --format=csv,noheader | tr -d '[:space:]')" +[[ "$uuid" == GPU-* ]] || { echo 'Cannot resolve GPU UUID' >&2; exit 2; } +export CUDA_VISIBLE_DEVICES="$uuid" DLB_ROOT="$root" ELF_PYTHON="$python_bin" +export ELF_SEED=42 ELF_COMPILE=0 ELF_TIMING_PROMPT=0 PYTHONDONTWRITEBYTECODE=1 +export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 WANDB_MODE=disabled +export ELF_RESULTS="$root/results/elf-timing-gpu${gpu}-$(date -u +%Y%m%dT%H%M%S)-$$" +mkdir -p "$ELF_RESULTS/logs" "$ELF_RESULTS/environment" +echo "TIMING OUTPUT: $ELF_RESULTS" +printf 'physical_gpu=%s\nuuid=%s\ngroup=%s\n' "$gpu" "$uuid" "$group" > "$ELF_RESULTS/environment/binding.txt" +git -C "$root" rev-parse HEAD > "$ELF_RESULTS/environment/project-commit.txt" +nvidia-smi -i "$uuid" > "$ELF_RESULTS/environment/gpu-start.txt" + +while read -r task split steps; do + # Check between cells; this cannot reserve the device against other users. + pids="$(nvidia-smi -i "$uuid" --query-compute-apps=pid --format=csv,noheader,nounits)" + if [[ -n "${pids//[[:space:]]/}" ]]; then + echo "GPU $gpu has compute processes ($pids); stop before $task/$split/$steps. Do not kill others." >&2 + exit 2 + fi + tag="${task}_${split}_s${steps}" + nvidia-smi -i "$uuid" > "$ELF_RESULTS/environment/${tag}-before.txt" + echo "TIMING GPU $gpu: $task/$split/$steps" + env ELF_STEPS="$steps" ELF_SPLIT="$split" \ + bash "$root/scripts/run_elf_suite.sh" timing "$task" 2>&1 | tee "$ELF_RESULTS/logs/$tag.log" + nvidia-smi -i "$uuid" > "$ELF_RESULTS/environment/${tag}-after.txt" +done < <(matrix) + +"$python_bin" "$root/scripts/summarize_elf.py" \ + --results "$ELF_RESULTS" --output "$ELF_RESULTS/summary.csv" +echo "FINISHED group=$group GPU=$gpu; $ELF_RESULTS/summary.csv" diff --git a/scripts/run_one.sh b/scripts/run_one.sh new file mode 100644 index 0000000000000000000000000000000000000000..afe37f55413f9e1833846ed198faabea0ed9fd8f --- /dev/null +++ b/scripts/run_one.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Run one registry-backed request without interpreting user input as shell code. +set -u + +script_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$script_dir/.." && pwd -P)}" + +model="" +dataset="" +steps="" +num_samples="" +seed="" +device="" +results_root="" + +usage() { + echo "usage: $0 --model MODEL --dataset {lm1b,owt} --steps N --num-samples N --seed N [--device DEVICE] [--results-root PATH]" >&2 +} + +while (( $# )); do + case "$1" in + --model|--dataset|--steps|--num-samples|--seed|--device|--results-root) + if (( $# < 2 )); then usage; exit 2; fi + case "$1" in + --model) model="$2" ;; + --dataset) dataset="$2" ;; + --steps) steps="$2" ;; + --num-samples) num_samples="$2" ;; + --seed) seed="$2" ;; + --device) device="$2" ;; + --results-root) results_root="$2" ;; + esac + shift 2 ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +if [[ -z "$model" || -z "$dataset" || -z "$steps" || -z "$num_samples" || -z "$seed" ]]; then + usage; exit 2 +fi +if [[ ! "$steps" =~ ^[0-9]+$ || ! "$num_samples" =~ ^[1-9][0-9]*$ || ! "$seed" =~ ^-?[0-9]+$ ]]; then + echo "steps, num-samples, and seed must be integers; num-samples must be positive" >&2 + exit 2 +fi + +DLB_PYTHON="${DLB_PYTHON:-python}" +if environment="$(PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$DLB_PYTHON" -m dlb.runner \ + --root "$DLB_ROOT" --model "$model" --dataset "$dataset" --steps "$steps" \ + --num-samples "$num_samples" --seed "$seed" --validate-only)"; then + : +else + exit $? +fi +if [[ ! "$environment" =~ ^[a-z][a-z0-9-]*$ ]]; then + echo "registry validation returned an unsafe environment name" >&2; exit 2 +fi + +runner_args=(--root "$DLB_ROOT" --model "$model" --dataset "$dataset" --steps "$steps" --num-samples "$num_samples" --seed "$seed") +if [[ -n "$device" ]]; then runner_args+=(--device "$device"); fi +if [[ -n "$results_root" ]]; then runner_args+=(--results-root "$results_root"); fi +exec "${DLB_CONDA:-conda}" run -n "$environment" env \ + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" \ + python -m dlb.runner "${runner_args[@]}" diff --git a/scripts/smoke_4gpu.sh b/scripts/smoke_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..9b909c710fc1fceab21ac38883c7ad0b909e9ce5 --- /dev/null +++ b/scripts/smoke_4gpu.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Four-GPU local smoke pass for GPU IDs 0,1,2,3 by default. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +SEED=42 +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--seed N] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/matrix/smoke.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.matrix \ + --root "$DLB_ROOT" --output "$MATRIX" --sample-count 1 --seed "$SEED" || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage smoke --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" "${DRY_RUN[@]}" diff --git a/scripts/smoke_all.sh b/scripts/smoke_all.sh new file mode 100644 index 0000000000000000000000000000000000000000..54583e3c0bc62a1474c277af21b6b06286d82053 --- /dev/null +++ b/scripts/smoke_all.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Server-side one-sample smoke pass. Its sample_count=1 identity cannot satisfy +# the production 1,024-sample aggregation contract. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_ROOT}/results/matrix/smoke.tsv" +if [[ "${1:-}" == "--help" ]]; then + printf 'Usage: %s\n' "$0" + exit 0 +fi +mkdir -p "$DLB_ROOT/results/matrix" "$DLB_ROOT/results/logs" +PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.matrix \ + --root "$DLB_ROOT" --output "$MATRIX" --sample-count 1 --seed 42 || exit $? + +status=0 +while IFS=$'\t' read -r task_id category model dataset steps sample_count seed environment adapter source provenance sample_dir metrics_path timing_path; do + [[ "$task_id" == "task_id" || -z "$task_id" ]] && continue + echo "SMOKE $task_id" + "$SCRIPT_DIR/run_one.sh" --model "$model" --dataset "$dataset" --steps "$steps" \ + --num-samples 1 --seed "$seed" --results-root "$DLB_ROOT/results/smoke" + if (($? != 0)); then status=1; fi +done < <(tail -n +3 "$MATRIX") +exit "$status" diff --git a/scripts/smoke_conditional_4gpu.sh b/scripts/smoke_conditional_4gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..a55197dcd4c4c17c9c717d4d1cd08f1d0843ec2b --- /dev/null +++ b/scripts/smoke_conditional_4gpu.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Four-GPU conditional smoke pass. It still uses the full C64 schedule. +set -uo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="${DLB_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd -P)}" +PYTHON_BIN="${DLB_PYTHON:-python}" +MATRIX="${DLB_CONDITIONAL_MATRIX:-}" +CUSTOM_MATRIX=0 +if [[ -n "$MATRIX" ]]; then CUSTOM_MATRIX=1; fi +GPUS="${DLB_GPUS:-0,1,2,3}" +MAX_JOBS="${DLB_MAX_JOBS:-4}" +DRY_RUN=() + +usage() { + printf 'Usage: %s [--root PATH] [--matrix PATH] [--gpus LIST] [--max-jobs N] [--dry-run]\n' "$0" +} + +while (($#)); do + case "$1" in + --root) DLB_ROOT="$2"; shift 2 ;; + --matrix) MATRIX="$2"; CUSTOM_MATRIX=1; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --max-jobs) MAX_JOBS="$2"; shift 2 ;; + --dry-run) DRY_RUN=(--dry-run); shift ;; + --help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac +done + +DLB_ROOT="$(CDPATH= cd -- "$DLB_ROOT" && pwd -P)" +if [[ -z "$MATRIX" ]]; then MATRIX="$DLB_ROOT/results/conditional/matrix/smoke.tsv"; fi +export DLB_ROOT +mkdir -p "$DLB_ROOT/results/conditional/matrix" "$DLB_ROOT/results/logs" +if (( CUSTOM_MATRIX )); then + if [[ ! -f "$MATRIX" ]]; then + echo "custom matrix does not exist: $MATRIX" >&2 + exit 2 + fi +else + PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.conditional_matrix \ + --root "$DLB_ROOT" --output "$MATRIX" || exit $? +fi +exec env PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" -m dlb.gpu_matrix \ + --root "$DLB_ROOT" --stage smoke --matrix "$MATRIX" \ + --gpus "$GPUS" --max-jobs "$MAX_JOBS" "${DRY_RUN[@]}" diff --git a/scripts/summarize_elf.py b/scripts/summarize_elf.py new file mode 100644 index 0000000000000000000000000000000000000000..36289fd68a53c47fc04e3374fbafedca5998ca64 --- /dev/null +++ b/scripts/summarize_elf.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Inventory ELF results without treating missing metrics/timing as successes.""" +import argparse +import csv +import json +from pathlib import Path + +from elf_common import ROOT, sha256 + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--results", type=Path, default=ROOT / "results/elf") +p.add_argument("--output", type=Path, default=ROOT / "results/elf/summary.csv") +args = p.parse_args() +records = [] +for path in sorted(args.results.rglob("request.json")): + request = json.loads(path.read_text()) + row = {"run": str(path.parent), "task": request["task"], "steps": request["steps"], + "seed": request["seed"], "compile": request["compile"], + "sampling": json.dumps(request["sampling"], sort_keys=True), + "split": (request.get("input") or {}).get("split", ""), + "status": "incomplete"} + for key in ("checkpoint_sha256", "source_commit", "gpu_name", "precision", "model_canvas_t5_tokens", + "elf_forward_calls_per_sample", "protocol", "batch_size", "condition_tokenization_policy"): + row[key] = request.get(key, "") + row["input_sha256"] = (request.get("input") or {}).get("sha256", "") + metrics_path, timing_path = path.parent / "metrics.json", path.parent / "timing.json" + if metrics_path.exists(): + m = json.loads(metrics_path.read_text()) + row["status"] = "quality_valid" if m.get("valid") else "quality_invalid" + if m["samples_sha256"] != sha256(path.parent / "samples.jsonl"): + raise ValueError(f"Samples changed: {path.parent}") + if m["generation_manifest_sha256"] != sha256(path.parent / "generation.json"): + raise ValueError(f"Generation manifest changed: {path.parent}") + for k, v in m["metrics"].items(): + if isinstance(v, (float, int)): + row[k] = v + elif isinstance(v, dict) and "perplexity" in v: + row[k] = v["perplexity"] + if timing_path.exists(): + t = json.loads(timing_path.read_text()) + row["status"] = "timing_only" + for k, v in t["results"].items(): + row[k + "_seconds"] = v["seconds_per_sample"] + row[k + "_std_seconds"] = v["standard_deviation_seconds"] + records.append(row) +args.output.parent.mkdir(parents=True, exist_ok=True) +with args.output.open("w") as f: + writer = csv.DictWriter(f, fieldnames=sorted({k for row in records for k in row})) + writer.writeheader() + writer.writerows(records) +print(f"Wrote {len(records)} run records to {args.output}; inventory only, not a completeness certificate") diff --git a/scripts/summarize_timing.py b/scripts/summarize_timing.py new file mode 100644 index 0000000000000000000000000000000000000000..a1dfeaf81d4e132e5688e1c2587137d4dd6c19f9 --- /dev/null +++ b/scripts/summarize_timing.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +"""Print available timing seconds_per_sample rows and missing diagnostics.""" + +from __future__ import annotations + +from dlb.timing_summary import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/train/candi.sh b/scripts/train/candi.sh new file mode 100644 index 0000000000000000000000000000000000000000..fcc335a6a93ada19097dc8b57ea6560323bf6840 --- /dev/null +++ b/scripts/train/candi.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)" +export PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" +PYTHON_BIN="${DLB_PYTHON:-python}" +exec "$PYTHON_BIN" -m dlb.recipes --root "$DLB_ROOT" --recipe candi "$@" diff --git a/scripts/train/mdlm.sh b/scripts/train/mdlm.sh new file mode 100644 index 0000000000000000000000000000000000000000..b60cf31dce5a97eedace8ee0da068f741caf21ec --- /dev/null +++ b/scripts/train/mdlm.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +DLB_ROOT="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)" +export PYTHONPATH="$DLB_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" +PYTHON_BIN="${DLB_PYTHON:-python}" +exec "$PYTHON_BIN" -m dlb.recipes --root "$DLB_ROOT" --recipe mdlm "$@" diff --git a/scripts/verify_checkpoints.py b/scripts/verify_checkpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..77475e1357c6da78a7b05afd413e0aa248263cdd --- /dev/null +++ b/scripts/verify_checkpoints.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Verify downloaded checkpoint files against the observed SHA-256 lock.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--lock", type=Path) + parser.add_argument("--manifest", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + root = args.root.resolve() + project_root = Path(__file__).resolve().parents[1] + sys.path.insert(0, str(project_root / "src")) + from dlb.checkpoints import verify_checkpoint_lock + + lock_path = args.lock or root / "artifacts" / "checkpoint_lock.json" + if not lock_path.is_file(): + print(f"ERROR missing checkpoint lock: {lock_path}", file=sys.stderr) + return 2 + lock = json.loads(lock_path.read_text(encoding="utf-8")) + manifest_path = args.manifest or root / "artifacts" / "checkpoints.yaml" + report = verify_checkpoint_lock(root, lock, manifest_path=manifest_path) + print(f"{str(report['manifest_status']).upper()} manifest") + print(f"{str(report['resource_set_status']).upper()} resource-set") + for resource_id, record in report["resources"].items(): + print(f"{str(record['status']).upper()} {resource_id}") + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_conditional_prompts.py b/scripts/verify_conditional_prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..b90057ef5ab5667d4ac04abdbe1cd1583c22826f --- /dev/null +++ b/scripts/verify_conditional_prompts.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Verify deterministic C64 conditional prompt artifacts before GPU work.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--config", type=Path) + parser.add_argument("--dataset", choices=("all", "lm1b", "owt"), default="all") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + root = args.root.resolve() + sys.path.insert(0, str(root / "src")) + from dlb.conditional_prompts import load_protocol, verify_prompts + + protocol = load_protocol(args.config or root / "configs" / "conditional.yaml") + datasets = ("lm1b", "owt") if args.dataset == "all" else (args.dataset,) + for dataset_id in datasets: + manifest = verify_prompts(root, dataset_id, protocol) + print(f"OK {dataset_id} prompts={manifest.prompt_count} sha256={manifest.prompt_file_sha256}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_data.py b/scripts/verify_data.py new file mode 100644 index 0000000000000000000000000000000000000000..75a2da194ff20e5ad33427b0afff6fd25f21fbb9 --- /dev/null +++ b/scripts/verify_data.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Verify processed datasets against pinned integrity manifests.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys + +import yaml + + +OUTPUT_NAMES = {"lm1b": "lm1b-bert-128", "owt": "owt-gpt2-1024"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--config", type=Path) + parser.add_argument("--dataset", choices=("all", "lm1b", "owt"), default="all") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + root = args.root.resolve() + sys.path.insert(0, str(root / "src")) + from dlb.data import build_processing_contract, verify_processed_dataset + from dlb.io import atomic_json_write + + config_path = args.config or root / "artifacts" / "data.yaml" + configuration = yaml.safe_load(config_path.read_text(encoding="utf-8")) + hf_home = root / "data" / "raw" / "huggingface" + hub_cache = hf_home / "hub" + os.environ["HF_HOME"] = str(hf_home) + + from datasets import load_from_disk + from huggingface_hub import snapshot_download + from transformers import AutoTokenizer + + names = ("lm1b", "owt") if args.dataset == "all" else (args.dataset,) + for name in names: + contract = build_processing_contract(configuration, name) + tokenizer_name = contract["tokenizer_id"] + tokenizer_revision = contract["tokenizer_revision"] + manifest_path = root / "data" / "manifests" / f"{name}.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + snapshot_path = snapshot_download( + repo_id=tokenizer_name, + revision=tokenizer_revision, + cache_dir=hub_cache, + local_files_only=True, + ) + tokenizer = AutoTokenizer.from_pretrained(snapshot_path, local_files_only=True) + output_dir = root / "data" / "processed" / OUTPUT_NAMES[name] + dataset = load_from_disk(str(output_dir)) + verified = verify_processed_dataset( + dataset=dataset, + manifest=manifest, + output_dir=output_dir, + tokenizer=tokenizer, + expected_contract=contract, + ) + atomic_json_write(manifest_path, verified) + print( + f"OK {name} documents={verified['document_counts']} " + f"packed={verified['packed_sequence_counts']} " + f"checked={verified['verification']['checked_sequences']} " + f"files={verified['verification']['files_checked']} verified=true" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_sources.py b/scripts/verify_sources.py new file mode 100644 index 0000000000000000000000000000000000000000..c34f30dac45e31df853017de53b5b3375e006681 --- /dev/null +++ b/scripts/verify_sources.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Verify the locally checked-out official source repositories.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path +from typing import Any + +import yaml + +from dlb.io import atomic_json_write, sha256_file +from dlb.registry import load_registry + + +def git_output(source_dir: Path, *args: str) -> str: + """Return stripped stdout for a Git command run inside *source_dir*.""" + + return subprocess.run( + ["git", "-C", str(source_dir), *args], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ).stdout.strip() + + +def load_manifest(path: Path, required_sources: set[str]) -> dict[str, dict[str, str]]: + """Load and validate the source manifest's shape and registry coverage.""" + + with path.open(encoding="utf-8") as manifest_file: + manifest: Any = yaml.safe_load(manifest_file) + if not isinstance(manifest, dict) or set(manifest) != required_sources: + raise ValueError("source manifest IDs must match registry sources") + for name, source in manifest.items(): + if ( + not isinstance(source, dict) + or set(source) != {"url", "commit"} + or not isinstance(source["url"], str) + or not isinstance(source["commit"], str) + or len(source["commit"]) != 40 + ): + raise ValueError(f"invalid manifest entry for {name}") + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + root = args.root.resolve() + manifest_path = root / "artifacts/sources.yaml" + + try: + registry = load_registry(root / "configs/experiments.yaml") + required_sources = {entry.source for entry in registry.models.values()} + manifest = load_manifest(manifest_path, required_sources) + except (OSError, ValueError, yaml.YAMLError) as error: + print(f"ERROR manifest {error}", file=sys.stderr) + return 1 + + verified: dict[str, dict[str, str]] = {} + failed = False + for name in sorted(required_sources): + source = manifest[name] + source_dir = root / "upstreams" / name + try: + if not (source_dir / ".git").is_dir(): + raise ValueError("repository is missing") + if git_output(source_dir, "remote", "get-url", "origin") != source["url"]: + raise ValueError("origin does not match manifest") + if git_output(source_dir, "rev-parse", "--abbrev-ref", "HEAD") != "HEAD": + raise ValueError("HEAD is not detached") + if git_output(source_dir, "rev-parse", "HEAD") != source["commit"]: + raise ValueError("HEAD does not match manifest") + if git_output(source_dir, "status", "--porcelain"): + raise ValueError("repository has uncommitted changes") + except (OSError, ValueError, subprocess.CalledProcessError) as error: + print(f"ERROR {name} {error}", file=sys.stderr) + failed = True + continue + + verified[name] = {"url": source["url"], "commit": source["commit"]} + print(f"OK {name} {source['commit']}") + + if failed: + return 1 + + atomic_json_write( + root / "artifacts/source_lock.json", + { + "manifest_sha256": sha256_file(manifest_path), + "sources": verified, + }, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())