Upload folder using huggingface_hub

README.md

@@ -0,0 +1,29 @@
+---
+tags: [data-portraits, bloom-filter, redpajama, code]
+---
+# Data Portrait: RedPajama-Data-1T GitHub
+
+A [Data Portrait](https://github.com/ruyimarone/data-portraits) of the GitHub subset of [togethercomputer/RedPajama-Data-1T](https://huggingface.co/datasets/togethercomputer/RedPajama-Data-1T): 98 jsonl files of source code
+(~59B tokens, served from data.together.xyz), the code part of the training data of [RedPajama-INCITE](https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Base) (Base 3B v1, 7B) and
+[OpenLLaMA v1](https://huggingface.co/openlm-research/open_llama_7b) (3B, 7B, 13B). Those models were
+trained on 0.8T to 1T of RedPajama's 1.2T tokens, so they need not have seen all of it. SlimPajama's GitHub part is
+a filtered, deduplicated (and NFC-normalised) subset of this data.
+
+Text is normalised as in data-portraits and split into 50-character n-grams with stride
+50, stored in 1 Bloom filter(s) sized to fit in RAM. A query ORs membership
+across filters: combined false-positive rate <= 0.01 (each filter 0.010000).
+
+Each partition is a separate revision of this repo (tag `p<i>of<k>`, branch `partition-<n>`):
+
+| tag | files | sources | size |
+|---|---|---|---|
+| [`p0of1`](https://huggingface.co/aalkaswan/redpajama-github-portrait/tree/p0of1) | 98 | github | 4.0 GiB |
+
+`partitions.json` lists the exact files (URL, size and ETag; URL list at a pinned Hub revision) in each partition. Query with
+`redpajama_github_portrait.query`:
+
+```python
+from redpajama_github_portrait.query import SequentialPortrait
+portrait = SequentialPortrait("aalkaswan/redpajama-github-portrait", ["p0of1"])
+report = portrait.contains_from_text([text], sort_chains_by_length=True)[0]
+```

partitions.json

@@ -0,0 +1,133 @@
+{
+ "revisions": {
+  "togethercomputer/RedPajama-Data-1T": "398f92572e94f4793e41c22ab7ea2a788d9e7de4"
+ },
+ "width": 50,
+ "stride": 50,
+ "num_partitions": 1,
+ "stage_partitions": {
+  "main": 1
+ },
+ "combined_error": 0.01,
+ "per_filter_error": 0.010000000000000009,
+ "est_unique_ngrams": {
+  "main": 2963451655
+ },
+ "skipped_sources": [],
+ "partitions": [
+  {
+   "index": 0,
+   "stage": "main",
+   "stage_index": 0,
+   "tag": "p0of1",
+   "key": "redpajama-github-p0of1.50-50.bf",
+   "error_rate": 0.010000000000000009,
+   "capacity": 3556141987,
+   "est_ngrams": 2963451655,
+   "est_gib": 3.968112699641199,
+   "families": {
+    "github": 98
+   },
+   "shards": [
+    "togethercomputer/RedPajama-Data-1T:github/filtered_08cdfa755e6d4d89b673d5bd1acee5f6.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_0f27d10d846a473b96070c3394832f32.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_0f979046c8e64e0fb5843d2634a9957d.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_10f129bfd0af45caa9cd72aa9d863ec5.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_11a1943edfa349c7939382799599eed6.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_17197bd2478044bebd9ff4634b6dfcee.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_1d750c0ce39d40c6bc20bad9469e5a99.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_21078cf63afb4d9eb4a7876f726a7226.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_216883d3a669406699428bc485a4c228.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_24278a707deb445b8e4f59c83dd67910.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_25989b8233a04ac791b0eccd502e0c7a.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_26a6fa61c5eb4bb885e7bc643e285f0e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_275100c1a44f4451b0343373ebc5637a.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_27f05c041a1c401783f90b9415e40e4b.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_28106e66bfd94978abbc15ec845aeddb.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_2afd093fefad4c8da76cc539e8fb6137.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_30d01c4edab64866bda8c609e90b4f4e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_34b0785b77814b7583433ddb27a61ae0.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_34e78793c1e94eeebd92852399097596.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_366012588bef4d749bbbea76ae701141.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_36c2a955ddd84672bbc778aa4ad2fbaf.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_3be6a5cc7428401393c23e5516a43537.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_3cc6cab9266746a6befa23648aa43119.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_3f09f384f0734a4b912bb75db3f812bc.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_478afe8aaccb43e6be2de7e34e041ef3.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_483f11d6bc864f7fbfbe63bdf3583ce2.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_4b6883dc304c4e799620ec95b96dc91a.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_4bebabdbd8544da7a2071864ccf81f2e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_4d44caf43c154ae4aaeab36eab0221c9.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_4f98dc136ba94eeaa1ba6c974814b33c.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_4ff3604761614a3db550ef758e6457b5.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_50423e84046948b4a2a70e7e4538e12d.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_58fad5994b4446c6bceb33453484acb4.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_5abf005e4e634f1dbfa8bd20b5687092.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_5f2b3517159b426bb1a9e81ca189abcd.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_610e908cafaa4d53958de50ad700822a.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_6353ab14cb8f4623a7d75678d9e7f44e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_64747f25102740bab0ab54559569342a.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_6742e99802894114a3ba44841f49b168.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_677e10f9b0af4c489e60670352e7e224.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_68534e6a093744fa9f38fa1a9cf51232.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_6adcf92fb2ee48059cb60579f2e931f7.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_70f0aa43987643a7874286bca4faa66b.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_7462f1a34f594e9f8334d9a0cbbf80e7.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_78a13a25258f4d24923702c07445e20e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_78e3afc1cfee41fbb7eaae2e5bfaa17b.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_791ed086a57f4879bb1596bed6d37bb3.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_7b5cc18857a34a0981b54f082de55cf8.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_7c54b908a2df4ec2ba316d2081fc674e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_7edd0af0b61c426e93e8bd3f549a8f78.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_7fa232e63a5d44a88a267181e9ac47b4.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_81138c01f3a84f7fa8b56bf3d8fa35ce.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_82a9a647b0eb4eb080d7ac15a13c765b.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_835ac26c7a70447b97f4ec38bcb969ed.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_8bc3c4fae78c41d999b2ae6d97cce96c.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_8dc430f8f7114f018440c7b3d990e602.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_8e1db2cf6c98420a88bad52fd57f4aa7.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_919156910e704e6ca52e0d4880cdbb63.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_944beac1347f491faa88f43c25d26fe4.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_96175b6e4c764abfbbf14e78d4fd6464.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_977d1fcdab92452d9dc38b2f4a99389b.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_9eea501af8544d0b88f0b002850829d4.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_a26c51ffe2924fd8ad6694c6aa0eacc5.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_a3a01a2f81ed4cd2afb30e175200b48f.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_a3d3e6f3f7d5495ca9ecf94d808fd350.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_a777da5620f1467f8df3616b17d533dc.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_a8fcae75b0c3410faabcff02f0056a36.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_aabc2d54d1c946908d3400228e0f238c.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_ad160286662849f49d1e6de27c0f1d15.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_adda41d791974289aff042e2e3d07ec3.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_ae1813abc63f4b1998dfa608e7fe5588.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_af7b386db97e4211a7378be08d7b3f4f.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_b15c575fa9f8465d98f70ba2f2f73c6e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_b40763d0b9ce4e0d8fb5f519f1f49f8c.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_b56453718c5f46efa9c46feb194b0d6e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_b6225e159b86432d9fa5bf226bb51393.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_b821f640b8f14ed588bf48ae13f44098.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_bfbcad9633f04601ba2f824d082eaacf.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_c6cfd16905814d7c955df1f4754a8b11.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_c82c6775f0b74dbdae4524bb9aebf0ef.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_ca73c69896b34adbbe468d78d9f134bc.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_cac864a472b948b0bfe18c8e9a19aeb5.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_cb330e8dc8ac411eba2dc8676c9c4403.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_d0a054a678fc4c38b496d10e91a2c735.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_d148857715424aabbd32b8ffe56c4082.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_d33701435f964c90a86c22e204dd5fde.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_d448c2553f474193a1224df1c38f74d4.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_d8d45c58819948c9b352d74383944c4a.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_db611a692a704f6db3c18e77f79fd2f0.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_dcaa5c8b729b4fb599399dbf4557e43e.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_dda863d833614d04b96bbe21b161768d.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_eabf427d56184fb89f9b5f27e73f7988.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_eee6980922ca4a14b0c3341fa8a904d9.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_f478ff72b57f4f4283c22ac22ae84134.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_f931feb0e85940879d194c0e20d9e28a.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_f9bbe6a065004a4c8e018c6ad63063b2.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_fbfc552e48164acda6605fa31fc2f563.sampled.jsonl",
+    "togethercomputer/RedPajama-Data-1T:github/filtered_fc817e4c16494957bcb89b166e91434f.sampled.jsonl"
+   ]
+  }
+ ]
+}