thefinalboss commited on
Commit
4be56b2
·
verified ·
1 Parent(s): a60d319

SPIRA Hashing: multi_lang_v2 scripts + manifest (train.bin sha256 35858aab..., not uploaded)

Browse files
results/hashing/multi_lang_v2/build.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SPIRA Hashing - multi_lang builder (v2: OUT dir, raw clones shared from multi_lang_v1/raw). Streams files; 64-byte rolling hash (verify_v3 hash) + byte confirmation vs all held-out eval windows."""
2
+ import os, sys, re, json, hashlib, subprocess, numpy as np
3
+ sys.path.insert(0, "/workspace/spira_shared/checker")
4
+ S = "/workspace/spira_shared"; O = os.environ.get("OUT", f"{S}/multi_lang_v2"); RAW = f"{S}/multi_lang_v1/raw"; K = 64; P = np.uint64(1099511628211)
5
+ CAP = 8 << 20
6
+ SEP = b"\n<|endoffile|>\n"
7
+ def hashes(b):
8
+ a = np.frombuffer(b, np.uint8).astype(np.uint64); n = len(a) - K + 1
9
+ if n <= 0: return np.zeros(0, np.uint64)
10
+ h = np.zeros(n, np.uint64)
11
+ with np.errstate(over="ignore"):
12
+ for j in range(K): h = h * P + a[j:j+n] + np.uint64(1)
13
+ return h
14
+ def windows():
15
+ W = {k: open(f"{S}/spira_m7/windows/{k}.bin","rb").read() for k in ("fresh_en","fresh_fr","neither_code","test20k","queries","freshcode")}
16
+ from windows_v2 import freshcode_v2; W["fc2"] = freshcode_v2()
17
+ for c in ("corpus_v3","broad/broad_local_v1","broad/broad_gutenberg_v1","broad/old_local_v1"):
18
+ for f in ("test","oneshot"): W[c.split("/")[-1]+"_"+f] = open(f"{S}/{c}/{f}.bin","rb").read()
19
+ return W # 'seen' deliberately excluded (training data)
20
+ EXT = {".py":"Python",".js":"JavaScript",".mjs":"JavaScript",".cjs":"JavaScript",".jsx":"JavaScript",".ts":"TypeScript",".tsx":"TypeScript",
21
+ ".rs":"Rust",".go":"Go",".c":"C",".h":"C",".cc":"C++",".cpp":"C++",".cxx":"C++",".hpp":"C++",".hh":"C++",".hxx":"C++",".java":"Java",
22
+ ".cs":"C#",".rb":"Ruby",".php":"PHP",".kt":"Kotlin",".kts":"Kotlin",".swift":"Swift",".scala":"Scala",".sh":"Shell",".bash":"Shell",".zsh":"Shell"}
23
+ BADDIR = {"node_modules","vendor","vendors","third_party","third-party","thirdparty","3rdparty","external","dist","build","out",".git",
24
+ "testdata","test_data","test-data","fixtures","fixture","__fixtures__","__snapshots__","snapshots","generated","gen"}
25
+ SECRET = re.compile(rb"AKIA[0-9A-Z]{16}|-----BEGIN|ghp_[A-Za-z0-9]{20,}|xox[bp]-|\bsk-[A-Za-z0-9_\-]{16,}|(?i:password)\s*[:=]\s*[\"'][^\"'\s]{1,}[\"']")
26
+ def reason(rel, b):
27
+ parts = rel.split("/")
28
+ if any(p.lower() in BADDIR for p in parts[:-1]): return "vendored_generated_or_fixture_dir"
29
+ n = parts[-1].lower()
30
+ if n.endswith((".min.js",".min.mjs")) or "lock" in n or n.endswith(("_pb2.py",".pb.go",".pb.cc",".pb.h",".generated.cs",".designer.cs")): return "minified_lock_or_generated_name"
31
+ if len(b) > 262144: return "too_large_256k"
32
+ if len(b) == 0: return "empty"
33
+ if b"\x00" in b: return "binary"
34
+ try: t = b.decode("utf-8")
35
+ except UnicodeDecodeError: return "non_utf8"
36
+ head = b[:2048]
37
+ if b"@generated" in head or b"DO NOT EDIT" in head or b"Code generated" in head or b"auto-generated" in head.lower(): return "generated_header"
38
+ L = b.split(b"\n")
39
+ if max(map(len, L)) > 3000 or len(b) / len(L) > 300: return "minified_long_lines"
40
+ if SECRET.search(b): return "secret_pattern"
41
+ return None
42
+ BAD = [("BUSL-1.1", r"Business Source License"), ("GPL", r"GNU (AFFERO |LESSER )?GENERAL PUBLIC LICENSE"), ("RPL-1.5", r"Reciprocal Public License"),
43
+ ("RSAL/SSPL", r"Source Available License|Server Side Public License")]
44
+ PERM = [("Apache-2.0", r"Apache License,?\s+Version 2\.0"), ("MIT", r"Permission is hereby granted, free of charge"), ("BSD-3-Clause", r"Neither the name|names of its\s+contributors may be used"),
45
+ ("BSD-2-Clause", r"Redistribution and use in source and binary forms"), ("zlib", r"This software is provided 'as-is'"),
46
+ ("curl (MIT/X derivative)", r"Permission to use, copy, modify, and distribute this software for any purpose")]
47
+ def license_of(d):
48
+ fs = [f for f in os.listdir(d) if os.path.isfile(f"{d}/{f}") and ("LICEN" in f.upper() or f.upper().startswith("COPYING")) and "THIRD" not in f.upper() and not f.startswith(".") and not f.endswith(".json")]
49
+ if not fs: return "NONE", None
50
+ fs.sort(key=lambda f: (0 if f.upper().startswith(("LICENSE","LICENCE")) else 1 if "LICEN" in f.upper() else 2, f))
51
+ t = "\n".join(open(f"{d}/{f}", errors="ignore").read() for f in fs)
52
+ for name, pat in BAD:
53
+ if re.search(pat, t): return name, ",".join(fs)
54
+ for name, pat in PERM:
55
+ if re.search(pat, t): return name, ",".join(fs)
56
+ return "UNKNOWN", ",".join(fs)
57
+ OK_LIC = ("Apache-2.0","MIT","BSD-3-Clause","BSD-2-Clause","zlib","curl (MIT/X derivative)")
58
+ if __name__ == "__main__":
59
+ W = windows(); names = sorted(W); wid = []; wpos = []; hs = []
60
+ for i, k in enumerate(names):
61
+ h = hashes(W[k]); hs.append(h); wid.append(np.full(len(h), i, np.int32)); wpos.append(np.arange(len(h), dtype=np.int32))
62
+ H = np.concatenate(hs); o = np.argsort(H, kind="stable"); H = H[o]; WID = np.concatenate(wid)[o]; WPOS = np.concatenate(wpos)[o]; del hs, wid, wpos, o
63
+ print("window hashes", len(H), flush=True)
64
+ repos = [l.rstrip("\n").split("\t") for l in open(os.environ.get("REPOS", f"{O}/repos.tsv"))]
65
+ drops = {}; seen_sha = set(); man = {"repos": [], "separator": SEP.decode(), "excluded_repos": []}
66
+ win_drop = {k: 0 for k in names}; hash_hit_files = 0
67
+ out = open(f"{O}/train.bin.tmp", "wb"); first = True
68
+ for lang, r in repos:
69
+ d = f"{RAW}/{r.replace('/','_')}"
70
+ if not os.path.isdir(d + "/.git"): man["excluded_repos"].append({"repo": r, "why": "clone_failed"}); continue
71
+ lic, lf = license_of(d)
72
+ sha = subprocess.check_output(["git","-C",d,"rev-parse","HEAD"]).decode().strip()
73
+ if lic not in OK_LIC:
74
+ man["excluded_repos"].append({"repo": r, "commit": sha, "why": f"license_not_permissive_or_unrecognized:{lic}:{lf}"}); continue
75
+ rec = {"repo": r, "url": f"https://github.com/{r}", "commit": sha, "license": lic, "license_file": lf, "intended_language": lang, "files_kept": 0, "bytes_kept": 0, "by_language": {}}
76
+ files = subprocess.check_output(["git","-C",d,"ls-files","-z"]).split(b"\0")
77
+ for rel in sorted(f.decode("utf-8","replace") for f in files if f):
78
+ lg = EXT.get(os.path.splitext(rel)[1].lower())
79
+ if lg is None: continue
80
+ p = f"{d}/{rel}"
81
+ if rec["bytes_kept"] >= CAP: why = "repo_cap_8MB_reached"; drops[why] = drops.get(why,0)+1; rec.setdefault("drops", {}); rec["drops"][why] = rec["drops"].get(why, 0) + 1; continue
82
+ if not os.path.isfile(p) or os.path.islink(p): drops["missing_blob_or_link(>1MB filtered at clone)"] = drops.get("missing_blob_or_link(>1MB filtered at clone)",0)+1; continue
83
+ b = open(p,"rb").read(); why = reason(rel, b)
84
+ if why is None:
85
+ hsh = hashlib.sha256(b).digest()
86
+ if hsh in seen_sha: why = "exact_dup_sha256"
87
+ else: seen_sha.add(hsh)
88
+ if why is None:
89
+ h = hashes(b); u = np.unique(h); i = np.minimum(np.searchsorted(H, u), len(H)-1); cand = u[H[i] == u]
90
+ if len(cand):
91
+ hash_hit_files += 1; hit = set()
92
+ for q in np.nonzero(np.isin(h, cand))[0]:
93
+ seg = b[q:q+K]; lo = np.searchsorted(H, h[q]); hi = np.searchsorted(H, h[q], "right")
94
+ for j in range(lo, hi):
95
+ k = names[WID[j]]
96
+ if W[k][WPOS[j]:WPOS[j]+K] == seg: hit.add(k)
97
+ if hit:
98
+ why = "eval_window_overlap64"
99
+ for k in hit: win_drop[k] += 1
100
+ if why: drops[why] = drops.get(why, 0) + 1; rec.setdefault("drops", {}); rec["drops"][why] = rec["drops"].get(why, 0) + 1; continue
101
+ if not first: out.write(SEP)
102
+ out.write(b); first = False
103
+ rec["files_kept"] += 1; rec["bytes_kept"] += len(b); bl = rec["by_language"].setdefault(lg, [0,0]); bl[0] += 1; bl[1] += len(b)
104
+ man["repos"].append(rec); print(r, lic, rec["files_kept"], rec["bytes_kept"], flush=True)
105
+ out.close(); os.replace(f"{O}/train.bin.tmp", f"{O}/train.bin")
106
+ man["drops"] = drops; man["files_with_window_hash_candidates"] = hash_hit_files; man["files_dropped_per_window_confirmed"] = win_drop; man["windows_used"] = {k: len(W[k]) for k in names}
107
+ json.dump(man, open(f"{O}/build_stage1.json","w"), indent=1); print("BUILD DONE", drops, win_drop)
results/hashing/multi_lang_v2/build_stage1.json ADDED
@@ -0,0 +1,1919 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repos": [
3
+ {
4
+ "repo": "psf/requests",
5
+ "url": "https://github.com/psf/requests",
6
+ "commit": "611c6162cbc4ac2020a2f91c7cfa4f3abf9bbb60",
7
+ "license": "Apache-2.0",
8
+ "license_file": "LICENSE",
9
+ "intended_language": "Python",
10
+ "files_kept": 34,
11
+ "bytes_kept": 334211,
12
+ "by_language": {
13
+ "Python": [
14
+ 34,
15
+ 334211
16
+ ]
17
+ },
18
+ "drops": {
19
+ "eval_window_overlap64": 1,
20
+ "secret_pattern": 1,
21
+ "empty": 1
22
+ }
23
+ },
24
+ {
25
+ "repo": "pallets/flask",
26
+ "url": "https://github.com/pallets/flask",
27
+ "commit": "d086db856be187255b8ec61ef409357393020f32",
28
+ "license": "BSD-3-Clause",
29
+ "license_file": "LICENSE.txt",
30
+ "intended_language": "Python",
31
+ "files_kept": 75,
32
+ "bytes_kept": 486917,
33
+ "by_language": {
34
+ "Shell": [
35
+ 1,
36
+ 165
37
+ ],
38
+ "Python": [
39
+ 74,
40
+ 486752
41
+ ]
42
+ },
43
+ "drops": {
44
+ "eval_window_overlap64": 5,
45
+ "secret_pattern": 1,
46
+ "empty": 3
47
+ }
48
+ },
49
+ {
50
+ "repo": "django/django",
51
+ "url": "https://github.com/django/django",
52
+ "commit": "0e128d524d0a7c55a955570d6e5da983ac6ce6c7",
53
+ "license": "BSD-3-Clause",
54
+ "license_file": "LICENSE,LICENSE.python",
55
+ "intended_language": "Python",
56
+ "files_kept": 1333,
57
+ "bytes_kept": 8413530,
58
+ "by_language": {
59
+ "JavaScript": [
60
+ 35,
61
+ 195616
62
+ ],
63
+ "Python": [
64
+ 1296,
65
+ 8214462
66
+ ],
67
+ "Shell": [
68
+ 2,
69
+ 3452
70
+ ]
71
+ },
72
+ "drops": {
73
+ "empty": 348,
74
+ "exact_dup_sha256": 7,
75
+ "eval_window_overlap64": 101,
76
+ "vendored_generated_or_fixture_dir": 68,
77
+ "minified_lock_or_generated_name": 1,
78
+ "generated_header": 1,
79
+ "secret_pattern": 46,
80
+ "too_large_256k": 1,
81
+ "repo_cap_8MB_reached": 1144
82
+ }
83
+ },
84
+ {
85
+ "repo": "scikit-learn/scikit-learn",
86
+ "url": "https://github.com/scikit-learn/scikit-learn",
87
+ "commit": "59106029d483678e79825f753bde67f8ade4332b",
88
+ "license": "BSD-3-Clause",
89
+ "license_file": "COPYING",
90
+ "intended_language": "Python",
91
+ "files_kept": 692,
92
+ "bytes_kept": 8424057,
93
+ "by_language": {
94
+ "Shell": [
95
+ 23,
96
+ 37357
97
+ ],
98
+ "Python": [
99
+ 664,
100
+ 8381084
101
+ ],
102
+ "JavaScript": [
103
+ 5,
104
+ 5616
105
+ ]
106
+ },
107
+ "drops": {
108
+ "eval_window_overlap64": 87,
109
+ "secret_pattern": 1,
110
+ "minified_lock_or_generated_name": 1,
111
+ "vendored_generated_or_fixture_dir": 1,
112
+ "empty": 46,
113
+ "exact_dup_sha256": 7,
114
+ "repo_cap_8MB_reached": 248
115
+ }
116
+ },
117
+ {
118
+ "repo": "encode/httpx",
119
+ "url": "https://github.com/encode/httpx",
120
+ "commit": "b5addb64f0161ff6bfe94c124ef76f6a1fba5254",
121
+ "license": "BSD-3-Clause",
122
+ "license_file": "LICENSE.md",
123
+ "intended_language": "Python",
124
+ "files_kept": 52,
125
+ "bytes_kept": 404301,
126
+ "by_language": {
127
+ "Python": [
128
+ 52,
129
+ 404301
130
+ ]
131
+ },
132
+ "drops": {
133
+ "eval_window_overlap64": 3,
134
+ "empty": 3,
135
+ "secret_pattern": 2
136
+ }
137
+ },
138
+ {
139
+ "repo": "expressjs/express",
140
+ "url": "https://github.com/expressjs/express",
141
+ "commit": "1c6f1b0fe4000977b9a7b4f34097e4d527dd9824",
142
+ "license": "MIT",
143
+ "license_file": "LICENSE",
144
+ "intended_language": "JavaScript",
145
+ "files_kept": 141,
146
+ "bytes_kept": 560914,
147
+ "by_language": {
148
+ "JavaScript": [
149
+ 141,
150
+ 560914
151
+ ]
152
+ },
153
+ "drops": {
154
+ "secret_pattern": 1
155
+ }
156
+ },
157
+ {
158
+ "repo": "lodash/lodash",
159
+ "url": "https://github.com/lodash/lodash",
160
+ "commit": "2b5e6f7399a7b48005140b5d5c6bc6c0e62919a8",
161
+ "license": "MIT",
162
+ "license_file": "LICENSE",
163
+ "intended_language": "JavaScript",
164
+ "files_kept": 8,
165
+ "bytes_kept": 13667,
166
+ "by_language": {
167
+ "JavaScript": [
168
+ 8,
169
+ 13667
170
+ ]
171
+ },
172
+ "drops": {
173
+ "vendored_generated_or_fixture_dir": 28,
174
+ "eval_window_overlap64": 17,
175
+ "too_large_256k": 2
176
+ }
177
+ },
178
+ {
179
+ "repo": "webpack/webpack",
180
+ "url": "https://github.com/webpack/webpack",
181
+ "commit": "7978462575e6f5a02b6b7a2c6d7b6b236ce76175",
182
+ "license": "MIT",
183
+ "license_file": "LICENSE",
184
+ "intended_language": "JavaScript",
185
+ "files_kept": 1240,
186
+ "bytes_kept": 8389112,
187
+ "by_language": {
188
+ "JavaScript": [
189
+ 1223,
190
+ 8124165
191
+ ],
192
+ "Shell": [
193
+ 2,
194
+ 557
195
+ ],
196
+ "TypeScript": [
197
+ 14,
198
+ 263673
199
+ ],
200
+ "C++": [
201
+ 1,
202
+ 717
203
+ ]
204
+ },
205
+ "drops": {
206
+ "exact_dup_sha256": 147,
207
+ "minified_long_lines": 27,
208
+ "vendored_generated_or_fixture_dir": 50,
209
+ "too_large_256k": 6,
210
+ "minified_lock_or_generated_name": 6,
211
+ "generated_header": 1,
212
+ "repo_cap_8MB_reached": 16117
213
+ }
214
+ },
215
+ {
216
+ "repo": "axios/axios",
217
+ "url": "https://github.com/axios/axios",
218
+ "commit": "aa54a5e1e0e50cd1a2561b854ae5490ff589d065",
219
+ "license": "MIT",
220
+ "license_file": "LICENSE",
221
+ "intended_language": "JavaScript",
222
+ "files_kept": 262,
223
+ "bytes_kept": 1026876,
224
+ "by_language": {
225
+ "TypeScript": [
226
+ 25,
227
+ 64591
228
+ ],
229
+ "JavaScript": [
230
+ 237,
231
+ 962285
232
+ ]
233
+ },
234
+ "drops": {
235
+ "secret_pattern": 14,
236
+ "exact_dup_sha256": 1
237
+ }
238
+ },
239
+ {
240
+ "repo": "mrdoob/three.js",
241
+ "url": "https://github.com/mrdoob/three.js",
242
+ "commit": "e27dfa95ae7f453b5d7a9a36ffdb563f7c9b529b",
243
+ "license": "MIT",
244
+ "license_file": "LICENSE",
245
+ "intended_language": "JavaScript",
246
+ "files_kept": 863,
247
+ "bytes_kept": 8396389,
248
+ "by_language": {
249
+ "JavaScript": [
250
+ 863,
251
+ 8396389
252
+ ]
253
+ },
254
+ "drops": {
255
+ "vendored_generated_or_fixture_dir": 7,
256
+ "minified_lock_or_generated_name": 6,
257
+ "minified_long_lines": 13,
258
+ "too_large_256k": 5,
259
+ "eval_window_overlap64": 2,
260
+ "empty": 1,
261
+ "repo_cap_8MB_reached": 864
262
+ }
263
+ },
264
+ {
265
+ "repo": "microsoft/vscode",
266
+ "url": "https://github.com/microsoft/vscode",
267
+ "commit": "1184ad4d45794110deb6b897ea83b15aac44ed89",
268
+ "license": "MIT",
269
+ "license_file": "LICENSE.txt",
270
+ "intended_language": "TypeScript",
271
+ "files_kept": 226,
272
+ "bytes_kept": 1378837,
273
+ "by_language": {
274
+ "Shell": [
275
+ 57,
276
+ 101458
277
+ ],
278
+ "TypeScript": [
279
+ 128,
280
+ 1174818
281
+ ],
282
+ "Python": [
283
+ 16,
284
+ 10028
285
+ ],
286
+ "JavaScript": [
287
+ 6,
288
+ 85559
289
+ ],
290
+ "Ruby": [
291
+ 2,
292
+ 570
293
+ ],
294
+ "Java": [
295
+ 1,
296
+ 599
297
+ ],
298
+ "PHP": [
299
+ 3,
300
+ 1014
301
+ ],
302
+ "Go": [
303
+ 2,
304
+ 652
305
+ ],
306
+ "Rust": [
307
+ 3,
308
+ 682
309
+ ],
310
+ "C++": [
311
+ 5,
312
+ 1491
313
+ ],
314
+ "C": [
315
+ 1,
316
+ 818
317
+ ],
318
+ "C#": [
319
+ 1,
320
+ 864
321
+ ],
322
+ "Swift": [
323
+ 1,
324
+ 284
325
+ ]
326
+ },
327
+ "drops": {
328
+ "eval_window_overlap64": 13775,
329
+ "generated_header": 72,
330
+ "vendored_generated_or_fixture_dir": 1647,
331
+ "minified_long_lines": 25,
332
+ "minified_lock_or_generated_name": 55,
333
+ "secret_pattern": 19,
334
+ "too_large_256k": 39,
335
+ "exact_dup_sha256": 25,
336
+ "empty": 3
337
+ }
338
+ },
339
+ {
340
+ "repo": "vuejs/core",
341
+ "url": "https://github.com/vuejs/core",
342
+ "commit": "0ff253bfba3ee8d3d65448bd44ad890fd01b0ecd",
343
+ "license": "MIT",
344
+ "license_file": "LICENSE",
345
+ "intended_language": "TypeScript",
346
+ "files_kept": 518,
347
+ "bytes_kept": 4533129,
348
+ "by_language": {
349
+ "JavaScript": [
350
+ 34,
351
+ 74741
352
+ ],
353
+ "TypeScript": [
354
+ 484,
355
+ 4458388
356
+ ]
357
+ },
358
+ "drops": {
359
+ "eval_window_overlap64": 4,
360
+ "exact_dup_sha256": 4
361
+ }
362
+ },
363
+ {
364
+ "repo": "angular/angular",
365
+ "url": "https://github.com/angular/angular",
366
+ "commit": "c27695f278a88ce5e74f7504320dae5a53862236",
367
+ "license": "MIT",
368
+ "license_file": "LICENSE",
369
+ "intended_language": "TypeScript",
370
+ "files_kept": 1736,
371
+ "bytes_kept": 8392070,
372
+ "by_language": {
373
+ "Shell": [
374
+ 6,
375
+ 6192
376
+ ],
377
+ "JavaScript": [
378
+ 56,
379
+ 75868
380
+ ],
381
+ "TypeScript": [
382
+ 1606,
383
+ 5921307
384
+ ],
385
+ "Rust": [
386
+ 68,
387
+ 2388703
388
+ ]
389
+ },
390
+ "drops": {
391
+ "too_large_256k": 6,
392
+ "minified_long_lines": 3,
393
+ "minified_lock_or_generated_name": 3,
394
+ "exact_dup_sha256": 231,
395
+ "empty": 1,
396
+ "eval_window_overlap64": 3,
397
+ "vendored_generated_or_fixture_dir": 8,
398
+ "repo_cap_8MB_reached": 5269
399
+ }
400
+ },
401
+ {
402
+ "repo": "nestjs/nest",
403
+ "url": "https://github.com/nestjs/nest",
404
+ "commit": "9a95605281355d6b89401eb06002c3bc64545838",
405
+ "license": "MIT",
406
+ "license_file": "LICENSE",
407
+ "intended_language": "TypeScript",
408
+ "files_kept": 1887,
409
+ "bytes_kept": 5616113,
410
+ "by_language": {
411
+ "JavaScript": [
412
+ 10,
413
+ 6522
414
+ ],
415
+ "TypeScript": [
416
+ 1873,
417
+ 5608810
418
+ ],
419
+ "Shell": [
420
+ 4,
421
+ 781
422
+ ]
423
+ },
424
+ "drops": {
425
+ "exact_dup_sha256": 160,
426
+ "secret_pattern": 17,
427
+ "vendored_generated_or_fixture_dir": 17,
428
+ "generated_header": 2
429
+ }
430
+ },
431
+ {
432
+ "repo": "BurntSushi/ripgrep",
433
+ "url": "https://github.com/BurntSushi/ripgrep",
434
+ "commit": "6892bc2b7efe5909cbbeec8f757ed2c4c76b1d30",
435
+ "license": "MIT",
436
+ "license_file": "LICENSE-MIT,UNLICENSE,COPYING",
437
+ "intended_language": "Rust",
438
+ "files_kept": 104,
439
+ "bytes_kept": 1876977,
440
+ "by_language": {
441
+ "Rust": [
442
+ 101,
443
+ 1872758
444
+ ],
445
+ "Shell": [
446
+ 2,
447
+ 3415
448
+ ],
449
+ "Ruby": [
450
+ 1,
451
+ 804
452
+ ]
453
+ },
454
+ "drops": {
455
+ "eval_window_overlap64": 1,
456
+ "generated_header": 1
457
+ }
458
+ },
459
+ {
460
+ "repo": "tokio-rs/tokio",
461
+ "url": "https://github.com/tokio-rs/tokio",
462
+ "commit": "8a2408edcfa2b7f19c00428a9d3a8ea6652a083f",
463
+ "license": "MIT",
464
+ "license_file": "LICENSE",
465
+ "intended_language": "Rust",
466
+ "files_kept": 784,
467
+ "bytes_kept": 5764849,
468
+ "by_language": {
469
+ "Rust": [
470
+ 784,
471
+ 5764849
472
+ ]
473
+ },
474
+ "drops": {
475
+ "eval_window_overlap64": 4,
476
+ "minified_lock_or_generated_name": 25,
477
+ "exact_dup_sha256": 1
478
+ }
479
+ },
480
+ {
481
+ "repo": "serde-rs/serde",
482
+ "url": "https://github.com/serde-rs/serde",
483
+ "commit": "0db9b3b504bcb0a7fa9cf001429869519afd3e9a",
484
+ "license": "Apache-2.0",
485
+ "license_file": "LICENSE-APACHE,LICENSE-MIT",
486
+ "intended_language": "Rust",
487
+ "files_kept": 207,
488
+ "bytes_kept": 1246203,
489
+ "by_language": {
490
+ "Rust": [
491
+ 207,
492
+ 1246203
493
+ ]
494
+ },
495
+ "drops": {
496
+ "exact_dup_sha256": 1
497
+ }
498
+ },
499
+ {
500
+ "repo": "rust-lang/cargo",
501
+ "url": "https://github.com/rust-lang/cargo",
502
+ "commit": "cad07cf8e597397fff5d0f6cb79c9f870c0b1dd2",
503
+ "license": "Apache-2.0",
504
+ "license_file": "LICENSE-APACHE,LICENSE-MIT",
505
+ "intended_language": "Rust",
506
+ "files_kept": 827,
507
+ "bytes_kept": 8403116,
508
+ "by_language": {
509
+ "Rust": [
510
+ 818,
511
+ 8361093
512
+ ],
513
+ "Shell": [
514
+ 6,
515
+ 14085
516
+ ],
517
+ "Python": [
518
+ 2,
519
+ 3560
520
+ ],
521
+ "JavaScript": [
522
+ 1,
523
+ 24378
524
+ ]
525
+ },
526
+ "drops": {
527
+ "exact_dup_sha256": 130,
528
+ "minified_lock_or_generated_name": 12,
529
+ "empty": 168,
530
+ "vendored_generated_or_fixture_dir": 130,
531
+ "repo_cap_8MB_reached": 121
532
+ }
533
+ },
534
+ {
535
+ "repo": "alacritty/alacritty",
536
+ "url": "https://github.com/alacritty/alacritty",
537
+ "commit": "542147067dd016fc5f49120ef0ba605a98a4e3a7",
538
+ "license": "Apache-2.0",
539
+ "license_file": "LICENSE-APACHE,LICENSE-MIT",
540
+ "intended_language": "Rust",
541
+ "files_kept": 91,
542
+ "bytes_kept": 1080257,
543
+ "by_language": {
544
+ "Shell": [
545
+ 6,
546
+ 26451
547
+ ],
548
+ "Rust": [
549
+ 85,
550
+ 1053806
551
+ ]
552
+ },
553
+ "drops": {
554
+ "eval_window_overlap64": 2,
555
+ "minified_lock_or_generated_name": 1
556
+ }
557
+ },
558
+ {
559
+ "repo": "gin-gonic/gin",
560
+ "url": "https://github.com/gin-gonic/gin",
561
+ "commit": "0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646",
562
+ "license": "MIT",
563
+ "license_file": "LICENSE",
564
+ "intended_language": "Go",
565
+ "files_kept": 98,
566
+ "bytes_kept": 697100,
567
+ "by_language": {
568
+ "Go": [
569
+ 98,
570
+ 697100
571
+ ]
572
+ },
573
+ "drops": {
574
+ "vendored_generated_or_fixture_dir": 1
575
+ }
576
+ },
577
+ {
578
+ "repo": "spf13/cobra",
579
+ "url": "https://github.com/spf13/cobra",
580
+ "commit": "adbc8813901bba65827259daa8e22ff94ec1f30e",
581
+ "license": "Apache-2.0",
582
+ "license_file": "LICENSE.txt",
583
+ "intended_language": "Go",
584
+ "files_kept": 36,
585
+ "bytes_kept": 512600,
586
+ "by_language": {
587
+ "Go": [
588
+ 36,
589
+ 512600
590
+ ]
591
+ }
592
+ },
593
+ {
594
+ "repo": "prometheus/prometheus",
595
+ "url": "https://github.com/prometheus/prometheus",
596
+ "commit": "a2fda0ef4638518a6a6fb27fc48be88fbd732f4b",
597
+ "license": "Apache-2.0",
598
+ "license_file": "LICENSE",
599
+ "intended_language": "Go",
600
+ "files_kept": 552,
601
+ "bytes_kept": 8396886,
602
+ "by_language": {
603
+ "Go": [
604
+ 540,
605
+ 8366827
606
+ ],
607
+ "Shell": [
608
+ 12,
609
+ 30059
610
+ ]
611
+ },
612
+ "drops": {
613
+ "secret_pattern": 8,
614
+ "eval_window_overlap64": 2,
615
+ "minified_lock_or_generated_name": 17,
616
+ "minified_long_lines": 1,
617
+ "generated_header": 5,
618
+ "too_large_256k": 3,
619
+ "exact_dup_sha256": 1,
620
+ "repo_cap_8MB_reached": 443
621
+ }
622
+ },
623
+ {
624
+ "repo": "gohugoio/hugo",
625
+ "url": "https://github.com/gohugoio/hugo",
626
+ "commit": "ed22b85c192dd750cf0bdfb2d2005d31ab3bc0a7",
627
+ "license": "Apache-2.0",
628
+ "license_file": "LICENSE",
629
+ "intended_language": "Go",
630
+ "files_kept": 933,
631
+ "bytes_kept": 6590268,
632
+ "by_language": {
633
+ "Go": [
634
+ 899,
635
+ 6245064
636
+ ],
637
+ "Shell": [
638
+ 10,
639
+ 5004
640
+ ],
641
+ "JavaScript": [
642
+ 20,
643
+ 174328
644
+ ],
645
+ "C": [
646
+ 4,
647
+ 165872
648
+ ]
649
+ },
650
+ "drops": {
651
+ "generated_header": 12,
652
+ "eval_window_overlap64": 3,
653
+ "too_large_256k": 1,
654
+ "vendored_generated_or_fixture_dir": 4,
655
+ "minified_lock_or_generated_name": 5,
656
+ "exact_dup_sha256": 1
657
+ }
658
+ },
659
+ {
660
+ "repo": "curl/curl",
661
+ "url": "https://github.com/curl/curl",
662
+ "commit": "178e5077b60be91a8dfcbaa22a6bf79e179009ff",
663
+ "license": "curl (MIT/X derivative)",
664
+ "license_file": "COPYING",
665
+ "intended_language": "C",
666
+ "files_kept": 969,
667
+ "bytes_kept": 8391247,
668
+ "by_language": {
669
+ "Shell": [
670
+ 13,
671
+ 15558
672
+ ],
673
+ "C": [
674
+ 953,
675
+ 8356082
676
+ ],
677
+ "C++": [
678
+ 2,
679
+ 8157
680
+ ],
681
+ "Python": [
682
+ 1,
683
+ 11450
684
+ ]
685
+ },
686
+ "drops": {
687
+ "eval_window_overlap64": 86,
688
+ "secret_pattern": 10,
689
+ "minified_lock_or_generated_name": 4,
690
+ "generated_header": 1,
691
+ "repo_cap_8MB_reached": 88
692
+ }
693
+ },
694
+ {
695
+ "repo": "libuv/libuv",
696
+ "url": "https://github.com/libuv/libuv",
697
+ "commit": "19bbf90e1a7a1526da6daa1c79eec2e3ff8b6c55",
698
+ "license": "MIT",
699
+ "license_file": "LICENSE,LICENSE-docs,LICENSE-extra",
700
+ "intended_language": "C",
701
+ "files_kept": 338,
702
+ "bytes_kept": 2645018,
703
+ "by_language": {
704
+ "Shell": [
705
+ 1,
706
+ 2489
707
+ ],
708
+ "C": [
709
+ 333,
710
+ 2626891
711
+ ],
712
+ "JavaScript": [
713
+ 1,
714
+ 511
715
+ ],
716
+ "Python": [
717
+ 3,
718
+ 15127
719
+ ]
720
+ },
721
+ "drops": {
722
+ "eval_window_overlap64": 34,
723
+ "minified_lock_or_generated_name": 2
724
+ }
725
+ },
726
+ {
727
+ "repo": "madler/zlib",
728
+ "url": "https://github.com/madler/zlib",
729
+ "commit": "767c4c947852e143f582c85f14cf573411df1b35",
730
+ "license": "zlib",
731
+ "license_file": "LICENSE",
732
+ "intended_language": "C",
733
+ "files_kept": 75,
734
+ "bytes_kept": 1029227,
735
+ "by_language": {
736
+ "C": [
737
+ 68,
738
+ 997753
739
+ ],
740
+ "C#": [
741
+ 2,
742
+ 10188
743
+ ],
744
+ "C++": [
745
+ 5,
746
+ 21286
747
+ ]
748
+ },
749
+ "drops": {
750
+ "non_utf8": 7,
751
+ "eval_window_overlap64": 6,
752
+ "too_large_256k": 1
753
+ }
754
+ },
755
+ {
756
+ "repo": "jqlang/jq",
757
+ "url": "https://github.com/jqlang/jq",
758
+ "commit": "da4da65c0cfffe0481ebf26ad64346eb45e4f34d",
759
+ "license": "MIT",
760
+ "license_file": "COPYING",
761
+ "intended_language": "C",
762
+ "files_kept": 58,
763
+ "bytes_kept": 667825,
764
+ "by_language": {
765
+ "Shell": [
766
+ 2,
767
+ 3348
768
+ ],
769
+ "Python": [
770
+ 5,
771
+ 13972
772
+ ],
773
+ "JavaScript": [
774
+ 1,
775
+ 1365
776
+ ],
777
+ "C": [
778
+ 48,
779
+ 639007
780
+ ],
781
+ "C++": [
782
+ 2,
783
+ 10133
784
+ ]
785
+ },
786
+ "drops": {
787
+ "eval_window_overlap64": 1,
788
+ "vendored_generated_or_fixture_dir": 30
789
+ }
790
+ },
791
+ {
792
+ "repo": "nlohmann/json",
793
+ "url": "https://github.com/nlohmann/json",
794
+ "commit": "ffcd392be06c871e9a75f7d6b4f4b9c2288bc19c",
795
+ "license": "MIT",
796
+ "license_file": "LICENSE.MIT",
797
+ "intended_language": "C++",
798
+ "files_kept": 573,
799
+ "bytes_kept": 5279605,
800
+ "by_language": {
801
+ "Swift": [
802
+ 2,
803
+ 967
804
+ ],
805
+ "C++": [
806
+ 561,
807
+ 5200900
808
+ ],
809
+ "Python": [
810
+ 9,
811
+ 75408
812
+ ],
813
+ "JavaScript": [
814
+ 1,
815
+ 2330
816
+ ]
817
+ },
818
+ "drops": {
819
+ "exact_dup_sha256": 21,
820
+ "eval_window_overlap64": 7,
821
+ "too_large_256k": 4,
822
+ "vendored_generated_or_fixture_dir": 6
823
+ }
824
+ },
825
+ {
826
+ "repo": "fmtlib/fmt",
827
+ "url": "https://github.com/fmtlib/fmt",
828
+ "commit": "0fc31633e7d53ac895ae0a688b83b7d44f0b0c1f",
829
+ "license": "MIT",
830
+ "license_file": "LICENSE",
831
+ "intended_language": "C++",
832
+ "files_kept": 72,
833
+ "bytes_kept": 874695,
834
+ "by_language": {
835
+ "JavaScript": [
836
+ 1,
837
+ 76
838
+ ],
839
+ "C": [
840
+ 21,
841
+ 405790
842
+ ],
843
+ "C++": [
844
+ 45,
845
+ 417553
846
+ ],
847
+ "Python": [
848
+ 4,
849
+ 49163
850
+ ],
851
+ "Shell": [
852
+ 1,
853
+ 2113
854
+ ]
855
+ },
856
+ "drops": {
857
+ "eval_window_overlap64": 5,
858
+ "too_large_256k": 3
859
+ }
860
+ },
861
+ {
862
+ "repo": "google/leveldb",
863
+ "url": "https://github.com/google/leveldb",
864
+ "commit": "bb74ef739973a70ca9f0d90788a1da89b788c0ad",
865
+ "license": "BSD-3-Clause",
866
+ "license_file": "LICENSE",
867
+ "intended_language": "C++",
868
+ "files_kept": 124,
869
+ "bytes_kept": 855586,
870
+ "by_language": {
871
+ "C++": [
872
+ 71,
873
+ 655720
874
+ ],
875
+ "C": [
876
+ 53,
877
+ 199866
878
+ ]
879
+ },
880
+ "drops": {
881
+ "minified_lock_or_generated_name": 8,
882
+ "eval_window_overlap64": 1
883
+ }
884
+ },
885
+ {
886
+ "repo": "opencv/opencv",
887
+ "url": "https://github.com/opencv/opencv",
888
+ "commit": "4058787c826610fdb1d6d2e18517d93e3ad9b184",
889
+ "license": "Apache-2.0",
890
+ "license_file": "LICENSE",
891
+ "intended_language": "C++",
892
+ "files_kept": 615,
893
+ "bytes_kept": 8392271,
894
+ "by_language": {
895
+ "C++": [
896
+ 467,
897
+ 7444738
898
+ ],
899
+ "Python": [
900
+ 17,
901
+ 268125
902
+ ],
903
+ "JavaScript": [
904
+ 8,
905
+ 27664
906
+ ],
907
+ "Java": [
908
+ 45,
909
+ 268778
910
+ ],
911
+ "Swift": [
912
+ 22,
913
+ 191850
914
+ ],
915
+ "C": [
916
+ 54,
917
+ 185526
918
+ ],
919
+ "Kotlin": [
920
+ 2,
921
+ 5590
922
+ ]
923
+ },
924
+ "drops": {
925
+ "vendored_generated_or_fixture_dir": 1293,
926
+ "eval_window_overlap64": 55,
927
+ "exact_dup_sha256": 1,
928
+ "minified_long_lines": 2,
929
+ "minified_lock_or_generated_name": 2,
930
+ "generated_header": 7,
931
+ "repo_cap_8MB_reached": 2661
932
+ }
933
+ },
934
+ {
935
+ "repo": "protocolbuffers/protobuf",
936
+ "url": "https://github.com/protocolbuffers/protobuf",
937
+ "commit": "5db636d202550a7c6efa49f5ea9742f87e32bab0",
938
+ "license": "BSD-3-Clause",
939
+ "license_file": "LICENSE",
940
+ "intended_language": "C++",
941
+ "files_kept": 758,
942
+ "bytes_kept": 8415537,
943
+ "by_language": {
944
+ "Python": [
945
+ 40,
946
+ 527533
947
+ ],
948
+ "Shell": [
949
+ 29,
950
+ 63077
951
+ ],
952
+ "C++": [
953
+ 69,
954
+ 800206
955
+ ],
956
+ "Java": [
957
+ 252,
958
+ 3878162
959
+ ],
960
+ "PHP": [
961
+ 58,
962
+ 580475
963
+ ],
964
+ "C": [
965
+ 106,
966
+ 736175
967
+ ],
968
+ "Rust": [
969
+ 1,
970
+ 4531
971
+ ],
972
+ "Ruby": [
973
+ 3,
974
+ 6830
975
+ ],
976
+ "C#": [
977
+ 170,
978
+ 1651726
979
+ ],
980
+ "Go": [
981
+ 4,
982
+ 8619
983
+ ],
984
+ "Kotlin": [
985
+ 23,
986
+ 132334
987
+ ],
988
+ "Swift": [
989
+ 3,
990
+ 25869
991
+ ]
992
+ },
993
+ "drops": {
994
+ "exact_dup_sha256": 7,
995
+ "generated_header": 149,
996
+ "too_large_256k": 21,
997
+ "eval_window_overlap64": 43,
998
+ "minified_lock_or_generated_name": 5,
999
+ "vendored_generated_or_fixture_dir": 2,
1000
+ "empty": 1,
1001
+ "repo_cap_8MB_reached": 1391
1002
+ }
1003
+ },
1004
+ {
1005
+ "repo": "google/guava",
1006
+ "url": "https://github.com/google/guava",
1007
+ "commit": "1b1d120f9cd79ed5dcb1eb1e355e8f48b2b1bc6f",
1008
+ "license": "Apache-2.0",
1009
+ "license_file": "LICENSE",
1010
+ "intended_language": "Java",
1011
+ "files_kept": 1054,
1012
+ "bytes_kept": 8390932,
1013
+ "by_language": {
1014
+ "Java": [
1015
+ 1054,
1016
+ 8390932
1017
+ ]
1018
+ },
1019
+ "drops": {
1020
+ "eval_window_overlap64": 1,
1021
+ "minified_lock_or_generated_name": 7,
1022
+ "repo_cap_8MB_reached": 2221
1023
+ }
1024
+ },
1025
+ {
1026
+ "repo": "spring-projects/spring-framework",
1027
+ "url": "https://github.com/spring-projects/spring-framework",
1028
+ "commit": "b671d1a36a42dbcba0ba3657d1d353d421aa64ba",
1029
+ "license": "Apache-2.0",
1030
+ "license_file": "LICENSE.txt",
1031
+ "intended_language": "Java",
1032
+ "files_kept": 1968,
1033
+ "bytes_kept": 8391477,
1034
+ "by_language": {
1035
+ "Java": [
1036
+ 1750,
1037
+ 8034624
1038
+ ],
1039
+ "Kotlin": [
1040
+ 218,
1041
+ 356853
1042
+ ]
1043
+ },
1044
+ "drops": {
1045
+ "vendored_generated_or_fixture_dir": 20,
1046
+ "eval_window_overlap64": 24,
1047
+ "minified_lock_or_generated_name": 6,
1048
+ "repo_cap_8MB_reached": 7685
1049
+ }
1050
+ },
1051
+ {
1052
+ "repo": "square/okhttp",
1053
+ "url": "https://github.com/square/okhttp",
1054
+ "commit": "c3179301fc7d0266c5bf154870c2b0b9da01e7e7",
1055
+ "license": "Apache-2.0",
1056
+ "license_file": "LICENSE.txt",
1057
+ "intended_language": "Java",
1058
+ "files_kept": 677,
1059
+ "bytes_kept": 4496384,
1060
+ "by_language": {
1061
+ "Kotlin": [
1062
+ 606,
1063
+ 4363381
1064
+ ],
1065
+ "Shell": [
1066
+ 4,
1067
+ 3047
1068
+ ],
1069
+ "Java": [
1070
+ 67,
1071
+ 129956
1072
+ ]
1073
+ },
1074
+ "drops": {
1075
+ "secret_pattern": 15,
1076
+ "minified_lock_or_generated_name": 2
1077
+ }
1078
+ },
1079
+ {
1080
+ "repo": "apache/commons-lang",
1081
+ "url": "https://github.com/apache/commons-lang",
1082
+ "commit": "b9b30efb01e861a16503232bb96b684107e73199",
1083
+ "license": "Apache-2.0",
1084
+ "license_file": "LICENSE.txt",
1085
+ "intended_language": "Java",
1086
+ "files_kept": 621,
1087
+ "bytes_kept": 7323827,
1088
+ "by_language": {
1089
+ "Java": [
1090
+ 621,
1091
+ 7323827
1092
+ ]
1093
+ },
1094
+ "drops": {
1095
+ "too_large_256k": 3,
1096
+ "secret_pattern": 1,
1097
+ "eval_window_overlap64": 1,
1098
+ "minified_lock_or_generated_name": 2,
1099
+ "binary": 1,
1100
+ "vendored_generated_or_fixture_dir": 1
1101
+ }
1102
+ },
1103
+ {
1104
+ "repo": "dotnet/aspnetcore",
1105
+ "url": "https://github.com/dotnet/aspnetcore",
1106
+ "commit": "ceb0eec49d8d2c77f4f7c26b85ddb6e96940dfed",
1107
+ "license": "MIT",
1108
+ "license_file": "LICENSE.txt",
1109
+ "intended_language": "C#",
1110
+ "files_kept": 1407,
1111
+ "bytes_kept": 8388820,
1112
+ "by_language": {
1113
+ "Shell": [
1114
+ 40,
1115
+ 147892
1116
+ ],
1117
+ "C#": [
1118
+ 1209,
1119
+ 6996777
1120
+ ],
1121
+ "JavaScript": [
1122
+ 25,
1123
+ 108159
1124
+ ],
1125
+ "Python": [
1126
+ 19,
1127
+ 403728
1128
+ ],
1129
+ "TypeScript": [
1130
+ 114,
1131
+ 732264
1132
+ ]
1133
+ },
1134
+ "drops": {
1135
+ "eval_window_overlap64": 4,
1136
+ "secret_pattern": 1,
1137
+ "exact_dup_sha256": 9,
1138
+ "minified_lock_or_generated_name": 39,
1139
+ "vendored_generated_or_fixture_dir": 32,
1140
+ "generated_header": 1,
1141
+ "repo_cap_8MB_reached": 10206
1142
+ }
1143
+ },
1144
+ {
1145
+ "repo": "JamesNK/Newtonsoft.Json",
1146
+ "url": "https://github.com/JamesNK/Newtonsoft.Json",
1147
+ "commit": "52fa3aef1f2cadcd3a3f874251eddc98d3efbbaa",
1148
+ "license": "MIT",
1149
+ "license_file": "LICENSE.md",
1150
+ "intended_language": "C#",
1151
+ "files_kept": 939,
1152
+ "bytes_kept": 6678579,
1153
+ "by_language": {
1154
+ "C#": [
1155
+ 939,
1156
+ 6678579
1157
+ ]
1158
+ },
1159
+ "drops": {
1160
+ "minified_long_lines": 1,
1161
+ "minified_lock_or_generated_name": 3,
1162
+ "eval_window_overlap64": 7,
1163
+ "too_large_256k": 1
1164
+ }
1165
+ },
1166
+ {
1167
+ "repo": "rails/rails",
1168
+ "url": "https://github.com/rails/rails",
1169
+ "commit": "a0fd31e1a4e96c80d3184998c956607004fb7f8c",
1170
+ "license": "MIT",
1171
+ "license_file": "MIT-LICENSE",
1172
+ "intended_language": "Ruby",
1173
+ "files_kept": 1421,
1174
+ "bytes_kept": 8394471,
1175
+ "by_language": {
1176
+ "Shell": [
1177
+ 1,
1178
+ 378
1179
+ ],
1180
+ "Ruby": [
1181
+ 1382,
1182
+ 8224947
1183
+ ],
1184
+ "JavaScript": [
1185
+ 38,
1186
+ 169146
1187
+ ]
1188
+ },
1189
+ "drops": {
1190
+ "secret_pattern": 17,
1191
+ "empty": 5,
1192
+ "exact_dup_sha256": 19,
1193
+ "generated_header": 5,
1194
+ "minified_lock_or_generated_name": 2,
1195
+ "vendored_generated_or_fixture_dir": 23,
1196
+ "too_large_256k": 1,
1197
+ "eval_window_overlap64": 2,
1198
+ "repo_cap_8MB_reached": 2049
1199
+ }
1200
+ },
1201
+ {
1202
+ "repo": "jekyll/jekyll",
1203
+ "url": "https://github.com/jekyll/jekyll",
1204
+ "commit": "97c3bdc822df9d5bea4c269b29a28de5a5e29d70",
1205
+ "license": "MIT",
1206
+ "license_file": "LICENSE",
1207
+ "intended_language": "Ruby",
1208
+ "files_kept": 160,
1209
+ "bytes_kept": 712533,
1210
+ "by_language": {
1211
+ "Ruby": [
1212
+ 157,
1213
+ 674265
1214
+ ],
1215
+ "JavaScript": [
1216
+ 2,
1217
+ 38178
1218
+ ],
1219
+ "PHP": [
1220
+ 1,
1221
+ 90
1222
+ ]
1223
+ },
1224
+ "drops": {
1225
+ "eval_window_overlap64": 3,
1226
+ "minified_lock_or_generated_name": 3,
1227
+ "generated_header": 1,
1228
+ "vendored_generated_or_fixture_dir": 1
1229
+ }
1230
+ },
1231
+ {
1232
+ "repo": "sinatra/sinatra",
1233
+ "url": "https://github.com/sinatra/sinatra",
1234
+ "commit": "cb22afd7902b566b6eaba6c4ea89739494a65d12",
1235
+ "license": "MIT",
1236
+ "license_file": "LICENSE",
1237
+ "intended_language": "Ruby",
1238
+ "files_kept": 146,
1239
+ "bytes_kept": 673565,
1240
+ "by_language": {
1241
+ "Ruby": [
1242
+ 146,
1243
+ 673565
1244
+ ]
1245
+ },
1246
+ "drops": {
1247
+ "eval_window_overlap64": 1
1248
+ }
1249
+ },
1250
+ {
1251
+ "repo": "laravel/framework",
1252
+ "url": "https://github.com/laravel/framework",
1253
+ "commit": "1de9acf2054cb360659200135e5c246320c7f330",
1254
+ "license": "MIT",
1255
+ "license_file": "LICENSE.md",
1256
+ "intended_language": "PHP",
1257
+ "files_kept": 1819,
1258
+ "bytes_kept": 8394381,
1259
+ "by_language": {
1260
+ "Shell": [
1261
+ 3,
1262
+ 7164
1263
+ ],
1264
+ "PHP": [
1265
+ 1814,
1266
+ 8383601
1267
+ ],
1268
+ "JavaScript": [
1269
+ 2,
1270
+ 3616
1271
+ ]
1272
+ },
1273
+ "drops": {
1274
+ "eval_window_overlap64": 22,
1275
+ "minified_lock_or_generated_name": 25,
1276
+ "secret_pattern": 2,
1277
+ "minified_long_lines": 1,
1278
+ "vendored_generated_or_fixture_dir": 10,
1279
+ "exact_dup_sha256": 4,
1280
+ "repo_cap_8MB_reached": 1291
1281
+ }
1282
+ },
1283
+ {
1284
+ "repo": "symfony/symfony",
1285
+ "url": "https://github.com/symfony/symfony",
1286
+ "commit": "03b10a6cd71262b770f6496511ff3c896a2de034",
1287
+ "license": "MIT",
1288
+ "license_file": "LICENSE",
1289
+ "intended_language": "PHP",
1290
+ "files_kept": 1645,
1291
+ "bytes_kept": 8390266,
1292
+ "by_language": {
1293
+ "PHP": [
1294
+ 1644,
1295
+ 8376452
1296
+ ],
1297
+ "JavaScript": [
1298
+ 1,
1299
+ 13814
1300
+ ]
1301
+ },
1302
+ "drops": {
1303
+ "eval_window_overlap64": 9,
1304
+ "minified_lock_or_generated_name": 27,
1305
+ "vendored_generated_or_fixture_dir": 345,
1306
+ "generated_header": 1,
1307
+ "exact_dup_sha256": 45,
1308
+ "secret_pattern": 1,
1309
+ "non_utf8": 1,
1310
+ "repo_cap_8MB_reached": 10576
1311
+ }
1312
+ },
1313
+ {
1314
+ "repo": "guzzle/guzzle",
1315
+ "url": "https://github.com/guzzle/guzzle",
1316
+ "commit": "b920e5fcaf81d761b8fabe600ca90999beaefdb7",
1317
+ "license": "MIT",
1318
+ "license_file": "LICENSE",
1319
+ "intended_language": "PHP",
1320
+ "files_kept": 135,
1321
+ "bytes_kept": 1779468,
1322
+ "by_language": {
1323
+ "PHP": [
1324
+ 135,
1325
+ 1779468
1326
+ ]
1327
+ },
1328
+ "drops": {
1329
+ "eval_window_overlap64": 7,
1330
+ "minified_lock_or_generated_name": 2,
1331
+ "too_large_256k": 1
1332
+ }
1333
+ },
1334
+ {
1335
+ "repo": "JetBrains/kotlin-web-site",
1336
+ "url": "https://github.com/JetBrains/kotlin-web-site",
1337
+ "commit": "4c0c682dfa2799ca98bf97e86640d757cdb7e11e",
1338
+ "license": "Apache-2.0",
1339
+ "license_file": "LICENSE",
1340
+ "intended_language": "Kotlin",
1341
+ "files_kept": 335,
1342
+ "bytes_kept": 894608,
1343
+ "by_language": {
1344
+ "Shell": [
1345
+ 3,
1346
+ 11765
1347
+ ],
1348
+ "Kotlin": [
1349
+ 71,
1350
+ 101646
1351
+ ],
1352
+ "JavaScript": [
1353
+ 31,
1354
+ 62832
1355
+ ],
1356
+ "TypeScript": [
1357
+ 229,
1358
+ 702829
1359
+ ],
1360
+ "Python": [
1361
+ 1,
1362
+ 15536
1363
+ ]
1364
+ },
1365
+ "drops": {
1366
+ "minified_lock_or_generated_name": 15,
1367
+ "minified_long_lines": 1,
1368
+ "vendored_generated_or_fixture_dir": 21
1369
+ }
1370
+ },
1371
+ {
1372
+ "repo": "square/kotlinpoet",
1373
+ "url": "https://github.com/square/kotlinpoet",
1374
+ "commit": "e9c64e7e33221d900473a283f1f17f1439c99753",
1375
+ "license": "Apache-2.0",
1376
+ "license_file": "LICENSE.txt",
1377
+ "intended_language": "Kotlin",
1378
+ "files_kept": 131,
1379
+ "bytes_kept": 1246878,
1380
+ "by_language": {
1381
+ "Kotlin": [
1382
+ 128,
1383
+ 1239697
1384
+ ],
1385
+ "Java": [
1386
+ 3,
1387
+ 7181
1388
+ ]
1389
+ },
1390
+ "drops": {
1391
+ "minified_lock_or_generated_name": 3
1392
+ }
1393
+ },
1394
+ {
1395
+ "repo": "Kotlin/kotlinx.coroutines",
1396
+ "url": "https://github.com/Kotlin/kotlinx.coroutines",
1397
+ "commit": "7e0d1b3e9ca83652427ae00dda7d1673f058a743",
1398
+ "license": "Apache-2.0",
1399
+ "license_file": "LICENSE.txt",
1400
+ "intended_language": "Kotlin",
1401
+ "files_kept": 1060,
1402
+ "bytes_kept": 3943739,
1403
+ "by_language": {
1404
+ "Kotlin": [
1405
+ 1037,
1406
+ 3902999
1407
+ ],
1408
+ "Python": [
1409
+ 1,
1410
+ 3107
1411
+ ],
1412
+ "Java": [
1413
+ 21,
1414
+ 34948
1415
+ ],
1416
+ "Shell": [
1417
+ 1,
1418
+ 2685
1419
+ ]
1420
+ },
1421
+ "drops": {
1422
+ "eval_window_overlap64": 9,
1423
+ "minified_lock_or_generated_name": 24,
1424
+ "empty": 1,
1425
+ "exact_dup_sha256": 12
1426
+ }
1427
+ },
1428
+ {
1429
+ "repo": "ktorio/ktor",
1430
+ "url": "https://github.com/ktorio/ktor",
1431
+ "commit": "d96280165fc9890217196ed0dffd45228e35987d",
1432
+ "license": "Apache-2.0",
1433
+ "license_file": "LICENSE",
1434
+ "intended_language": "Kotlin",
1435
+ "files_kept": 2045,
1436
+ "bytes_kept": 8389113,
1437
+ "by_language": {
1438
+ "Python": [
1439
+ 1,
1440
+ 4977
1441
+ ],
1442
+ "Kotlin": [
1443
+ 2028,
1444
+ 8153707
1445
+ ],
1446
+ "JavaScript": [
1447
+ 2,
1448
+ 1653
1449
+ ],
1450
+ "C": [
1451
+ 5,
1452
+ 179946
1453
+ ],
1454
+ "Rust": [
1455
+ 8,
1456
+ 46487
1457
+ ],
1458
+ "Java": [
1459
+ 1,
1460
+ 2343
1461
+ ]
1462
+ },
1463
+ "drops": {
1464
+ "vendored_generated_or_fixture_dir": 15,
1465
+ "secret_pattern": 12,
1466
+ "exact_dup_sha256": 38,
1467
+ "eval_window_overlap64": 6,
1468
+ "minified_long_lines": 1,
1469
+ "binary": 1,
1470
+ "minified_lock_or_generated_name": 12,
1471
+ "repo_cap_8MB_reached": 503
1472
+ }
1473
+ },
1474
+ {
1475
+ "repo": "Alamofire/Alamofire",
1476
+ "url": "https://github.com/Alamofire/Alamofire",
1477
+ "commit": "bda9ed57d72988a3a2ada33d824583541f86eac6",
1478
+ "license": "MIT",
1479
+ "license_file": "LICENSE",
1480
+ "intended_language": "Swift",
1481
+ "files_kept": 79,
1482
+ "bytes_kept": 1141239,
1483
+ "by_language": {
1484
+ "Swift": [
1485
+ 76,
1486
+ 1065948
1487
+ ],
1488
+ "JavaScript": [
1489
+ 3,
1490
+ 75291
1491
+ ]
1492
+ },
1493
+ "drops": {
1494
+ "eval_window_overlap64": 22,
1495
+ "secret_pattern": 3,
1496
+ "minified_lock_or_generated_name": 4,
1497
+ "exact_dup_sha256": 3
1498
+ }
1499
+ },
1500
+ {
1501
+ "repo": "apple/swift-nio",
1502
+ "url": "https://github.com/apple/swift-nio",
1503
+ "commit": "108daca35da6a1b4ed8da29c1dfbd19668b9b8fc",
1504
+ "license": "Apache-2.0",
1505
+ "license_file": "LICENSE.txt",
1506
+ "intended_language": "Swift",
1507
+ "files_kept": 7,
1508
+ "bytes_kept": 72733,
1509
+ "by_language": {
1510
+ "Swift": [
1511
+ 2,
1512
+ 6166
1513
+ ],
1514
+ "C": [
1515
+ 5,
1516
+ 66567
1517
+ ]
1518
+ },
1519
+ "drops": {
1520
+ "eval_window_overlap64": 627,
1521
+ "exact_dup_sha256": 13,
1522
+ "too_large_256k": 1,
1523
+ "minified_lock_or_generated_name": 7,
1524
+ "generated_header": 1
1525
+ }
1526
+ },
1527
+ {
1528
+ "repo": "vapor/vapor",
1529
+ "url": "https://github.com/vapor/vapor",
1530
+ "commit": "1ccd882a17841f7ae1202b80795259c2c0082354",
1531
+ "license": "MIT",
1532
+ "license_file": "LICENSE",
1533
+ "intended_language": "Swift",
1534
+ "files_kept": 273,
1535
+ "bytes_kept": 1642178,
1536
+ "by_language": {
1537
+ "Swift": [
1538
+ 272,
1539
+ 1639745
1540
+ ],
1541
+ "Shell": [
1542
+ 1,
1543
+ 2433
1544
+ ]
1545
+ },
1546
+ "drops": {
1547
+ "secret_pattern": 2,
1548
+ "eval_window_overlap64": 2,
1549
+ "minified_long_lines": 1
1550
+ }
1551
+ },
1552
+ {
1553
+ "repo": "scala/scala",
1554
+ "url": "https://github.com/scala/scala",
1555
+ "commit": "65f01dbd3131320033301b9db4b83a919a4e7da7",
1556
+ "license": "Apache-2.0",
1557
+ "license_file": "LICENSE",
1558
+ "intended_language": "Scala",
1559
+ "files_kept": 962,
1560
+ "bytes_kept": 8422876,
1561
+ "by_language": {
1562
+ "Shell": [
1563
+ 3,
1564
+ 3261
1565
+ ],
1566
+ "Scala": [
1567
+ 914,
1568
+ 8325919
1569
+ ],
1570
+ "JavaScript": [
1571
+ 2,
1572
+ 5547
1573
+ ],
1574
+ "Java": [
1575
+ 43,
1576
+ 88149
1577
+ ]
1578
+ },
1579
+ "drops": {
1580
+ "generated_header": 104,
1581
+ "minified_long_lines": 1,
1582
+ "eval_window_overlap64": 32,
1583
+ "too_large_256k": 1,
1584
+ "minified_lock_or_generated_name": 1,
1585
+ "repo_cap_8MB_reached": 8949
1586
+ }
1587
+ },
1588
+ {
1589
+ "repo": "playframework/playframework",
1590
+ "url": "https://github.com/playframework/playframework",
1591
+ "commit": "f3eb3222f8911f5126ea5b127d86931518421f36",
1592
+ "license": "Apache-2.0",
1593
+ "license_file": "LICENSE",
1594
+ "intended_language": "Scala",
1595
+ "files_kept": 1626,
1596
+ "bytes_kept": 7612801,
1597
+ "by_language": {
1598
+ "Java": [
1599
+ 789,
1600
+ 3039844
1601
+ ],
1602
+ "Scala": [
1603
+ 815,
1604
+ 4444881
1605
+ ],
1606
+ "JavaScript": [
1607
+ 20,
1608
+ 123593
1609
+ ],
1610
+ "Kotlin": [
1611
+ 2,
1612
+ 4483
1613
+ ]
1614
+ },
1615
+ "drops": {
1616
+ "exact_dup_sha256": 17,
1617
+ "eval_window_overlap64": 23,
1618
+ "secret_pattern": 5,
1619
+ "generated_header": 1,
1620
+ "minified_lock_or_generated_name": 6,
1621
+ "vendored_generated_or_fixture_dir": 4
1622
+ }
1623
+ },
1624
+ {
1625
+ "repo": "ohmyzsh/ohmyzsh",
1626
+ "url": "https://github.com/ohmyzsh/ohmyzsh",
1627
+ "commit": "2100c630ba4bd67bb48802c3e7da893a4fdaa1a0",
1628
+ "license": "MIT",
1629
+ "license_file": "LICENSE.txt",
1630
+ "intended_language": "Shell",
1631
+ "files_kept": 372,
1632
+ "bytes_kept": 932312,
1633
+ "by_language": {
1634
+ "Python": [
1635
+ 7,
1636
+ 36459
1637
+ ],
1638
+ "Shell": [
1639
+ 365,
1640
+ 895853
1641
+ ]
1642
+ },
1643
+ "drops": {
1644
+ "eval_window_overlap64": 42,
1645
+ "minified_lock_or_generated_name": 1,
1646
+ "too_large_256k": 1,
1647
+ "generated_header": 1,
1648
+ "empty": 2
1649
+ }
1650
+ },
1651
+ {
1652
+ "repo": "nvm-sh/nvm",
1653
+ "url": "https://github.com/nvm-sh/nvm",
1654
+ "commit": "e780535240ee3ba67048b434cec6f7f509ee7b31",
1655
+ "license": "MIT",
1656
+ "license_file": "LICENSE.md",
1657
+ "intended_language": "Shell",
1658
+ "files_kept": 9,
1659
+ "bytes_kept": 238163,
1660
+ "by_language": {
1661
+ "Shell": [
1662
+ 8,
1663
+ 238125
1664
+ ],
1665
+ "JavaScript": [
1666
+ 1,
1667
+ 38
1668
+ ]
1669
+ }
1670
+ },
1671
+ {
1672
+ "repo": "dylanaraps/neofetch",
1673
+ "url": "https://github.com/dylanaraps/neofetch",
1674
+ "commit": "ccd5d9f52609bbdcd5d8fa78c4fdb0f12954125f",
1675
+ "license": "MIT",
1676
+ "license_file": "LICENSE.md",
1677
+ "intended_language": "Shell",
1678
+ "files_kept": 0,
1679
+ "bytes_kept": 0,
1680
+ "by_language": {}
1681
+ },
1682
+ {
1683
+ "repo": "sbt/sbt",
1684
+ "url": "https://github.com/sbt/sbt",
1685
+ "commit": "6fcb936dc56b9e020c88bb138879ce9b10e2b29f",
1686
+ "license": "Apache-2.0",
1687
+ "license_file": "LICENSE",
1688
+ "intended_language": "Scala",
1689
+ "files_kept": 1738,
1690
+ "bytes_kept": 4732831,
1691
+ "by_language": {
1692
+ "Shell": [
1693
+ 18,
1694
+ 19480
1695
+ ],
1696
+ "Scala": [
1697
+ 1578,
1698
+ 4554781
1699
+ ],
1700
+ "Java": [
1701
+ 135,
1702
+ 157091
1703
+ ],
1704
+ "C++": [
1705
+ 2,
1706
+ 508
1707
+ ],
1708
+ "C": [
1709
+ 5,
1710
+ 971
1711
+ ]
1712
+ },
1713
+ "drops": {
1714
+ "eval_window_overlap64": 8,
1715
+ "generated_header": 458,
1716
+ "exact_dup_sha256": 165,
1717
+ "secret_pattern": 1,
1718
+ "minified_lock_or_generated_name": 19,
1719
+ "empty": 5,
1720
+ "vendored_generated_or_fixture_dir": 1
1721
+ }
1722
+ },
1723
+ {
1724
+ "repo": "apple/swift-argument-parser",
1725
+ "url": "https://github.com/apple/swift-argument-parser",
1726
+ "commit": "bd10e57565e21be23b144f3770af7f2bfb068ecf",
1727
+ "license": "Apache-2.0",
1728
+ "license_file": "LICENSE.txt",
1729
+ "intended_language": "Swift",
1730
+ "files_kept": 0,
1731
+ "bytes_kept": 0,
1732
+ "by_language": {},
1733
+ "drops": {
1734
+ "eval_window_overlap64": 172,
1735
+ "vendored_generated_or_fixture_dir": 8
1736
+ }
1737
+ },
1738
+ {
1739
+ "repo": "ReactiveX/RxSwift",
1740
+ "url": "https://github.com/ReactiveX/RxSwift",
1741
+ "commit": "3e33f90c1bcd3cdea25bdb49bd4a594a50c3ab84",
1742
+ "license": "MIT",
1743
+ "license_file": "LICENSE.md",
1744
+ "intended_language": "Swift",
1745
+ "files_kept": 596,
1746
+ "bytes_kept": 3189661,
1747
+ "by_language": {
1748
+ "Swift": [
1749
+ 578,
1750
+ 3162406
1751
+ ],
1752
+ "C": [
1753
+ 8,
1754
+ 3335
1755
+ ],
1756
+ "Shell": [
1757
+ 9,
1758
+ 23259
1759
+ ],
1760
+ "Ruby": [
1761
+ 1,
1762
+ 661
1763
+ ]
1764
+ },
1765
+ "drops": {
1766
+ "minified_lock_or_generated_name": 12,
1767
+ "eval_window_overlap64": 4,
1768
+ "exact_dup_sha256": 8
1769
+ }
1770
+ },
1771
+ {
1772
+ "repo": "SwiftyJSON/SwiftyJSON",
1773
+ "url": "https://github.com/SwiftyJSON/SwiftyJSON",
1774
+ "commit": "3d2544101a03f22b2217d464dc7d359eeb86e159",
1775
+ "license": "MIT",
1776
+ "license_file": "LICENSE",
1777
+ "intended_language": "Swift",
1778
+ "files_kept": 24,
1779
+ "bytes_kept": 165999,
1780
+ "by_language": {
1781
+ "Swift": [
1782
+ 23,
1783
+ 164610
1784
+ ],
1785
+ "C": [
1786
+ 1,
1787
+ 1389
1788
+ ]
1789
+ }
1790
+ },
1791
+ {
1792
+ "repo": "Moya/Moya",
1793
+ "url": "https://github.com/Moya/Moya",
1794
+ "commit": "24c26671a2f0fc26b53c9fa2410c97987fb0f05a",
1795
+ "license": "MIT",
1796
+ "license_file": "License.md",
1797
+ "intended_language": "Swift",
1798
+ "files_kept": 68,
1799
+ "bytes_kept": 329973,
1800
+ "by_language": {
1801
+ "Swift": [
1802
+ 58,
1803
+ 324915
1804
+ ],
1805
+ "C": [
1806
+ 3,
1807
+ 1183
1808
+ ],
1809
+ "Shell": [
1810
+ 7,
1811
+ 3875
1812
+ ]
1813
+ },
1814
+ "drops": {
1815
+ "exact_dup_sha256": 1,
1816
+ "minified_long_lines": 2,
1817
+ "eval_window_overlap64": 1,
1818
+ "secret_pattern": 2
1819
+ }
1820
+ },
1821
+ {
1822
+ "repo": "bats-core/bats-core",
1823
+ "url": "https://github.com/bats-core/bats-core",
1824
+ "commit": "5b61d8f22b46abf001e218cdd63f2938c356fa93",
1825
+ "license": "MIT",
1826
+ "license_file": "LICENSE.md",
1827
+ "intended_language": "Shell",
1828
+ "files_kept": 18,
1829
+ "bytes_kept": 60392,
1830
+ "by_language": {
1831
+ "Shell": [
1832
+ 17,
1833
+ 60255
1834
+ ],
1835
+ "Python": [
1836
+ 1,
1837
+ 137
1838
+ ]
1839
+ },
1840
+ "drops": {
1841
+ "eval_window_overlap64": 2,
1842
+ "vendored_generated_or_fixture_dir": 34
1843
+ }
1844
+ }
1845
+ ],
1846
+ "separator": "\n<|endoffile|>\n",
1847
+ "excluded_repos": [
1848
+ {
1849
+ "repo": "hashicorp/terraform",
1850
+ "commit": "55988303ba5651e41e2c9097e208468c8eb3da4b",
1851
+ "why": "license_not_permissive_or_unrecognized:BUSL-1.1:LICENSE"
1852
+ },
1853
+ {
1854
+ "repo": "redis/redis",
1855
+ "commit": "81d653f5dc8d472a0d4f7dd328be720021580731",
1856
+ "why": "license_not_permissive_or_unrecognized:GPL:LICENSE.txt"
1857
+ },
1858
+ {
1859
+ "repo": "AutoMapper/AutoMapper",
1860
+ "commit": "f0ecba7a97d1403973825fb3cbe2733e6216ff9a",
1861
+ "why": "license_not_permissive_or_unrecognized:RPL-1.5:LICENSE.md"
1862
+ },
1863
+ {
1864
+ "repo": "akka/akka",
1865
+ "commit": "838fe7279217fb185a098abb84808c134f299161",
1866
+ "why": "license_not_permissive_or_unrecognized:BUSL-1.1:LICENSE,COPYING.protobuf"
1867
+ }
1868
+ ],
1869
+ "drops": {
1870
+ "eval_window_overlap64": 15279,
1871
+ "secret_pattern": 182,
1872
+ "empty": 588,
1873
+ "exact_dup_sha256": 1079,
1874
+ "vendored_generated_or_fixture_dir": 3807,
1875
+ "minified_lock_or_generated_name": 375,
1876
+ "generated_header": 824,
1877
+ "too_large_256k": 101,
1878
+ "repo_cap_8MB_reached": 71826,
1879
+ "minified_long_lines": 79,
1880
+ "missing_blob_or_link(>1MB filtered at clone)": 442,
1881
+ "non_utf8": 8,
1882
+ "binary": 2
1883
+ },
1884
+ "files_with_window_hash_candidates": 15279,
1885
+ "files_dropped_per_window_confirmed": {
1886
+ "broad_gutenberg_v1_oneshot": 0,
1887
+ "broad_gutenberg_v1_test": 294,
1888
+ "broad_local_v1_oneshot": 0,
1889
+ "broad_local_v1_test": 294,
1890
+ "corpus_v3_oneshot": 0,
1891
+ "corpus_v3_test": 0,
1892
+ "fc2": 7,
1893
+ "fresh_en": 0,
1894
+ "fresh_fr": 0,
1895
+ "freshcode": 0,
1896
+ "neither_code": 15038,
1897
+ "old_local_v1_oneshot": 0,
1898
+ "old_local_v1_test": 276,
1899
+ "queries": 0,
1900
+ "test20k": 276
1901
+ },
1902
+ "windows_used": {
1903
+ "broad_gutenberg_v1_oneshot": 119763,
1904
+ "broad_gutenberg_v1_test": 894718,
1905
+ "broad_local_v1_oneshot": 119763,
1906
+ "broad_local_v1_test": 894718,
1907
+ "corpus_v3_oneshot": 119763,
1908
+ "corpus_v3_test": 3486653,
1909
+ "fc2": 199792,
1910
+ "fresh_en": 119763,
1911
+ "fresh_fr": 1088688,
1912
+ "freshcode": 200000,
1913
+ "neither_code": 546036,
1914
+ "old_local_v1_oneshot": 119763,
1915
+ "old_local_v1_test": 319940,
1916
+ "queries": 27800,
1917
+ "test20k": 319940
1918
+ }
1919
+ }
results/hashing/multi_lang_v2/make_manifest.py ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SPIRA Hashing - manifest for multi_lang_v2 (streams train.bin; nothing estimated)."""
2
+ import json, hashlib, re, os
3
+ O = "/workspace/spira_shared/multi_lang_v2"; V1 = "/workspace/spira_shared/multi_lang_v1"
4
+ b = json.load(open(f"{O}/build_stage1.json")); v = json.load(open(f"{O}/verify_final.json"))
5
+ sha = hashlib.sha256(); ntok = 0; n = 0; prev_ws = True; WS = b" \t\n\r\x0b\x0c"
6
+ with open(f"{O}/train.bin","rb") as f:
7
+ while True:
8
+ c = f.read(1 << 24)
9
+ if not c: break
10
+ sha.update(c); n += len(c); t = c.split()
11
+ ntok += len(t)
12
+ if t and not prev_ws and c[0] not in WS: ntok -= 1 # token straddling chunk boundary
13
+ prev_ws = c[-1] in WS
14
+ assert sha.hexdigest() == v["train_sha256"] and n == v["train_bytes"]
15
+ lang = {}
16
+ for r in b["repos"]:
17
+ for lg, (fc, by) in r["by_language"].items():
18
+ L = lang.setdefault(lg, {"files": 0, "bytes": 0}); L["files"] += fc; L["bytes"] += by
19
+ v1repos = {l.split("\t")[1].strip() for l in open(f"{V1}/repos.tsv")} if False else None
20
+ v1 = json.load(open(f"{V1}/build_stage1.json")); v1set = {r["repo"] for r in v1["repos"]} | {r["repo"] for r in v1["excluded_repos"]}
21
+ files = sum(r["files_kept"] for r in b["repos"]); sepb = len(b["separator"].encode()) * (files - 1)
22
+ m = {"name": "multi_lang_v2", "builder": "SPIRA Hashing", "purpose": "multi_lang_v1 + more Swift (RxSwift, SwiftyJSON, Moya) and Shell (bats-core) after strict eval dedup left Swift/Shell thin in v1; v1 (feef51d5) frozen and untouched",
23
+ "train_bin": {"path": f"{O}/train.bin", "bytes": n, "sha256": sha.hexdigest()},
24
+ "format": {"separator": b["separator"], "separator_note": "files concatenated in repo order (repos.tsv) then sorted path order, joined by the separator (no separator at start/end)", "content_bytes_excl_separators": n - sepb, "separator_bytes": sepb},
25
+ "tokens": {"bytes": n, "whitespace_tokens": ntok, "definition": "Python bytes.split() with no argument over the whole train.bin (ASCII whitespace runs as delimiters), separators included; computed streaming in 16 MiB chunks with boundary correction. No subword tokenizer/EPT was run."},
26
+ "files_kept": files, "repos_kept": len(b["repos"]), "repos_with_zero_files": [r["repo"] for r in b["repos"] if r["files_kept"] == 0],
27
+ "per_language": dict(sorted(lang.items(), key=lambda x: -x[1]["bytes"])),
28
+ "repos": b["repos"], "excluded_repos": b["excluded_repos"], "repos_added_vs_v1": [r["repo"] for r in b["repos"] + b["excluded_repos"] if r["repo"] not in v1set],
29
+ "filters": {"extensions": "see build.py EXT", "max_file_bytes": 262144, "per_repo_cap_bytes": 8 << 20, "dirs_dropped": "node_modules, vendor(s), third_party, 3rdparty, external, dist, build, out, testdata, test_data, test-data, fixtures, __snapshots__, snapshots, generated, gen", "names_dropped": "*.min.js, *lock*, *_pb2.py, *.pb.go/.pb.cc/.pb.h, *.generated.cs, *.designer.cs", "generated_header": "@generated / DO NOT EDIT / Code generated / auto-generated in first 2 KB", "minified": "any line > 3000 bytes or mean line > 300 bytes", "binary_or_non_utf8": "NUL byte or failed UTF-8 decode", "secret_regex": r"AKIA[0-9A-Z]{16} | -----BEGIN | ghp_[A-Za-z0-9]{20,} | xox[bp]- | \bsk-[A-Za-z0-9_-]{16,} | password\s*[:=]\s*'literal'/\"literal\" (case-insensitive)", "exact_dedup": "sha256 per file across all repos", "clone": "git clone --depth 1 --filter=blob:limit=1m", "license_gate": "MIT, Apache-2.0, BSD-2/3-Clause, zlib, curl only (repo LICENSE/COPYING text match)"},
30
+ "drop_counts": b["drops"], "files_dropped_for_eval_overlap_by_window": b["files_dropped_per_window_confirmed"], "files_with_window_hash_candidates": b["files_with_window_hash_candidates"],
31
+ "dedup_verification_final": {"method": "verify_v3 hash (64-byte polynomial, P=1099511628211) on ALL positions of train.bin, separators included, byte-for-byte confirmation; 'seen' window excluded (training data)", "windows": v["windows"], "positions_checked": v["positions_checked"], "valid_zero_overlap": v["valid_zero_overlap"]},
32
+ "checker_recount": "PENDING - Checker recounts v2 before any training", "test_bin": "none (no hold-out produced)"}
33
+ outs = {}
34
+ for fn in ("build.py", "verify_final.py", "repos.tsv", "build_stage1.json", "verify_final.json"):
35
+ outs[fn] = hashlib.sha256(open(f"{O}/{fn}","rb").read()).hexdigest()
36
+ m["sha256_outputs"] = {"train.bin": sha.hexdigest(), **outs}
37
+ json.dump(m, open(f"{O}/manifest.json","w"), indent=1, ensure_ascii=False)
38
+ print(json.dumps({k: m[k] for k in ("train_bin","tokens","files_kept","repos_kept","repos_with_zero_files","per_language","drop_counts","repos_added_vs_v1")}, indent=1))
39
+ print({k: (w["train_positions_hash_hits"], w["confirmed_train_positions"]) for k, w in v["windows"].items()}, v["valid_zero_overlap"])
results/hashing/multi_lang_v2/manifest.json ADDED
@@ -0,0 +1,2130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "multi_lang_v2",
3
+ "builder": "SPIRA Hashing",
4
+ "purpose": "multi_lang_v1 + more Swift (RxSwift, SwiftyJSON, Moya) and Shell (bats-core) after strict eval dedup left Swift/Shell thin in v1; v1 (feef51d5) frozen and untouched",
5
+ "train_bin": {
6
+ "path": "/workspace/spira_shared/multi_lang_v2/train.bin",
7
+ "bytes": 243083814,
8
+ "sha256": "35858aab67bdcbff3a4a852560d9550b85eb06f262958e280b959c962c079c9f"
9
+ },
10
+ "format": {
11
+ "separator": "\n<|endoffile|>\n",
12
+ "separator_note": "files concatenated in repo order (repos.tsv) then sorted path order, joined by the separator (no separator at start/end)",
13
+ "content_bytes_excl_separators": 242519589,
14
+ "separator_bytes": 564225
15
+ },
16
+ "tokens": {
17
+ "bytes": 243083814,
18
+ "whitespace_tokens": 23706778,
19
+ "definition": "Python bytes.split() with no argument over the whole train.bin (ASCII whitespace runs as delimiters), separators included; computed streaming in 16 MiB chunks with boundary correction. No subword tokenizer/EPT was run."
20
+ },
21
+ "files_kept": 37616,
22
+ "repos_kept": 62,
23
+ "repos_with_zero_files": [
24
+ "dylanaraps/neofetch",
25
+ "apple/swift-argument-parser"
26
+ ],
27
+ "per_language": {
28
+ "Java": {
29
+ "files": 4782,
30
+ "bytes": 31356434
31
+ },
32
+ "Rust": {
33
+ "files": 2075,
34
+ "bytes": 20739112
35
+ },
36
+ "JavaScript": {
37
+ "files": 2778,
38
+ "bytes": 19333861
39
+ },
40
+ "Python": {
41
+ "files": 2247,
42
+ "bytes": 19259120
43
+ },
44
+ "PHP": {
45
+ "files": 3655,
46
+ "bytes": 19121100
47
+ },
48
+ "TypeScript": {
49
+ "files": 4473,
50
+ "bytes": 18926680
51
+ },
52
+ "Kotlin": {
53
+ "files": 4115,
54
+ "bytes": 18260690
55
+ },
56
+ "Scala": {
57
+ "files": 3307,
58
+ "bytes": 17325581
59
+ },
60
+ "Go": {
61
+ "files": 1579,
62
+ "bytes": 15830862
63
+ },
64
+ "C#": {
65
+ "files": 2321,
66
+ "bytes": 15338134
67
+ },
68
+ "C": {
69
+ "files": 1668,
70
+ "bytes": 14567171
71
+ },
72
+ "C++": {
73
+ "files": 1230,
74
+ "bytes": 14561409
75
+ },
76
+ "Ruby": {
77
+ "files": 1692,
78
+ "bytes": 9581642
79
+ },
80
+ "Swift": {
81
+ "files": 1037,
82
+ "bytes": 6582760
83
+ },
84
+ "Shell": {
85
+ "files": 657,
86
+ "bytes": 1735033
87
+ }
88
+ },
89
+ "repos": [
90
+ {
91
+ "repo": "psf/requests",
92
+ "url": "https://github.com/psf/requests",
93
+ "commit": "611c6162cbc4ac2020a2f91c7cfa4f3abf9bbb60",
94
+ "license": "Apache-2.0",
95
+ "license_file": "LICENSE",
96
+ "intended_language": "Python",
97
+ "files_kept": 34,
98
+ "bytes_kept": 334211,
99
+ "by_language": {
100
+ "Python": [
101
+ 34,
102
+ 334211
103
+ ]
104
+ },
105
+ "drops": {
106
+ "eval_window_overlap64": 1,
107
+ "secret_pattern": 1,
108
+ "empty": 1
109
+ }
110
+ },
111
+ {
112
+ "repo": "pallets/flask",
113
+ "url": "https://github.com/pallets/flask",
114
+ "commit": "d086db856be187255b8ec61ef409357393020f32",
115
+ "license": "BSD-3-Clause",
116
+ "license_file": "LICENSE.txt",
117
+ "intended_language": "Python",
118
+ "files_kept": 75,
119
+ "bytes_kept": 486917,
120
+ "by_language": {
121
+ "Shell": [
122
+ 1,
123
+ 165
124
+ ],
125
+ "Python": [
126
+ 74,
127
+ 486752
128
+ ]
129
+ },
130
+ "drops": {
131
+ "eval_window_overlap64": 5,
132
+ "secret_pattern": 1,
133
+ "empty": 3
134
+ }
135
+ },
136
+ {
137
+ "repo": "django/django",
138
+ "url": "https://github.com/django/django",
139
+ "commit": "0e128d524d0a7c55a955570d6e5da983ac6ce6c7",
140
+ "license": "BSD-3-Clause",
141
+ "license_file": "LICENSE,LICENSE.python",
142
+ "intended_language": "Python",
143
+ "files_kept": 1333,
144
+ "bytes_kept": 8413530,
145
+ "by_language": {
146
+ "JavaScript": [
147
+ 35,
148
+ 195616
149
+ ],
150
+ "Python": [
151
+ 1296,
152
+ 8214462
153
+ ],
154
+ "Shell": [
155
+ 2,
156
+ 3452
157
+ ]
158
+ },
159
+ "drops": {
160
+ "empty": 348,
161
+ "exact_dup_sha256": 7,
162
+ "eval_window_overlap64": 101,
163
+ "vendored_generated_or_fixture_dir": 68,
164
+ "minified_lock_or_generated_name": 1,
165
+ "generated_header": 1,
166
+ "secret_pattern": 46,
167
+ "too_large_256k": 1,
168
+ "repo_cap_8MB_reached": 1144
169
+ }
170
+ },
171
+ {
172
+ "repo": "scikit-learn/scikit-learn",
173
+ "url": "https://github.com/scikit-learn/scikit-learn",
174
+ "commit": "59106029d483678e79825f753bde67f8ade4332b",
175
+ "license": "BSD-3-Clause",
176
+ "license_file": "COPYING",
177
+ "intended_language": "Python",
178
+ "files_kept": 692,
179
+ "bytes_kept": 8424057,
180
+ "by_language": {
181
+ "Shell": [
182
+ 23,
183
+ 37357
184
+ ],
185
+ "Python": [
186
+ 664,
187
+ 8381084
188
+ ],
189
+ "JavaScript": [
190
+ 5,
191
+ 5616
192
+ ]
193
+ },
194
+ "drops": {
195
+ "eval_window_overlap64": 87,
196
+ "secret_pattern": 1,
197
+ "minified_lock_or_generated_name": 1,
198
+ "vendored_generated_or_fixture_dir": 1,
199
+ "empty": 46,
200
+ "exact_dup_sha256": 7,
201
+ "repo_cap_8MB_reached": 248
202
+ }
203
+ },
204
+ {
205
+ "repo": "encode/httpx",
206
+ "url": "https://github.com/encode/httpx",
207
+ "commit": "b5addb64f0161ff6bfe94c124ef76f6a1fba5254",
208
+ "license": "BSD-3-Clause",
209
+ "license_file": "LICENSE.md",
210
+ "intended_language": "Python",
211
+ "files_kept": 52,
212
+ "bytes_kept": 404301,
213
+ "by_language": {
214
+ "Python": [
215
+ 52,
216
+ 404301
217
+ ]
218
+ },
219
+ "drops": {
220
+ "eval_window_overlap64": 3,
221
+ "empty": 3,
222
+ "secret_pattern": 2
223
+ }
224
+ },
225
+ {
226
+ "repo": "expressjs/express",
227
+ "url": "https://github.com/expressjs/express",
228
+ "commit": "1c6f1b0fe4000977b9a7b4f34097e4d527dd9824",
229
+ "license": "MIT",
230
+ "license_file": "LICENSE",
231
+ "intended_language": "JavaScript",
232
+ "files_kept": 141,
233
+ "bytes_kept": 560914,
234
+ "by_language": {
235
+ "JavaScript": [
236
+ 141,
237
+ 560914
238
+ ]
239
+ },
240
+ "drops": {
241
+ "secret_pattern": 1
242
+ }
243
+ },
244
+ {
245
+ "repo": "lodash/lodash",
246
+ "url": "https://github.com/lodash/lodash",
247
+ "commit": "2b5e6f7399a7b48005140b5d5c6bc6c0e62919a8",
248
+ "license": "MIT",
249
+ "license_file": "LICENSE",
250
+ "intended_language": "JavaScript",
251
+ "files_kept": 8,
252
+ "bytes_kept": 13667,
253
+ "by_language": {
254
+ "JavaScript": [
255
+ 8,
256
+ 13667
257
+ ]
258
+ },
259
+ "drops": {
260
+ "vendored_generated_or_fixture_dir": 28,
261
+ "eval_window_overlap64": 17,
262
+ "too_large_256k": 2
263
+ }
264
+ },
265
+ {
266
+ "repo": "webpack/webpack",
267
+ "url": "https://github.com/webpack/webpack",
268
+ "commit": "7978462575e6f5a02b6b7a2c6d7b6b236ce76175",
269
+ "license": "MIT",
270
+ "license_file": "LICENSE",
271
+ "intended_language": "JavaScript",
272
+ "files_kept": 1240,
273
+ "bytes_kept": 8389112,
274
+ "by_language": {
275
+ "JavaScript": [
276
+ 1223,
277
+ 8124165
278
+ ],
279
+ "Shell": [
280
+ 2,
281
+ 557
282
+ ],
283
+ "TypeScript": [
284
+ 14,
285
+ 263673
286
+ ],
287
+ "C++": [
288
+ 1,
289
+ 717
290
+ ]
291
+ },
292
+ "drops": {
293
+ "exact_dup_sha256": 147,
294
+ "minified_long_lines": 27,
295
+ "vendored_generated_or_fixture_dir": 50,
296
+ "too_large_256k": 6,
297
+ "minified_lock_or_generated_name": 6,
298
+ "generated_header": 1,
299
+ "repo_cap_8MB_reached": 16117
300
+ }
301
+ },
302
+ {
303
+ "repo": "axios/axios",
304
+ "url": "https://github.com/axios/axios",
305
+ "commit": "aa54a5e1e0e50cd1a2561b854ae5490ff589d065",
306
+ "license": "MIT",
307
+ "license_file": "LICENSE",
308
+ "intended_language": "JavaScript",
309
+ "files_kept": 262,
310
+ "bytes_kept": 1026876,
311
+ "by_language": {
312
+ "TypeScript": [
313
+ 25,
314
+ 64591
315
+ ],
316
+ "JavaScript": [
317
+ 237,
318
+ 962285
319
+ ]
320
+ },
321
+ "drops": {
322
+ "secret_pattern": 14,
323
+ "exact_dup_sha256": 1
324
+ }
325
+ },
326
+ {
327
+ "repo": "mrdoob/three.js",
328
+ "url": "https://github.com/mrdoob/three.js",
329
+ "commit": "e27dfa95ae7f453b5d7a9a36ffdb563f7c9b529b",
330
+ "license": "MIT",
331
+ "license_file": "LICENSE",
332
+ "intended_language": "JavaScript",
333
+ "files_kept": 863,
334
+ "bytes_kept": 8396389,
335
+ "by_language": {
336
+ "JavaScript": [
337
+ 863,
338
+ 8396389
339
+ ]
340
+ },
341
+ "drops": {
342
+ "vendored_generated_or_fixture_dir": 7,
343
+ "minified_lock_or_generated_name": 6,
344
+ "minified_long_lines": 13,
345
+ "too_large_256k": 5,
346
+ "eval_window_overlap64": 2,
347
+ "empty": 1,
348
+ "repo_cap_8MB_reached": 864
349
+ }
350
+ },
351
+ {
352
+ "repo": "microsoft/vscode",
353
+ "url": "https://github.com/microsoft/vscode",
354
+ "commit": "1184ad4d45794110deb6b897ea83b15aac44ed89",
355
+ "license": "MIT",
356
+ "license_file": "LICENSE.txt",
357
+ "intended_language": "TypeScript",
358
+ "files_kept": 226,
359
+ "bytes_kept": 1378837,
360
+ "by_language": {
361
+ "Shell": [
362
+ 57,
363
+ 101458
364
+ ],
365
+ "TypeScript": [
366
+ 128,
367
+ 1174818
368
+ ],
369
+ "Python": [
370
+ 16,
371
+ 10028
372
+ ],
373
+ "JavaScript": [
374
+ 6,
375
+ 85559
376
+ ],
377
+ "Ruby": [
378
+ 2,
379
+ 570
380
+ ],
381
+ "Java": [
382
+ 1,
383
+ 599
384
+ ],
385
+ "PHP": [
386
+ 3,
387
+ 1014
388
+ ],
389
+ "Go": [
390
+ 2,
391
+ 652
392
+ ],
393
+ "Rust": [
394
+ 3,
395
+ 682
396
+ ],
397
+ "C++": [
398
+ 5,
399
+ 1491
400
+ ],
401
+ "C": [
402
+ 1,
403
+ 818
404
+ ],
405
+ "C#": [
406
+ 1,
407
+ 864
408
+ ],
409
+ "Swift": [
410
+ 1,
411
+ 284
412
+ ]
413
+ },
414
+ "drops": {
415
+ "eval_window_overlap64": 13775,
416
+ "generated_header": 72,
417
+ "vendored_generated_or_fixture_dir": 1647,
418
+ "minified_long_lines": 25,
419
+ "minified_lock_or_generated_name": 55,
420
+ "secret_pattern": 19,
421
+ "too_large_256k": 39,
422
+ "exact_dup_sha256": 25,
423
+ "empty": 3
424
+ }
425
+ },
426
+ {
427
+ "repo": "vuejs/core",
428
+ "url": "https://github.com/vuejs/core",
429
+ "commit": "0ff253bfba3ee8d3d65448bd44ad890fd01b0ecd",
430
+ "license": "MIT",
431
+ "license_file": "LICENSE",
432
+ "intended_language": "TypeScript",
433
+ "files_kept": 518,
434
+ "bytes_kept": 4533129,
435
+ "by_language": {
436
+ "JavaScript": [
437
+ 34,
438
+ 74741
439
+ ],
440
+ "TypeScript": [
441
+ 484,
442
+ 4458388
443
+ ]
444
+ },
445
+ "drops": {
446
+ "eval_window_overlap64": 4,
447
+ "exact_dup_sha256": 4
448
+ }
449
+ },
450
+ {
451
+ "repo": "angular/angular",
452
+ "url": "https://github.com/angular/angular",
453
+ "commit": "c27695f278a88ce5e74f7504320dae5a53862236",
454
+ "license": "MIT",
455
+ "license_file": "LICENSE",
456
+ "intended_language": "TypeScript",
457
+ "files_kept": 1736,
458
+ "bytes_kept": 8392070,
459
+ "by_language": {
460
+ "Shell": [
461
+ 6,
462
+ 6192
463
+ ],
464
+ "JavaScript": [
465
+ 56,
466
+ 75868
467
+ ],
468
+ "TypeScript": [
469
+ 1606,
470
+ 5921307
471
+ ],
472
+ "Rust": [
473
+ 68,
474
+ 2388703
475
+ ]
476
+ },
477
+ "drops": {
478
+ "too_large_256k": 6,
479
+ "minified_long_lines": 3,
480
+ "minified_lock_or_generated_name": 3,
481
+ "exact_dup_sha256": 231,
482
+ "empty": 1,
483
+ "eval_window_overlap64": 3,
484
+ "vendored_generated_or_fixture_dir": 8,
485
+ "repo_cap_8MB_reached": 5269
486
+ }
487
+ },
488
+ {
489
+ "repo": "nestjs/nest",
490
+ "url": "https://github.com/nestjs/nest",
491
+ "commit": "9a95605281355d6b89401eb06002c3bc64545838",
492
+ "license": "MIT",
493
+ "license_file": "LICENSE",
494
+ "intended_language": "TypeScript",
495
+ "files_kept": 1887,
496
+ "bytes_kept": 5616113,
497
+ "by_language": {
498
+ "JavaScript": [
499
+ 10,
500
+ 6522
501
+ ],
502
+ "TypeScript": [
503
+ 1873,
504
+ 5608810
505
+ ],
506
+ "Shell": [
507
+ 4,
508
+ 781
509
+ ]
510
+ },
511
+ "drops": {
512
+ "exact_dup_sha256": 160,
513
+ "secret_pattern": 17,
514
+ "vendored_generated_or_fixture_dir": 17,
515
+ "generated_header": 2
516
+ }
517
+ },
518
+ {
519
+ "repo": "BurntSushi/ripgrep",
520
+ "url": "https://github.com/BurntSushi/ripgrep",
521
+ "commit": "6892bc2b7efe5909cbbeec8f757ed2c4c76b1d30",
522
+ "license": "MIT",
523
+ "license_file": "LICENSE-MIT,UNLICENSE,COPYING",
524
+ "intended_language": "Rust",
525
+ "files_kept": 104,
526
+ "bytes_kept": 1876977,
527
+ "by_language": {
528
+ "Rust": [
529
+ 101,
530
+ 1872758
531
+ ],
532
+ "Shell": [
533
+ 2,
534
+ 3415
535
+ ],
536
+ "Ruby": [
537
+ 1,
538
+ 804
539
+ ]
540
+ },
541
+ "drops": {
542
+ "eval_window_overlap64": 1,
543
+ "generated_header": 1
544
+ }
545
+ },
546
+ {
547
+ "repo": "tokio-rs/tokio",
548
+ "url": "https://github.com/tokio-rs/tokio",
549
+ "commit": "8a2408edcfa2b7f19c00428a9d3a8ea6652a083f",
550
+ "license": "MIT",
551
+ "license_file": "LICENSE",
552
+ "intended_language": "Rust",
553
+ "files_kept": 784,
554
+ "bytes_kept": 5764849,
555
+ "by_language": {
556
+ "Rust": [
557
+ 784,
558
+ 5764849
559
+ ]
560
+ },
561
+ "drops": {
562
+ "eval_window_overlap64": 4,
563
+ "minified_lock_or_generated_name": 25,
564
+ "exact_dup_sha256": 1
565
+ }
566
+ },
567
+ {
568
+ "repo": "serde-rs/serde",
569
+ "url": "https://github.com/serde-rs/serde",
570
+ "commit": "0db9b3b504bcb0a7fa9cf001429869519afd3e9a",
571
+ "license": "Apache-2.0",
572
+ "license_file": "LICENSE-APACHE,LICENSE-MIT",
573
+ "intended_language": "Rust",
574
+ "files_kept": 207,
575
+ "bytes_kept": 1246203,
576
+ "by_language": {
577
+ "Rust": [
578
+ 207,
579
+ 1246203
580
+ ]
581
+ },
582
+ "drops": {
583
+ "exact_dup_sha256": 1
584
+ }
585
+ },
586
+ {
587
+ "repo": "rust-lang/cargo",
588
+ "url": "https://github.com/rust-lang/cargo",
589
+ "commit": "cad07cf8e597397fff5d0f6cb79c9f870c0b1dd2",
590
+ "license": "Apache-2.0",
591
+ "license_file": "LICENSE-APACHE,LICENSE-MIT",
592
+ "intended_language": "Rust",
593
+ "files_kept": 827,
594
+ "bytes_kept": 8403116,
595
+ "by_language": {
596
+ "Rust": [
597
+ 818,
598
+ 8361093
599
+ ],
600
+ "Shell": [
601
+ 6,
602
+ 14085
603
+ ],
604
+ "Python": [
605
+ 2,
606
+ 3560
607
+ ],
608
+ "JavaScript": [
609
+ 1,
610
+ 24378
611
+ ]
612
+ },
613
+ "drops": {
614
+ "exact_dup_sha256": 130,
615
+ "minified_lock_or_generated_name": 12,
616
+ "empty": 168,
617
+ "vendored_generated_or_fixture_dir": 130,
618
+ "repo_cap_8MB_reached": 121
619
+ }
620
+ },
621
+ {
622
+ "repo": "alacritty/alacritty",
623
+ "url": "https://github.com/alacritty/alacritty",
624
+ "commit": "542147067dd016fc5f49120ef0ba605a98a4e3a7",
625
+ "license": "Apache-2.0",
626
+ "license_file": "LICENSE-APACHE,LICENSE-MIT",
627
+ "intended_language": "Rust",
628
+ "files_kept": 91,
629
+ "bytes_kept": 1080257,
630
+ "by_language": {
631
+ "Shell": [
632
+ 6,
633
+ 26451
634
+ ],
635
+ "Rust": [
636
+ 85,
637
+ 1053806
638
+ ]
639
+ },
640
+ "drops": {
641
+ "eval_window_overlap64": 2,
642
+ "minified_lock_or_generated_name": 1
643
+ }
644
+ },
645
+ {
646
+ "repo": "gin-gonic/gin",
647
+ "url": "https://github.com/gin-gonic/gin",
648
+ "commit": "0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646",
649
+ "license": "MIT",
650
+ "license_file": "LICENSE",
651
+ "intended_language": "Go",
652
+ "files_kept": 98,
653
+ "bytes_kept": 697100,
654
+ "by_language": {
655
+ "Go": [
656
+ 98,
657
+ 697100
658
+ ]
659
+ },
660
+ "drops": {
661
+ "vendored_generated_or_fixture_dir": 1
662
+ }
663
+ },
664
+ {
665
+ "repo": "spf13/cobra",
666
+ "url": "https://github.com/spf13/cobra",
667
+ "commit": "adbc8813901bba65827259daa8e22ff94ec1f30e",
668
+ "license": "Apache-2.0",
669
+ "license_file": "LICENSE.txt",
670
+ "intended_language": "Go",
671
+ "files_kept": 36,
672
+ "bytes_kept": 512600,
673
+ "by_language": {
674
+ "Go": [
675
+ 36,
676
+ 512600
677
+ ]
678
+ }
679
+ },
680
+ {
681
+ "repo": "prometheus/prometheus",
682
+ "url": "https://github.com/prometheus/prometheus",
683
+ "commit": "a2fda0ef4638518a6a6fb27fc48be88fbd732f4b",
684
+ "license": "Apache-2.0",
685
+ "license_file": "LICENSE",
686
+ "intended_language": "Go",
687
+ "files_kept": 552,
688
+ "bytes_kept": 8396886,
689
+ "by_language": {
690
+ "Go": [
691
+ 540,
692
+ 8366827
693
+ ],
694
+ "Shell": [
695
+ 12,
696
+ 30059
697
+ ]
698
+ },
699
+ "drops": {
700
+ "secret_pattern": 8,
701
+ "eval_window_overlap64": 2,
702
+ "minified_lock_or_generated_name": 17,
703
+ "minified_long_lines": 1,
704
+ "generated_header": 5,
705
+ "too_large_256k": 3,
706
+ "exact_dup_sha256": 1,
707
+ "repo_cap_8MB_reached": 443
708
+ }
709
+ },
710
+ {
711
+ "repo": "gohugoio/hugo",
712
+ "url": "https://github.com/gohugoio/hugo",
713
+ "commit": "ed22b85c192dd750cf0bdfb2d2005d31ab3bc0a7",
714
+ "license": "Apache-2.0",
715
+ "license_file": "LICENSE",
716
+ "intended_language": "Go",
717
+ "files_kept": 933,
718
+ "bytes_kept": 6590268,
719
+ "by_language": {
720
+ "Go": [
721
+ 899,
722
+ 6245064
723
+ ],
724
+ "Shell": [
725
+ 10,
726
+ 5004
727
+ ],
728
+ "JavaScript": [
729
+ 20,
730
+ 174328
731
+ ],
732
+ "C": [
733
+ 4,
734
+ 165872
735
+ ]
736
+ },
737
+ "drops": {
738
+ "generated_header": 12,
739
+ "eval_window_overlap64": 3,
740
+ "too_large_256k": 1,
741
+ "vendored_generated_or_fixture_dir": 4,
742
+ "minified_lock_or_generated_name": 5,
743
+ "exact_dup_sha256": 1
744
+ }
745
+ },
746
+ {
747
+ "repo": "curl/curl",
748
+ "url": "https://github.com/curl/curl",
749
+ "commit": "178e5077b60be91a8dfcbaa22a6bf79e179009ff",
750
+ "license": "curl (MIT/X derivative)",
751
+ "license_file": "COPYING",
752
+ "intended_language": "C",
753
+ "files_kept": 969,
754
+ "bytes_kept": 8391247,
755
+ "by_language": {
756
+ "Shell": [
757
+ 13,
758
+ 15558
759
+ ],
760
+ "C": [
761
+ 953,
762
+ 8356082
763
+ ],
764
+ "C++": [
765
+ 2,
766
+ 8157
767
+ ],
768
+ "Python": [
769
+ 1,
770
+ 11450
771
+ ]
772
+ },
773
+ "drops": {
774
+ "eval_window_overlap64": 86,
775
+ "secret_pattern": 10,
776
+ "minified_lock_or_generated_name": 4,
777
+ "generated_header": 1,
778
+ "repo_cap_8MB_reached": 88
779
+ }
780
+ },
781
+ {
782
+ "repo": "libuv/libuv",
783
+ "url": "https://github.com/libuv/libuv",
784
+ "commit": "19bbf90e1a7a1526da6daa1c79eec2e3ff8b6c55",
785
+ "license": "MIT",
786
+ "license_file": "LICENSE,LICENSE-docs,LICENSE-extra",
787
+ "intended_language": "C",
788
+ "files_kept": 338,
789
+ "bytes_kept": 2645018,
790
+ "by_language": {
791
+ "Shell": [
792
+ 1,
793
+ 2489
794
+ ],
795
+ "C": [
796
+ 333,
797
+ 2626891
798
+ ],
799
+ "JavaScript": [
800
+ 1,
801
+ 511
802
+ ],
803
+ "Python": [
804
+ 3,
805
+ 15127
806
+ ]
807
+ },
808
+ "drops": {
809
+ "eval_window_overlap64": 34,
810
+ "minified_lock_or_generated_name": 2
811
+ }
812
+ },
813
+ {
814
+ "repo": "madler/zlib",
815
+ "url": "https://github.com/madler/zlib",
816
+ "commit": "767c4c947852e143f582c85f14cf573411df1b35",
817
+ "license": "zlib",
818
+ "license_file": "LICENSE",
819
+ "intended_language": "C",
820
+ "files_kept": 75,
821
+ "bytes_kept": 1029227,
822
+ "by_language": {
823
+ "C": [
824
+ 68,
825
+ 997753
826
+ ],
827
+ "C#": [
828
+ 2,
829
+ 10188
830
+ ],
831
+ "C++": [
832
+ 5,
833
+ 21286
834
+ ]
835
+ },
836
+ "drops": {
837
+ "non_utf8": 7,
838
+ "eval_window_overlap64": 6,
839
+ "too_large_256k": 1
840
+ }
841
+ },
842
+ {
843
+ "repo": "jqlang/jq",
844
+ "url": "https://github.com/jqlang/jq",
845
+ "commit": "da4da65c0cfffe0481ebf26ad64346eb45e4f34d",
846
+ "license": "MIT",
847
+ "license_file": "COPYING",
848
+ "intended_language": "C",
849
+ "files_kept": 58,
850
+ "bytes_kept": 667825,
851
+ "by_language": {
852
+ "Shell": [
853
+ 2,
854
+ 3348
855
+ ],
856
+ "Python": [
857
+ 5,
858
+ 13972
859
+ ],
860
+ "JavaScript": [
861
+ 1,
862
+ 1365
863
+ ],
864
+ "C": [
865
+ 48,
866
+ 639007
867
+ ],
868
+ "C++": [
869
+ 2,
870
+ 10133
871
+ ]
872
+ },
873
+ "drops": {
874
+ "eval_window_overlap64": 1,
875
+ "vendored_generated_or_fixture_dir": 30
876
+ }
877
+ },
878
+ {
879
+ "repo": "nlohmann/json",
880
+ "url": "https://github.com/nlohmann/json",
881
+ "commit": "ffcd392be06c871e9a75f7d6b4f4b9c2288bc19c",
882
+ "license": "MIT",
883
+ "license_file": "LICENSE.MIT",
884
+ "intended_language": "C++",
885
+ "files_kept": 573,
886
+ "bytes_kept": 5279605,
887
+ "by_language": {
888
+ "Swift": [
889
+ 2,
890
+ 967
891
+ ],
892
+ "C++": [
893
+ 561,
894
+ 5200900
895
+ ],
896
+ "Python": [
897
+ 9,
898
+ 75408
899
+ ],
900
+ "JavaScript": [
901
+ 1,
902
+ 2330
903
+ ]
904
+ },
905
+ "drops": {
906
+ "exact_dup_sha256": 21,
907
+ "eval_window_overlap64": 7,
908
+ "too_large_256k": 4,
909
+ "vendored_generated_or_fixture_dir": 6
910
+ }
911
+ },
912
+ {
913
+ "repo": "fmtlib/fmt",
914
+ "url": "https://github.com/fmtlib/fmt",
915
+ "commit": "0fc31633e7d53ac895ae0a688b83b7d44f0b0c1f",
916
+ "license": "MIT",
917
+ "license_file": "LICENSE",
918
+ "intended_language": "C++",
919
+ "files_kept": 72,
920
+ "bytes_kept": 874695,
921
+ "by_language": {
922
+ "JavaScript": [
923
+ 1,
924
+ 76
925
+ ],
926
+ "C": [
927
+ 21,
928
+ 405790
929
+ ],
930
+ "C++": [
931
+ 45,
932
+ 417553
933
+ ],
934
+ "Python": [
935
+ 4,
936
+ 49163
937
+ ],
938
+ "Shell": [
939
+ 1,
940
+ 2113
941
+ ]
942
+ },
943
+ "drops": {
944
+ "eval_window_overlap64": 5,
945
+ "too_large_256k": 3
946
+ }
947
+ },
948
+ {
949
+ "repo": "google/leveldb",
950
+ "url": "https://github.com/google/leveldb",
951
+ "commit": "bb74ef739973a70ca9f0d90788a1da89b788c0ad",
952
+ "license": "BSD-3-Clause",
953
+ "license_file": "LICENSE",
954
+ "intended_language": "C++",
955
+ "files_kept": 124,
956
+ "bytes_kept": 855586,
957
+ "by_language": {
958
+ "C++": [
959
+ 71,
960
+ 655720
961
+ ],
962
+ "C": [
963
+ 53,
964
+ 199866
965
+ ]
966
+ },
967
+ "drops": {
968
+ "minified_lock_or_generated_name": 8,
969
+ "eval_window_overlap64": 1
970
+ }
971
+ },
972
+ {
973
+ "repo": "opencv/opencv",
974
+ "url": "https://github.com/opencv/opencv",
975
+ "commit": "4058787c826610fdb1d6d2e18517d93e3ad9b184",
976
+ "license": "Apache-2.0",
977
+ "license_file": "LICENSE",
978
+ "intended_language": "C++",
979
+ "files_kept": 615,
980
+ "bytes_kept": 8392271,
981
+ "by_language": {
982
+ "C++": [
983
+ 467,
984
+ 7444738
985
+ ],
986
+ "Python": [
987
+ 17,
988
+ 268125
989
+ ],
990
+ "JavaScript": [
991
+ 8,
992
+ 27664
993
+ ],
994
+ "Java": [
995
+ 45,
996
+ 268778
997
+ ],
998
+ "Swift": [
999
+ 22,
1000
+ 191850
1001
+ ],
1002
+ "C": [
1003
+ 54,
1004
+ 185526
1005
+ ],
1006
+ "Kotlin": [
1007
+ 2,
1008
+ 5590
1009
+ ]
1010
+ },
1011
+ "drops": {
1012
+ "vendored_generated_or_fixture_dir": 1293,
1013
+ "eval_window_overlap64": 55,
1014
+ "exact_dup_sha256": 1,
1015
+ "minified_long_lines": 2,
1016
+ "minified_lock_or_generated_name": 2,
1017
+ "generated_header": 7,
1018
+ "repo_cap_8MB_reached": 2661
1019
+ }
1020
+ },
1021
+ {
1022
+ "repo": "protocolbuffers/protobuf",
1023
+ "url": "https://github.com/protocolbuffers/protobuf",
1024
+ "commit": "5db636d202550a7c6efa49f5ea9742f87e32bab0",
1025
+ "license": "BSD-3-Clause",
1026
+ "license_file": "LICENSE",
1027
+ "intended_language": "C++",
1028
+ "files_kept": 758,
1029
+ "bytes_kept": 8415537,
1030
+ "by_language": {
1031
+ "Python": [
1032
+ 40,
1033
+ 527533
1034
+ ],
1035
+ "Shell": [
1036
+ 29,
1037
+ 63077
1038
+ ],
1039
+ "C++": [
1040
+ 69,
1041
+ 800206
1042
+ ],
1043
+ "Java": [
1044
+ 252,
1045
+ 3878162
1046
+ ],
1047
+ "PHP": [
1048
+ 58,
1049
+ 580475
1050
+ ],
1051
+ "C": [
1052
+ 106,
1053
+ 736175
1054
+ ],
1055
+ "Rust": [
1056
+ 1,
1057
+ 4531
1058
+ ],
1059
+ "Ruby": [
1060
+ 3,
1061
+ 6830
1062
+ ],
1063
+ "C#": [
1064
+ 170,
1065
+ 1651726
1066
+ ],
1067
+ "Go": [
1068
+ 4,
1069
+ 8619
1070
+ ],
1071
+ "Kotlin": [
1072
+ 23,
1073
+ 132334
1074
+ ],
1075
+ "Swift": [
1076
+ 3,
1077
+ 25869
1078
+ ]
1079
+ },
1080
+ "drops": {
1081
+ "exact_dup_sha256": 7,
1082
+ "generated_header": 149,
1083
+ "too_large_256k": 21,
1084
+ "eval_window_overlap64": 43,
1085
+ "minified_lock_or_generated_name": 5,
1086
+ "vendored_generated_or_fixture_dir": 2,
1087
+ "empty": 1,
1088
+ "repo_cap_8MB_reached": 1391
1089
+ }
1090
+ },
1091
+ {
1092
+ "repo": "google/guava",
1093
+ "url": "https://github.com/google/guava",
1094
+ "commit": "1b1d120f9cd79ed5dcb1eb1e355e8f48b2b1bc6f",
1095
+ "license": "Apache-2.0",
1096
+ "license_file": "LICENSE",
1097
+ "intended_language": "Java",
1098
+ "files_kept": 1054,
1099
+ "bytes_kept": 8390932,
1100
+ "by_language": {
1101
+ "Java": [
1102
+ 1054,
1103
+ 8390932
1104
+ ]
1105
+ },
1106
+ "drops": {
1107
+ "eval_window_overlap64": 1,
1108
+ "minified_lock_or_generated_name": 7,
1109
+ "repo_cap_8MB_reached": 2221
1110
+ }
1111
+ },
1112
+ {
1113
+ "repo": "spring-projects/spring-framework",
1114
+ "url": "https://github.com/spring-projects/spring-framework",
1115
+ "commit": "b671d1a36a42dbcba0ba3657d1d353d421aa64ba",
1116
+ "license": "Apache-2.0",
1117
+ "license_file": "LICENSE.txt",
1118
+ "intended_language": "Java",
1119
+ "files_kept": 1968,
1120
+ "bytes_kept": 8391477,
1121
+ "by_language": {
1122
+ "Java": [
1123
+ 1750,
1124
+ 8034624
1125
+ ],
1126
+ "Kotlin": [
1127
+ 218,
1128
+ 356853
1129
+ ]
1130
+ },
1131
+ "drops": {
1132
+ "vendored_generated_or_fixture_dir": 20,
1133
+ "eval_window_overlap64": 24,
1134
+ "minified_lock_or_generated_name": 6,
1135
+ "repo_cap_8MB_reached": 7685
1136
+ }
1137
+ },
1138
+ {
1139
+ "repo": "square/okhttp",
1140
+ "url": "https://github.com/square/okhttp",
1141
+ "commit": "c3179301fc7d0266c5bf154870c2b0b9da01e7e7",
1142
+ "license": "Apache-2.0",
1143
+ "license_file": "LICENSE.txt",
1144
+ "intended_language": "Java",
1145
+ "files_kept": 677,
1146
+ "bytes_kept": 4496384,
1147
+ "by_language": {
1148
+ "Kotlin": [
1149
+ 606,
1150
+ 4363381
1151
+ ],
1152
+ "Shell": [
1153
+ 4,
1154
+ 3047
1155
+ ],
1156
+ "Java": [
1157
+ 67,
1158
+ 129956
1159
+ ]
1160
+ },
1161
+ "drops": {
1162
+ "secret_pattern": 15,
1163
+ "minified_lock_or_generated_name": 2
1164
+ }
1165
+ },
1166
+ {
1167
+ "repo": "apache/commons-lang",
1168
+ "url": "https://github.com/apache/commons-lang",
1169
+ "commit": "b9b30efb01e861a16503232bb96b684107e73199",
1170
+ "license": "Apache-2.0",
1171
+ "license_file": "LICENSE.txt",
1172
+ "intended_language": "Java",
1173
+ "files_kept": 621,
1174
+ "bytes_kept": 7323827,
1175
+ "by_language": {
1176
+ "Java": [
1177
+ 621,
1178
+ 7323827
1179
+ ]
1180
+ },
1181
+ "drops": {
1182
+ "too_large_256k": 3,
1183
+ "secret_pattern": 1,
1184
+ "eval_window_overlap64": 1,
1185
+ "minified_lock_or_generated_name": 2,
1186
+ "binary": 1,
1187
+ "vendored_generated_or_fixture_dir": 1
1188
+ }
1189
+ },
1190
+ {
1191
+ "repo": "dotnet/aspnetcore",
1192
+ "url": "https://github.com/dotnet/aspnetcore",
1193
+ "commit": "ceb0eec49d8d2c77f4f7c26b85ddb6e96940dfed",
1194
+ "license": "MIT",
1195
+ "license_file": "LICENSE.txt",
1196
+ "intended_language": "C#",
1197
+ "files_kept": 1407,
1198
+ "bytes_kept": 8388820,
1199
+ "by_language": {
1200
+ "Shell": [
1201
+ 40,
1202
+ 147892
1203
+ ],
1204
+ "C#": [
1205
+ 1209,
1206
+ 6996777
1207
+ ],
1208
+ "JavaScript": [
1209
+ 25,
1210
+ 108159
1211
+ ],
1212
+ "Python": [
1213
+ 19,
1214
+ 403728
1215
+ ],
1216
+ "TypeScript": [
1217
+ 114,
1218
+ 732264
1219
+ ]
1220
+ },
1221
+ "drops": {
1222
+ "eval_window_overlap64": 4,
1223
+ "secret_pattern": 1,
1224
+ "exact_dup_sha256": 9,
1225
+ "minified_lock_or_generated_name": 39,
1226
+ "vendored_generated_or_fixture_dir": 32,
1227
+ "generated_header": 1,
1228
+ "repo_cap_8MB_reached": 10206
1229
+ }
1230
+ },
1231
+ {
1232
+ "repo": "JamesNK/Newtonsoft.Json",
1233
+ "url": "https://github.com/JamesNK/Newtonsoft.Json",
1234
+ "commit": "52fa3aef1f2cadcd3a3f874251eddc98d3efbbaa",
1235
+ "license": "MIT",
1236
+ "license_file": "LICENSE.md",
1237
+ "intended_language": "C#",
1238
+ "files_kept": 939,
1239
+ "bytes_kept": 6678579,
1240
+ "by_language": {
1241
+ "C#": [
1242
+ 939,
1243
+ 6678579
1244
+ ]
1245
+ },
1246
+ "drops": {
1247
+ "minified_long_lines": 1,
1248
+ "minified_lock_or_generated_name": 3,
1249
+ "eval_window_overlap64": 7,
1250
+ "too_large_256k": 1
1251
+ }
1252
+ },
1253
+ {
1254
+ "repo": "rails/rails",
1255
+ "url": "https://github.com/rails/rails",
1256
+ "commit": "a0fd31e1a4e96c80d3184998c956607004fb7f8c",
1257
+ "license": "MIT",
1258
+ "license_file": "MIT-LICENSE",
1259
+ "intended_language": "Ruby",
1260
+ "files_kept": 1421,
1261
+ "bytes_kept": 8394471,
1262
+ "by_language": {
1263
+ "Shell": [
1264
+ 1,
1265
+ 378
1266
+ ],
1267
+ "Ruby": [
1268
+ 1382,
1269
+ 8224947
1270
+ ],
1271
+ "JavaScript": [
1272
+ 38,
1273
+ 169146
1274
+ ]
1275
+ },
1276
+ "drops": {
1277
+ "secret_pattern": 17,
1278
+ "empty": 5,
1279
+ "exact_dup_sha256": 19,
1280
+ "generated_header": 5,
1281
+ "minified_lock_or_generated_name": 2,
1282
+ "vendored_generated_or_fixture_dir": 23,
1283
+ "too_large_256k": 1,
1284
+ "eval_window_overlap64": 2,
1285
+ "repo_cap_8MB_reached": 2049
1286
+ }
1287
+ },
1288
+ {
1289
+ "repo": "jekyll/jekyll",
1290
+ "url": "https://github.com/jekyll/jekyll",
1291
+ "commit": "97c3bdc822df9d5bea4c269b29a28de5a5e29d70",
1292
+ "license": "MIT",
1293
+ "license_file": "LICENSE",
1294
+ "intended_language": "Ruby",
1295
+ "files_kept": 160,
1296
+ "bytes_kept": 712533,
1297
+ "by_language": {
1298
+ "Ruby": [
1299
+ 157,
1300
+ 674265
1301
+ ],
1302
+ "JavaScript": [
1303
+ 2,
1304
+ 38178
1305
+ ],
1306
+ "PHP": [
1307
+ 1,
1308
+ 90
1309
+ ]
1310
+ },
1311
+ "drops": {
1312
+ "eval_window_overlap64": 3,
1313
+ "minified_lock_or_generated_name": 3,
1314
+ "generated_header": 1,
1315
+ "vendored_generated_or_fixture_dir": 1
1316
+ }
1317
+ },
1318
+ {
1319
+ "repo": "sinatra/sinatra",
1320
+ "url": "https://github.com/sinatra/sinatra",
1321
+ "commit": "cb22afd7902b566b6eaba6c4ea89739494a65d12",
1322
+ "license": "MIT",
1323
+ "license_file": "LICENSE",
1324
+ "intended_language": "Ruby",
1325
+ "files_kept": 146,
1326
+ "bytes_kept": 673565,
1327
+ "by_language": {
1328
+ "Ruby": [
1329
+ 146,
1330
+ 673565
1331
+ ]
1332
+ },
1333
+ "drops": {
1334
+ "eval_window_overlap64": 1
1335
+ }
1336
+ },
1337
+ {
1338
+ "repo": "laravel/framework",
1339
+ "url": "https://github.com/laravel/framework",
1340
+ "commit": "1de9acf2054cb360659200135e5c246320c7f330",
1341
+ "license": "MIT",
1342
+ "license_file": "LICENSE.md",
1343
+ "intended_language": "PHP",
1344
+ "files_kept": 1819,
1345
+ "bytes_kept": 8394381,
1346
+ "by_language": {
1347
+ "Shell": [
1348
+ 3,
1349
+ 7164
1350
+ ],
1351
+ "PHP": [
1352
+ 1814,
1353
+ 8383601
1354
+ ],
1355
+ "JavaScript": [
1356
+ 2,
1357
+ 3616
1358
+ ]
1359
+ },
1360
+ "drops": {
1361
+ "eval_window_overlap64": 22,
1362
+ "minified_lock_or_generated_name": 25,
1363
+ "secret_pattern": 2,
1364
+ "minified_long_lines": 1,
1365
+ "vendored_generated_or_fixture_dir": 10,
1366
+ "exact_dup_sha256": 4,
1367
+ "repo_cap_8MB_reached": 1291
1368
+ }
1369
+ },
1370
+ {
1371
+ "repo": "symfony/symfony",
1372
+ "url": "https://github.com/symfony/symfony",
1373
+ "commit": "03b10a6cd71262b770f6496511ff3c896a2de034",
1374
+ "license": "MIT",
1375
+ "license_file": "LICENSE",
1376
+ "intended_language": "PHP",
1377
+ "files_kept": 1645,
1378
+ "bytes_kept": 8390266,
1379
+ "by_language": {
1380
+ "PHP": [
1381
+ 1644,
1382
+ 8376452
1383
+ ],
1384
+ "JavaScript": [
1385
+ 1,
1386
+ 13814
1387
+ ]
1388
+ },
1389
+ "drops": {
1390
+ "eval_window_overlap64": 9,
1391
+ "minified_lock_or_generated_name": 27,
1392
+ "vendored_generated_or_fixture_dir": 345,
1393
+ "generated_header": 1,
1394
+ "exact_dup_sha256": 45,
1395
+ "secret_pattern": 1,
1396
+ "non_utf8": 1,
1397
+ "repo_cap_8MB_reached": 10576
1398
+ }
1399
+ },
1400
+ {
1401
+ "repo": "guzzle/guzzle",
1402
+ "url": "https://github.com/guzzle/guzzle",
1403
+ "commit": "b920e5fcaf81d761b8fabe600ca90999beaefdb7",
1404
+ "license": "MIT",
1405
+ "license_file": "LICENSE",
1406
+ "intended_language": "PHP",
1407
+ "files_kept": 135,
1408
+ "bytes_kept": 1779468,
1409
+ "by_language": {
1410
+ "PHP": [
1411
+ 135,
1412
+ 1779468
1413
+ ]
1414
+ },
1415
+ "drops": {
1416
+ "eval_window_overlap64": 7,
1417
+ "minified_lock_or_generated_name": 2,
1418
+ "too_large_256k": 1
1419
+ }
1420
+ },
1421
+ {
1422
+ "repo": "JetBrains/kotlin-web-site",
1423
+ "url": "https://github.com/JetBrains/kotlin-web-site",
1424
+ "commit": "4c0c682dfa2799ca98bf97e86640d757cdb7e11e",
1425
+ "license": "Apache-2.0",
1426
+ "license_file": "LICENSE",
1427
+ "intended_language": "Kotlin",
1428
+ "files_kept": 335,
1429
+ "bytes_kept": 894608,
1430
+ "by_language": {
1431
+ "Shell": [
1432
+ 3,
1433
+ 11765
1434
+ ],
1435
+ "Kotlin": [
1436
+ 71,
1437
+ 101646
1438
+ ],
1439
+ "JavaScript": [
1440
+ 31,
1441
+ 62832
1442
+ ],
1443
+ "TypeScript": [
1444
+ 229,
1445
+ 702829
1446
+ ],
1447
+ "Python": [
1448
+ 1,
1449
+ 15536
1450
+ ]
1451
+ },
1452
+ "drops": {
1453
+ "minified_lock_or_generated_name": 15,
1454
+ "minified_long_lines": 1,
1455
+ "vendored_generated_or_fixture_dir": 21
1456
+ }
1457
+ },
1458
+ {
1459
+ "repo": "square/kotlinpoet",
1460
+ "url": "https://github.com/square/kotlinpoet",
1461
+ "commit": "e9c64e7e33221d900473a283f1f17f1439c99753",
1462
+ "license": "Apache-2.0",
1463
+ "license_file": "LICENSE.txt",
1464
+ "intended_language": "Kotlin",
1465
+ "files_kept": 131,
1466
+ "bytes_kept": 1246878,
1467
+ "by_language": {
1468
+ "Kotlin": [
1469
+ 128,
1470
+ 1239697
1471
+ ],
1472
+ "Java": [
1473
+ 3,
1474
+ 7181
1475
+ ]
1476
+ },
1477
+ "drops": {
1478
+ "minified_lock_or_generated_name": 3
1479
+ }
1480
+ },
1481
+ {
1482
+ "repo": "Kotlin/kotlinx.coroutines",
1483
+ "url": "https://github.com/Kotlin/kotlinx.coroutines",
1484
+ "commit": "7e0d1b3e9ca83652427ae00dda7d1673f058a743",
1485
+ "license": "Apache-2.0",
1486
+ "license_file": "LICENSE.txt",
1487
+ "intended_language": "Kotlin",
1488
+ "files_kept": 1060,
1489
+ "bytes_kept": 3943739,
1490
+ "by_language": {
1491
+ "Kotlin": [
1492
+ 1037,
1493
+ 3902999
1494
+ ],
1495
+ "Python": [
1496
+ 1,
1497
+ 3107
1498
+ ],
1499
+ "Java": [
1500
+ 21,
1501
+ 34948
1502
+ ],
1503
+ "Shell": [
1504
+ 1,
1505
+ 2685
1506
+ ]
1507
+ },
1508
+ "drops": {
1509
+ "eval_window_overlap64": 9,
1510
+ "minified_lock_or_generated_name": 24,
1511
+ "empty": 1,
1512
+ "exact_dup_sha256": 12
1513
+ }
1514
+ },
1515
+ {
1516
+ "repo": "ktorio/ktor",
1517
+ "url": "https://github.com/ktorio/ktor",
1518
+ "commit": "d96280165fc9890217196ed0dffd45228e35987d",
1519
+ "license": "Apache-2.0",
1520
+ "license_file": "LICENSE",
1521
+ "intended_language": "Kotlin",
1522
+ "files_kept": 2045,
1523
+ "bytes_kept": 8389113,
1524
+ "by_language": {
1525
+ "Python": [
1526
+ 1,
1527
+ 4977
1528
+ ],
1529
+ "Kotlin": [
1530
+ 2028,
1531
+ 8153707
1532
+ ],
1533
+ "JavaScript": [
1534
+ 2,
1535
+ 1653
1536
+ ],
1537
+ "C": [
1538
+ 5,
1539
+ 179946
1540
+ ],
1541
+ "Rust": [
1542
+ 8,
1543
+ 46487
1544
+ ],
1545
+ "Java": [
1546
+ 1,
1547
+ 2343
1548
+ ]
1549
+ },
1550
+ "drops": {
1551
+ "vendored_generated_or_fixture_dir": 15,
1552
+ "secret_pattern": 12,
1553
+ "exact_dup_sha256": 38,
1554
+ "eval_window_overlap64": 6,
1555
+ "minified_long_lines": 1,
1556
+ "binary": 1,
1557
+ "minified_lock_or_generated_name": 12,
1558
+ "repo_cap_8MB_reached": 503
1559
+ }
1560
+ },
1561
+ {
1562
+ "repo": "Alamofire/Alamofire",
1563
+ "url": "https://github.com/Alamofire/Alamofire",
1564
+ "commit": "bda9ed57d72988a3a2ada33d824583541f86eac6",
1565
+ "license": "MIT",
1566
+ "license_file": "LICENSE",
1567
+ "intended_language": "Swift",
1568
+ "files_kept": 79,
1569
+ "bytes_kept": 1141239,
1570
+ "by_language": {
1571
+ "Swift": [
1572
+ 76,
1573
+ 1065948
1574
+ ],
1575
+ "JavaScript": [
1576
+ 3,
1577
+ 75291
1578
+ ]
1579
+ },
1580
+ "drops": {
1581
+ "eval_window_overlap64": 22,
1582
+ "secret_pattern": 3,
1583
+ "minified_lock_or_generated_name": 4,
1584
+ "exact_dup_sha256": 3
1585
+ }
1586
+ },
1587
+ {
1588
+ "repo": "apple/swift-nio",
1589
+ "url": "https://github.com/apple/swift-nio",
1590
+ "commit": "108daca35da6a1b4ed8da29c1dfbd19668b9b8fc",
1591
+ "license": "Apache-2.0",
1592
+ "license_file": "LICENSE.txt",
1593
+ "intended_language": "Swift",
1594
+ "files_kept": 7,
1595
+ "bytes_kept": 72733,
1596
+ "by_language": {
1597
+ "Swift": [
1598
+ 2,
1599
+ 6166
1600
+ ],
1601
+ "C": [
1602
+ 5,
1603
+ 66567
1604
+ ]
1605
+ },
1606
+ "drops": {
1607
+ "eval_window_overlap64": 627,
1608
+ "exact_dup_sha256": 13,
1609
+ "too_large_256k": 1,
1610
+ "minified_lock_or_generated_name": 7,
1611
+ "generated_header": 1
1612
+ }
1613
+ },
1614
+ {
1615
+ "repo": "vapor/vapor",
1616
+ "url": "https://github.com/vapor/vapor",
1617
+ "commit": "1ccd882a17841f7ae1202b80795259c2c0082354",
1618
+ "license": "MIT",
1619
+ "license_file": "LICENSE",
1620
+ "intended_language": "Swift",
1621
+ "files_kept": 273,
1622
+ "bytes_kept": 1642178,
1623
+ "by_language": {
1624
+ "Swift": [
1625
+ 272,
1626
+ 1639745
1627
+ ],
1628
+ "Shell": [
1629
+ 1,
1630
+ 2433
1631
+ ]
1632
+ },
1633
+ "drops": {
1634
+ "secret_pattern": 2,
1635
+ "eval_window_overlap64": 2,
1636
+ "minified_long_lines": 1
1637
+ }
1638
+ },
1639
+ {
1640
+ "repo": "scala/scala",
1641
+ "url": "https://github.com/scala/scala",
1642
+ "commit": "65f01dbd3131320033301b9db4b83a919a4e7da7",
1643
+ "license": "Apache-2.0",
1644
+ "license_file": "LICENSE",
1645
+ "intended_language": "Scala",
1646
+ "files_kept": 962,
1647
+ "bytes_kept": 8422876,
1648
+ "by_language": {
1649
+ "Shell": [
1650
+ 3,
1651
+ 3261
1652
+ ],
1653
+ "Scala": [
1654
+ 914,
1655
+ 8325919
1656
+ ],
1657
+ "JavaScript": [
1658
+ 2,
1659
+ 5547
1660
+ ],
1661
+ "Java": [
1662
+ 43,
1663
+ 88149
1664
+ ]
1665
+ },
1666
+ "drops": {
1667
+ "generated_header": 104,
1668
+ "minified_long_lines": 1,
1669
+ "eval_window_overlap64": 32,
1670
+ "too_large_256k": 1,
1671
+ "minified_lock_or_generated_name": 1,
1672
+ "repo_cap_8MB_reached": 8949
1673
+ }
1674
+ },
1675
+ {
1676
+ "repo": "playframework/playframework",
1677
+ "url": "https://github.com/playframework/playframework",
1678
+ "commit": "f3eb3222f8911f5126ea5b127d86931518421f36",
1679
+ "license": "Apache-2.0",
1680
+ "license_file": "LICENSE",
1681
+ "intended_language": "Scala",
1682
+ "files_kept": 1626,
1683
+ "bytes_kept": 7612801,
1684
+ "by_language": {
1685
+ "Java": [
1686
+ 789,
1687
+ 3039844
1688
+ ],
1689
+ "Scala": [
1690
+ 815,
1691
+ 4444881
1692
+ ],
1693
+ "JavaScript": [
1694
+ 20,
1695
+ 123593
1696
+ ],
1697
+ "Kotlin": [
1698
+ 2,
1699
+ 4483
1700
+ ]
1701
+ },
1702
+ "drops": {
1703
+ "exact_dup_sha256": 17,
1704
+ "eval_window_overlap64": 23,
1705
+ "secret_pattern": 5,
1706
+ "generated_header": 1,
1707
+ "minified_lock_or_generated_name": 6,
1708
+ "vendored_generated_or_fixture_dir": 4
1709
+ }
1710
+ },
1711
+ {
1712
+ "repo": "ohmyzsh/ohmyzsh",
1713
+ "url": "https://github.com/ohmyzsh/ohmyzsh",
1714
+ "commit": "2100c630ba4bd67bb48802c3e7da893a4fdaa1a0",
1715
+ "license": "MIT",
1716
+ "license_file": "LICENSE.txt",
1717
+ "intended_language": "Shell",
1718
+ "files_kept": 372,
1719
+ "bytes_kept": 932312,
1720
+ "by_language": {
1721
+ "Python": [
1722
+ 7,
1723
+ 36459
1724
+ ],
1725
+ "Shell": [
1726
+ 365,
1727
+ 895853
1728
+ ]
1729
+ },
1730
+ "drops": {
1731
+ "eval_window_overlap64": 42,
1732
+ "minified_lock_or_generated_name": 1,
1733
+ "too_large_256k": 1,
1734
+ "generated_header": 1,
1735
+ "empty": 2
1736
+ }
1737
+ },
1738
+ {
1739
+ "repo": "nvm-sh/nvm",
1740
+ "url": "https://github.com/nvm-sh/nvm",
1741
+ "commit": "e780535240ee3ba67048b434cec6f7f509ee7b31",
1742
+ "license": "MIT",
1743
+ "license_file": "LICENSE.md",
1744
+ "intended_language": "Shell",
1745
+ "files_kept": 9,
1746
+ "bytes_kept": 238163,
1747
+ "by_language": {
1748
+ "Shell": [
1749
+ 8,
1750
+ 238125
1751
+ ],
1752
+ "JavaScript": [
1753
+ 1,
1754
+ 38
1755
+ ]
1756
+ }
1757
+ },
1758
+ {
1759
+ "repo": "dylanaraps/neofetch",
1760
+ "url": "https://github.com/dylanaraps/neofetch",
1761
+ "commit": "ccd5d9f52609bbdcd5d8fa78c4fdb0f12954125f",
1762
+ "license": "MIT",
1763
+ "license_file": "LICENSE.md",
1764
+ "intended_language": "Shell",
1765
+ "files_kept": 0,
1766
+ "bytes_kept": 0,
1767
+ "by_language": {}
1768
+ },
1769
+ {
1770
+ "repo": "sbt/sbt",
1771
+ "url": "https://github.com/sbt/sbt",
1772
+ "commit": "6fcb936dc56b9e020c88bb138879ce9b10e2b29f",
1773
+ "license": "Apache-2.0",
1774
+ "license_file": "LICENSE",
1775
+ "intended_language": "Scala",
1776
+ "files_kept": 1738,
1777
+ "bytes_kept": 4732831,
1778
+ "by_language": {
1779
+ "Shell": [
1780
+ 18,
1781
+ 19480
1782
+ ],
1783
+ "Scala": [
1784
+ 1578,
1785
+ 4554781
1786
+ ],
1787
+ "Java": [
1788
+ 135,
1789
+ 157091
1790
+ ],
1791
+ "C++": [
1792
+ 2,
1793
+ 508
1794
+ ],
1795
+ "C": [
1796
+ 5,
1797
+ 971
1798
+ ]
1799
+ },
1800
+ "drops": {
1801
+ "eval_window_overlap64": 8,
1802
+ "generated_header": 458,
1803
+ "exact_dup_sha256": 165,
1804
+ "secret_pattern": 1,
1805
+ "minified_lock_or_generated_name": 19,
1806
+ "empty": 5,
1807
+ "vendored_generated_or_fixture_dir": 1
1808
+ }
1809
+ },
1810
+ {
1811
+ "repo": "apple/swift-argument-parser",
1812
+ "url": "https://github.com/apple/swift-argument-parser",
1813
+ "commit": "bd10e57565e21be23b144f3770af7f2bfb068ecf",
1814
+ "license": "Apache-2.0",
1815
+ "license_file": "LICENSE.txt",
1816
+ "intended_language": "Swift",
1817
+ "files_kept": 0,
1818
+ "bytes_kept": 0,
1819
+ "by_language": {},
1820
+ "drops": {
1821
+ "eval_window_overlap64": 172,
1822
+ "vendored_generated_or_fixture_dir": 8
1823
+ }
1824
+ },
1825
+ {
1826
+ "repo": "ReactiveX/RxSwift",
1827
+ "url": "https://github.com/ReactiveX/RxSwift",
1828
+ "commit": "3e33f90c1bcd3cdea25bdb49bd4a594a50c3ab84",
1829
+ "license": "MIT",
1830
+ "license_file": "LICENSE.md",
1831
+ "intended_language": "Swift",
1832
+ "files_kept": 596,
1833
+ "bytes_kept": 3189661,
1834
+ "by_language": {
1835
+ "Swift": [
1836
+ 578,
1837
+ 3162406
1838
+ ],
1839
+ "C": [
1840
+ 8,
1841
+ 3335
1842
+ ],
1843
+ "Shell": [
1844
+ 9,
1845
+ 23259
1846
+ ],
1847
+ "Ruby": [
1848
+ 1,
1849
+ 661
1850
+ ]
1851
+ },
1852
+ "drops": {
1853
+ "minified_lock_or_generated_name": 12,
1854
+ "eval_window_overlap64": 4,
1855
+ "exact_dup_sha256": 8
1856
+ }
1857
+ },
1858
+ {
1859
+ "repo": "SwiftyJSON/SwiftyJSON",
1860
+ "url": "https://github.com/SwiftyJSON/SwiftyJSON",
1861
+ "commit": "3d2544101a03f22b2217d464dc7d359eeb86e159",
1862
+ "license": "MIT",
1863
+ "license_file": "LICENSE",
1864
+ "intended_language": "Swift",
1865
+ "files_kept": 24,
1866
+ "bytes_kept": 165999,
1867
+ "by_language": {
1868
+ "Swift": [
1869
+ 23,
1870
+ 164610
1871
+ ],
1872
+ "C": [
1873
+ 1,
1874
+ 1389
1875
+ ]
1876
+ }
1877
+ },
1878
+ {
1879
+ "repo": "Moya/Moya",
1880
+ "url": "https://github.com/Moya/Moya",
1881
+ "commit": "24c26671a2f0fc26b53c9fa2410c97987fb0f05a",
1882
+ "license": "MIT",
1883
+ "license_file": "License.md",
1884
+ "intended_language": "Swift",
1885
+ "files_kept": 68,
1886
+ "bytes_kept": 329973,
1887
+ "by_language": {
1888
+ "Swift": [
1889
+ 58,
1890
+ 324915
1891
+ ],
1892
+ "C": [
1893
+ 3,
1894
+ 1183
1895
+ ],
1896
+ "Shell": [
1897
+ 7,
1898
+ 3875
1899
+ ]
1900
+ },
1901
+ "drops": {
1902
+ "exact_dup_sha256": 1,
1903
+ "minified_long_lines": 2,
1904
+ "eval_window_overlap64": 1,
1905
+ "secret_pattern": 2
1906
+ }
1907
+ },
1908
+ {
1909
+ "repo": "bats-core/bats-core",
1910
+ "url": "https://github.com/bats-core/bats-core",
1911
+ "commit": "5b61d8f22b46abf001e218cdd63f2938c356fa93",
1912
+ "license": "MIT",
1913
+ "license_file": "LICENSE.md",
1914
+ "intended_language": "Shell",
1915
+ "files_kept": 18,
1916
+ "bytes_kept": 60392,
1917
+ "by_language": {
1918
+ "Shell": [
1919
+ 17,
1920
+ 60255
1921
+ ],
1922
+ "Python": [
1923
+ 1,
1924
+ 137
1925
+ ]
1926
+ },
1927
+ "drops": {
1928
+ "eval_window_overlap64": 2,
1929
+ "vendored_generated_or_fixture_dir": 34
1930
+ }
1931
+ }
1932
+ ],
1933
+ "excluded_repos": [
1934
+ {
1935
+ "repo": "hashicorp/terraform",
1936
+ "commit": "55988303ba5651e41e2c9097e208468c8eb3da4b",
1937
+ "why": "license_not_permissive_or_unrecognized:BUSL-1.1:LICENSE"
1938
+ },
1939
+ {
1940
+ "repo": "redis/redis",
1941
+ "commit": "81d653f5dc8d472a0d4f7dd328be720021580731",
1942
+ "why": "license_not_permissive_or_unrecognized:GPL:LICENSE.txt"
1943
+ },
1944
+ {
1945
+ "repo": "AutoMapper/AutoMapper",
1946
+ "commit": "f0ecba7a97d1403973825fb3cbe2733e6216ff9a",
1947
+ "why": "license_not_permissive_or_unrecognized:RPL-1.5:LICENSE.md"
1948
+ },
1949
+ {
1950
+ "repo": "akka/akka",
1951
+ "commit": "838fe7279217fb185a098abb84808c134f299161",
1952
+ "why": "license_not_permissive_or_unrecognized:BUSL-1.1:LICENSE,COPYING.protobuf"
1953
+ }
1954
+ ],
1955
+ "repos_added_vs_v1": [
1956
+ "ReactiveX/RxSwift",
1957
+ "SwiftyJSON/SwiftyJSON",
1958
+ "Moya/Moya",
1959
+ "bats-core/bats-core"
1960
+ ],
1961
+ "filters": {
1962
+ "extensions": "see build.py EXT",
1963
+ "max_file_bytes": 262144,
1964
+ "per_repo_cap_bytes": 8388608,
1965
+ "dirs_dropped": "node_modules, vendor(s), third_party, 3rdparty, external, dist, build, out, testdata, test_data, test-data, fixtures, __snapshots__, snapshots, generated, gen",
1966
+ "names_dropped": "*.min.js, *lock*, *_pb2.py, *.pb.go/.pb.cc/.pb.h, *.generated.cs, *.designer.cs",
1967
+ "generated_header": "@generated / DO NOT EDIT / Code generated / auto-generated in first 2 KB",
1968
+ "minified": "any line > 3000 bytes or mean line > 300 bytes",
1969
+ "binary_or_non_utf8": "NUL byte or failed UTF-8 decode",
1970
+ "secret_regex": "AKIA[0-9A-Z]{16} | -----BEGIN | ghp_[A-Za-z0-9]{20,} | xox[bp]- | \\bsk-[A-Za-z0-9_-]{16,} | password\\s*[:=]\\s*'literal'/\\\"literal\\\" (case-insensitive)",
1971
+ "exact_dedup": "sha256 per file across all repos",
1972
+ "clone": "git clone --depth 1 --filter=blob:limit=1m",
1973
+ "license_gate": "MIT, Apache-2.0, BSD-2/3-Clause, zlib, curl only (repo LICENSE/COPYING text match)"
1974
+ },
1975
+ "drop_counts": {
1976
+ "eval_window_overlap64": 15279,
1977
+ "secret_pattern": 182,
1978
+ "empty": 588,
1979
+ "exact_dup_sha256": 1079,
1980
+ "vendored_generated_or_fixture_dir": 3807,
1981
+ "minified_lock_or_generated_name": 375,
1982
+ "generated_header": 824,
1983
+ "too_large_256k": 101,
1984
+ "repo_cap_8MB_reached": 71826,
1985
+ "minified_long_lines": 79,
1986
+ "missing_blob_or_link(>1MB filtered at clone)": 442,
1987
+ "non_utf8": 8,
1988
+ "binary": 2
1989
+ },
1990
+ "files_dropped_for_eval_overlap_by_window": {
1991
+ "broad_gutenberg_v1_oneshot": 0,
1992
+ "broad_gutenberg_v1_test": 294,
1993
+ "broad_local_v1_oneshot": 0,
1994
+ "broad_local_v1_test": 294,
1995
+ "corpus_v3_oneshot": 0,
1996
+ "corpus_v3_test": 0,
1997
+ "fc2": 7,
1998
+ "fresh_en": 0,
1999
+ "fresh_fr": 0,
2000
+ "freshcode": 0,
2001
+ "neither_code": 15038,
2002
+ "old_local_v1_oneshot": 0,
2003
+ "old_local_v1_test": 276,
2004
+ "queries": 0,
2005
+ "test20k": 276
2006
+ },
2007
+ "files_with_window_hash_candidates": 15279,
2008
+ "dedup_verification_final": {
2009
+ "method": "verify_v3 hash (64-byte polynomial, P=1099511628211) on ALL positions of train.bin, separators included, byte-for-byte confirmation; 'seen' window excluded (training data)",
2010
+ "windows": {
2011
+ "broad_gutenberg_v1_oneshot": {
2012
+ "window_bytes": 119763,
2013
+ "n_64grams": 119700,
2014
+ "train_positions_hash_hits": 0,
2015
+ "confirmed_train_positions": 0,
2016
+ "confirmed_window_positions": 0
2017
+ },
2018
+ "broad_gutenberg_v1_test": {
2019
+ "window_bytes": 894718,
2020
+ "n_64grams": 894655,
2021
+ "train_positions_hash_hits": 0,
2022
+ "confirmed_train_positions": 0,
2023
+ "confirmed_window_positions": 0
2024
+ },
2025
+ "broad_local_v1_oneshot": {
2026
+ "window_bytes": 119763,
2027
+ "n_64grams": 119700,
2028
+ "train_positions_hash_hits": 0,
2029
+ "confirmed_train_positions": 0,
2030
+ "confirmed_window_positions": 0
2031
+ },
2032
+ "broad_local_v1_test": {
2033
+ "window_bytes": 894718,
2034
+ "n_64grams": 894655,
2035
+ "train_positions_hash_hits": 0,
2036
+ "confirmed_train_positions": 0,
2037
+ "confirmed_window_positions": 0
2038
+ },
2039
+ "corpus_v3_oneshot": {
2040
+ "window_bytes": 119763,
2041
+ "n_64grams": 119700,
2042
+ "train_positions_hash_hits": 0,
2043
+ "confirmed_train_positions": 0,
2044
+ "confirmed_window_positions": 0
2045
+ },
2046
+ "corpus_v3_test": {
2047
+ "window_bytes": 3486653,
2048
+ "n_64grams": 3486590,
2049
+ "train_positions_hash_hits": 0,
2050
+ "confirmed_train_positions": 0,
2051
+ "confirmed_window_positions": 0
2052
+ },
2053
+ "fc2": {
2054
+ "window_bytes": 199792,
2055
+ "n_64grams": 199729,
2056
+ "train_positions_hash_hits": 0,
2057
+ "confirmed_train_positions": 0,
2058
+ "confirmed_window_positions": 0
2059
+ },
2060
+ "fresh_en": {
2061
+ "window_bytes": 119763,
2062
+ "n_64grams": 119700,
2063
+ "train_positions_hash_hits": 0,
2064
+ "confirmed_train_positions": 0,
2065
+ "confirmed_window_positions": 0
2066
+ },
2067
+ "fresh_fr": {
2068
+ "window_bytes": 1088688,
2069
+ "n_64grams": 1088625,
2070
+ "train_positions_hash_hits": 0,
2071
+ "confirmed_train_positions": 0,
2072
+ "confirmed_window_positions": 0
2073
+ },
2074
+ "freshcode": {
2075
+ "window_bytes": 200000,
2076
+ "n_64grams": 199937,
2077
+ "train_positions_hash_hits": 0,
2078
+ "confirmed_train_positions": 0,
2079
+ "confirmed_window_positions": 0
2080
+ },
2081
+ "neither_code": {
2082
+ "window_bytes": 546036,
2083
+ "n_64grams": 545973,
2084
+ "train_positions_hash_hits": 0,
2085
+ "confirmed_train_positions": 0,
2086
+ "confirmed_window_positions": 0
2087
+ },
2088
+ "old_local_v1_oneshot": {
2089
+ "window_bytes": 119763,
2090
+ "n_64grams": 119700,
2091
+ "train_positions_hash_hits": 0,
2092
+ "confirmed_train_positions": 0,
2093
+ "confirmed_window_positions": 0
2094
+ },
2095
+ "old_local_v1_test": {
2096
+ "window_bytes": 319940,
2097
+ "n_64grams": 319877,
2098
+ "train_positions_hash_hits": 0,
2099
+ "confirmed_train_positions": 0,
2100
+ "confirmed_window_positions": 0
2101
+ },
2102
+ "queries": {
2103
+ "window_bytes": 27800,
2104
+ "n_64grams": 27737,
2105
+ "train_positions_hash_hits": 0,
2106
+ "confirmed_train_positions": 0,
2107
+ "confirmed_window_positions": 0
2108
+ },
2109
+ "test20k": {
2110
+ "window_bytes": 319940,
2111
+ "n_64grams": 319877,
2112
+ "train_positions_hash_hits": 0,
2113
+ "confirmed_train_positions": 0,
2114
+ "confirmed_window_positions": 0
2115
+ }
2116
+ },
2117
+ "positions_checked": 243083751,
2118
+ "valid_zero_overlap": true
2119
+ },
2120
+ "checker_recount": "PENDING - Checker recounts v2 before any training",
2121
+ "test_bin": "none (no hold-out produced)",
2122
+ "sha256_outputs": {
2123
+ "train.bin": "35858aab67bdcbff3a4a852560d9550b85eb06f262958e280b959c962c079c9f",
2124
+ "build.py": "bab4221a8f7163dda3b267a3c0776aebd5d073d58ea30606bdcdc0d15ac0fa6e",
2125
+ "verify_final.py": "3f1bd510fe22fd828cf51d84fae9da50a26c197937265e18c80189d1ba357f11",
2126
+ "repos.tsv": "31deb83e91c6e3512c5b1aaadeb61a5b305dc2c53c6d7819a2efb659e0c257c9",
2127
+ "build_stage1.json": "591321a767c2dba022a9712a1de251ee9bc01f7e2d0fa1cdd37d638aeb1f99a5",
2128
+ "verify_final.json": "075a0dbdfbdd0ab24634d1622f1c2f3ea43a5340290364565f0492446c7df8cd"
2129
+ }
2130
+ }
results/hashing/multi_lang_v2/repos.tsv ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Python psf/requests
2
+ Python pallets/flask
3
+ Python django/django
4
+ Python scikit-learn/scikit-learn
5
+ Python encode/httpx
6
+ JavaScript expressjs/express
7
+ JavaScript lodash/lodash
8
+ JavaScript webpack/webpack
9
+ JavaScript axios/axios
10
+ JavaScript mrdoob/three.js
11
+ TypeScript microsoft/vscode
12
+ TypeScript vuejs/core
13
+ TypeScript angular/angular
14
+ TypeScript nestjs/nest
15
+ Rust BurntSushi/ripgrep
16
+ Rust tokio-rs/tokio
17
+ Rust serde-rs/serde
18
+ Rust rust-lang/cargo
19
+ Rust alacritty/alacritty
20
+ Go gin-gonic/gin
21
+ Go spf13/cobra
22
+ Go prometheus/prometheus
23
+ Go hashicorp/terraform
24
+ Go gohugoio/hugo
25
+ C redis/redis
26
+ C curl/curl
27
+ C libuv/libuv
28
+ C madler/zlib
29
+ C jqlang/jq
30
+ C++ nlohmann/json
31
+ C++ fmtlib/fmt
32
+ C++ google/leveldb
33
+ C++ opencv/opencv
34
+ C++ protocolbuffers/protobuf
35
+ Java google/guava
36
+ Java spring-projects/spring-framework
37
+ Java square/okhttp
38
+ Java apache/commons-lang
39
+ C# dotnet/aspnetcore
40
+ C# JamesNK/Newtonsoft.Json
41
+ C# AutoMapper/AutoMapper
42
+ Ruby rails/rails
43
+ Ruby jekyll/jekyll
44
+ Ruby sinatra/sinatra
45
+ PHP laravel/framework
46
+ PHP symfony/symfony
47
+ PHP guzzle/guzzle
48
+ Kotlin JetBrains/kotlin-web-site
49
+ Kotlin square/kotlinpoet
50
+ Kotlin Kotlin/kotlinx.coroutines
51
+ Kotlin ktorio/ktor
52
+ Swift Alamofire/Alamofire
53
+ Swift apple/swift-nio
54
+ Swift vapor/vapor
55
+ Scala akka/akka
56
+ Scala scala/scala
57
+ Scala playframework/playframework
58
+ Shell ohmyzsh/ohmyzsh
59
+ Shell nvm-sh/nvm
60
+ Shell dylanaraps/neofetch
61
+ Scala sbt/sbt
62
+ Swift apple/swift-argument-parser
63
+ Swift ReactiveX/RxSwift
64
+ Swift SwiftyJSON/SwiftyJSON
65
+ Swift Moya/Moya
66
+ Shell bats-core/bats-core
results/hashing/multi_lang_v2/run.sh ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ cd /workspace/spira_shared/multi_lang_v2
2
+ while [ $(awk '/MemAvailable/{print $2}' /proc/meminfo) -lt 4500000 ]; do echo "$(date) mem gate wait"; sleep 60; done
3
+ OUT=/workspace/spira_shared/multi_lang_v2 OMP_NUM_THREADS=1 PYTHONHASHSEED=0 nice -n 10 /workspace/SPIRA-venv/bin/python -c "
4
+ import runpy,resource,sys; sys.argv=['build.py']
5
+ try: runpy.run_path('build.py', run_name='__main__')
6
+ finally: print('maxrss MB', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024)" > build.log 2>&1
7
+ while [ $(awk '/MemAvailable/{print $2}' /proc/meminfo) -lt 4500000 ]; do echo "$(date) mem gate wait"; sleep 60; done
8
+ OUT=/workspace/spira_shared/multi_lang_v2 OMP_NUM_THREADS=1 PYTHONHASHSEED=0 nice -n 10 /workspace/SPIRA-venv/bin/python -c "
9
+ import runpy,resource,sys; sys.argv=['verify_final.py']
10
+ try: runpy.run_path('verify_final.py', run_name='__main__')
11
+ finally: print('maxrss MB', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024)" > verify.log 2>&1
12
+ echo ALLDONE
results/hashing/multi_lang_v2/verify_final.json ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train_bytes": 243083814,
3
+ "train_sha256": "35858aab67bdcbff3a4a852560d9550b85eb06f262958e280b959c962c079c9f",
4
+ "positions_checked": 243083751,
5
+ "windows": {
6
+ "broad_gutenberg_v1_oneshot": {
7
+ "window_bytes": 119763,
8
+ "n_64grams": 119700,
9
+ "train_positions_hash_hits": 0,
10
+ "confirmed_train_positions": 0,
11
+ "confirmed_window_positions": 0
12
+ },
13
+ "broad_gutenberg_v1_test": {
14
+ "window_bytes": 894718,
15
+ "n_64grams": 894655,
16
+ "train_positions_hash_hits": 0,
17
+ "confirmed_train_positions": 0,
18
+ "confirmed_window_positions": 0
19
+ },
20
+ "broad_local_v1_oneshot": {
21
+ "window_bytes": 119763,
22
+ "n_64grams": 119700,
23
+ "train_positions_hash_hits": 0,
24
+ "confirmed_train_positions": 0,
25
+ "confirmed_window_positions": 0
26
+ },
27
+ "broad_local_v1_test": {
28
+ "window_bytes": 894718,
29
+ "n_64grams": 894655,
30
+ "train_positions_hash_hits": 0,
31
+ "confirmed_train_positions": 0,
32
+ "confirmed_window_positions": 0
33
+ },
34
+ "corpus_v3_oneshot": {
35
+ "window_bytes": 119763,
36
+ "n_64grams": 119700,
37
+ "train_positions_hash_hits": 0,
38
+ "confirmed_train_positions": 0,
39
+ "confirmed_window_positions": 0
40
+ },
41
+ "corpus_v3_test": {
42
+ "window_bytes": 3486653,
43
+ "n_64grams": 3486590,
44
+ "train_positions_hash_hits": 0,
45
+ "confirmed_train_positions": 0,
46
+ "confirmed_window_positions": 0
47
+ },
48
+ "fc2": {
49
+ "window_bytes": 199792,
50
+ "n_64grams": 199729,
51
+ "train_positions_hash_hits": 0,
52
+ "confirmed_train_positions": 0,
53
+ "confirmed_window_positions": 0
54
+ },
55
+ "fresh_en": {
56
+ "window_bytes": 119763,
57
+ "n_64grams": 119700,
58
+ "train_positions_hash_hits": 0,
59
+ "confirmed_train_positions": 0,
60
+ "confirmed_window_positions": 0
61
+ },
62
+ "fresh_fr": {
63
+ "window_bytes": 1088688,
64
+ "n_64grams": 1088625,
65
+ "train_positions_hash_hits": 0,
66
+ "confirmed_train_positions": 0,
67
+ "confirmed_window_positions": 0
68
+ },
69
+ "freshcode": {
70
+ "window_bytes": 200000,
71
+ "n_64grams": 199937,
72
+ "train_positions_hash_hits": 0,
73
+ "confirmed_train_positions": 0,
74
+ "confirmed_window_positions": 0
75
+ },
76
+ "neither_code": {
77
+ "window_bytes": 546036,
78
+ "n_64grams": 545973,
79
+ "train_positions_hash_hits": 0,
80
+ "confirmed_train_positions": 0,
81
+ "confirmed_window_positions": 0
82
+ },
83
+ "old_local_v1_oneshot": {
84
+ "window_bytes": 119763,
85
+ "n_64grams": 119700,
86
+ "train_positions_hash_hits": 0,
87
+ "confirmed_train_positions": 0,
88
+ "confirmed_window_positions": 0
89
+ },
90
+ "old_local_v1_test": {
91
+ "window_bytes": 319940,
92
+ "n_64grams": 319877,
93
+ "train_positions_hash_hits": 0,
94
+ "confirmed_train_positions": 0,
95
+ "confirmed_window_positions": 0
96
+ },
97
+ "queries": {
98
+ "window_bytes": 27800,
99
+ "n_64grams": 27737,
100
+ "train_positions_hash_hits": 0,
101
+ "confirmed_train_positions": 0,
102
+ "confirmed_window_positions": 0
103
+ },
104
+ "test20k": {
105
+ "window_bytes": 319940,
106
+ "n_64grams": 319877,
107
+ "train_positions_hash_hits": 0,
108
+ "confirmed_train_positions": 0,
109
+ "confirmed_window_positions": 0
110
+ }
111
+ },
112
+ "seen_excluded": true,
113
+ "valid_zero_overlap": true
114
+ }
results/hashing/multi_lang_v2/verify_final.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """SPIRA Hashing - final re-verification of multi_lang_v1/train.bin: ALL positions, 64-byte hash (verify_v3), byte confirmation, streamed in 4M-position chunks."""
2
+ import sys, json, hashlib, mmap, numpy as np
3
+ import os; sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from build import windows, hashes, K
4
+ O = os.environ.get("OUT", "/workspace/spira_shared/multi_lang_v2"); CH = 1 << 22
5
+ W = windows(); names = sorted(W); hs=[]; wid=[]; wpos=[]
6
+ for i, k in enumerate(names):
7
+ h = hashes(W[k]); hs.append(h); wid.append(np.full(len(h), i, np.int32)); wpos.append(np.arange(len(h), dtype=np.int32))
8
+ H = np.concatenate(hs); o = np.argsort(H, kind="stable"); H = H[o]; WID = np.concatenate(wid)[o]; WPOS = np.concatenate(wpos)[o]; del hs, wid, wpos, o
9
+ f = open(f"{O}/train.bin","rb"); tr = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
10
+ sha = hashlib.sha256()
11
+ for s in range(0, len(tr), 1<<24): sha.update(tr[s:s+(1<<24)])
12
+ R = {k: {"window_bytes": len(W[k]), "n_64grams": max(0, len(W[k])-K+1), "train_positions_hash_hits": 0, "confirmed_train_positions": 0, "confirmed_window_positions": set()} for k in names}
13
+ n = len(tr) - K + 1
14
+ for s in range(0, n, CH):
15
+ e = min(n, s+CH); h = hashes(tr[s:e+K-1]); i = np.minimum(np.searchsorted(H, h), len(H)-1); m = H[i] == h
16
+ for q in np.nonzero(m)[0]:
17
+ seg = tr[s+q:s+q+K]; lo = np.searchsorted(H, h[q]); hi = np.searchsorted(H, h[q], "right"); hitk = set()
18
+ for j in range(lo, hi):
19
+ k = names[WID[j]]; R[k]["train_positions_hash_hits"] += 1 if k not in hitk else 0; hitk.add(k)
20
+ if W[k][WPOS[j]:WPOS[j]+K] == seg: R[k]["confirmed_window_positions"].add(int(WPOS[j]))
21
+ for k in {names[WID[j]] for j in range(lo, hi) if W[names[WID[j]]][WPOS[j]:WPOS[j]+K] == seg}: R[k]["confirmed_train_positions"] += 1
22
+ print(f"chunk {s//CH+1}/{(n+CH-1)//CH}", flush=True)
23
+ for k in names: R[k]["confirmed_window_positions"] = len(R[k]["confirmed_window_positions"])
24
+ out = {"train_bytes": len(tr), "train_sha256": sha.hexdigest(), "positions_checked": n, "windows": R, "seen_excluded": True,
25
+ "valid_zero_overlap": all(v["confirmed_train_positions"] == 0 for v in R.values())}
26
+ json.dump(out, open(f"{O}/verify_final.json","w"), indent=1); print(json.dumps(out, indent=1))