| """Text normalization for de-duplication (spec §3.3.5).""" |
| from __future__ import annotations |
|
|
| import re |
|
|
| _LINE_COMMENT_RE = re.compile(r"//.*") |
| _BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) |
| _WHITESPACE_RE = re.compile(r"\s+") |
|
|
|
|
| def strip_comments_and_whitespace(code: str) -> str: |
| """Strips C/C++ comments then collapses all whitespace, for dedup hashing |
| and shingling. Not a full lexer (doesn't special-case string/char literals |
| containing `//` or `/*`) — acceptable for a dedup signal, not for |
| correctness-sensitive code transforms.""" |
| code = _BLOCK_COMMENT_RE.sub(" ", code) |
| code = _LINE_COMMENT_RE.sub(" ", code) |
| code = _WHITESPACE_RE.sub(" ", code).strip() |
| return code |
|
|
|
|
| def shingles(text: str, n: int = 5) -> set[str]: |
| """Word n-grams ("5-gram") over the normalized text, for Jaccard/MinHash.""" |
| tokens = text.split(" ") |
| if len(tokens) < n: |
| return {" ".join(tokens)} if tokens else set() |
| return {" ".join(tokens[i : i + n]) for i in range(len(tokens) - n + 1)} |
|
|
