File size: 1,037 Bytes
5bfc214 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | """Text normalization for de-duplication (spec §3.3.5)."""
from __future__ import annotations
import re
_LINE_COMMENT_RE = re.compile(r"//.*")
_BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL)
_WHITESPACE_RE = re.compile(r"\s+")
def strip_comments_and_whitespace(code: str) -> str:
"""Strips C/C++ comments then collapses all whitespace, for dedup hashing
and shingling. Not a full lexer (doesn't special-case string/char literals
containing `//` or `/*`) — acceptable for a dedup signal, not for
correctness-sensitive code transforms."""
code = _BLOCK_COMMENT_RE.sub(" ", code)
code = _LINE_COMMENT_RE.sub(" ", code)
code = _WHITESPACE_RE.sub(" ", code).strip()
return code
def shingles(text: str, n: int = 5) -> set[str]:
"""Word n-grams ("5-gram") over the normalized text, for Jaccard/MinHash."""
tokens = text.split(" ")
if len(tokens) < n:
return {" ".join(tokens)} if tokens else set()
return {" ".join(tokens[i : i + n]) for i in range(len(tokens) - n + 1)}
|