Spaces:

siddhm11
/

ResearchIT

Running

ResearchIT / app /readability.py

siddhm11

Claude Opus 5

feat(paper): put comprehension on the page that was built for it

c3db102 2 months ago

6.51 kB

"""
How dense is this abstract to read?
WHAT THIS IS, AND WHAT IT IS NOT
--------------------------------
doc 01 names paper difficulty ratings as "a complete gap" that nothing fills,
and it also says why the obvious approach fails: standard readability formulas
(Flesch-Kincaid, FORCAST) are calibrated on general prose and are inadequate for
academic writing full of domain jargon and mathematical notation. A paper can be
conceptually simple and score as unreadable, or conceptually brutal and score as
plain.
So this deliberately does NOT claim to measure how hard a paper is to
UNDERSTAND. That would need a model trained on researcher judgements, which
does not exist here. It measures how dense the abstract is to READ — notation,
sentence length, jargon rate — which is a real and separate thing, is honestly
computable from text alone, and is what a reader deciding whether to open
something actually wants to know.
The UI wording follows from that: "dense with notation", not "hard".
CALIBRATION
-----------
The band thresholds are the terciles of the real corpus, not round numbers
chosen by eye. A scale where 90% of papers land in one band tells a reader
nothing; terciles guarantee the label discriminates. See
`scripts/calibrate_readability.py` for the derivation and
tests/test_readability.py for the distribution assertion.
No LLM, no network, no dependencies. It runs in microseconds, so it can be
computed inline on any surface.
"""
from __future__ import annotations

import re

# ── Signals ──────────────────────────────────────────────────────────────────

# Mathematical notation: LaTeX delimiters, sub/superscripts, operators, and the
# Greek letters that survive into plain-text abstracts.
_MATH = re.compile(r"[$\\^_{}]|\\[a-zA-Z]+|[=<>≤≥≈∈∀∃∑∏∫∇×·±→←↔⊗⊕]|"
                   r"[αβγδεζηθικλμνξπρστυφχψωΓΔΘΛΞΠΣΦΨΩ]")

# An acronym: two or more capitals in a row, optionally with digits (BERT, GAN,
# SU(2), LSTM). These are the single clearest marker of assumed background.
_ACRONYM = re.compile(r"\b[A-Z]{2,}[0-9]*\b")

_SENTENCE = re.compile(r"[.!?](?:\s|$)")
_WORD = re.compile(r"[A-Za-z][A-Za-z\-']*")

# Long words stand in for morphological complexity — "convolutional",
# "regularisation", "heteroskedasticity". Cheaper than syllable counting and
# more robust on technical vocabulary, where syllable heuristics do badly.
_LONG_WORD_CHARS = 11

# Terciles of the real corpus (n=525 cached abstracts), not round numbers.
# My first guess at these was 0.34 / 0.50, which put 82.5% of papers in a
# single band — a label that never discriminates is worse than no label, since
# it costs the reader attention and returns nothing.
_TECHNICAL_AT = 0.181
_SPECIALIST_AT = 0.259

LEVELS = ("accessible", "technical", "specialist")

_LABEL = {
    "accessible": "Plainly written",
    "technical": "Technical",
    "specialist": "Dense with notation",
}

# Wording is deliberately COMPARATIVE. The bands are terciles of this corpus,
# so the only claim they can honestly support is "relative to other arXiv
# abstracts" — not an absolute statement about difficulty.
_BLURB = {
    "accessible": "Lighter on notation and acronyms than most arXiv abstracts.",
    "technical": "About as dense as a typical arXiv abstract.",
    "specialist": "Denser than most arXiv abstracts — heavy notation or "
                  "assumed vocabulary.",
}


def signals(text: str) -> dict:
    """The four raw measurements, each normalised to roughly 0-1."""
    text = (text or "").strip()
    words = _WORD.findall(text)
    n_words = len(words)
    if n_words < 20:
        # Too little text to say anything. Returning zeros rather than
        # extrapolating from a fragment keeps the caller honest.
        return {"math": 0.0, "acronym": 0.0, "sentence": 0.0, "long_word": 0.0,
                "n_words": n_words}

    n_sentences = max(1, len(_SENTENCE.findall(text)))
    long_words = sum(1 for w in words if len(w) >= _LONG_WORD_CHARS)

    return {
        # Notation marks per word, saturating at 1 mark every 5 words.
        "math": min(1.0, len(_MATH.findall(text)) / n_words * 5),
        # Acronyms per word, saturating at 1 in 12.
        "acronym": min(1.0, len(_ACRONYM.findall(text)) / n_words * 12),
        # Mean sentence length, mapped so 12 words -> 0 and 40 -> 1.
        "sentence": min(1.0, max(0.0, (n_words / n_sentences - 12) / 28)),
        # Long-word share, saturating at 25%.
        "long_word": min(1.0, long_words / n_words * 4),
        "n_words": n_words,
    }


def score(text: str) -> float:
    """A single 0-1 density score.
    Weights are a judgement, not a fit: notation and assumed vocabulary are what
    actually stop a reader who lacks the background, while long sentences are
    merely tiring. There is no labelled data to fit against — saying so is more
    useful than implying a precision this does not have.
    """
    s = signals(text)
    if s["n_words"] < 20:
        return 0.0
    return (
        0.35 * s["math"]
        + 0.30 * s["acronym"]
        + 0.20 * s["long_word"]
        + 0.15 * s["sentence"]
    )


def level(text: str) -> str | None:
    """One of LEVELS, or None when there is too little text to judge."""
    s = signals(text)
    if s["n_words"] < 20:
        return None
    v = score(text)
    if v >= _SPECIALIST_AT:
        return "specialist"
    if v >= _TECHNICAL_AT:
        return "technical"
    return "accessible"


def assess(text: str) -> dict | None:
    """Everything the UI needs, or None when the text is too short to judge."""
    lvl = level(text)
    if lvl is None:
        return None
    s = signals(text)
    return {
        "level": lvl,
        "label": _LABEL[lvl],
        "blurb": _BLURB[lvl],
        "score": round(score(text), 3),
        # What actually drove it, so the badge is explainable rather than
        # oracular. Only the signals that are genuinely elevated.
        "drivers": [
            name for name, threshold in (
                ("notation", 0.35), ("acronyms", 0.35),
                ("long words", 0.5), ("long sentences", 0.5),
            )
            if s[{"notation": "math", "acronyms": "acronym",
                  "long words": "long_word",
                  "long sentences": "sentence"}[name]] >= threshold
        ],
    }

	"""
	How dense is this abstract to read?

	WHAT THIS IS, AND WHAT IT IS NOT
	--------------------------------
	doc 01 names paper difficulty ratings as "a complete gap" that nothing fills,
	and it also says why the obvious approach fails: standard readability formulas
	(Flesch-Kincaid, FORCAST) are calibrated on general prose and are inadequate for
	academic writing full of domain jargon and mathematical notation. A paper can be
	conceptually simple and score as unreadable, or conceptually brutal and score as
	plain.

	So this deliberately does NOT claim to measure how hard a paper is to
	UNDERSTAND. That would need a model trained on researcher judgements, which
	does not exist here. It measures how dense the abstract is to READ — notation,
	sentence length, jargon rate — which is a real and separate thing, is honestly
	computable from text alone, and is what a reader deciding whether to open
	something actually wants to know.

	The UI wording follows from that: "dense with notation", not "hard".

	CALIBRATION
	-----------
	The band thresholds are the terciles of the real corpus, not round numbers
	chosen by eye. A scale where 90% of papers land in one band tells a reader
	nothing; terciles guarantee the label discriminates. See
	`scripts/calibrate_readability.py` for the derivation and
	tests/test_readability.py for the distribution assertion.

	No LLM, no network, no dependencies. It runs in microseconds, so it can be
	computed inline on any surface.
	"""
	from __future__ import annotations

	import re

	# ── Signals ──────────────────────────────────────────────────────────────────

	# Mathematical notation: LaTeX delimiters, sub/superscripts, operators, and the
	# Greek letters that survive into plain-text abstracts.
	_MATH = re.compile(r"[$\\^_{}]\|\\[a-zA-Z]+\|[=<>≤≥≈∈∀∃∑∏∫∇×·±→←↔⊗⊕]\|"
	r"[αβγδεζηθικλμνξπρστυφχψωΓΔΘΛΞΠΣΦΨΩ]")

	# An acronym: two or more capitals in a row, optionally with digits (BERT, GAN,
	# SU(2), LSTM). These are the single clearest marker of assumed background.
	_ACRONYM = re.compile(r"\b[A-Z]{2,}[0-9]*\b")

	_SENTENCE = re.compile(r"[.!?](?:\s\|$)")
	_WORD = re.compile(r"[A-Za-z][A-Za-z\-']*")

	# Long words stand in for morphological complexity — "convolutional",
	# "regularisation", "heteroskedasticity". Cheaper than syllable counting and
	# more robust on technical vocabulary, where syllable heuristics do badly.
	_LONG_WORD_CHARS = 11

	# Terciles of the real corpus (n=525 cached abstracts), not round numbers.
	# My first guess at these was 0.34 / 0.50, which put 82.5% of papers in a
	# single band — a label that never discriminates is worse than no label, since
	# it costs the reader attention and returns nothing.
	_TECHNICAL_AT = 0.181
	_SPECIALIST_AT = 0.259

	LEVELS = ("accessible", "technical", "specialist")

	_LABEL = {
	"accessible": "Plainly written",
	"technical": "Technical",
	"specialist": "Dense with notation",
	}

	# Wording is deliberately COMPARATIVE. The bands are terciles of this corpus,
	# so the only claim they can honestly support is "relative to other arXiv
	# abstracts" — not an absolute statement about difficulty.
	_BLURB = {
	"accessible": "Lighter on notation and acronyms than most arXiv abstracts.",
	"technical": "About as dense as a typical arXiv abstract.",
	"specialist": "Denser than most arXiv abstracts — heavy notation or "
	"assumed vocabulary.",
	}


	def signals(text: str) -> dict:
	"""The four raw measurements, each normalised to roughly 0-1."""
	text = (text or "").strip()
	words = _WORD.findall(text)
	n_words = len(words)
	if n_words < 20:
	# Too little text to say anything. Returning zeros rather than
	# extrapolating from a fragment keeps the caller honest.
	return {"math": 0.0, "acronym": 0.0, "sentence": 0.0, "long_word": 0.0,
	"n_words": n_words}

	n_sentences = max(1, len(_SENTENCE.findall(text)))
	long_words = sum(1 for w in words if len(w) >= _LONG_WORD_CHARS)

	return {
	# Notation marks per word, saturating at 1 mark every 5 words.
	"math": min(1.0, len(_MATH.findall(text)) / n_words * 5),
	# Acronyms per word, saturating at 1 in 12.
	"acronym": min(1.0, len(_ACRONYM.findall(text)) / n_words * 12),
	# Mean sentence length, mapped so 12 words -> 0 and 40 -> 1.
	"sentence": min(1.0, max(0.0, (n_words / n_sentences - 12) / 28)),
	# Long-word share, saturating at 25%.
	"long_word": min(1.0, long_words / n_words * 4),
	"n_words": n_words,
	}


	def score(text: str) -> float:
	"""A single 0-1 density score.

	Weights are a judgement, not a fit: notation and assumed vocabulary are what
	actually stop a reader who lacks the background, while long sentences are
	merely tiring. There is no labelled data to fit against — saying so is more
	useful than implying a precision this does not have.
	"""
	s = signals(text)
	if s["n_words"] < 20:
	return 0.0
	return (
	0.35 * s["math"]
	+ 0.30 * s["acronym"]
	+ 0.20 * s["long_word"]
	+ 0.15 * s["sentence"]
	)


	def level(text: str) -> str \| None:
	"""One of LEVELS, or None when there is too little text to judge."""
	s = signals(text)
	if s["n_words"] < 20:
	return None
	v = score(text)
	if v >= _SPECIALIST_AT:
	return "specialist"
	if v >= _TECHNICAL_AT:
	return "technical"
	return "accessible"


	def assess(text: str) -> dict \| None:
	"""Everything the UI needs, or None when the text is too short to judge."""
	lvl = level(text)
	if lvl is None:
	return None
	s = signals(text)
	return {
	"level": lvl,
	"label": _LABEL[lvl],
	"blurb": _BLURB[lvl],
	"score": round(score(text), 3),
	# What actually drove it, so the badge is explainable rather than
	# oracular. Only the signals that are genuinely elevated.
	"drivers": [
	name for name, threshold in (
	("notation", 0.35), ("acronyms", 0.35),
	("long words", 0.5), ("long sentences", 0.5),
	)
	if s[{"notation": "math", "acronyms": "acronym",
	"long words": "long_word",
	"long sentences": "sentence"}[name]] >= threshold
	],
	}