Hugging Face's logo

Spaces:
huggingface
/
paperswithcode-mcp
Running on CPU Upgrade

App Files Community
paperswithcode-mcp / standalone_cli /tests /test_cli.py
nielsr's picture
nielsr HF Staff
Prototype PwC MCP Space from pwc-cli@6a547659
ee0b555 verified
Raw History Blame Contribute Delete
106 kB
from __future__ import annotations
import argparse
import importlib.util
import io
import json
import sys
import urllib.error
import urllib.request
from contextlib import redirect_stderr, redirect_stdout
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from pwc_cli.cli import UsageError, build_parser, main # noqa: E402
from pwc_cli.skills import build_skill_md # noqa: E402
from pwc_cli.transport import ( # noqa: E402
Client,
HTTPStatusError,
Response,
ResponseError,
)
INSTALLER_SPEC = importlib.util.spec_from_file_location(
"pwc_cli_installer", ROOT / "install.py"
)
assert INSTALLER_SPEC and INSTALLER_SPEC.loader
installer = importlib.util.module_from_spec(INSTALLER_SPEC)
INSTALLER_SPEC.loader.exec_module(installer)
def test_complete_parser_omits_administrative_commands():
help_text = build_parser().format_help()
assert (
"{search,paper,task,method,conference,organization,framework,benchmark,skills,version,auth}"
in help_text
)
for command in ("add-external", "cron", "embedding", "github-issue"):
assert command not in help_text
assert build_parser().parse_args(["paper", "lineage", "list", "2501.1"]).paper
assert (
build_parser()
.parse_args(["paper", "info", "2501.1", "--include-evals"])
.include_evals
)
paper_filters = build_parser().parse_args(
[
"paper",
"list",
"--task",
"Image Classification",
"--method",
"Vision Transformer",
"--conference",
"CVPR 2026",
"--framework",
"vLLM",
"--organization",
"Qwen",
]
)
assert paper_filters.task == "Image Classification"
assert paper_filters.method == "Vision Transformer"
assert paper_filters.conference == "CVPR 2026"
assert paper_filters.framework == "vLLM"
assert paper_filters.organization == "Qwen"
assert (
build_parser().parse_args(["task", "--name", "scene-text-recognition"]).name
== "scene-text-recognition"
)
assert build_parser().parse_args(["task", "list", "--area", "Vision"]).area
assert build_parser().parse_args(["task", "list", "--group-by-area"]).group_by_area
assert build_parser().parse_args(["method", "list", "--area", "Audio"]).area
assert build_parser().parse_args(["method", "--name", "Transformer"]).name
assert build_parser().parse_args(["conference", "list", "--year", "2025"]).year
assert build_parser().parse_args(["conference", "--name", "CVPR 2025"]).name
assert build_parser().parse_args(["organization", "--name", "NVIDIA"]).name
assert (
build_parser()
.parse_args(["organization", "list", "--featured-only"])
.featured_only
)
assert build_parser().parse_args(["framework", "--name", "vLLM"]).name
assert (
build_parser().parse_args(["framework", "list", "--platform", "gpu"]).platform
== "gpu"
)
def test_generated_skill_matches_installed_cli_version_and_commands():
skill = build_skill_md()
assert "name: pwc-cli" in skill
assert "Generated with `pwc v0.4.6`" in skill
assert "`pwc search QUERY" in skill
assert "--include-evals" in skill
assert "[--organization ORGANIZATION]" in skill
assert "[--author AUTHOR]" in skill
assert "When the user identifies an author" in skill
assert "`pwc organization [--name NAME] [--json]`" in skill
assert "`pwc framework list [--domain DOMAIN]" in skill
assert "`pwc benchmark --name NAME" in skill
assert "[--max-parameters SIZE]" in skill
assert "`pwc skills add" in skill
assert "Publication date ranges are inclusive" in skill
assert "must not be later than `--end-date`" in skill
def test_published_skill_documents_x_thread_edits():
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
assert "x_threads/x_threads" in skill
assert "authors or their organization" in skill
def test_published_skill_advertises_inclusive_date_ranges():
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
assert "--start-date YYYY-MM-DD --end-date YYYY-MM-DD" in skill
assert "--published-after" not in skill
assert "--time today|week|month|all_time" not in skill
assert "--max-parameters SIZE" in skill
assert "inclusive parameter" in skill
def test_skills_add_installs_project_skill(monkeypatch, tmp_path, capsys):
monkeypatch.chdir(tmp_path)
assert main(["skills", "add"]) == 0
skill_file = tmp_path / ".agents" / "skills" / "pwc-cli" / "SKILL.md"
assert skill_file.read_text() == build_skill_md()
assert str(skill_file.parent) in capsys.readouterr().out
def test_skills_add_global_and_claude_uses_user_directories(
monkeypatch, tmp_path, capsys
):
monkeypatch.setenv("HOME", str(tmp_path))
assert main(["skills", "add", "--global", "--claude"]) == 0
central = tmp_path / ".agents" / "skills" / "pwc-cli"
claude = tmp_path / ".claude" / "skills" / "pwc-cli"
assert (central / "SKILL.md").is_file()
assert claude.is_symlink()
assert claude.resolve() == central
output = capsys.readouterr().out
assert str(central) in output
assert str(claude) in output
def test_skills_add_protects_existing_install_and_force_replaces_it(
monkeypatch, tmp_path, capsys
):
monkeypatch.chdir(tmp_path)
existing = tmp_path / ".agents" / "skills" / "pwc-cli"
existing.mkdir(parents=True)
(existing / "old.txt").write_text("keep unless forced")
assert main(["skills", "add"]) == 2
assert (existing / "old.txt").is_file()
assert "rerun with --force" in capsys.readouterr().err
assert main(["skills", "add", "--force"]) == 0
assert not (existing / "old.txt").exists()
assert (existing / "SKILL.md").is_file()
def test_skills_add_supports_custom_harness_directory_and_rejects_mixed_scope(
tmp_path, capsys
):
custom = tmp_path / "agent-skills"
assert main(["skills", "add", "--dest", str(custom)]) == 0
assert (custom / "pwc-cli" / "SKILL.md").is_file()
assert main(["skills", "add", "--dest", str(custom), "--global"]) == 2
assert "--dest cannot be combined" in capsys.readouterr().err
def test_installer_downloads_from_this_repository_release_origin():
source = (ROOT / "install.py").read_text()
assert "github.com/huggingface/pwc-cli/releases/download" in source
assert "github.com/paperswithcode/paperswithcode.co" not in source
def test_installer_version_pin_is_optional_and_explicit_pin_skips_lookup(monkeypatch):
assert installer.parse_args([]).version is None
monkeypatch.setattr(
installer, "_open", lambda _url: (_ for _ in ()).throw(AssertionError())
)
assert installer.resolve_version("1.2.3", "unused") == "1.2.3"
def test_installer_resolves_valid_latest_release_tag(monkeypatch):
class Response(io.BytesIO):
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
monkeypatch.setattr(
installer,
"_open",
lambda _url: Response(b'{"tag_name": "pwc-cli-1.2.3"}'),
)
assert installer.resolve_version(None, "https://example.test/latest") == "1.2.3"
def test_installer_release_lookup_network_error_is_one_line_exit_3(monkeypatch, capsys):
monkeypatch.setattr(
installer.urllib.request,
"urlopen",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
urllib.error.URLError("offline\nnow")
),
)
assert installer.main([]) == 3
assert capsys.readouterr().err == "pwc installer: network error: offline now\n"
def test_installer_artifact_and_checksum_network_errors_exit_3(
monkeypatch, tmp_path, capsys
):
monkeypatch.setattr(installer.platform, "system", lambda: "Linux")
monkeypatch.setattr(installer.platform, "machine", lambda: "x86_64")
monkeypatch.setattr(
installer,
"_download",
lambda *_args: (_ for _ in ()).throw(installer.NetworkError("artifact")),
)
assert installer.main(["--version", "1.2.3", "--prefix", str(tmp_path)]) == 3
assert capsys.readouterr().err == "pwc installer: network error: artifact\n"
monkeypatch.setattr(
installer, "_download", lambda _url, destination: destination.write_bytes(b"x")
)
monkeypatch.setattr(
installer,
"_read_checksum",
lambda *_args: (_ for _ in ()).throw(installer.NetworkError("checksum")),
)
assert installer.main(["--version", "1.2.3", "--prefix", str(tmp_path)]) == 3
assert capsys.readouterr().err == "pwc installer: network error: checksum\n"
def test_installer_reports_path_and_persistent_instruction(tmp_path, capsys):
bin_directory = tmp_path / ".local" / "bin"
installer.report_path(bin_directory, f"/usr/bin:{bin_directory}")
captured = capsys.readouterr()
assert captured.out == f"{bin_directory} is on PATH\n"
assert captured.err == ""
installer.report_path(bin_directory, "/usr/bin")
captured = capsys.readouterr()
assert captured.out == ""
assert captured.err == (
f"warning: {bin_directory} is not on PATH\n"
f'Add this to ~/.profile: export PATH={bin_directory}:"$PATH"\n'
)
def test_installer_default_path_instruction_uses_expandable_home(capsys):
installer.report_path(Path.home() / ".local" / "bin", "/usr/bin")
assert capsys.readouterr().err == (
"warning: ~/.local/bin is not on PATH\n"
'Add this to ~/.profile: export PATH="$HOME/.local/bin:$PATH"\n'
)
def test_installer_replaces_binary_atomically(tmp_path):
artifact = tmp_path / "artifact"
artifact.write_bytes(b"new executable")
destination = tmp_path / "bin" / "pwc"
destination.parent.mkdir()
destination.write_bytes(b"old executable")
installer._install_atomically(artifact, destination)
assert destination.read_bytes() == b"new executable"
assert destination.stat().st_mode & 0o111
assert list(destination.parent.iterdir()) == [destination]
def test_benchmark_limit_alias_preserves_page_size_destination():
parser = build_parser()
assert parser.parse_args(["benchmark", "list", "--limit", "7"]).page_size == 7
assert parser.parse_args(["benchmark", "list", "--page-size", "8"]).page_size == 8
def test_paper_list_forwards_composable_catalog_filters(monkeypatch):
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
return Response(
b'{"count":0,"results":[],"applied_filters":'
b'{"task":"Image Classification","method":"Vision Transformer",'
b'"conference":"CVPR 2026","framework":"vLLM",'
b'"organization":"Qwen","authors":['
b'{"id":"264","name":"Kaiming He","hf_username":null},'
b'{"id":"26405","name":"Yilun Du","hf_username":"yilundu"}]}}',
{},
)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert (
main(
[
"paper",
"list",
"--task",
"Image Classification",
"--method",
"Vision Transformer",
"--conference",
"CVPR 2026",
"--framework",
"vLLM",
"--organization",
"Qwen",
"--author",
"Kaiming He",
"--author",
"@yilundu",
]
)
== 0
)
assert calls == [
(
"papers/",
{
"page": 1,
"page_size": 20,
"search": None,
"start_date": None,
"end_date": None,
"task": "Image Classification",
"method": "Vision Transformer",
"conference": "CVPR 2026",
"framework": "vLLM",
"organization": "Qwen",
"author": ["Kaiming He", "@yilundu"],
"latest_only": True,
"order_by": "trending",
"order_dir": "desc",
"include_resources": False,
},
)
]
def test_paper_list_rejects_more_than_ten_authors_before_request(monkeypatch, capsys):
class Client:
def __init__(self):
pass
def get(self, path, params=None):
raise AssertionError("author limit must be checked before the request")
monkeypatch.setattr("pwc_cli.cli.Client", Client)
arguments = ["paper", "list"]
for author_id in range(1, 12):
arguments.extend(("--author", str(author_id)))
assert main(arguments) == 2
assert "at most 10 author filters" in capsys.readouterr().err.lower()
def test_search_and_paper_list_forward_inclusive_date_ranges(monkeypatch):
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
return Response(
b'{"count":0,"results":[],"applied_filters":'
b'{"start_date":"2026-08-01","end_date":"2026-08-21"}}',
{},
)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert (
main(
[
"search",
"date filtering",
"--start-date",
"2026-08-01",
"--end-date",
"2026-08-21",
]
)
== 0
)
assert (
main(
[
"paper",
"list",
"--start-date",
"2026-08-01",
"--end-date",
"2026-08-21",
]
)
== 0
)
assert calls[0][1]["start_date"].isoformat() == "2026-08-01"
assert calls[0][1]["end_date"].isoformat() == "2026-08-21"
assert calls[1][1]["start_date"].isoformat() == "2026-08-01"
assert calls[1][1]["end_date"].isoformat() == "2026-08-21"
def test_date_range_rejects_an_end_before_its_start(capsys):
assert (
main(
[
"search",
"date filtering",
"--start-date",
"2026-08-21",
"--end-date",
"2026-08-01",
]
)
== 2
)
assert "start date must be on or before end date" in capsys.readouterr().err
def test_paper_list_rejects_server_that_does_not_confirm_filters(monkeypatch, capsys):
class Client:
def __init__(self):
pass
def get(self, path, params=None):
return Response(b'{"count":20,"results":[]}', {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["paper", "list", "--organization", "Qwen"]) == 4
assert "server must be upgraded" in capsys.readouterr().err
def test_paper_list_rejects_server_that_does_not_confirm_authors(monkeypatch, capsys):
class Client:
def __init__(self):
pass
def get(self, path, params=None):
return Response(b'{"count":20,"results":[],"applied_filters":{}}', {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["paper", "list", "--author", "Kaiming He"]) == 4
assert "did not confirm" in capsys.readouterr().err
def test_paper_list_rejects_partial_author_confirmation(monkeypatch, capsys):
class Client:
def __init__(self):
pass
def get(self, path, params=None):
return Response(
b'{"count":0,"results":[],"applied_filters":{"authors":['
b'{"id":"264","name":"Kaiming He","hf_username":null}]}}',
{},
)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert (
main(
[
"paper",
"list",
"--author",
"Kaiming He",
"--author",
"@yilundu",
]
)
== 4
)
assert "did not confirm" in capsys.readouterr().err
def test_paper_list_accepts_canonical_confirmation_for_zero_padded_id(
monkeypatch, capsys
):
class Client:
def __init__(self):
pass
def get(self, path, params=None):
return Response(
b'{"count":0,"results":[],"applied_filters":{"authors":['
b'{"id":"264","name":"Kaiming He","hf_username":null}]}}',
{},
)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["paper", "list", "--author", "000264"]) == 0
assert capsys.readouterr().err == ""
def test_paper_list_reports_ambiguous_author_as_usage_error(monkeypatch, capsys):
class Client:
def __init__(self):
pass
def get(self, path, params=None):
raise HTTPStatusError(
409,
json.dumps(
{
"detail": {
"code": "ambiguous_author",
"reference": "Alex Kim",
"candidates": [
{
"id": "41",
"name": "Alex Kim",
"hf_username": "alex-one",
},
{
"id": "72",
"name": "Alex Kim",
"hf_username": None,
},
],
}
}
),
)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["paper", "list", "--author", "Alex Kim"]) == 2
error = capsys.readouterr().err
assert "Alex Kim is ambiguous" in error
assert "--author 41" in error
assert "@alex-one" in error
assert "--author 72" in error
def test_benchmark_detail_parser_matches_requested_command_shape():
args = build_parser().parse_args(
[
"benchmark",
"--name",
"SWE-Bench Pro",
"--limit",
"5",
"--require-metrics",
"Resolved,Latency",
"--min",
"Resolved=50",
"--max",
"Latency=2.5",
"--sort",
"Resolved:desc",
"--pareto",
"Resolved:higher,Latency:lower",
"--max-parameters",
"3B",
]
)
assert args.name == "SWE-Bench Pro"
assert args.limit == 5
assert args.require_metrics == ("Resolved", "Latency")
assert args.minimum_metrics == [("Resolved", 50)]
assert args.maximum_metrics == [("Latency", 2.5)]
assert args.sort_metric == ("Resolved", "desc")
assert args.pareto == (("Resolved", "higher"), ("Latency", "lower"))
assert args.max_parameters == 3_000_000_000
def test_benchmark_detail_parser_accepts_human_parameter_sizes():
parser = build_parser()
assert (
parser.parse_args(
["benchmark", "--name", "LM", "--max-parameters", "500m"]
).max_parameters
== 500_000_000
)
assert (
parser.parse_args(
["benchmark", "--name", "LM", "--max-parameters", "1.5B"]
).max_parameters
== 1_500_000_000
)
assert (
parser.parse_args(
["benchmark", "--name", "LM", "--max-parameters", "3_000_000_000"]
).max_parameters
== 3_000_000_000
)
for invalid in (
"0",
"-1",
"1.5",
"3K",
"3B parameters",
"_3B",
"3_B",
"3B_",
"1__000",
"1_.5B",
):
errors = io.StringIO()
with redirect_stderr(errors):
try:
parser.parse_args(
["benchmark", "--name", "LM", "--max-parameters", invalid]
)
except SystemExit as error:
assert error.code == 2
else:
raise AssertionError(f"accepted invalid parameter size: {invalid}")
assert "--max-parameters" in errors.getvalue()
def test_benchmark_list_rejects_parent_parameter_filter(monkeypatch, capsys):
class Client:
def __init__(self):
pass
def get(self, _path, _params=None):
return Response(b'{"count":0,"results":[]}', {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["benchmark", "--max-parameters", "3B", "list"]) == 2
assert capsys.readouterr().err == (
"pwc: --max-parameters is only valid with benchmark --name\n"
)
def test_task_benchmark_list_uses_frontend_ranking_and_aligned_output(monkeypatch):
calls = []
payload = {
"task_id": "18",
"task_slug": "ocr",
"recent_days": 90,
"results": [
{
"id": "2",
"slug": "recent-ocr",
"name": "Recent OCR",
"description": "Recent OCR benchmark.",
"hf_url": "https://huggingface.co/datasets/example/recent-ocr",
"recent_paper_count": 4,
"all_time_paper_count": 5,
"trend_score": 1.3333,
"support_weighted_score": 3.8627,
"best_model_name": "Reader 2",
"best_paper_title": "Recent OCR Paper",
"best_code_url": "https://github.com/example/reader-2",
},
{
"id": "1",
"slug": "classic-ocr",
"name": "Classic OCR",
"description": None,
"hf_url": None,
"recent_paper_count": 1,
"all_time_paper_count": 20,
"trend_score": 0.3333,
"support_weighted_score": 0.5941,
"best_model_name": "Reader 1",
"best_paper_title": "Classic OCR Paper",
"best_code_url": None,
},
],
}
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["benchmark", "list", "--task", "OCR"]) == 0
assert calls[0][0] == "tasks/OCR/trending-benchmarks"
assert calls[0][1]["min_recent_papers"] == 0
rows = output.getvalue().splitlines()
assert rows[0].split() == [
"id",
"slug",
"name",
"description",
"hf_url",
"recent_papers",
"ranking_score",
"best_model",
"paper_title",
"code",
]
assert "Recent OCR benchmark." in rows[1]
assert "https://huggingface.co/datasets/example/recent-ocr" in rows[1]
assert "Classic OCR" in rows[2]
def test_task_filter_takes_precedence_over_grouped_benchmark_display(
monkeypatch, capsys
):
payload = {
"count": 1,
"results": [
{
"id": "2",
"slug": "recent-ocr",
"name": "Recent OCR",
"recent_paper_count": 4,
"trend_score": 1.3333,
"best_model_name": "Reader 2",
"best_paper_title": "Recent OCR Paper",
}
],
}
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["benchmark", "list", "--task", "OCR", "--group-by-area"]) == 0
captured = capsys.readouterr()
assert calls[0][0] == "tasks/OCR/trending-benchmarks"
assert captured.out.startswith("id slug name")
assert captured.err == (
"# filters select task-scoped flat output; --group-by-area ignored\n"
)
def test_flat_benchmark_list_aligns_captured_column_widths(monkeypatch, capsys):
payload = {
"count": 2,
"results": [
{
"id": "2",
"name": "Short",
"description": "Short description",
"hf_url": None,
"full_name": "S",
"paper_count": 3,
},
{
"id": "10",
"name": "Longer Name",
"description": "Longer benchmark description",
"hf_url": "https://huggingface.co/datasets/example/longer",
"full_name": "Full benchmark name",
"paper_count": 12,
},
],
}
class Client:
def __init__(self):
pass
def get(self, _path, _params):
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["benchmark", "list", "--flat"]) == 0
rows = capsys.readouterr().out.splitlines()
assert rows[0].split() == [
"id",
"name",
"description",
"hf_url",
"full_name",
"evals",
]
assert "Short description" in rows[1]
assert "Longer benchmark description" in rows[2]
assert "https://huggingface.co/datasets/example/longer" in rows[2]
def test_benchmark_list_groups_top_benchmarks_as_markdown(monkeypatch):
calls = []
payload = {
"results": [
{
"id": "1",
"name": "Vision",
"tasks": [
{
"id": "10",
"name": "Image Classification",
"slug": "image-classification",
"paper_count": 100,
"benchmarks": [
{
"id": "20",
"name": "ImageNet [1K]",
"slug": "imagenet",
"full_name": "ImageNet Large Scale Visual Recognition",
"description": "Image classification benchmark.",
"hf_url": "https://huggingface.co/datasets/imagenet-1k",
"evaluation_count": 1234,
"featured_rank": 1,
}
],
}
],
}
]
}
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert (
main(
[
"benchmark",
"list",
"--group-by-area",
"--area",
"Vision",
"--benchmarks-per-task",
"5",
]
)
== 0
)
assert calls == [
(
"areas/with-task-benchmarks/",
{"benchmarks_per_task": 5, "min_eval_count": 2},
)
]
assert output.getvalue() == (
"# Area: Vision\n"
"\n"
"Inspect a benchmark with `pwc benchmark --name SLUG`.\n"
"\n"
"## Image Classification (`image-classification`)\n"
"\n"
"- ImageNet [1K] — description: Image classification benchmark.; HF: "
"https://huggingface.co/datasets/imagenet-1k; full name: ImageNet Large Scale Visual Recognition; "
"slug: `imagenet`; ID: `20`; 1,234 evaluations\n"
)
def test_task_list_resolves_area_name_and_renders_compact_output(monkeypatch):
calls = []
areas = {
"count": 2,
"results": [{"id": "1", "name": "Vision"}, {"id": "6", "name": "Audio"}],
}
tasks = {
"count": 1,
"results": [
{
"id": "10",
"slug": "image-classification",
"name": "Image Classification",
"area_id": "1",
"level": 0,
"paper_count": 123,
"benchmark_count": 4,
"evaluation_count": 567,
}
],
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
payload = areas if path == "areas/" else tasks
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert (
main(
[
"task",
"list",
"--area",
"vision",
"--search",
"image",
"--page-size",
"1",
"--order-by",
"paper_count",
"--order-dir",
"desc",
]
)
== 0
)
assert calls == [
("areas/", {"page": 1, "page_size": 500, "ordering": "name"}),
(
"tasks/",
{
"page": 1,
"page_size": 1,
"q": "image",
"area_id": "1",
"level": None,
"visible_only": False,
"ordering": "-paper_count",
},
),
]
assert output.getvalue() == (
"id\tslug\tname\tarea\tlevel\tpapers\tbenchmarks\tevals\n"
"10\timage-classification\tImage Classification\tVision\t0\t123\t4\t567\n"
)
def test_task_list_groups_complete_taxonomy_in_interactive_terminal(monkeypatch):
payload = {
"results": [
{
"id": "6",
"name": "Audio",
"tasks": [
{
"id": "259",
"slug": "audio-classification",
"name": "Audio Classification",
"paper_count": 63,
"benchmark_count": 1,
"evaluation_count": 8,
}
],
},
{
"id": "1",
"name": "Vision",
"tasks": [
{
"id": "13",
"slug": "3d-generation",
"name": "3D generation",
"paper_count": 2389,
"benchmark_count": 12,
"evaluation_count": 345,
},
{
"id": "1",
"slug": "image-classification",
"name": "Image Classification",
"paper_count": 1,
"benchmark_count": 1,
"evaluation_count": 1,
},
],
},
]
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
assert (path, params) == ("areas/with-tasks/", None)
return Response(json.dumps(payload).encode(), {})
class TerminalOutput(io.StringIO):
def isatty(self):
return True
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = TerminalOutput()
with redirect_stdout(output):
assert main(["task", "list"]) == 0
assert output.getvalue() == (
"# Area: Audio\n"
"- **Audio Classification** (`audio-classification`) — "
"63 papers · 1 benchmark · 8 evals\n"
"\n"
"# Area: Vision\n"
"- **3D generation** (`3d-generation`) — "
"2,389 papers · 12 benchmarks · 345 evals\n"
"- **Image Classification** (`image-classification`) — "
"1 paper · 1 benchmark · 1 eval\n"
)
assert "(`13`)" not in output.getvalue()
def test_task_list_can_force_grouped_area_output_when_captured(monkeypatch):
payload = {
"results": [
{"id": "6", "name": "Audio", "tasks": []},
{
"id": "1",
"name": "Vision",
"tasks": [
{
"id": "13",
"slug": "3d-generation",
"name": "3D generation",
"paper_count": 2389,
"benchmark_count": 12,
"evaluation_count": 345,
}
],
},
]
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
assert (path, params) == ("areas/with-tasks/", None)
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["task", "list", "--group-by-area", "--area", "vision"]) == 0
assert output.getvalue() == (
"# Area: Vision\n- **3D generation** (`3d-generation`) — "
"2,389 papers · 12 benchmarks · 345 evals\n"
)
def test_task_detail_renders_task_page_sections(monkeypatch):
calls = []
search = {
"count": 1,
"results": [
{
"id": "17",
"slug": "scene-text-recognition",
"name": "Scene Text Recognition",
"area_id": "1",
"level": 1,
"paper_count": 204,
"benchmark_count": 21,
"evaluation_count": 401,
}
],
}
page = {
"task": {
"id": "17",
"slug": "scene-text-recognition",
"name": "Scene Text Recognition",
"description": "Recognizing text in natural images.",
"research_trends": ["Vision-language pretraining"],
"area_id": "1",
"level": 1,
"paper_count": 204,
},
"parents": [{"id": "2", "slug": "ocr", "name": "OCR"}],
"sisters": [{"id": "18", "slug": "text-detection", "name": "Text Detection"}],
"recommended_frameworks": [
{
"id": "44",
"url": "https://github.com/huggingface/transformers",
"owner": "huggingface",
"name": "transformers",
"num_stars": 150000,
}
],
"children": [{"id": "19", "slug": "handwritten-text", "name": "Handwriting"}],
"benchmarks": [
{
"id": "30",
"slug": "iiit5k",
"name": "IIIT5K",
"paper_count": 12,
"best_model_name": "Model A",
},
{
"id": "32",
"slug": "svt",
"name": "SVT",
"paper_count": 4,
"best_model_name": "Model B",
},
],
"subtask_benchmarks": [
{
"subtask_id": "19",
"benchmarks": [{"id": "31", "slug": "iam", "name": "IAM"}],
}
],
}
papers = {
"count": 204,
"results": [
{
"id": "100",
"arxiv_id": "2501.00001",
"title": "A Better Text Recognizer",
"published": "2025-01-01",
"citation_count": 10,
"methods": [{"id": "7", "slug": "transformer", "name": "Transformer"}],
}
],
}
benchmark_trends = {
"support_prior_papers": 2.0,
"results": [
{
"id": "32",
"slug": "svt",
"name": "SVT",
"support_weighted_score": 2.5,
},
{
"id": "30",
"slug": "iiit5k",
"name": "IIIT5K",
"support_weighted_score": 1.25,
},
],
}
areas = {"count": 1, "results": [{"id": "1", "name": "Vision"}]}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
payloads = {
"tasks/": search,
"tasks/17/page": page,
"tasks/17/trending-benchmarks": benchmark_trends,
"tasks/17/papers": papers,
"areas/": areas,
}
return Response(json.dumps(payloads[path]).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["task", "--name", "scene-text-recognition"]) == 0
rendered = output.getvalue()
assert "# Scene Text Recognition" in rendered
assert (
"Area: Vision · Level: 1 · 204 papers · 21 benchmarks · 401 evaluations"
in rendered
)
assert "## Recommended frameworks" in rendered
assert (
"[huggingface/transformers](https://github.com/huggingface/transformers)"
in rendered
)
assert "## Sister tasks" in rendered
assert "## Common methods (from 1 sampled papers)" in rendered
assert "## Benchmarks by subtask" in rendered
assert "## Trending papers (top 1 of 204)" in rendered
assert rendered.index("SVT") < rendered.index("IIIT5K")
assert "trend score: 2.50" in rendered
assert calls == [
("tasks/", {"q": "scene-text-recognition", "page": 1, "page_size": 100}),
("tasks/17/page", None),
(
"tasks/17/trending-benchmarks",
{"limit": 100, "min_recent_papers": 0},
),
(
"tasks/17/papers",
{
"page": 1,
"page_size": 100,
"order_by": "trending",
"order_dir": "desc",
"include_resources": True,
},
),
("areas/", {"page": 1, "page_size": 500, "ordering": "name"}),
]
def test_task_detail_json_has_stable_composed_payload(monkeypatch, capsys):
payloads = {
"tasks/": {
"count": 1,
"results": [
{
"id": "17",
"slug": "scene-text-recognition",
"name": "Scene Text Recognition",
"area_id": "1",
}
],
},
"tasks/17/page": {
"task": {"id": "17", "name": "Scene Text Recognition"},
"recommended_frameworks": [],
"benchmarks": [],
},
"tasks/17/trending-benchmarks": {"results": []},
"tasks/17/papers": {"count": 0, "results": []},
"tasks/17": {
"id": "17",
"slug": "scene-text-recognition",
"name": "Scene Text Recognition",
"area_id": "1",
},
"areas/": {"count": 1, "results": [{"id": "1", "name": "Vision"}]},
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
return Response(json.dumps(payloads[path]).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["task", "--name", "Scene Text Recognition", "--json"]) == 0
result = json.loads(capsys.readouterr().out)
assert result["schema_version"] == "v1"
assert result["data"]["task"]["id"] == "17"
assert result["data"]["area"]["name"] == "Vision"
assert result["data"]["paper_sample_size"] == 0
assert main(["task", "--name", "17", "--json"]) == 0
result = json.loads(capsys.readouterr().out)
assert result["data"]["task"]["id"] == "17"
def test_method_list_accepts_area_id_and_year_filter(monkeypatch):
calls = []
areas = {
"count": 1,
"results": [{"id": "6", "name": "Audio"}],
}
methods = {
"count": 1,
"results": [
{
"id": "20",
"slug": "wav2vec-2",
"name": "wav2vec 2.0",
"full_name": "wav2vec 2.0",
"area_id": "6",
"introduced_year": 2020,
"paper_count": 45,
}
],
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
payload = areas if path == "areas/" else methods
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert (
main(
[
"method",
"list",
"--area",
"6",
"--search",
"wav2vec",
"--introduced-year",
"2020",
]
)
== 0
)
assert calls[1] == (
"methods/",
{
"page": 1,
"page_size": 50,
"q": "wav2vec",
"area_id": "6",
"introduced_year": 2020,
"ordering": "name",
},
)
assert output.getvalue() == (
"id\tslug\tname\tfull_name\tarea\tintroduced\tpapers\n"
"20\twav2vec-2\twav2vec 2.0\twav2vec 2.0\tAudio\t2020\t45\n"
)
def test_method_detail_resolves_exact_name_and_renders_source(monkeypatch):
calls = []
payloads = {
"methods/": {
"count": 1,
"results": [
{
"id": "2",
"slug": "transformer",
"name": "Transformer",
"full_name": "Transformer",
}
],
},
"methods/2": {
"id": "2",
"slug": "transformer",
"name": "Transformer",
"description": "Attention-only architecture.",
"area_id": "3",
"introduced_year": 2017,
"paper_count": 13086,
"source_url": "/paper/1706.03762",
"source_title": "Attention Is All You Need",
},
"areas/": {"count": 1, "results": [{"id": "3", "name": "General"}]},
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
return Response(json.dumps(payloads[path]).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["method", "--name", "Transformer"]) == 0
rendered = output.getvalue()
assert "# Transformer" in rendered
assert "Area: General · Introduced: 2017 · 13,086 papers" in rendered
assert (
"[Attention Is All You Need](https://paperswithcode.co/paper/1706.03762)"
in rendered
)
assert calls[0] == (
"methods/",
{"q": "Transformer", "page": 1, "page_size": 100},
)
calls.clear()
with redirect_stdout(io.StringIO()):
assert main(["method", "--name", "2"]) == 0
assert calls == [
("methods/2", None),
("areas/", {"page": 1, "page_size": 500, "ordering": "name"}),
]
def _keyword_method_client(monkeypatch, catalog):
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
if path == "methods/" and params.get("q"):
payload = {"count": 0, "results": []}
elif path == "methods/":
payload = {"count": len(catalog), "results": catalog}
elif path == "areas/":
payload = {"count": 0, "results": []}
else:
method_id = path.split("/")[1]
payload = next(m for m in catalog if m["id"] == method_id)
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
return calls
def test_method_detail_resolves_keyword_alias(monkeypatch):
catalog = [
{
"id": "1205",
"slug": "gbdt",
"name": "Gradient Boosted Decision Trees",
"full_name": "Gradient Boosted Decision Trees (GBDT)",
"keywords": "GBDT, GBDTs, GDBT, gradient boosting machine",
},
{
"id": "9",
"slug": "pi0",
"name": "pi0",
"full_name": "pi0",
"keywords": "pi-0, pi zero",
},
]
calls = _keyword_method_client(monkeypatch, catalog)
for alias, slug in (("gdbt", "gbdt"), ("GDBT", "gbdt"), ("pi 0", "pi0")):
output = io.StringIO()
with redirect_stdout(output):
assert main(["method", "--name", alias, "--json"]) == 0
assert json.loads(output.getvalue())["data"]["method"]["slug"] == slug
assert ("methods/", {"page": 1, "page_size": 500}) in calls
def test_method_detail_keyword_alias_must_be_unique(monkeypatch, capsys):
catalog = [
{"id": "1", "slug": "a", "name": "Alpha", "keywords": "shared, one"},
{"id": "2", "slug": "b", "name": "Beta", "keywords": "Shared"},
]
_keyword_method_client(monkeypatch, catalog)
assert main(["method", "--name", "shared"]) != 0
assert "Method not found: shared" in capsys.readouterr().err
def test_conference_list_filters_complete_response_by_year(monkeypatch):
payload = {
"count": 2,
"results": [
{
"slug": "cvpr-2025",
"name": "CVPR 2025",
"year": 2025,
"paper_count": 1834,
"location": "Nashville, USA",
},
{
"slug": "cvpr-2026",
"name": "CVPR 2026",
"year": 2026,
"paper_count": 2200,
"location": "Denver, USA",
},
],
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
assert (path, params) == ("conferences/", None)
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["conference", "list", "--year", "2025"]) == 0
assert output.getvalue() == (
"slug\tname\tyear\tpapers\tlocation\n"
"cvpr-2025\tCVPR 2025\t2025\t1834\tNashville, USA\n"
)
def test_conference_detail_accepts_name_slug_or_id(monkeypatch, capsys):
listing = {
"count": 1,
"results": [
{"id": "7", "slug": "cvpr-2026", "name": "CVPR 2026", "year": 2026}
],
}
detail = {
"id": "7",
"slug": "cvpr-2026",
"name": "CVPR 2026",
"year": 2026,
"paper_count": 2677,
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
payload = listing if path == "conferences/" else detail
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
for reference in ("CVPR 2026", "cvpr-2026", "7"):
assert main(["conference", "--name", reference, "--json"]) == 0
result = json.loads(capsys.readouterr().out)
assert result["data"]["slug"] == "cvpr-2026"
def test_conference_list_aligns_columns_in_interactive_terminal(monkeypatch):
payload = {
"count": 2,
"results": [
{
"slug": "cvpr-2025",
"name": "CVPR 2025",
"year": 2025,
"paper_count": 1834,
"location": "Nashville, USA",
},
{
"slug": "neurips-2025",
"name": "NeurIPS 2025",
"year": 2025,
"paper_count": 2763,
"location": "San Diego, USA",
},
],
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
assert (path, params) == ("conferences/", None)
return Response(json.dumps(payload).encode(), {})
class TerminalOutput(io.StringIO):
def isatty(self):
return True
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = TerminalOutput()
with redirect_stdout(output):
assert main(["conference", "list"]) == 0
assert output.getvalue() == (
"slug name year papers location\n"
"cvpr-2025 CVPR 2025 2025 1834 Nashville, USA\n"
"neurips-2025 NeurIPS 2025 2025 2763 San Diego, USA\n"
)
def test_organization_detail_and_featured_list(monkeypatch, capsys):
listing = {
"count": 1,
"results": [
{
"id": "1",
"slug": "nvidia",
"name": "NVIDIA",
"paper_count": 158,
"is_featured": True,
"trending_rank": 1,
}
],
}
detail = {
**listing["results"][0],
"hf_url": "https://huggingface.co/nvidia",
"website_url": "https://www.nvidia.com/",
}
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
payload = detail if path == "organizations/nvidia" else listing
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["organization", "--name", "NVIDIA", "--json"]) == 0
result = json.loads(capsys.readouterr().out)
assert result["data"]["slug"] == "nvidia"
assert main(["organization", "list", "--featured-only"]) == 0
assert "1\tnvidia\tNVIDIA\t158\tyes\t1" in capsys.readouterr().out
assert calls == [
("organizations/", {"featured_only": False}),
("organizations/nvidia", None),
("organizations/", {"featured_only": True}),
]
def test_framework_detail_and_filtered_list_flatten_catalog(monkeypatch, capsys):
catalog = {
"domains": [
{
"slug": "llm",
"name": "Large Language Models",
"categories": [
{
"slug": "inference",
"name": "Inference & Serving",
"frameworks": [
{
"id": "11",
"slug": "vllm",
"name": "vLLM",
"description": "High-throughput serving.",
"platforms": ["gpu"],
"paper_count": 73,
},
{
"id": "9",
"slug": "llama-cpp",
"name": "llama.cpp",
"description": "Portable inference.",
"platforms": ["cpu", "apple"],
"paper_count": 2,
},
],
}
],
}
]
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
assert (path, params) == ("frameworks/", None)
return Response(json.dumps(catalog).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["framework", "--name", "11", "--json"]) == 0
result = json.loads(capsys.readouterr().out)
assert result["data"]["slug"] == "vllm"
assert result["data"]["domain_slug"] == "llm"
assert main(["framework", "list", "--platform", "apple"]) == 0
output = capsys.readouterr().out
assert "llama-cpp" in output
assert "vllm" not in output
def test_unknown_area_lists_valid_choices(monkeypatch):
areas = {
"count": 2,
"results": [{"id": "1", "name": "Vision"}, {"id": "6", "name": "Audio"}],
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
assert path == "areas/"
return Response(json.dumps(areas).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
errors = io.StringIO()
with redirect_stderr(errors):
assert main(["task", "list", "--area", "Robotics"]) == 4
assert errors.getvalue() == (
"pwc: Area not found: Robotics; available areas: Audio, Vision\n"
)
def test_benchmark_detail_renders_merged_markdown_leaderboard(monkeypatch):
calls = []
benchmark_payload = {
"count": 1,
"results": [
{
"id": "42",
"name": "SWE-Bench Pro",
"slug": "swe-bench-pro",
"description": "A coding-agent benchmark.",
}
],
}
evaluations_payload = {
"count": 3,
"results": [
{
"id": "1",
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Agent | One",
"metrics": {"Resolved": 55.5},
"best_metric": "Resolved",
"best_rank": 1,
"num_parameters": 10_000_000_000,
"paper_title": "An Agent Paper",
"paper_arxiv_id": "2601.12345",
"paper_published_date": "2026-01-20",
"task_name": "Coding Agents",
"is_open": False,
},
{
"id": "2",
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Agent | One",
"metrics": {"Pass@1": 50},
"best_metric": "Pass@1",
"best_rank": 2,
"num_parameters": 10_000_000_000,
"paper_title": "An Agent Paper",
"paper_arxiv_id": "2601.12345",
"paper_published_date": "2026-01-20",
"task_name": "Coding Agents",
"is_open": False,
},
{
"id": "3",
"paper_id": "9",
"task_id": "8",
"dataset_id": "42",
"model_name": "Agent | One",
"metrics": {"Resolved": 55.5},
"best_metric": "Resolved",
"best_rank": 4,
"num_parameters": 10_000_000_000,
"paper_title": "An Agent Paper",
"paper_arxiv_id": "2601.12345",
"paper_published_date": "2026-01-20",
"task_name": "Software Engineering",
"task_slug": "software-engineering",
"is_open": False,
},
],
}
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
payload = benchmark_payload if path == "datasets/" else evaluations_payload
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["benchmark", "--name", "SWE-Bench Pro"]) == 0
rendered = output.getvalue()
assert "# SWE-Bench Pro" in rendered
assert "Agent \\| One" in rendered
assert "| 1 | Agent \\| One | 10B |" in rendered
assert "Resolved: 55.5, Pass@1: 50" in rendered
assert "[An Agent Paper](https://paperswithcode.co/paper/2601.12345)" in rendered
assert "2026-01-20" in rendered
assert calls[1] == (
"datasets/42/evaluations/",
{
"page": 1,
"page_size": 20,
"ordering": "best_rank",
"is_open": None,
},
)
json_output = io.StringIO()
with redirect_stdout(json_output):
assert main(["benchmark", "--name", "SWE-Bench Pro", "--json"]) == 0
rows = json.loads(json_output.getvalue())["data"]["results"]
assert len(rows) == 1
assert rows[0]["rank_scopes"] == [
{
"task_id": "3",
"task_name": "Coding Agents",
"task_slug": None,
"rank": 1,
},
{
"task_id": "8",
"task_name": "Software Engineering",
"task_slug": "software-engineering",
"rank": 4,
},
]
def test_benchmark_detail_renders_aligned_table_in_terminal(monkeypatch):
benchmark_payload = {
"count": 1,
"results": [
{
"id": "42",
"name": "SWE-Bench Pro",
"slug": "swe-bench-pro",
"description": "A coding-agent benchmark.",
}
],
}
evaluations_payload = {
"count": 1,
"results": [
{
"id": "1",
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Agent One",
"metrics": {"Resolved": 55.5},
"best_metric": "Resolved",
"best_rank": 1,
"paper_title": "An Agent Paper",
"paper_arxiv_id": "2601.12345",
"paper_published_date": "2026-01-20",
"task_name": "Coding Agents",
"is_open": False,
}
],
}
class Client:
def __init__(self):
pass
def get(self, path, params):
payload = benchmark_payload if path == "datasets/" else evaluations_payload
return Response(json.dumps(payload).encode(), {})
class TerminalBuffer(io.StringIO):
def isatty(self):
return True
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = TerminalBuffer()
with redirect_stdout(output):
assert main(["benchmark", "--name", "SWE-Bench Pro"]) == 0
rendered = output.getvalue()
assert rendered.startswith("SWE-Bench Pro\n\nA coding-agent benchmark.\n")
assert (
"View benchmark: https://paperswithcode.co/benchmark/swe-bench-pro" in rendered
)
assert "Rank Model Parameters Scores Task" in rendered
assert " 1 Agent One — Resolved: 55.5 Coding Agents" in rendered
assert "An Agent Paper [2601.12345]" in rendered
assert "| ---:" not in rendered
def _parameter_filter_client(monkeypatch, evaluations_payload):
benchmark_payload = {
"count": 1,
"results": [{"id": "42", "name": "Language Model Benchmark", "slug": "lmb"}],
}
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
payload = benchmark_payload if path == "datasets/" else evaluations_payload
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
return calls
def test_benchmark_detail_filters_by_inclusive_max_parameters(monkeypatch, capsys):
evaluations_payload = {
"count": 1,
"next_page": None,
"previous_page": None,
"parameter_coverage_known": 1,
"parameter_coverage_total": 2,
"results": [
{
"id": "1",
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Edge 3B",
"metrics": {"Perplexity": 12.5},
"best_metric": "Perplexity",
"best_rank": 4,
"num_parameters": 3_000_000_000,
"is_open": True,
}
],
}
calls = _parameter_filter_client(monkeypatch, evaluations_payload)
assert (
main(
[
"benchmark",
"--name",
"Language Model Benchmark",
"--is-open",
"true",
"--max-parameters",
"3B",
"--json",
]
)
== 0
)
assert calls[1] == (
"evaluations/",
{
"page": 1,
"page_size": 20,
"dataset_id": "42",
"ordering": "best_rank",
"is_open": "true",
"max_parameters_exclusive": 3_000_000_001,
},
)
result = json.loads(capsys.readouterr().out)
assert result["data"]["matched_count"] == 1
assert result["data"]["results"][0]["model_name"] == "Edge 3B"
assert result["data"]["results"][0]["num_parameters"] == 3_000_000_000
assert (
main(
[
"benchmark",
"--name",
"Language Model Benchmark",
"--max-parameters",
"3B",
]
)
== 0
)
rendered = capsys.readouterr().out
assert "| Rank | Model | Parameters | Scores |" in rendered
assert "| 4 | Edge 3B | 3B | Perplexity: 12.5 |" in rendered
def test_benchmark_detail_parameter_filter_fails_closed_without_api_support(
monkeypatch, capsys
):
unsupported_payload = {"count": 0, "results": []}
_parameter_filter_client(monkeypatch, unsupported_payload)
assert (
main(
[
"benchmark",
"--name",
"Language Model Benchmark",
"--max-parameters",
"3B",
]
)
== 4
)
assert capsys.readouterr().err == (
"pwc: parameter filtering requires a compatible API; server must be upgraded\n"
)
def test_benchmark_detail_parameter_filter_rejects_out_of_bound_response(
monkeypatch, capsys
):
invalid_payload = {
"count": 1,
"parameter_coverage_known": 1,
"parameter_coverage_total": 1,
"results": [
{
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Too Large",
"metrics": {"Perplexity": 10},
"num_parameters": 3_000_000_001,
}
],
}
_parameter_filter_client(monkeypatch, invalid_payload)
assert (
main(
[
"benchmark",
"--name",
"Language Model Benchmark",
"--max-parameters",
"3B",
]
)
== 4
)
assert capsys.readouterr().err == (
"pwc: API returned a model outside the requested parameter limit\n"
)
def test_benchmark_detail_parameter_filter_rejects_conflicting_sibling_counts(
monkeypatch, capsys
):
invalid_payload = {
"count": 1,
"parameter_coverage_known": 1,
"parameter_coverage_total": 1,
"results": [
{
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Conflicting",
"metrics": {"Perplexity": 10},
"num_parameters": 2_000_000_000,
},
{
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Conflicting",
"metrics": {"Accuracy": 90},
"num_parameters": 2_500_000_000,
},
],
}
_parameter_filter_client(monkeypatch, invalid_payload)
assert (
main(
[
"benchmark",
"--name",
"Language Model Benchmark",
"--max-parameters",
"3B",
]
)
== 4
)
assert capsys.readouterr().err == (
"pwc: API returned a model outside the requested parameter limit\n"
)
def test_benchmark_detail_parameter_filter_rejects_incomplete_sibling_counts(
monkeypatch, capsys
):
invalid_payload = {
"count": 1,
"parameter_coverage_known": 1,
"parameter_coverage_total": 1,
"results": [
{
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Incomplete",
"metrics": {"Perplexity": 10},
"num_parameters": 2_000_000_000,
},
{
"paper_id": "9",
"task_id": "3",
"dataset_id": "42",
"model_name": "Incomplete",
"metrics": {"Accuracy": 90},
"num_parameters": None,
},
],
}
_parameter_filter_client(monkeypatch, invalid_payload)
assert (
main(
[
"benchmark",
"--name",
"Language Model Benchmark",
"--max-parameters",
"3B",
]
)
== 4
)
assert capsys.readouterr().err == (
"pwc: API returned a model outside the requested parameter limit\n"
)
def test_benchmark_detail_parameter_filter_composes_with_metric_pagination(
monkeypatch, capsys
):
benchmark_payload = {
"count": 1,
"results": [{"id": "42", "name": "Language Model Benchmark"}],
}
pages = {
1: {
"count": 2,
"next_page": 2,
"parameter_coverage_known": 2,
"parameter_coverage_total": 3,
"results": [
{
"paper_id": "1",
"task_id": "3",
"dataset_id": "42",
"model_name": "Model A",
"metrics": {"Accuracy": 80},
"num_parameters": 1_000_000_000,
},
{
"paper_id": "1",
"task_id": "3",
"dataset_id": "42",
"model_name": "Model A",
"metrics": {"Latency": 5},
"num_parameters": 1_000_000_000,
},
],
},
2: {
"count": 2,
"next_page": None,
"parameter_coverage_known": 2,
"parameter_coverage_total": 3,
"results": [
{
"paper_id": "2",
"task_id": "3",
"dataset_id": "42",
"model_name": "Model B",
"metrics": {"Accuracy": 90, "Latency": 7},
"num_parameters": 2_000_000_000,
}
],
},
}
evaluation_calls = []
class Client:
def __init__(self):
pass
def get(self, path, params):
if path == "datasets/":
return Response(json.dumps(benchmark_payload).encode(), {})
evaluation_calls.append(params)
return Response(json.dumps(pages[params["page"]]).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert (
main(
[
"benchmark",
"--name",
"Language Model Benchmark",
"--max-parameters",
"2B",
"--require-metrics",
"Accuracy,Latency",
"--sort",
"Accuracy:desc",
"--json",
]
)
== 0
)
result = json.loads(capsys.readouterr().out)
assert [row["model_name"] for row in result["data"]["results"]] == [
"Model B",
"Model A",
]
assert [call["page"] for call in evaluation_calls] == [1, 2]
def test_benchmark_detail_filters_sorts_and_computes_pareto_frontier(
monkeypatch, capsys
):
benchmark_payload = {
"count": 1,
"results": [{"id": "6", "name": "COCO", "slug": "coco"}],
}
evaluation_pages = {
1: {
"count": 3,
"results": [
{
"id": "1",
"paper_id": "1",
"task_id": "3",
"dataset_id": "6",
"model_name": "Accurate",
"metrics": {"mAP": 60, "FPS": 30},
"best_rank": 1,
},
{
"id": "2",
"paper_id": "2",
"task_id": "3",
"dataset_id": "6",
"model_name": "Fast",
"metrics": {"mAP": "58", "FPS": "100"},
"best_rank": 2,
},
],
},
2: {
"count": 3,
"results": [
{
"id": "3",
"paper_id": "3",
"task_id": "3",
"dataset_id": "6",
"model_name": "Dominated",
"metrics": {"mAP": 57, "FPS": 50},
"best_rank": 3,
}
],
},
}
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
if path == "datasets/":
return Response(json.dumps(benchmark_payload).encode(), {})
return Response(json.dumps(evaluation_pages[params["page"]]).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert (
main(
[
"benchmark",
"--name",
"COCO",
"--pareto",
"mAP:higher,FPS:higher",
"--sort",
"FPS:desc",
]
)
== 0
)
captured = capsys.readouterr()
assert captured.err == ""
assert (
"Showing 2 of 2 matching merged rows from 3 stored evaluation rows."
in captured.out
)
assert captured.out.index("Fast") < captured.out.index("Accurate")
assert "Dominated" not in captured.out
assert [params["page"] for path, params in calls if path != "datasets/"] == [1, 2]
def test_benchmark_detail_metric_thresholds_and_unknown_metric(monkeypatch, capsys):
benchmark_payload = {
"count": 1,
"results": [{"id": "6", "name": "COCO", "slug": "coco"}],
}
evaluations_payload = {
"count": 2,
"results": [
{
"paper_id": "1",
"task_id": "3",
"dataset_id": "6",
"model_name": "Fast",
"metrics": {"mAP": 58, "FPS": 100},
},
{
"paper_id": "2",
"task_id": "3",
"dataset_id": "6",
"model_name": "Slow",
"metrics": {"mAP": 62, "FPS": 20},
},
],
}
class Client:
def __init__(self):
pass
def get(self, path, _params):
payload = benchmark_payload if path == "datasets/" else evaluations_payload
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert (
main(
[
"benchmark",
"--name",
"COCO",
"--require-metrics",
"mAP,FPS",
"--min",
"FPS=60",
"--max",
"mAP=60",
]
)
== 0
)
captured = capsys.readouterr()
assert "Fast" in captured.out
assert "Slow" not in captured.out
assert main(["benchmark", "--name", "COCO", "--sort", "Latency"]) == 2
captured = capsys.readouterr()
assert captured.out == ""
assert "unknown metric(s): latency; available metrics: FPS, mAP" in captured.err
def test_top_level_version_is_offline_and_stable():
output = io.StringIO()
with redirect_stdout(output):
try:
build_parser().parse_args(["--version"])
except SystemExit as error:
assert error.code == 0
assert output.getvalue() == "pwc 0.4.6\tapi v1\n"
def test_search_default_output_is_compact_deterministic_tsv(monkeypatch):
payload = {
"count": 1,
"results": [
{
"arxiv_id": "2501.01234",
"title": "Line one\nline two",
"published": "2025-01-03",
"citation_count": 42,
"has_official_implementation": True,
"code_repository_count": 3,
}
],
}
class Client:
def __init__(self):
pass
def get(self, _path, _params):
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["search", "vision"]) == 0
assert output.getvalue() == (
"id\ttitle\tyear\tcitations\n2501.01234\tLine one\\nline two\t2025\t42\n"
)
output = io.StringIO()
with redirect_stdout(output):
assert main(["search", "vision", "--implementation-coverage"]) == 0
assert output.getvalue() == (
"id\ttitle\tyear\tcitations\tofficial_implementation\tcode_repositories\n"
"2501.01234\tLine one\\nline two\t2025\t42\ttrue\t3\n"
)
def test_search_official_implementation_filter_is_forwarded_and_fail_closed(
monkeypatch, capsys
):
calls = []
payloads = [
{"results": [], "applied_filters": {}},
{
"results": [],
"applied_filters": {"has_official_implementation": "true"},
},
]
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
return Response(json.dumps(payloads.pop(0)).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
command = ["search", "vision", "--has-official-implementation"]
assert main(command) == 4
assert "did not confirm has_official_implementation" in capsys.readouterr().err
assert main(command) == 0
capsys.readouterr()
assert calls[0][0] == "papers/search"
assert calls[0][1]["has_official_implementation"] is True
def test_paper_list_forwards_confirmed_official_implementation_filter(monkeypatch):
calls = []
payload = {
"results": [],
"applied_filters": {"has_official_implementation": "true"},
}
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
assert main(["paper", "list", "--has-official-implementation"]) == 0
assert calls[0][0] == "papers/"
assert calls[0][1]["has_official_implementation"] is True
def test_all_paper_commands_resolve_exact_titles(monkeypatch):
title = "Transformers Are RNNs"
resolved = "2006.16236"
search_payload = {
"count": 1,
"results": [{"arxiv_id": resolved, "title": title}],
}
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params=None):
calls.append((path, params))
if path == "papers/search":
return Response(json.dumps(search_payload).encode(), {})
if path == f"papers/{resolved}":
return Response(
json.dumps({"arxiv_id": resolved, "title": title}).encode(), {}
)
if path == f"research/papers/{resolved}/read":
return Response(b"# Paper", {})
if path == f"papers/{resolved}/related":
return Response(b'{"count":0,"results":[]}', {})
if path == f"research/papers/{resolved}/lineage":
return Response(
json.dumps(
{
"paper": {"reference": resolved, "title": title},
"predecessors": [],
"successors": [],
}
).encode(),
{},
)
raise AssertionError(path)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
commands = (
(
["paper", "info", title.lower()],
(f"papers/{resolved}", {"include_resources": False}),
),
(
["paper", "read", title.lower()],
(f"research/papers/{resolved}/read", None),
),
(
["paper", "related", title.lower()],
(f"papers/{resolved}/related", {"limit": 4}),
),
(
["paper", "lineage", "list", title.lower()],
(f"research/papers/{resolved}/lineage", None),
),
)
for argv, expected_request in commands:
calls.clear()
with redirect_stdout(io.StringIO()):
assert main(argv) == 0
assert calls == [
(
"papers/search",
{
"q": title.lower(),
"page": 1,
"page_size": 20,
"mode": "keyword",
},
),
expected_request,
]
def test_ambiguous_paper_title_is_rejected_with_matches(monkeypatch):
title = "A Shared Title"
payload = {
"count": 2,
"results": [
{"arxiv_id": "2501.00001", "title": title},
{"arxiv_id": "2501.00002", "title": title},
],
}
class Client:
def __init__(self):
pass
def get(self, path, params=None):
assert path == "papers/search"
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
errors = io.StringIO()
with redirect_stderr(errors):
assert main(["paper", "info", title]) == 4
assert errors.getvalue() == (
"pwc: Paper title is ambiguous: A Shared Title; "
"A Shared Title (2501.00001); A Shared Title (2501.00002)\n"
)
def test_paper_info_renders_metadata_and_abstract_sections(monkeypatch):
payload = {
"arxiv_id": "2501.01234",
"title": "A Paper",
"abstract": "First line.\nSecond line.",
"published": "2025-01-03",
"authors": ["Ada Researcher", "Grace Scientist"],
"organizations": [{"id": "1", "name": "Example AI", "slug": "example"}],
"citation_count": 42,
"has_official_implementation": False,
"code_repository_count": 2,
"url_abs": "https://paperswithcode.co/paper/2501.01234",
}
class Client:
def __init__(self):
pass
def get(self, _path, _params):
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "info", "2501.01234"]) == 0
assert output.getvalue() == (
"id: 2501.01234\n"
"title: A Paper\n"
"published: 2025-01-03\n"
"authors: Ada Researcher, Grace Scientist\n"
"organizations: Example AI\n"
"citations: 42\n"
"official_implementation: false\n"
"code_repositories: 2\n"
"url: https://paperswithcode.co/paper/2501.01234\n"
"\n"
"## Abstract\n"
"\n"
"First line.\n"
"Second line.\n"
"\n"
"## Paper lineage\n"
"\n"
"### Predecessors\n"
"\n"
"- None\n"
"\n"
"### Successors\n"
"\n"
"- None\n"
)
def test_paper_info_renders_linked_lineage_sections(monkeypatch):
payload = {
"arxiv_id": "2501.01234",
"title": "A Paper",
"predecessor_papers": [
{
"title": "Earlier [Paper]",
"route_identifier": "2401.00001",
"published": "2024-01-02",
}
],
"successor_papers": [
{
"title": "Later Paper",
"route_identifier": "2601.00001",
"published": None,
}
],
}
class Client:
def __init__(self):
pass
def get(self, _path, _params):
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "info", "2501.01234"]) == 0
assert output.getvalue().endswith(
"\n## Paper lineage\n\n"
"### Predecessors\n\n"
"- [Earlier \\[Paper\\]](https://paperswithcode.co/paper/2401.00001)"
" (2024-01-02)\n\n"
"### Successors\n\n"
"- [Later Paper](https://paperswithcode.co/paper/2601.00001)\n"
)
def test_paper_info_renders_linked_resources_when_requested(monkeypatch):
calls = []
payload = {
"arxiv_id": "2501.01234",
"title": "A Paper",
"has_official_implementation": True,
"code_repository_count": 2,
"repositories": [
{"url": "https://github.com/example/implementation", "is_official": False},
{"url": "https://github.com/example/official", "is_official": True},
],
"project_pages": [
{"url": "https://example.org/community", "is_official": False},
{"url": "https://example.org/project", "is_official": True},
],
"hf_models": ["https://huggingface.co/example/model"],
"hf_datasets": ["https://huggingface.co/datasets/example/data"],
"hf_spaces": ["https://huggingface.co/spaces/example/demo"],
"x_threads": [
{
"url": "https://x.com/author/status/123",
"author_name": "Ada Author",
"author_handle": "author",
}
],
}
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "info", "2501.01234", "--include-resources"]) == 0
assert calls == [
("papers/2501.01234", {"include_resources": True}),
]
assert output.getvalue() == (
"id: 2501.01234\n"
"title: A Paper\n"
"official_implementation: true\n"
"code_repositories: 2\n"
"\n"
"## Paper lineage\n"
"\n"
"### Predecessors\n"
"\n"
"- None\n"
"\n"
"### Successors\n"
"\n"
"- None\n"
"\n"
"## Repositories\n"
"\n"
"- **Official:** https://github.com/example/official\n"
"- https://github.com/example/implementation\n"
"\n"
"## Project pages\n"
"\n"
"- **Official:** https://example.org/project\n"
"- https://example.org/community\n"
"\n"
"## Hugging Face artifacts\n"
"\n"
"- **Model:** https://huggingface.co/example/model\n"
"- **Dataset:** https://huggingface.co/datasets/example/data\n"
"- **Space:** https://huggingface.co/spaces/example/demo\n"
"\n"
"## X threads\n"
"\n"
"- **Ada Author (@author):** https://x.com/author/status/123\n"
)
def test_paper_info_include_evals_renders_paginated_markdown_table(monkeypatch):
calls = []
paper_payload = {
"id": "9",
"arxiv_id": "2607.24653",
"title": "An Evaluated Paper",
}
evaluation_rows = [
{
"id": "1",
"paper_id": "9",
"model_name": "Agent | One",
"metrics": {"Resolved": 55.5},
"best_metric": "Resolved",
"best_rank": 1,
"task_name": "Coding Agents",
"dataset_name": "SWE-Bench Pro",
"is_open": True,
"result_url": "https://example.test/result-one",
},
{
"id": "2",
"paper_id": "9",
"model_name": "Agent Two",
"harness": "verified",
"metrics": {"Accuracy": 50},
"best_metric": "Accuracy",
"best_rank": None,
"task_name": "Agents",
"dataset_name": "Toolathlon",
"is_open": False,
"source_url": "https://example.test/result-two",
},
]
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
if path.startswith("papers/"):
payload = paper_payload
else:
payload = {
"count": 2,
"results": [evaluation_rows[params["page"] - 1]],
}
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "info", "2607.24653", "--include-evals"]) == 0
assert calls == [
("papers/2607.24653", {"include_resources": False}),
(
"evaluations/",
{
"page": 1,
"page_size": 100,
"paper_id": "9",
"ordering": "-benchmark_popularity",
},
),
(
"evaluations/",
{
"page": 2,
"page_size": 100,
"paper_id": "9",
"ordering": "-benchmark_popularity",
},
),
]
rendered = output.getvalue()
assert "\n## Evaluations\n" in rendered
assert "| Benchmark | Model | Scores | Rank | Task | Open | Source |" in rendered
assert "Agent \\| One" in rendered
assert "Resolved: 55.5" in rendered
assert "Agent Two (verified)" in rendered
assert (
"| — | Agents | no | [source](<https://example.test/result-two>) |" in rendered
)
def test_paper_info_include_evals_adds_structured_json(monkeypatch):
calls = []
class Client:
def __init__(self):
pass
def get(self, path, params):
calls.append((path, params))
payload = (
{"id": "9", "arxiv_id": "2607.24653", "title": "Paper"}
if path.startswith("papers/")
else {"count": 0, "results": []}
)
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "info", "2607.24653", "--include-evals", "--json"]) == 0
data = json.loads(output.getvalue())["data"]
assert data["evaluations"] == {"count": 0, "results": []}
assert calls[-1][0] == "evaluations/"
def test_paper_lineage_renders_linked_markdown_sections(monkeypatch):
payload = {
"paper": {
"reference": "2307.09288",
"title": "Llama 2: Open Foundation and Fine-Tuned Chat Models",
},
"predecessors": [
{
"reference": "2302.13971",
"title": "LLaMA: Open and Efficient Foundation Language Models",
}
],
"successors": [],
}
class Client:
def __init__(self):
pass
def get(self, path):
assert path == "research/papers/2307.09288/lineage"
return Response(json.dumps(payload).encode(), {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "lineage", "list", "2307.09288"]) == 0
assert output.getvalue() == (
"# Paper lineage\n"
"\n"
"## Paper\n"
"\n"
"- [Llama 2: Open Foundation and Fine-Tuned Chat Models]"
"(https://paperswithcode.co/paper/2307.09288) (`2307.09288`)\n"
"\n"
"## Predecessors\n"
"\n"
"- [LLaMA: Open and Efficient Foundation Language Models]"
"(https://paperswithcode.co/paper/2302.13971) (`2302.13971`)\n"
"\n"
"## Successors\n"
"\n"
"- None\n"
)
def test_json_schema_and_errors_use_stable_streams(monkeypatch):
class Client:
def __init__(self):
pass
def get(self, _path, _params):
return Response(b'{"count":0,"results":[]}', {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
errors = io.StringIO()
with redirect_stdout(output), redirect_stderr(errors):
assert main(["search", "none", "--json"]) == 0
assert json.loads(output.getvalue())["schema_version"] == "v1"
assert errors.getvalue() == ""
def test_paper_read_supports_markdown_and_versioned_json(monkeypatch):
paths = []
class Client:
def __init__(self):
pass
def get(self, path, params=None):
paths.append((path, params))
return Response(b"# Paper\n\nBody", {})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "read", "2501.01234"]) == 0
assert output.getvalue() == "# Paper\n\nBody\n"
output = io.StringIO()
with redirect_stdout(output):
assert main(["paper", "read", "2501.01234", "--json"]) == 0
payload = json.loads(output.getvalue())
assert payload == {
"schema_version": "v1",
"data": {"paper": "2501.01234", "markdown": "# Paper\n\nBody"},
}
assert paths == [
("research/papers/2501.01234/read", None),
("research/papers/2501.01234/read", None),
]
def test_transport_is_anonymous_by_default(monkeypatch):
requests = []
class HTTPResponse(io.BytesIO):
headers = {}
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
def fake_urlopen(request, *, timeout):
requests.append(request)
assert timeout == 30
return HTTPResponse(b"# Public paper\n")
monkeypatch.delenv("PWC_API_URL", raising=False)
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
response = Client("https://example.test/api/v1").get(
"research/papers/2501.01234/read"
)
assert response.body == b"# Public paper\n"
assert requests[0].full_url.endswith("/api/v1/research/papers/2501.01234/read")
assert requests[0].get_header("Authorization") is None
def test_transport_timeout_is_configurable_without_changing_the_default(monkeypatch):
observed = []
class HTTPResponse(io.BytesIO):
headers = {}
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
def fake_urlopen(_request, *, timeout):
observed.append(timeout)
return HTTPResponse(b"{}")
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
Client("https://example.test/api/v1", timeout=25).get("health")
Client("https://example.test/api/v1").get("health")
assert observed == [25, 30]
def test_transport_encodes_repeated_query_parameters(monkeypatch):
requests = []
class HTTPResponse(io.BytesIO):
headers = {}
def __enter__(self):
return self
def __exit__(self, *_args):
self.close()
def fake_urlopen(request, *, timeout):
requests.append(request)
return HTTPResponse(b'{"count":0,"results":[]}')
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
Client("https://example.test/api/v1").get(
"papers/", {"author": ["Kaiming He", "@yilundu"]}
)
assert requests[0].full_url.endswith(
"/api/v1/papers/?author=Kaiming+He&author=%40yilundu"
)
def test_transport_reports_read_timeouts_and_socket_errors_as_transport_errors(
monkeypatch,
):
from pwc_cli.transport import TransportError
failures = iter([TimeoutError("The read operation timed out"), ConnectionResetError(104, "reset")])
def fake_urlopen(_request, *, timeout):
raise next(failures)
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
client = Client("https://example.test/api/v1")
with pytest.raises(TransportError, match="timed out"):
client.get("health")
with pytest.raises(TransportError, match="API request failed"):
client.get("health")
def test_closest_results_rank_by_shared_words_not_search_order():
from pwc_cli.cli import _closest_names, _exact_entity_match
candidates = [
{"name": "Spoken Language Identification"},
{"name": "Face Recognition"},
{"name": "Person Re-Identification (Video)"},
{"name": "Unsupervised Person Re-Identification"},
{"name": "Image Classification"},
]
assert _closest_names("person re-identification", candidates) == (
"Person Re-Identification (Video), Unsupervised Person Re-Identification"
)
with pytest.raises(ResponseError, match="closest results: Person Re-Identification"):
_exact_entity_match("person re-identification", candidates, label="Task")
def test_closest_results_omit_unrelated_candidates():
from pwc_cli.cli import _closest_names, _exact_entity_match
# Production chat: "FaceNet" suggested WaveNet, and "person
# re-identification" suggested Spoken Language Identification.
assert _closest_names("FaceNet", [{"name": "WaveNet"}, {"name": "ResNet"}]) == ""
assert (
_closest_names(
"person re-identification",
[{"name": "Spoken Language Identification"}, {"name": "Face Recognition"}],
)
== ""
)
with pytest.raises(ResponseError) as error:
_exact_entity_match("FaceNet", [{"name": "WaveNet"}], label="Method")
assert str(error.value) == "Method not found: FaceNet"
# Near spellings and containment still produce a hint.
assert _closest_names("Transfomer", [{"name": "Transformer"}]) == "Transformer"
assert _closest_names("ViT", [{"name": "ViTDet"}]) == ""
assert _closest_names("Swin", [{"name": "Swin Transformer"}]) == "Swin Transformer"
def test_metric_requests_resolve_through_case_and_common_aliases():
from pwc_cli.cli import _resolve_metric_name, _select_metric_rows
available = {"map": "mAP", "fps": "FPS", "top 1 accuracy": "Top 1 Accuracy"}
assert _resolve_metric_name("AP", available) == "mAP"
assert _resolve_metric_name("box ap", available) == "mAP"
assert _resolve_metric_name("top1", available) == "Top 1 Accuracy"
assert _resolve_metric_name("Top-1 Accuracy", available) == "Top 1 Accuracy"
assert _resolve_metric_name("fps", available) == "FPS"
assert _resolve_metric_name("latency", available) is None
rows = [
{"model_name": "Fast", "metrics": {"mAP": 58, "FPS": 100}},
{"model_name": "Slow", "metrics": {"mAP": 62, "FPS": 20}},
]
args = argparse.Namespace(
require_metrics=["ap"],
minimum_metrics=[("AP", 60)],
maximum_metrics=[],
sort_metric=("ap", "desc"),
pareto=[],
)
assert [row["model_name"] for row in _select_metric_rows(rows, args)] == ["Slow"]
assert args.sort_metric == ("mAP", "desc")
assert args.minimum_metrics == [("mAP", 60)]
unknown = argparse.Namespace(
require_metrics=[],
minimum_metrics=[],
maximum_metrics=[],
sort_metric=("latency", "desc"),
pareto=[],
)
with pytest.raises(UsageError, match="unknown metric\\(s\\): latency; available metrics: FPS, mAP"):
_select_metric_rows(rows, unknown)
def test_metric_requests_resolve_by_containment_and_alias_containment():
from pwc_cli.cli import _resolve_metric_name
# Requests seen from chat agents against production leaderboards.
assert (
_resolve_metric_name("D4RL Normalized Score", {"normalized score": "Normalized Score"})
== "Normalized Score"
)
assert _resolve_metric_name("Overall", {"geneval score": "GenEval Score"}) == "GenEval Score"
assert _resolve_metric_name("Score", {"dpg-bench score": "DPG-Bench Score"}) == "DPG-Bench Score"
assert _resolve_metric_name("Pass@1", {"accuracy": "Accuracy"}) == "Accuracy"
assert _resolve_metric_name("Pass Rate", {"success rate": "Success Rate"}) == "Success Rate"
mvtec = {
"detection auroc": "Detection AUROC",
"segmentation auroc": "Segmentation AUROC",
"detection ap": "Detection AP",
}
assert _resolve_metric_name("image-level AUROC", mvtec) == "Detection AUROC"
assert _resolve_metric_name("pixel-level AUROC", mvtec) == "Segmentation AUROC"
# Ambiguous or unrelated requests still surface the leaderboard's names.
assert _resolve_metric_name("AUROC", mvtec) is None
assert (
_resolve_metric_name(
"F-score",
{"mean f1 (advanced)": "Mean F1 (Advanced)", "mean f1 (intermediate)": "Mean F1 (Intermediate)"},
)
is None
)
assert _resolve_metric_name("latency", {"map": "mAP"}) is None
def test_entity_names_match_ignoring_punctuation_and_trailing_acronyms():
from pwc_cli.cli import _exact_entity_match, _task_match, _without_abbreviation
methods = [
{"name": "Mixture-of-Experts (MoE)", "slug": "moe"},
{"name": "Vision Transformer (ViT)", "slug": "vit"},
]
assert _exact_entity_match("mixture of experts", methods, label="Method")["slug"] == "moe"
assert _exact_entity_match("vision transformer", methods, label="Method")["slug"] == "vit"
with pytest.raises(ResponseError, match="Method not found: experts"):
_exact_entity_match("experts", methods, label="Method")
tasks = [
{"name": "Time Series Forecasting", "slug": "time-series-forecasting"},
{"name": "Person Re-Identification (Video)", "slug": "person-re-identification-video"},
{"name": "Unsupervised Person Re-Identification", "slug": "unsupervised-person-re-identification"},
]
assert _task_match("time-series forecasting", tasks)["slug"] == "time-series-forecasting"
# "(Video)" qualifies the task instead of abbreviating it, so the bare name
# must not silently pick the video variant.
assert _task_match("person re-identification", tasks) is None
# Two candidates that normalize alike stay unresolved.
assert (
_task_match(
"image-classification",
[{"name": "Image Classification"}, {"name": "Image, Classification"}],
)
is None
)
assert (
_without_abbreviation("Physics-Informed Neural Networks (PINNs)")
== "Physics-Informed Neural Networks"
)
assert _without_abbreviation("Aspect-Based Sentiment Analysis (ABSA)") == "Aspect-Based Sentiment Analysis"
assert _without_abbreviation("Person Re-Identification (Video)") is None
assert _without_abbreviation("MVTec AD (Unified Multi-Class)") is None
assert _without_abbreviation("Knowledge Distillation") is None
PUBLIC_METADATA = {
"in_catalog": False,
"catalog_id": None,
"catalog_url": None,
"metadata_source": "huggingface",
"note": "Not indexed on Papers With Code.",
"id": "2610.18869",
"arxiv_id": "2610.18869",
"doi": None,
"title": "A Paper Outside The Catalog",
"authors": ["Ada Lovelace", "Alan Turing"],
"abstract": "An abstract.",
"published": "2026-10-12",
"url": "https://arxiv.org/abs/2610.18869",
"pdf_url": "https://arxiv.org/pdf/2610.18869",
"hf_url": "https://huggingface.co/papers/2610.18869",
"repositories": [],
"project_pages": [],
"available_at": [
{"label": "arXiv", "url": "https://arxiv.org/abs/2610.18869", "kind": "abstract"},
{"label": "Hugging Face Papers", "url": "https://huggingface.co/papers/2610.18869", "kind": "huggingface"},
],
}
def _outside_catalog_client(public):
paths = []
class Client:
def __init__(self):
pass
def get(self, path, params=None):
paths.append(path)
if path.endswith("/public-metadata"):
if path in public:
return Response(json.dumps(public[path]).encode(), {})
raise HTTPStatusError(404, '{"detail":"Paper not found"}')
if path in {"papers/1706.03762", "research/papers/1706.03762/read"}:
if path.endswith("/read"):
return Response(b"# Attention\n", {})
return Response(
json.dumps({"id": "755", "title": "Attention Is All You Need"}).encode(),
{},
)
if path == "papers/search":
return Response(json.dumps({"results": []}).encode(), {})
raise HTTPStatusError(404, '{"detail":"Paper not found"}')
return Client, paths
@pytest.mark.parametrize("command", ["info", "read"])
@pytest.mark.parametrize("json_flag", [[], ["--json"]])
def test_paper_outside_the_catalog_shows_where_it_is_available(
monkeypatch, command, json_flag
):
Client, paths = _outside_catalog_client(
{"papers/2610.18869/public-metadata": PUBLIC_METADATA}
)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output, errors = io.StringIO(), io.StringIO()
with redirect_stdout(output), redirect_stderr(errors):
# The exit status and stdout of a miss are unchanged for scripts.
assert main(["paper", command, "2610.18869", *json_flag]) == 3
assert output.getvalue() == ""
assert errors.getvalue() == (
"Not indexed on Papers With Code — available at:\n"
"- arXiv: https://arxiv.org/abs/2610.18869\n"
"- Hugging Face Papers: https://huggingface.co/papers/2610.18869\n"
"\n"
"title: A Paper Outside The Catalog\n"
"published: 2026-10-12\n"
"authors: Ada Lovelace, Alan Turing\n"
"metadata_source: huggingface\n"
"\n"
"## Abstract\n"
"\n"
"An abstract.\n"
"\n"
'pwc: API request failed (404): {"detail":"Paper not found"}\n'
)
assert paths[-1] == "papers/2610.18869/public-metadata"
def test_paper_title_outside_the_catalog_keeps_its_exit_status(monkeypatch):
path = "papers/Some%20Unindexed%20Title/public-metadata"
Client, _paths = _outside_catalog_client({path: PUBLIC_METADATA})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
errors = io.StringIO()
with redirect_stdout(io.StringIO()), redirect_stderr(errors):
assert main(["paper", "info", "Some Unindexed Title"]) == 4
assert errors.getvalue().startswith("Not indexed on Papers With Code — available at:\n")
assert errors.getvalue().endswith(
"pwc: Paper title not found: Some Unindexed Title\n"
)
def test_paper_unknown_everywhere_keeps_the_plain_error(monkeypatch):
Client, paths = _outside_catalog_client({})
monkeypatch.setattr("pwc_cli.cli.Client", Client)
errors = io.StringIO()
with redirect_stdout(io.StringIO()), redirect_stderr(errors):
assert main(["paper", "info", "2610.99999"]) == 3
assert errors.getvalue() == (
'pwc: API request failed (404): {"detail":"Paper not found"}\n'
)
assert paths == ["papers/2610.99999", "papers/2610.99999/public-metadata"]
@pytest.mark.parametrize(
("command", "expected"),
[("info", "title: Attention Is All You Need\n"), ("read", "# Attention\n")],
)
def test_paper_reference_the_api_finds_in_the_catalog_is_retried(
monkeypatch, command, expected
):
# The CLI resolves only IDs and titles; the API also maps arXiv URLs.
url = "https://arxiv.org/abs/1706.03762v7"
record = {**PUBLIC_METADATA, "in_catalog": True, "catalog_id": "755",
"arxiv_id": "1706.03762"}
Client, _paths = _outside_catalog_client(
{f"papers/{url}/public-metadata": record}
)
monkeypatch.setattr("pwc_cli.cli.Client", Client)
output, errors = io.StringIO(), io.StringIO()
with redirect_stdout(output), redirect_stderr(errors):
assert main(["paper", command, url]) == 0
assert expected in output.getvalue()
assert errors.getvalue() == ""
def test_embedded_paper_queries_do_not_call_the_public_metadata_api():
from pwc_cli.queries import query
Client, paths = _outside_catalog_client(
{"papers/2610.18869/public-metadata": PUBLIC_METADATA}
)
with pytest.raises(HTTPStatusError):
query(("paper", "info"), {"paper": "2610.18869"}, Client())
assert not any(path.endswith("/public-metadata") for path in paths)