Buckets:

download
raw
5.75 kB
"""Tests for JSONL export functionality."""
import gzip
import json
from io import StringIO
from pathlib import Path
import pytest
from chunker.export import JSONLExporter, SchemaType
from chunker.types import CodeChunk
@pytest.fixture
def sample_chunks():
"""Create sample chunks with relationships."""
chunks = [
CodeChunk(
language="python",
file_path="test.py",
node_type="class_definition",
start_line=1,
end_line=10,
byte_start=0,
byte_end=200,
parent_context="",
content="class TestClass:\n pass",
chunk_id="chunk1",
parent_chunk_id=None,
metadata={"kind": "class", "semantic_text": "class TestClass"},
),
CodeChunk(
language="python",
file_path="test.py",
node_type="method_definition",
start_line=3,
end_line=5,
byte_start=50,
byte_end=100,
parent_context="class_definition",
content="def method(self):\n pass",
chunk_id="chunk2",
parent_chunk_id="chunk1",
metadata={"kind": "method", "semantic_text": "method TestClass.method"},
),
]
return chunks
def test_jsonl_export_flat_schema(sample_chunks):
"""Test JSONL export with flat schema."""
exporter = JSONLExporter(SchemaType.FLAT)
output = StringIO()
exporter.export(sample_chunks, output)
lines = output.getvalue().strip().split("\n")
assert len(lines) == 2
# Parse each line
chunk1 = json.loads(lines[0])
chunk2 = json.loads(lines[1])
assert chunk1["chunk_id"] == "chunk1"
assert chunk2["parent_chunk_id"] == "chunk1"
def test_jsonl_export_full_schema(sample_chunks):
"""Test JSONL export with full schema."""
exporter = JSONLExporter(SchemaType.FULL)
output = StringIO()
exporter.export(sample_chunks, output)
lines = output.getvalue().strip().split("\n")
assert len(lines) == 4 # metadata + 2 chunks + relationships
# Parse each line
metadata = json.loads(lines[0])
chunk1 = json.loads(lines[1])
json.loads(lines[2])
relationships = json.loads(lines[3])
assert metadata["type"] == "metadata"
assert metadata["data"]["total_chunks"] == 2
assert chunk1["type"] == "chunk"
assert chunk1["data"]["chunk_id"] == "chunk1"
assert relationships["type"] == "relationships"
assert len(relationships["data"]["parent_child"]) == 1
def test_jsonl_export_to_file(sample_chunks, tmp_path):
"""Test JSONL export to file."""
exporter = JSONLExporter(SchemaType.FLAT)
output_path = tmp_path / "output.jsonl"
exporter.export(sample_chunks, output_path)
assert output_path.exists()
with Path(output_path).open(
"r",
encoding="utf-8",
) as f:
lines = f.readlines()
assert len(lines) == 2
chunk1 = json.loads(lines[0])
assert chunk1["chunk_id"] == "chunk1"
def test_jsonl_export_compressed(sample_chunks, tmp_path):
"""Test JSONL export with compression."""
exporter = JSONLExporter(SchemaType.FLAT)
output_path = tmp_path / "output.jsonl"
exporter.export(sample_chunks, output_path, compress=True)
compressed_path = Path(f"{output_path}.gz")
assert compressed_path.exists()
with gzip.open(compressed_path, "rt") as f:
lines = f.readlines()
assert len(lines) == 2
chunk1 = json.loads(lines[0])
assert chunk1["chunk_id"] == "chunk1"
def test_jsonl_stream_export(sample_chunks, tmp_path):
"""Test JSONL streaming export."""
exporter = JSONLExporter(SchemaType.FLAT)
output_path = tmp_path / "output.jsonl"
# Generator function
def chunk_generator():
yield from sample_chunks
exporter.stream_export(chunk_generator(), output_path)
assert output_path.exists()
with Path(output_path).open(
"r",
encoding="utf-8",
) as f:
lines = f.readlines()
assert len(lines) == 2
chunk1 = json.loads(lines[0])
assert chunk1["chunk_id"] == "chunk1"
def test_jsonl_stream_export_compressed(sample_chunks, tmp_path):
"""Test JSONL streaming export with compression."""
exporter = JSONLExporter(SchemaType.FLAT)
output_path = tmp_path / "output.jsonl"
# Generator function
def chunk_generator():
yield from sample_chunks
exporter.stream_export(chunk_generator(), output_path, compress=True)
compressed_path = Path(f"{output_path}.gz")
assert compressed_path.exists()
with gzip.open(compressed_path, "rt") as f:
lines = f.readlines()
assert len(lines) == 2
chunk1 = json.loads(lines[0])
assert chunk1["chunk_id"] == "chunk1"
def test_jsonl_minimal_schema(sample_chunks):
"""Test JSONL export with minimal schema."""
exporter = JSONLExporter(SchemaType.MINIMAL)
output = StringIO()
exporter.export(sample_chunks, output)
lines = output.getvalue().strip().split("\n")
assert len(lines) == 2
chunk1 = json.loads(lines[0])
assert "id" in chunk1
assert "type" in chunk1
assert "content" in chunk1
assert "chunk_id" not in chunk1 # Minimal schema
def test_jsonl_export_preserves_metadata(sample_chunks):
"""Test JSONL export retains chunk metadata when present."""
exporter = JSONLExporter(SchemaType.FLAT)
output = StringIO()
exporter.export(sample_chunks, output)
lines = output.getvalue().strip().split("\n")
chunk1 = json.loads(lines[0])
chunk2 = json.loads(lines[1])
assert chunk1["metadata"]["kind"] == "class"
assert chunk2["metadata"]["semantic_text"] == "method TestClass.method"

Xet Storage Details

Size:
5.75 kB
·
Xet hash:
db743d7ddeb167b009b8ea2ce06fbb5ab3d38f01023e01c0240c521650544551

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.