Buckets:

download
raw
35.8 kB
"""Comprehensive tests for Python-specific language features.""" from chunker import chunk_file class TestPythonAsyncFunctions: """Test async function detection and handling.""" @staticmethod def test_simple_async_function(tmp_path): """Test basic async function detection.""" src = tmp_path / "async.py" src.write_text("\nasync def fetch_data():\n return await some_api_call()\n") chunks = chunk_file(src, "python") assert len(chunks) == 1 assert chunks[0].node_type == "function_definition" assert "async def fetch_data" in chunks[0].content assert chunks[0].start_line == 2 assert chunks[0].end_line == 3 @staticmethod def test_async_function_with_docstring(tmp_path): """Test async function with docstring.""" src = tmp_path / "async_doc.py" src.write_text( """ async def process_items(items): '''Process items asynchronously. Args: items: List of items to process Returns: Processed results ''' results = [] for item in items: result = await process_single(item) results.append(result) return results """, ) chunks = chunk_file(src, "python") assert len(chunks) == 1 assert chunks[0].node_type == "function_definition" assert "Process items asynchronously" in chunks[0].content @staticmethod def test_nested_async_functions(tmp_path): """Test nested async function definitions.""" src = tmp_path / "nested_async.py" src.write_text( """ async def outer_async(): async def inner_async(): return await something() return await inner_async() """, ) chunks = chunk_file(src, "python") assert len(chunks) == 2 assert all(c.node_type == "function_definition" for c in chunks) next( c for c in chunks if "outer_async" in c.content and "inner_async" in c.content ) inner = next( c for c in chunks if "inner_async" in c.content and "outer_async" not in c.content ) assert inner.parent_context == "function_definition" class TestPythonDecorators: """Test decorator handling.""" @staticmethod def test_simple_decorator(tmp_path): """Test function with single decorator.""" src = tmp_path / "decorated.py" src.write_text( """ @lru_cache def fibonacci(n): if n < 2: return n return fibonacci(n-1) + fibonacci(n-2) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 2 decorated = next(c for c in chunks if c.node_type == "decorated_definition") assert "@lru_cache" in decorated.content assert decorated.start_line == 2 func = next(c for c in chunks if c.node_type == "function_definition") assert func.parent_context == "decorated_definition" @staticmethod def test_multiple_decorators(tmp_path): """Test function with multiple decorators.""" src = tmp_path / "multi_decorated.py" src.write_text( """ @app.route('/api/users') @require_auth @validate_params def get_users(request): return {'users': User.query.all()} """, ) chunks = chunk_file(src, "python") assert len(chunks) == 2 decorated = next(c for c in chunks if c.node_type == "decorated_definition") assert "@app.route" in decorated.content assert "@require_auth" in decorated.content assert "@validate_params" in decorated.content func = next(c for c in chunks if c.node_type == "function_definition") assert func.parent_context == "decorated_definition" @staticmethod def test_decorated_class(tmp_path): """Test decorated class definition.""" src = tmp_path / "decorated_class.py" src.write_text( """ @dataclass @frozen class Point: x: float y: float def distance(self, other): return ((self.x - other.x)**2 + (self.y - other.y)**2)**0.5 """, ) chunks = chunk_file(src, "python") assert len(chunks) == 3 decorated = next(c for c in chunks if c.node_type == "decorated_definition") assert "@dataclass" in decorated.content assert "@frozen" in decorated.content class_chunk = next(c for c in chunks if c.node_type == "class_definition") assert class_chunk.parent_context == "decorated_definition" method_chunk = next( c for c in chunks if c.node_type == "function_definition" and "def distance" in c.content ) assert method_chunk.parent_context == "class_definition" @staticmethod def test_decorator_with_arguments(tmp_path): """Test decorators with arguments.""" src = tmp_path / "decorator_args.py" src.write_text( """ @app.route('/user/<int:id>', methods=['GET', 'POST']) @cache.memoize(timeout=300) @retry(max_attempts=3, backoff=2.0) async def get_user(id: int): return await db.fetch_user(id) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 2 decorated = next(c for c in chunks if c.node_type == "decorated_definition") assert "methods=['GET', 'POST']" in decorated.content assert "timeout=300" in decorated.content assert "max_attempts=3" in decorated.content func = next(c for c in chunks if c.node_type == "function_definition") assert func.parent_context == "decorated_definition" class TestPythonNestedClasses: """Test nested class definitions.""" @staticmethod def test_simple_nested_class(tmp_path): """Test basic nested class.""" src = tmp_path / "nested_class.py" src.write_text( """ class Outer: class Inner: def inner_method(self): pass def outer_method(self): return self.Inner() """, ) chunks = chunk_file(src, "python") assert len(chunks) == 4 next( c for c in chunks if c.node_type == "class_definition" and c.content.strip().startswith("class Outer:") ) inner = next( c for c in chunks if c.node_type == "class_definition" and c.content.strip().startswith("class Inner:") ) inner_method = next( c for c in chunks if c.node_type == "function_definition" and c.content.strip().startswith("def inner_method") ) outer_method = next( c for c in chunks if c.node_type == "function_definition" and c.content.strip().startswith("def outer_method") ) assert inner.parent_context == "class_definition" assert inner_method.parent_context == "class_definition" assert outer_method.parent_context == "class_definition" @staticmethod def test_deeply_nested_classes(tmp_path): """Test deeply nested class hierarchies.""" src = tmp_path / "deep_nested.py" src.write_text( """ class Level1: class Level2: class Level3: class Level4: def deep_method(self): return "deep" def level3_method(self): return self.Level4() def level2_method(self): return self.Level3() def level1_method(self): return self.Level2() """, ) chunks = chunk_file(src, "python") classes = [c for c in chunks if c.node_type == "class_definition"] methods = [c for c in chunks if c.node_type == "function_definition"] assert len(classes) == 4 assert len(methods) == 4 level4 = None for c in classes: if c.content.strip().startswith("class Level4:"): level4 = c break assert level4 is not None, "Could not find Level4 class chunk" assert level4.parent_context == "class_definition" class TestPythonLambdaExpressions: """Test lambda expression handling.""" @staticmethod def test_simple_lambda(tmp_path): """Test standalone lambda expressions.""" src = tmp_path / "lambda.py" src.write_text("\nsquare = lambda x: x ** 2\nadd = lambda x, y: x + y\n") chunks = chunk_file(src, "python") assert len(chunks) == 4 lambda_chunks = [c for c in chunks if c.node_type == "lambda"] assert len(lambda_chunks) == 4 @staticmethod def test_lambda_in_function(tmp_path): """Test lambdas inside functions.""" src = tmp_path / "lambda_func.py" src.write_text( """ def process_numbers(numbers): squared = map(lambda x: x ** 2, numbers) filtered = filter(lambda x: x > 10, squared) return list(filtered) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 5 func_chunk = next(c for c in chunks if c.node_type == "function_definition") assert "lambda x: x ** 2" in func_chunk.content lambda_chunks = [ c for c in chunks if c.node_type == "lambda" and c.parent_context == "function_definition" ] assert len(lambda_chunks) == 2 @staticmethod def test_complex_lambda(tmp_path): """Test complex lambda with conditional logic.""" src = tmp_path / "complex_lambda.py" src.write_text( """ def sort_data(data): return sorted( data, key=lambda item: ( item['priority'] if 'priority' in item else 999, item['name'].lower(), -item['score'] ) ) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 3 func_chunk = next(c for c in chunks if c.node_type == "function_definition") assert "lambda item:" in func_chunk.content lambda_chunk = next( c for c in chunks if c.node_type == "lambda" and "lambda item:" in c.content ) assert lambda_chunk.parent_context == "function_definition" class TestPythonComprehensions: """Test list, dict, set comprehensions and generator expressions.""" @staticmethod def test_list_comprehension(tmp_path): """Test list comprehension in functions.""" src = tmp_path / "list_comp.py" src.write_text( """ def process_data(items): # Simple list comprehension squares = [x ** 2 for x in items] # Conditional comprehension evens = [x for x in items if x % 2 == 0] # Nested comprehension matrix = [[i * j for j in range(5)] for i in range(5)] return squares, evens, matrix """, ) chunks = chunk_file(src, "python") assert len(chunks) == 1 assert "[x ** 2 for x in items]" in chunks[0].content assert "[x for x in items if x % 2 == 0]" in chunks[0].content @staticmethod def test_dict_comprehension(tmp_path): """Test dictionary comprehension.""" src = tmp_path / "dict_comp.py" src.write_text( """ def create_mappings(keys, values): # Basic dict comprehension mapping = {k: v for k, v in zip(keys, values)} # Conditional dict comprehension filtered = {k: v for k, v in mapping.items() if v > 0} # Complex transformation transformed = { k.upper(): v ** 2 for k, v in mapping.items() if isinstance(k, str) and isinstance(v, (int, float)) } return transformed """, ) chunks = chunk_file(src, "python") assert len(chunks) == 1 assert "{k: v for k, v in zip(keys, values)}" in chunks[0].content @staticmethod def test_generator_expression(tmp_path): """Test generator expressions.""" src = tmp_path / "gen_exp.py" src.write_text( """ def sum_of_squares(n): # Generator expression return sum(x ** 2 for x in range(n)) def lazy_processing(data): # Multiple generator expressions processed = ( item.strip().lower() for item in data if item and not item.isspace() ) return list(processed) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 2 assert any("x ** 2 for x in range(n)" in c.content for c in chunks) class TestPythonTypeAnnotations: """Test type annotation handling.""" @staticmethod def test_function_annotations(tmp_path): """Test function parameter and return type annotations.""" src = tmp_path / "type_annotations.py" src.write_text( """ def add(a: int, b: int) -> int: return a + b def process_list(items: list[str]) -> dict[str, int]: return {item: len(item) for item in items} def complex_types( data: dict[str, list[tuple[int, str]]], callback: Callable[[str], bool] | None = None ) -> Iterator[tuple[str, int]]: for key, values in data.items(): if callback is None or callback(key): for idx, val in values: yield (val, idx) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 3 add_chunk = next(c for c in chunks if "def add" in c.content) assert "a: int" in add_chunk.content assert "-> int:" in add_chunk.content complex_chunk = next(c for c in chunks if "complex_types" in c.content) assert "dict[str, list[tuple[int, str]]]" in complex_chunk.content assert "Callable[[str], bool] | None" in complex_chunk.content @staticmethod def test_class_annotations(tmp_path): """Test class variable and method annotations.""" src = tmp_path / "class_annotations.py" src.write_text( """ class User: name: str age: int emails: list[str] metadata: dict[str, Any] = {} def __init__(self, name: str, age: int) -> None: self.name = name self.age = age self.emails = [] def add_email(self, email: str) -> None: self.emails.append(email) def get_info(self) -> dict[str, Union[str, int, list[str]]]: return { 'name': self.name, 'age': self.age, 'emails': self.emails } """, ) chunks = chunk_file(src, "python") class_chunk = next(c for c in chunks if c.node_type == "class_definition") assert "name: str" in class_chunk.content assert "emails: list[str]" in class_chunk.content init_chunk = next(c for c in chunks if "__init__" in c.content) assert "name: str, age: int" in init_chunk.content assert "-> None:" in init_chunk.content @staticmethod def test_generic_annotations(tmp_path): """Test generic type annotations.""" src = tmp_path / "generics.py" src.write_text( """ from typing import TypeVar, Generic, Protocol T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') class Container(Generic[T]): def __init__(self, value: T) -> None: self._value = value def get(self) -> T: return self._value def transform(self, func: Callable[[T], K]) -> 'Container[K]': return Container(func(self._value)) class Comparable(Protocol): def __lt__(self, other: 'Comparable') -> bool: ... def sort_items(items: list[Comparable]) -> list[Comparable]: return sorted(items) """, ) chunks = chunk_file(src, "python") container_chunk = next(c for c in chunks if "class Container" in c.content) assert "Generic[T]" in container_chunk.content transform_chunk = next(c for c in chunks if "def transform" in c.content) assert "Callable[[T], K]" in transform_chunk.content assert "'Container[K]'" in transform_chunk.content class TestPythonDocstrings: """Test docstring extraction and handling.""" @staticmethod def test_function_docstrings(tmp_path): """Test various docstring formats in functions.""" src = tmp_path / "docstrings.py" src.write_text( """ def single_line_doc(): ""\"This is a single line docstring.""\" pass def multi_line_doc(): ""\" This is a multi-line docstring. It has multiple paragraphs and provides detailed documentation about the function. ""\" pass def google_style_doc(param1: str, param2: int) -> bool: ""\"Summary line. Extended description of function. Args: param1: Description of param1 param2: Description of param2 Returns: Description of return value Raises: ValueError: If param2 is negative ""\" if param2 < 0: raise ValueError("param2 must be non-negative") return len(param1) > param2 def numpy_style_doc(x, y): ""\" Calculate something. Parameters ---------- x : array_like Input array y : array_like Another input array Returns ------- result : ndarray The calculated result ""\" return x + y """, ) chunks = chunk_file(src, "python") assert len(chunks) == 4 single_chunk = next(c for c in chunks if "single_line_doc" in c.content) assert "This is a single line docstring." in single_chunk.content google_chunk = next(c for c in chunks if "google_style_doc" in c.content) assert "Args:" in google_chunk.content assert "Returns:" in google_chunk.content assert "Raises:" in google_chunk.content @staticmethod def test_class_docstrings(tmp_path): """Test class and method docstrings.""" src = tmp_path / "class_docs.py" src.write_text( """ class DocumentedClass: ""\" A well-documented class. This class demonstrates proper documentation with class-level and method-level docstrings. Attributes: name: The name of the instance value: The current value ""\" def __init__(self, name: str, value: float): ""\" Initialize the instance. Args: name: Instance name value: Initial value ""\" self.name = name self.value = value def calculate(self, factor: float) -> float: ""\"Calculate adjusted value. Multiplies the current value by the given factor. Args: factor: Multiplication factor Returns: The adjusted value Example: >>> obj = DocumentedClass("test", 10.0) >>> obj.calculate(1.5) 15.0 ""\" return self.value * factor """, ) chunks = chunk_file(src, "python") class_chunk = next(c for c in chunks if c.node_type == "class_definition") assert "A well-documented class." in class_chunk.content assert "Attributes:" in class_chunk.content calc_chunk = next( c for c in chunks if "calculate" in c.content and "def" in c.content ) assert "Example:" in calc_chunk.content assert ">>> obj.calculate(1.5)" in calc_chunk.content @staticmethod def test_raw_docstrings(tmp_path): """Test raw docstrings with special characters.""" src = tmp_path / "raw_docs.py" src.write_text( """ def regex_function(): r""\" Process text with regex patterns. This function uses patterns like: - \\d+ for digits - \\w+ for words - \\s* for optional whitespace The backslashes are preserved in raw strings. ""\" pass def path_function(): r""\" Handle Windows paths. Example paths: C:\\Users\\Name\\Documents \\\\server\\share\\folder ""\" pass """, ) chunks = chunk_file(src, "python") regex_chunk = next(c for c in chunks if "regex_function" in c.content) assert "\\d+ for digits" in regex_chunk.content assert "\\w+ for words" in regex_chunk.content class TestPythonEdgeCases: """Test edge cases and complex Python constructs.""" @staticmethod def test_walrus_operator(tmp_path): """Test assignment expressions (walrus operator).""" src = tmp_path / "walrus.py" src.write_text( """ def process_with_walrus(items): # Walrus in if statement if (n := len(items)) > 10: print(f"Processing {n} items") # Walrus in while loop while (item := get_next_item()) is not None: process_item(item) # Walrus in comprehension filtered = [y for x in items if (y := transform(x)) is not None] return filtered """, ) chunks = chunk_file(src, "python") assert len(chunks) == 1 assert ":=" in chunks[0].content assert "n := len(items)" in chunks[0].content @staticmethod def test_match_statement(tmp_path): """Test pattern matching (Python 3.10+).""" src = tmp_path / "pattern_match.py" src.write_text( """ def handle_command(command): match command.split(): case ["quit"]: return "Goodbye" case ["load", filename]: return f"Loading {filename}" case ["save", filename, *rest]: options = rest or [] return f"Saving {filename} with {options}" case ["move", x, y] if x.isdigit() and y.isdigit(): return f"Moving to ({x}, {y})" case _: return "Unknown command\" """, ) chunks = chunk_file(src, "python") assert len(chunks) == 1 assert "match command.split():" in chunks[0].content assert 'case ["quit"]:' in chunks[0].content assert "case _:" in chunks[0].content @staticmethod def test_async_context_managers(tmp_path): """Test async with statements.""" src = tmp_path / "async_context.py" src.write_text( """ async def process_async_resource(): async with get_connection() as conn: async with conn.transaction(): result = await conn.execute("SELECT * FROM users") async for row in result: yield process_row(row) async with AsyncExitStack() as stack: files = [ await stack.enter_async_context(aiofiles.Path(f).open("r", )) for f in filenames ] return await process_files(files) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 1 assert "async with" in chunks[0].content assert "async for" in chunks[0].content @staticmethod def test_complex_decorators(tmp_path): """Test complex decorator patterns.""" src = tmp_path / "complex_decorators.py" src.write_text( """ @decorator_factory(param="value") @another_decorator @property def decorated_property(self): return self._value @contextmanager def my_context(): setup() try: yield resource finally: cleanup() @overload def process(x: int) -> str: ... @overload def process(x: str) -> int: ... def process(x): if isinstance(x, int): return str(x) return int(x) """, ) chunks = chunk_file(src, "python") property_chunk = next(c for c in chunks if "decorated_property" in c.content) assert "@property" in property_chunk.content assert "@decorator_factory" in property_chunk.content @staticmethod def test_metaclass_usage(tmp_path): """Test metaclass definitions and usage.""" src = tmp_path / "metaclass.py" src.write_text( """ class SingletonMeta(type): _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super().__call__(*args, **kwargs) return cls._instances[cls] class Database(metaclass=SingletonMeta): def __init__(self, connection_string): self.connection = connection_string def query(self, sql): return f"Executing: {sql}" class AbstractBase(metaclass=ABCMeta): @abstractmethod def must_implement(self): pass """, ) chunks = chunk_file(src, "python") meta_chunk = next( c for c in chunks if "SingletonMeta" in c.content and "class" in c.content ) assert "type" in meta_chunk.content db_chunk = next(c for c in chunks if "class Database" in c.content) assert "metaclass=SingletonMeta" in db_chunk.content @staticmethod def test_deeply_nested_structures(tmp_path): """Test deeply nested functions and classes.""" src = tmp_path / "deep_nesting.py" src.write_text( """ def outer_function(): def level1(): def level2(): def level3(): class InnerClass: def inner_method(self): lambda x: x * 2 def deepest(): return "very deep" return deepest() return InnerClass() return level3() return level2() return level1() """, ) chunks = chunk_file(src, "python") function_chunks = [c for c in chunks if c.node_type == "function_definition"] class_chunks = [c for c in chunks if c.node_type == "class_definition"] assert len(function_chunks) >= 5 assert len(class_chunks) == 1 deepest = None for c in chunks: if c.node_type == "function_definition" and c.content.strip().startswith( "def deepest", ): deepest = c break assert deepest is not None, "Could not find deepest function chunk" assert deepest.parent_context == "function_definition" class TestPythonModernFeatures: """Test modern Python features (3.8+).""" @staticmethod def test_positional_only_params(tmp_path): """Test positional-only parameters.""" src = tmp_path / "pos_only.py" src.write_text( """ def positional_only(a, b, /, c, d): return a + b + c + d def mixed_params(pos_only, /, standard, *, kw_only): return (pos_only, standard, kw_only) def all_types(a, /, b, c, *args, d, e, **kwargs): return { 'positional_only': a, 'standard': (b, c), 'varargs': args, 'keyword_only': (d, e), 'kwargs': kwargs } """, ) chunks = chunk_file(src, "python") assert len(chunks) == 3 pos_chunk = next( c for c in chunks if "positional_only" in c.content and "def" in c.content ) assert "a, b, /" in pos_chunk.content @staticmethod def test_type_union_operator(tmp_path): """Test type union operator (|) from Python 3.10.""" src = tmp_path / "union_types.py" src.write_text( """ def process_value(val: int | float | str) -> str | None: if isinstance(val, (int, float)): return str(val) elif isinstance(val, str): return val.upper() return None def handle_optional(data: dict[str, Any] | None = None) -> list[str] | None: if data is None: return None return list(data.keys()) class FlexibleContainer: value: int | str | list[int] | None def set_value(self, v: int | str | list[int]) -> None: self.value = v """, ) chunks = chunk_file(src, "python") process_chunk = next(c for c in chunks if "process_value" in c.content) assert "int | float | str" in process_chunk.content assert "str | None" in process_chunk.content @staticmethod def test_dataclass_advanced(tmp_path): """Test advanced dataclass features.""" src = tmp_path / "dataclass_advanced.py" src.write_text( """ from dataclasses import dataclass, field, InitVar from typing import ClassVar @dataclass(frozen=True, slots=True) class ImmutablePoint: x: float y: float _id: ClassVar[int] = 0 def distance_from_origin(self) -> float: return (self.x ** 2 + self.y ** 2) ** 0.5 @dataclass class ConfigurableClass: name: str values: list[int] = field(default_factory=list) metadata: dict[str, Any] = field(default_factory=dict) _init_value: InitVar[int] = 0 def __post_init__(self, _init_value: int): if _init_value > 0: self.values = list(range(_init_value)) def add_value(self, val: int) -> None: self.values.append(val) """, ) chunks = chunk_file(src, "python") immutable_chunk = next(c for c in chunks if "ImmutablePoint" in c.content) assert "@dataclass(frozen=True, slots=True)" in immutable_chunk.content assert "ClassVar[int]" in immutable_chunk.content config_chunk = next(c for c in chunks if "ConfigurableClass" in c.content) assert "field(default_factory=list)" in config_chunk.content assert "InitVar[int]" in config_chunk.content @staticmethod def test_exception_groups(tmp_path): """Test exception groups (Python 3.11+).""" src = tmp_path / "exception_groups.py" src.write_text( """ def process_multiple_errors(): errors = [] try: # Simulate multiple operations for operation in operations: try: operation() except (IOError, IndexError, KeyError) as e: errors.append(e) if errors: raise ExceptionGroup("Multiple errors occurred", errors) except* ValueError as eg: for e in eg.exceptions: log_value_error(e) except* TypeError as eg: for e in eg.exceptions: log_type_error(e) except* Exception as eg: for e in eg.exceptions: log_generic_error(e) """, ) chunks = chunk_file(src, "python") assert len(chunks) == 1 assert "ExceptionGroup" in chunks[0].content assert "except*" in chunks[0].content class TestPythonSpecialMethods: """Test special methods and protocols.""" @staticmethod def test_dunder_methods(tmp_path): """Test various dunder methods.""" src = tmp_path / "dunder_methods.py" src.write_text( """ class Vector: def __init__(self, x, y): self.x = x self.y = y def __repr__(self): return f"Vector({self.x}, {self.y})" def __str__(self): return f"<{self.x}, {self.y}>" def __add__(self, other): return Vector(self.x + other.x, self.y + other.y) def __mul__(self, scalar): return Vector(self.x * scalar, self.y * scalar) def __eq__(self, other): return self.x == other.x and self.y == other.y def __len__(self): return 2 def __getitem__(self, index): return (self.x, self.y)[index] def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass """, ) chunks = chunk_file(src, "python") assert ( len( [c for c in chunks if c.node_type == "function_definition"], ) >= 10 ) dunder_names = [ "__init__", "__repr__", "__str__", "__add__", "__mul__", "__eq__", "__len__", "__getitem__", "__enter__", "__exit__", ] for name in dunder_names: assert any(name in c.content for c in chunks) @staticmethod def test_async_iteration_protocol(tmp_path): """Test async iteration protocol methods.""" src = tmp_path / "async_iter.py" src.write_text( """ class AsyncRange: def __init__(self, start, stop): self.start = start self.stop = stop def __aiter__(self): self.current = self.start return self async def __anext__(self): if self.current < self.stop: await asyncio.sleep(0.1) value = self.current self.current += 1 return value raise StopAsyncIteration async def use_async_range(): async for i in AsyncRange(0, 5): print(i) """, ) chunks = chunk_file(src, "python") aiter_chunk = next(c for c in chunks if "__aiter__" in c.content) anext_chunk = next(c for c in chunks if "__anext__" in c.content) assert aiter_chunk is not None assert anext_chunk is not None assert "async def __anext__" in anext_chunk.content class TestPythonImportStatements: """Test various import patterns (though typically not chunked).""" @staticmethod def test_import_patterns(tmp_path): """Test that imports are handled correctly within chunks.""" src = tmp_path / "imports.py" src.write_text( """ # Standard library imports import os import sys from pathlib import Path from typing import List, Dict, Optional # Third party imports import numpy as np import pandas as pd from sklearn.model_selection import train_test_split # Local imports from .utils import helper_function from ..parent_module import ParentClass def process_data(file_path: Path) -> pd.DataFrame: '''Process data from file.''' df = pd.read_csv(file_path) return df class DataProcessor: '''Class using imported modules.''' def __init__(self): self.data_dir = Path.home() / 'data' self.arrays = [] def load_array(self, name: str) -> np.ndarray: return np.load(self.data_dir / f"{name}.npy") """, ) chunks = chunk_file(src, "python") import_chunks = [c for c in chunks if c.node_type == "import_statement"] assert len(import_chunks) == 0 func_chunk = next(c for c in chunks if "process_data" in c.content) class_chunk = next( c for c in chunks if "DataProcessor" in c.content and "class" in c.content ) assert func_chunk is not None assert class_chunk is not None def test_empty_file(tmp_path): """Test handling of empty Python file.""" src = tmp_path / "empty.py" src.write_text("") chunks = chunk_file(src, "python") assert len(chunks) == 0 def test_syntax_error_handling(tmp_path): """Test handling of files with syntax errors.""" src = tmp_path / "syntax_error.py" src.write_text( """ def broken_function( # Missing closing parenthesis pass class IncompleteClass # Missing colon def method(self): return """, ) chunks = chunk_file(src, "python") assert len(chunks) >= 0 def test_unicode_and_encoding(tmp_path): """Test handling of Unicode in Python code.""" src = tmp_path / "unicode.py" src.write_text( """ def process_unicode(): '''Process Unicode strings. 处理Unicode字符串。''' emoji = "🐍 Python rocks! 🚀" chinese = "你好世界" arabic = "مرحبا بالعالم" return { 'emoji': emoji, 'chinese': chinese, 'arabic': arabic } class 多语言类: '''A class with non-ASCII name.''' def 获取信息(self): return "信息\" """, ) chunks = chunk_file(src, "python") assert any("🐍 Python rocks! 🚀" in c.content for c in chunks) assert any("你好世界" in c.content for c in chunks) assert any("多语言类" in c.content for c in chunks)
"""Comprehensive tests for Python-specific language features."""
from chunker import chunk_file
class TestPythonAsyncFunctions:
"""Test async function detection and handling."""
@staticmethod
def test_simple_async_function(tmp_path):
"""Test basic async function detection."""
src = tmp_path / "async.py"
src.write_text("\nasync def fetch_data():\n return await some_api_call()\n")
chunks = chunk_file(src, "python")
assert len(chunks) == 1
assert chunks[0].node_type == "function_definition"
assert "async def fetch_data" in chunks[0].content
assert chunks[0].start_line == 2
assert chunks[0].end_line == 3
@staticmethod
def test_async_function_with_docstring(tmp_path):
"""Test async function with docstring."""
src = tmp_path / "async_doc.py"
src.write_text(
"""
async def process_items(items):
'''Process items asynchronously.
Args:
items: List of items to process
Returns:
Processed results
'''
results = []
for item in items:
result = await process_single(item)
results.append(result)
return results
""",
)
chunks = chunk_file(src, "python")
assert len(chunks) == 1
assert chunks[0].node_type == "function_definition"
assert "Process items asynchronously" in chunks[0].content
@staticmethod
def test_nested_async_functions(tmp_path):
"""Test nested async function definitions."""
src = tmp_path / "nested_async.py"
src.write_text(
"""
async def outer_async():
async def inner_async():
return await something()
return await inner_async()
""",
)
chunks = chunk_file(src, "python")
assert len(chunks) == 2
assert all(c.node_type == "function_definition" for c in chunks)
next(
c
for c in chunks
if "outer_async" in c.content and "inner_async" in c.content
)
inner = next(
c
for c in chunks
if "inner_async" in c.content and "outer_async" not in c.content
)
assert inner.parent_context == "function_definition"
class TestPythonDecorators:
"""Test decorator handling."""
@staticmethod
def test_simple_decorator(tmp_path):
"""Test function with single decorator."""
src = tmp_path / "decorated.py"
src.write_text(
"""
@lru_cache
def fibonacci(n):
if n < 2:
return n
return fibonacci(n-1) + fibonacci(n-2)
""",
)
chunks = chunk_file(src, "python")
assert len(chunks) == 2
decorated = next(c for c in chunks if c.node_type == "decorated_definition")
assert "@lru_cache" in decorated.content
assert decorated.start_line == 2
func = next(c for c in chunks if c.node_type == "function_definition")
assert func.parent_context == "decorated_definition"

Xet Storage Details

Size:
35.8 kB
·
Xet hash:
15c722dc5439aeef4de257b53b03177bb48937b1ac36b776a0263eecd9c84d15

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.