Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Consiliency_treesitter-chunker /examples /custom_rules_demo.py
| """Demo of custom chunking rules.""" | |
| from chunker.parser import get_parser | |
| from chunker.rules import ( | |
| DefaultRuleEngine, | |
| DocumentationBlockRule, | |
| HeaderCommentRule, | |
| MetadataRule, | |
| RegionMarkerRule, | |
| SeparatorLineRule, | |
| TodoCommentRule, | |
| ) | |
| def demo_custom_rules(): | |
| """Demonstrate custom rule usage.""" | |
| # Sample code with various patterns | |
| source = b'''# Copyright 2024 Example Corp | |
| # Licensed under MIT License | |
| """ | |
| Module documentation with structure: | |
| Features: | |
| - Custom chunking rules | |
| - Pattern-based extraction | |
| - Comment block analysis | |
| """ | |
| import os | |
| import sys | |
| # ======================================== | |
| # Configuration Section | |
| # ======================================== | |
| DEBUG = True | |
| #region Helper Functions | |
| def helper1(): | |
| """Helper function 1.""" | |
| # TODO: Optimize this function | |
| return 42 | |
| def helper2(): | |
| """Helper function 2.""" | |
| # FIXME: Handle edge cases | |
| return "result" | |
| #endregion | |
| # ======================================== | |
| # Main Logic | |
| # ======================================== | |
| def main(): | |
| """Main entry point.""" | |
| # TODO: Implement main logic | |
| print("Starting application...") | |
| result = helper1() + len(helper2()) | |
| return result | |
| if __name__ == "__main__": | |
| main() | |
| ''' | |
| # Create rule engine | |
| engine = DefaultRuleEngine() | |
| # Add various rules with different priorities | |
| engine.add_rule(MetadataRule(priority=100)) | |
| engine.add_rule(HeaderCommentRule(priority=90)) | |
| engine.add_rule(RegionMarkerRule(priority=80)) | |
| engine.add_rule(DocumentationBlockRule(priority=70)) | |
| engine.add_rule(TodoCommentRule(priority=60)) | |
| engine.add_rule(SeparatorLineRule(priority=40)) | |
| # Parse the source | |
| parser = get_parser("python") | |
| tree = parser.parse(source) | |
| # Apply rules | |
| chunks = engine.apply_rules(tree, source, "demo.py") | |
| # Display results | |
| print(f"Found {len(chunks)} chunks:\n") | |
| for i, chunk in enumerate(chunks, 1): | |
| print(f"{i}. {chunk.node_type} (lines {chunk.start_line}-{chunk.end_line})") | |
| print( | |
| f" Priority: {engine._priorities.get(chunk.node_type.split('_')[-1], 'N/A')}", | |
| ) | |
| if len(chunk.content) > 100: | |
| print(f" Content: {chunk.content[:100]}...") | |
| else: | |
| print(f" Content: {chunk.content}") | |
| print() | |
| # List all registered rules | |
| print("\nRegistered Rules:") | |
| for rule_info in engine.list_rules(): | |
| print( | |
| f"- {rule_info['name']}: {rule_info['description']} (priority: {rule_info['priority']})", | |
| ) | |
| if __name__ == "__main__": | |
| demo_custom_rules() | |
Xet Storage Details
- Size:
- 2.63 kB
- Xet hash:
- e3f99101e1135e09466816c34d49fd121f78d6e4860eebbbddb1edb97234ad93
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.