Duplicate from peculiar-ragdoll/Qwen-Sharp-Chat-Templates

c68f953 about 1 month ago

3.67 kB

import re
import sys
import argparse

def minify_jinja(content):
    """
    Minifies a Jinja2 template by removing comments and collapsing whitespace.
    
    This function is designed to be "safe" for chat templates:
    1. It removes all Jinja2 comments.
    2. It replaces newlines with spaces to ensure words don't merge (e.g., "Hello\nWorld" -> "Hello World").
    3. It collapses multiple spaces into one to keep the file size small.
    4. It removes spaces between adjacent tags, as the template's use of 
       white-space stripping markers (e.g., '{%-') ensures this is safe.
    
    Args:
        content (str): The raw Jinja2 template content.
        
    Returns:
        str: The minified, single-line template.
    """
    # Remove Jinja2 comments: {# ... #}
    content = re.sub(r'\{#.*?#\}', '', content, flags=re.DOTALL)

    # PATCH (Sharp, 2026-08-10): protect literal text inside {% set %}...{% endset %}.
    # Upstream's template has no such block, so collapsing newlines was safe there. Ours
    # carries the terseness prompt as a four-line {%- set _terse %} body, and turning those
    # newlines into spaces silently emits it as one run-on paragraph -- a different system
    # prompt from the one that was benchmarked. Stash those blocks, minify around them, and
    # restore them verbatim.
    stash = []

    def _stash(m):
        stash.append(m.group(0))
        return f"\x00SET{len(stash) - 1}\x00"

    content = re.sub(r'\{%-?\s*set\s+\w+\s*%\}.*?\{%-?\s*endset\s*-?%\}',
                     _stash, content, flags=re.DOTALL)

    # Replace newlines and tabs with spaces. This is a "safe" minification
    # strategy that prevents content from merging incorrectly.
    content = content.replace('\n', ' ').replace('\t', ' ')

    # Collapse multiple spaces into a single space.
    content = re.sub(r' +', ' ', content)

    # Remove spaces between Jinja tags. This is generally safe in templates 
    # that use white-space stripping (the '-' in '{%-') and significantly
    # reduces the token count for the tokenizer_config.json.
    content = content.replace('%} {%', '%}{%').replace('%} {{', '%}{{')
    content = content.replace('}} {%', '}}{%').replace('}} {{', '}}{{')

    out = content.strip()
    for i, block in enumerate(stash):
        out = out.replace(f"\x00SET{i}\x00", block)
    return out

def main():
    # Setup argument parser for a better command-line interface
    parser = argparse.ArgumentParser(
        description="Minify a Jinja2 chat template for use in tokenizer_config.json"
    )
    parser.add_argument("input", help="Path to the source .jinja file")
    parser.add_argument("output", help="Path to save the minified output")
    
    args = parser.parse_args()
    
    try:
        # Read the input template
        with open(args.input, 'r', encoding='utf-8') as f:
            content = f.read()
        
        # Perform minification
        minified = minify_jinja(content)
        
        # Write the result to the output file
        with open(args.output, 'w', encoding='utf-8') as f:
            f.write(minified)
            
        # Provide feedback on the process
        print(f"Minification complete: '{args.input}' -> '{args.output}'")
        print(f"Original size: {len(content)} bytes")
        print(f"Minified size: {len(minified)} bytes")
        print(f"Reduction: {100 - (len(minified) / len(content) * 100):.1f}%")
        
    except FileNotFoundError:
        print(f"Error: The file '{args.input}' was not found.")
        sys.exit(1)
    except Exception as e:
        print(f"An error occurred: {e}")
        sys.exit(1)

if __name__ == '__main__':
    main()

	import re
	import sys
	import argparse

	def minify_jinja(content):
	"""
	Minifies a Jinja2 template by removing comments and collapsing whitespace.

	This function is designed to be "safe" for chat templates:
	1. It removes all Jinja2 comments.
	2. It replaces newlines with spaces to ensure words don't merge (e.g., "Hello\nWorld" -> "Hello World").
	3. It collapses multiple spaces into one to keep the file size small.
	4. It removes spaces between adjacent tags, as the template's use of
	white-space stripping markers (e.g., '{%-') ensures this is safe.

	Args:
	content (str): The raw Jinja2 template content.

	Returns:
	str: The minified, single-line template.
	"""
	# Remove Jinja2 comments: {# ... #}
	content = re.sub(r'\{#.*?#\}', '', content, flags=re.DOTALL)

	# PATCH (Sharp, 2026-08-10): protect literal text inside {% set %}...{% endset %}.
	# Upstream's template has no such block, so collapsing newlines was safe there. Ours
	# carries the terseness prompt as a four-line {%- set _terse %} body, and turning those
	# newlines into spaces silently emits it as one run-on paragraph -- a different system
	# prompt from the one that was benchmarked. Stash those blocks, minify around them, and
	# restore them verbatim.
	stash = []

	def _stash(m):
	stash.append(m.group(0))
	return f"\x00SET{len(stash) - 1}\x00"

	content = re.sub(r'\{%-?\sset\s+\w+\s%\}.?\{%-?\sendset\s*-?%\}',
	_stash, content, flags=re.DOTALL)

	# Replace newlines and tabs with spaces. This is a "safe" minification
	# strategy that prevents content from merging incorrectly.
	content = content.replace('\n', ' ').replace('\t', ' ')

	# Collapse multiple spaces into a single space.
	content = re.sub(r' +', ' ', content)

	# Remove spaces between Jinja tags. This is generally safe in templates
	# that use white-space stripping (the '-' in '{%-') and significantly
	# reduces the token count for the tokenizer_config.json.
	content = content.replace('%} {%', '%}{%').replace('%} {{', '%}{{')
	content = content.replace('}} {%', '}}{%').replace('}} {{', '}}{{')

	out = content.strip()
	for i, block in enumerate(stash):
	out = out.replace(f"\x00SET{i}\x00", block)
	return out

	def main():
	# Setup argument parser for a better command-line interface
	parser = argparse.ArgumentParser(
	description="Minify a Jinja2 chat template for use in tokenizer_config.json"
	)
	parser.add_argument("input", help="Path to the source .jinja file")
	parser.add_argument("output", help="Path to save the minified output")

	args = parser.parse_args()

	try:
	# Read the input template
	with open(args.input, 'r', encoding='utf-8') as f:
	content = f.read()

	# Perform minification
	minified = minify_jinja(content)

	# Write the result to the output file
	with open(args.output, 'w', encoding='utf-8') as f:
	f.write(minified)

	# Provide feedback on the process
	print(f"Minification complete: '{args.input}' -> '{args.output}'")
	print(f"Original size: {len(content)} bytes")
	print(f"Minified size: {len(minified)} bytes")
	print(f"Reduction: {100 - (len(minified) / len(content) * 100):.1f}%")

	except FileNotFoundError:
	print(f"Error: The file '{args.input}' was not found.")
	sys.exit(1)
	except Exception as e:
	print(f"An error occurred: {e}")
	sys.exit(1)

	if __name__ == '__main__':
	main()