Instructions to use ayjays132/Phillnet-2 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use ayjays132/Phillnet-2 with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="ayjays132/Phillnet-2", trust_remote_code=True)
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("ayjays132/Phillnet-2", trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained("ayjays132/Phillnet-2", trust_remote_code=True, device_map="auto")
messages = [
    {"role": "user", "content": "Who are you?"},
]
inputs = tokenizer.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=256)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use ayjays132/Phillnet-2 with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "ayjays132/Phillnet-2"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "ayjays132/Phillnet-2",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/ayjays132/Phillnet-2

SGLang

How to use ayjays132/Phillnet-2 with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "ayjays132/Phillnet-2" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "ayjays132/Phillnet-2",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "ayjays132/Phillnet-2" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "ayjays132/Phillnet-2",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Docker Model Runner
How to use ayjays132/Phillnet-2 with Docker Model Runner:
```
docker model run hf.co/ayjays132/Phillnet-2
```

Phillnet-2 / memory_optimization /cleanup.py

ayjays132

Upload 478 files

101858b verified 6 months ago

Raw History Blame Contribute Delete

3.27 kB

"""
Memory Cleanup Module
Adaptive memory cleanup and optimization utilities.
"""
import torch
import gc
import logging
from typing import Optional

logger = logging.getLogger(__name__)

class MemoryCleanup:
    """
    Memory cleanup and optimization utilities.
    """
    def __init__(self, memory_threshold: float = 0.85, cleanup_threshold: float = 0.75):
        self.memory_threshold = memory_threshold
        self.cleanup_threshold = cleanup_threshold
        self.memory_pressure_level = 0
        
        logger.debug("MemoryCleanup initialized")
    
    def check_memory_pressure(self) -> bool:
        """
        Check if memory usage is above threshold.
        
        Returns:
            True if memory pressure is high
        """
        if not torch.cuda.is_available():
            return False
        
        try:
            memory_allocated = torch.cuda.memory_allocated()
            max_memory = torch.cuda.max_memory_allocated()
            
            # Avoid division by zero
            if max_memory == 0:
                return False
            
            memory_ratio = memory_allocated / max_memory
            return memory_ratio > self.memory_threshold
        except Exception:
            return False
    
    def adaptive_cleanup(self, tensor_pool=None) -> None:
        """
        Perform adaptive memory cleanup based on usage patterns.
        
        Args:
            tensor_pool: Optional tensor pool to clean
        """
        if not torch.cuda.is_available():
            return
        
        # Clear unused tensor pools
        if tensor_pool is not None:
            tensor_pool.clear_pool(keep_ratio=0.5)
        
        # Clear cache if memory pressure is high
        if self.check_memory_pressure():
            torch.cuda.empty_cache()
            gc.collect()
            logger.debug("[CLEANUP] Adaptive cleanup performed")
    
    def emergency_cleanup(self, tensor_pool=None) -> None:
        """
        Perform emergency memory cleanup.
        
        Args:
            tensor_pool: Optional tensor pool to clear
        """
        logger.warning("[CLEANUP] Performing emergency memory cleanup")
        
        # Clear tensor pools
        if tensor_pool is not None:
            tensor_pool.clear_all()
        
        # Clear PyTorch cache
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        
        # Force garbage collection
        gc.collect()
        
        logger.info("[CLEANUP] Emergency cleanup completed")
    
    def get_memory_stats(self) -> dict:
        """Get current memory statistics."""
        stats = {
            'memory_pressure_level': self.memory_pressure_level,
            'memory_threshold': self.memory_threshold
        }
        
        if torch.cuda.is_available():
            stats.update({
                'cuda_allocated': torch.cuda.memory_allocated(),
                'cuda_reserved': torch.cuda.memory_reserved(),
                'cuda_max_allocated': torch.cuda.max_memory_allocated(),
                'cuda_max_reserved': torch.cuda.max_memory_reserved()
            })
        
        return stats

	"""
	Memory Cleanup Module
	Adaptive memory cleanup and optimization utilities.
	"""
	import torch
	import gc
	import logging
	from typing import Optional

	logger = logging.getLogger(__name__)

	class MemoryCleanup:
	"""
	Memory cleanup and optimization utilities.
	"""
	def __init__(self, memory_threshold: float = 0.85, cleanup_threshold: float = 0.75):
	self.memory_threshold = memory_threshold
	self.cleanup_threshold = cleanup_threshold
	self.memory_pressure_level = 0

	logger.debug("MemoryCleanup initialized")

	def check_memory_pressure(self) -> bool:
	"""
	Check if memory usage is above threshold.

	Returns:
	True if memory pressure is high
	"""
	if not torch.cuda.is_available():
	return False

	try:
	memory_allocated = torch.cuda.memory_allocated()
	max_memory = torch.cuda.max_memory_allocated()

	# Avoid division by zero
	if max_memory == 0:
	return False

	memory_ratio = memory_allocated / max_memory
	return memory_ratio > self.memory_threshold
	except Exception:
	return False

	def adaptive_cleanup(self, tensor_pool=None) -> None:
	"""
	Perform adaptive memory cleanup based on usage patterns.

	Args:
	tensor_pool: Optional tensor pool to clean
	"""
	if not torch.cuda.is_available():
	return

	# Clear unused tensor pools
	if tensor_pool is not None:
	tensor_pool.clear_pool(keep_ratio=0.5)

	# Clear cache if memory pressure is high
	if self.check_memory_pressure():
	torch.cuda.empty_cache()
	gc.collect()
	logger.debug("[CLEANUP] Adaptive cleanup performed")

	def emergency_cleanup(self, tensor_pool=None) -> None:
	"""
	Perform emergency memory cleanup.

	Args:
	tensor_pool: Optional tensor pool to clear
	"""
	logger.warning("[CLEANUP] Performing emergency memory cleanup")

	# Clear tensor pools
	if tensor_pool is not None:
	tensor_pool.clear_all()

	# Clear PyTorch cache
	if torch.cuda.is_available():
	torch.cuda.empty_cache()

	# Force garbage collection
	gc.collect()

	logger.info("[CLEANUP] Emergency cleanup completed")

	def get_memory_stats(self) -> dict:
	"""Get current memory statistics."""
	stats = {
	'memory_pressure_level': self.memory_pressure_level,
	'memory_threshold': self.memory_threshold
	}

	if torch.cuda.is_available():
	stats.update({
	'cuda_allocated': torch.cuda.memory_allocated(),
	'cuda_reserved': torch.cuda.memory_reserved(),
	'cuda_max_allocated': torch.cuda.max_memory_allocated(),
	'cuda_max_reserved': torch.cuda.max_memory_reserved()
	})

	return stats